aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/Kconfig.preempt25
-rw-r--r--kernel/Makefile11
-rw-r--r--kernel/acct.c7
-rw-r--r--kernel/audit.c32
-rw-r--r--kernel/audit.h5
-rw-r--r--kernel/audit_tree.c3
-rw-r--r--kernel/auditfilter.c325
-rw-r--r--kernel/auditsc.c950
-rw-r--r--kernel/capability.c286
-rw-r--r--kernel/cgroup.c34
-rw-r--r--kernel/compat.c49
-rw-r--r--kernel/cpu.c153
-rw-r--r--kernel/cpuset.c6
-rw-r--r--kernel/cred-internals.h21
-rw-r--r--kernel/cred.c588
-rw-r--r--kernel/delayacct.c2
-rw-r--r--kernel/exit.c32
-rw-r--r--kernel/extable.c21
-rw-r--r--kernel/fork.c96
-rw-r--r--kernel/futex.c20
-rw-r--r--kernel/futex_compat.c7
-rw-r--r--kernel/hrtimer.c332
-rw-r--r--kernel/irq/Makefile1
-rw-r--r--kernel/irq/chip.c18
-rw-r--r--kernel/irq/handle.c205
-rw-r--r--kernel/irq/internals.h7
-rw-r--r--kernel/irq/manage.c118
-rw-r--r--kernel/irq/migration.c25
-rw-r--r--kernel/irq/numa_migrate.c119
-rw-r--r--kernel/irq/proc.c63
-rw-r--r--kernel/kallsyms.c16
-rw-r--r--kernel/kexec.c2
-rw-r--r--kernel/kmod.c30
-rw-r--r--kernel/kthread.c3
-rw-r--r--kernel/latencytop.c2
-rw-r--r--kernel/lockdep.c65
-rw-r--r--kernel/lockdep_proc.c28
-rw-r--r--kernel/marker.c192
-rw-r--r--kernel/module.c72
-rw-r--r--kernel/mutex.c10
-rw-r--r--kernel/notifier.c8
-rw-r--r--kernel/nsproxy.c15
-rw-r--r--kernel/panic.c33
-rw-r--r--kernel/posix-cpu-timers.c12
-rw-r--r--kernel/posix-timers.c46
-rw-r--r--kernel/power/disk.c13
-rw-r--r--kernel/power/main.c5
-rw-r--r--kernel/power/poweroff.c2
-rw-r--r--kernel/power/swap.c2
-rw-r--r--kernel/printk.c2
-rw-r--r--kernel/profile.c44
-rw-r--r--kernel/ptrace.c45
-rw-r--r--kernel/rcuclassic.c36
-rw-r--r--kernel/rcupdate.c11
-rw-r--r--kernel/rcupreempt.c40
-rw-r--r--kernel/rcupreempt_trace.c10
-rw-r--r--kernel/rcutorture.c91
-rw-r--r--kernel/rcutree.c1532
-rw-r--r--kernel/rcutree_trace.c271
-rw-r--r--kernel/relay.c7
-rw-r--r--kernel/resource.c9
-rw-r--r--kernel/sched.c1473
-rw-r--r--kernel/sched_clock.c6
-rw-r--r--kernel/sched_cpupri.c39
-rw-r--r--kernel/sched_cpupri.h5
-rw-r--r--kernel/sched_debug.c57
-rw-r--r--kernel/sched_fair.c41
-rw-r--r--kernel/sched_rt.c83
-rw-r--r--kernel/sched_stats.h8
-rw-r--r--kernel/signal.c62
-rw-r--r--kernel/smp.c145
-rw-r--r--kernel/softirq.c41
-rw-r--r--kernel/softlockup.c14
-rw-r--r--kernel/stacktrace.c11
-rw-r--r--kernel/stop_machine.c63
-rw-r--r--kernel/sys.c588
-rw-r--r--kernel/sysctl.c46
-rw-r--r--kernel/sysctl_check.c1
-rw-r--r--kernel/taskstats.c41
-rw-r--r--kernel/time/clockevents.c2
-rw-r--r--kernel/time/clocksource.c9
-rw-r--r--kernel/time/ntp.c4
-rw-r--r--kernel/time/tick-broadcast.c113
-rw-r--r--kernel/time/tick-common.c18
-rw-r--r--kernel/time/tick-sched.c66
-rw-r--r--kernel/time/timekeeping.c22
-rw-r--r--kernel/timer.c23
-rw-r--r--kernel/trace/Kconfig115
-rw-r--r--kernel/trace/Makefile9
-rw-r--r--kernel/trace/ftrace.c929
-rw-r--r--kernel/trace/ring_buffer.c787
-rw-r--r--kernel/trace/trace.c1043
-rw-r--r--kernel/trace/trace.h265
-rw-r--r--kernel/trace/trace_boot.c160
-rw-r--r--kernel/trace/trace_branch.c342
-rw-r--r--kernel/trace/trace_functions.c30
-rw-r--r--kernel/trace/trace_functions_graph.c669
-rw-r--r--kernel/trace/trace_hw_branches.c195
-rw-r--r--kernel/trace/trace_irqsoff.c61
-rw-r--r--kernel/trace/trace_mmiotrace.c49
-rw-r--r--kernel/trace/trace_nop.c65
-rw-r--r--kernel/trace/trace_power.c179
-rw-r--r--kernel/trace/trace_sched_switch.c121
-rw-r--r--kernel/trace/trace_sched_wakeup.c72
-rw-r--r--kernel/trace/trace_selftest.c173
-rw-r--r--kernel/trace/trace_stack.c94
-rw-r--r--kernel/trace/trace_sysprof.c45
-rw-r--r--kernel/tracepoint.c295
-rw-r--r--kernel/tsacct.c6
-rw-r--r--kernel/uid16.c31
-rw-r--r--kernel/user.c98
-rw-r--r--kernel/user_namespace.c65
-rw-r--r--kernel/workqueue.c34
113 files changed, 11038 insertions, 4090 deletions
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index 9fdba03dc1fc..bf987b95b356 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -52,28 +52,3 @@ config PREEMPT
52 52
53endchoice 53endchoice
54 54
55config PREEMPT_RCU
56 bool "Preemptible RCU"
57 depends on PREEMPT
58 default n
59 help
60 This option reduces the latency of the kernel by making certain
61 RCU sections preemptible. Normally RCU code is non-preemptible, if
62 this option is selected then read-only RCU sections become
63 preemptible. This helps latency, but may expose bugs due to
64 now-naive assumptions about each RCU read-side critical section
65 remaining on a given CPU through its execution.
66
67 Say N if you are unsure.
68
69config RCU_TRACE
70 bool "Enable tracing for RCU - currently stats in debugfs"
71 depends on PREEMPT_RCU
72 select DEBUG_FS
73 default y
74 help
75 This option provides tracing in RCU which presents stats
76 in debugfs for debugging RCU implementation.
77
78 Say Y here if you want to enable RCU tracing
79 Say N if you are unsure.
diff --git a/kernel/Makefile b/kernel/Makefile
index 19fad003b19d..e1c5bf3365c0 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -9,7 +9,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o \
9 rcupdate.o extable.o params.o posix-timers.o \ 9 rcupdate.o extable.o params.o posix-timers.o \
10 kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \ 10 kthread.o wait.o kfifo.o sys_ni.o posix-cpu-timers.o mutex.o \
11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \ 11 hrtimer.o rwsem.o nsproxy.o srcu.o semaphore.o \
12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o 12 notifier.o ksysfs.o pm_qos_params.o sched_clock.o cred.o
13 13
14ifdef CONFIG_FUNCTION_TRACER 14ifdef CONFIG_FUNCTION_TRACER
15# Do not trace debug files and internal ftrace files 15# Do not trace debug files and internal ftrace files
@@ -19,7 +19,6 @@ CFLAGS_REMOVE_mutex-debug.o = -pg
19CFLAGS_REMOVE_rtmutex-debug.o = -pg 19CFLAGS_REMOVE_rtmutex-debug.o = -pg
20CFLAGS_REMOVE_cgroup-debug.o = -pg 20CFLAGS_REMOVE_cgroup-debug.o = -pg
21CFLAGS_REMOVE_sched_clock.o = -pg 21CFLAGS_REMOVE_sched_clock.o = -pg
22CFLAGS_REMOVE_sched.o = -pg
23endif 22endif
24 23
25obj-$(CONFIG_FREEZER) += freezer.o 24obj-$(CONFIG_FREEZER) += freezer.o
@@ -74,10 +73,10 @@ obj-$(CONFIG_GENERIC_HARDIRQS) += irq/
74obj-$(CONFIG_SECCOMP) += seccomp.o 73obj-$(CONFIG_SECCOMP) += seccomp.o
75obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o 74obj-$(CONFIG_RCU_TORTURE_TEST) += rcutorture.o
76obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o 75obj-$(CONFIG_CLASSIC_RCU) += rcuclassic.o
76obj-$(CONFIG_TREE_RCU) += rcutree.o
77obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o 77obj-$(CONFIG_PREEMPT_RCU) += rcupreempt.o
78ifeq ($(CONFIG_PREEMPT_RCU),y) 78obj-$(CONFIG_TREE_RCU_TRACE) += rcutree_trace.o
79obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o 79obj-$(CONFIG_PREEMPT_RCU_TRACE) += rcupreempt_trace.o
80endif
81obj-$(CONFIG_RELAY) += relay.o 80obj-$(CONFIG_RELAY) += relay.o
82obj-$(CONFIG_SYSCTL) += utsname_sysctl.o 81obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
83obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o 82obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
@@ -90,7 +89,7 @@ obj-$(CONFIG_FUNCTION_TRACER) += trace/
90obj-$(CONFIG_TRACING) += trace/ 89obj-$(CONFIG_TRACING) += trace/
91obj-$(CONFIG_SMP) += sched_cpupri.o 90obj-$(CONFIG_SMP) += sched_cpupri.o
92 91
93ifneq ($(CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER),y) 92ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y)
94# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is 93# According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is
95# needed for x86 only. Why this used to be enabled for all architectures is beyond 94# needed for x86 only. Why this used to be enabled for all architectures is beyond
96# me. I suspect most platforms don't need this, but until we know that for sure 95# me. I suspect most platforms don't need this, but until we know that for sure
diff --git a/kernel/acct.c b/kernel/acct.c
index f6006a60df5d..d57b7cbb98b6 100644
--- a/kernel/acct.c
+++ b/kernel/acct.c
@@ -530,15 +530,14 @@ static void do_acct_process(struct bsd_acct_struct *acct,
530 do_div(elapsed, AHZ); 530 do_div(elapsed, AHZ);
531 ac.ac_btime = get_seconds() - elapsed; 531 ac.ac_btime = get_seconds() - elapsed;
532 /* we really need to bite the bullet and change layout */ 532 /* we really need to bite the bullet and change layout */
533 ac.ac_uid = current->uid; 533 current_uid_gid(&ac.ac_uid, &ac.ac_gid);
534 ac.ac_gid = current->gid;
535#if ACCT_VERSION==2 534#if ACCT_VERSION==2
536 ac.ac_ahz = AHZ; 535 ac.ac_ahz = AHZ;
537#endif 536#endif
538#if ACCT_VERSION==1 || ACCT_VERSION==2 537#if ACCT_VERSION==1 || ACCT_VERSION==2
539 /* backward-compatible 16 bit fields */ 538 /* backward-compatible 16 bit fields */
540 ac.ac_uid16 = current->uid; 539 ac.ac_uid16 = ac.ac_uid;
541 ac.ac_gid16 = current->gid; 540 ac.ac_gid16 = ac.ac_gid;
542#endif 541#endif
543#if ACCT_VERSION==3 542#if ACCT_VERSION==3
544 ac.ac_pid = task_tgid_nr_ns(current, ns); 543 ac.ac_pid = task_tgid_nr_ns(current, ns);
diff --git a/kernel/audit.c b/kernel/audit.c
index 4414e93d8750..ce6d8ea3131e 100644
--- a/kernel/audit.c
+++ b/kernel/audit.c
@@ -61,8 +61,11 @@
61 61
62#include "audit.h" 62#include "audit.h"
63 63
64/* No auditing will take place until audit_initialized != 0. 64/* No auditing will take place until audit_initialized == AUDIT_INITIALIZED.
65 * (Initialization happens after skb_init is called.) */ 65 * (Initialization happens after skb_init is called.) */
66#define AUDIT_DISABLED -1
67#define AUDIT_UNINITIALIZED 0
68#define AUDIT_INITIALIZED 1
66static int audit_initialized; 69static int audit_initialized;
67 70
68#define AUDIT_OFF 0 71#define AUDIT_OFF 0
@@ -965,6 +968,9 @@ static int __init audit_init(void)
965{ 968{
966 int i; 969 int i;
967 970
971 if (audit_initialized == AUDIT_DISABLED)
972 return 0;
973
968 printk(KERN_INFO "audit: initializing netlink socket (%s)\n", 974 printk(KERN_INFO "audit: initializing netlink socket (%s)\n",
969 audit_default ? "enabled" : "disabled"); 975 audit_default ? "enabled" : "disabled");
970 audit_sock = netlink_kernel_create(&init_net, NETLINK_AUDIT, 0, 976 audit_sock = netlink_kernel_create(&init_net, NETLINK_AUDIT, 0,
@@ -976,7 +982,7 @@ static int __init audit_init(void)
976 982
977 skb_queue_head_init(&audit_skb_queue); 983 skb_queue_head_init(&audit_skb_queue);
978 skb_queue_head_init(&audit_skb_hold_queue); 984 skb_queue_head_init(&audit_skb_hold_queue);
979 audit_initialized = 1; 985 audit_initialized = AUDIT_INITIALIZED;
980 audit_enabled = audit_default; 986 audit_enabled = audit_default;
981 audit_ever_enabled |= !!audit_default; 987 audit_ever_enabled |= !!audit_default;
982 988
@@ -999,13 +1005,21 @@ __initcall(audit_init);
999static int __init audit_enable(char *str) 1005static int __init audit_enable(char *str)
1000{ 1006{
1001 audit_default = !!simple_strtol(str, NULL, 0); 1007 audit_default = !!simple_strtol(str, NULL, 0);
1002 printk(KERN_INFO "audit: %s%s\n", 1008 if (!audit_default)
1003 audit_default ? "enabled" : "disabled", 1009 audit_initialized = AUDIT_DISABLED;
1004 audit_initialized ? "" : " (after initialization)"); 1010
1005 if (audit_initialized) { 1011 printk(KERN_INFO "audit: %s", audit_default ? "enabled" : "disabled");
1012
1013 if (audit_initialized == AUDIT_INITIALIZED) {
1006 audit_enabled = audit_default; 1014 audit_enabled = audit_default;
1007 audit_ever_enabled |= !!audit_default; 1015 audit_ever_enabled |= !!audit_default;
1016 } else if (audit_initialized == AUDIT_UNINITIALIZED) {
1017 printk(" (after initialization)");
1018 } else {
1019 printk(" (until reboot)");
1008 } 1020 }
1021 printk("\n");
1022
1009 return 1; 1023 return 1;
1010} 1024}
1011 1025
@@ -1107,9 +1121,7 @@ unsigned int audit_serial(void)
1107static inline void audit_get_stamp(struct audit_context *ctx, 1121static inline void audit_get_stamp(struct audit_context *ctx,
1108 struct timespec *t, unsigned int *serial) 1122 struct timespec *t, unsigned int *serial)
1109{ 1123{
1110 if (ctx) 1124 if (!ctx || !auditsc_get_stamp(ctx, t, serial)) {
1111 auditsc_get_stamp(ctx, t, serial);
1112 else {
1113 *t = CURRENT_TIME; 1125 *t = CURRENT_TIME;
1114 *serial = audit_serial(); 1126 *serial = audit_serial();
1115 } 1127 }
@@ -1146,7 +1158,7 @@ struct audit_buffer *audit_log_start(struct audit_context *ctx, gfp_t gfp_mask,
1146 int reserve; 1158 int reserve;
1147 unsigned long timeout_start = jiffies; 1159 unsigned long timeout_start = jiffies;
1148 1160
1149 if (!audit_initialized) 1161 if (audit_initialized != AUDIT_INITIALIZED)
1150 return NULL; 1162 return NULL;
1151 1163
1152 if (unlikely(audit_filter_type(type))) 1164 if (unlikely(audit_filter_type(type)))
diff --git a/kernel/audit.h b/kernel/audit.h
index 9d6717412fec..16f18cac661b 100644
--- a/kernel/audit.h
+++ b/kernel/audit.h
@@ -159,11 +159,8 @@ static inline int audit_signal_info(int sig, struct task_struct *t)
159 return __audit_signal_info(sig, t); 159 return __audit_signal_info(sig, t);
160 return 0; 160 return 0;
161} 161}
162extern enum audit_state audit_filter_inodes(struct task_struct *, 162extern void audit_filter_inodes(struct task_struct *, struct audit_context *);
163 struct audit_context *);
164extern void audit_set_auditable(struct audit_context *);
165#else 163#else
166#define audit_signal_info(s,t) AUDIT_DISABLED 164#define audit_signal_info(s,t) AUDIT_DISABLED
167#define audit_filter_inodes(t,c) AUDIT_DISABLED 165#define audit_filter_inodes(t,c) AUDIT_DISABLED
168#define audit_set_auditable(c)
169#endif 166#endif
diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c
index 8b509441f49a..8ad9545b8db9 100644
--- a/kernel/audit_tree.c
+++ b/kernel/audit_tree.c
@@ -450,6 +450,7 @@ static void kill_rules(struct audit_tree *tree)
450 audit_log_end(ab); 450 audit_log_end(ab);
451 rule->tree = NULL; 451 rule->tree = NULL;
452 list_del_rcu(&entry->list); 452 list_del_rcu(&entry->list);
453 list_del(&entry->rule.list);
453 call_rcu(&entry->rcu, audit_free_rule_rcu); 454 call_rcu(&entry->rcu, audit_free_rule_rcu);
454 } 455 }
455 } 456 }
@@ -617,7 +618,7 @@ int audit_make_tree(struct audit_krule *rule, char *pathname, u32 op)
617 618
618 if (pathname[0] != '/' || 619 if (pathname[0] != '/' ||
619 rule->listnr != AUDIT_FILTER_EXIT || 620 rule->listnr != AUDIT_FILTER_EXIT ||
620 op & ~AUDIT_EQUAL || 621 op != Audit_equal ||
621 rule->inode_f || rule->watch || rule->tree) 622 rule->inode_f || rule->watch || rule->tree)
622 return -EINVAL; 623 return -EINVAL;
623 rule->tree = alloc_tree(pathname); 624 rule->tree = alloc_tree(pathname);
diff --git a/kernel/auditfilter.c b/kernel/auditfilter.c
index 9fd85a4640a0..fbf24d121d97 100644
--- a/kernel/auditfilter.c
+++ b/kernel/auditfilter.c
@@ -86,6 +86,14 @@ struct list_head audit_filter_list[AUDIT_NR_FILTERS] = {
86#error Fix audit_filter_list initialiser 86#error Fix audit_filter_list initialiser
87#endif 87#endif
88}; 88};
89static struct list_head audit_rules_list[AUDIT_NR_FILTERS] = {
90 LIST_HEAD_INIT(audit_rules_list[0]),
91 LIST_HEAD_INIT(audit_rules_list[1]),
92 LIST_HEAD_INIT(audit_rules_list[2]),
93 LIST_HEAD_INIT(audit_rules_list[3]),
94 LIST_HEAD_INIT(audit_rules_list[4]),
95 LIST_HEAD_INIT(audit_rules_list[5]),
96};
89 97
90DEFINE_MUTEX(audit_filter_mutex); 98DEFINE_MUTEX(audit_filter_mutex);
91 99
@@ -244,7 +252,8 @@ static inline int audit_to_inode(struct audit_krule *krule,
244 struct audit_field *f) 252 struct audit_field *f)
245{ 253{
246 if (krule->listnr != AUDIT_FILTER_EXIT || 254 if (krule->listnr != AUDIT_FILTER_EXIT ||
247 krule->watch || krule->inode_f || krule->tree) 255 krule->watch || krule->inode_f || krule->tree ||
256 (f->op != Audit_equal && f->op != Audit_not_equal))
248 return -EINVAL; 257 return -EINVAL;
249 258
250 krule->inode_f = f; 259 krule->inode_f = f;
@@ -262,7 +271,7 @@ static int audit_to_watch(struct audit_krule *krule, char *path, int len,
262 271
263 if (path[0] != '/' || path[len-1] == '/' || 272 if (path[0] != '/' || path[len-1] == '/' ||
264 krule->listnr != AUDIT_FILTER_EXIT || 273 krule->listnr != AUDIT_FILTER_EXIT ||
265 op & ~AUDIT_EQUAL || 274 op != Audit_equal ||
266 krule->inode_f || krule->watch || krule->tree) 275 krule->inode_f || krule->watch || krule->tree)
267 return -EINVAL; 276 return -EINVAL;
268 277
@@ -412,12 +421,32 @@ exit_err:
412 return ERR_PTR(err); 421 return ERR_PTR(err);
413} 422}
414 423
424static u32 audit_ops[] =
425{
426 [Audit_equal] = AUDIT_EQUAL,
427 [Audit_not_equal] = AUDIT_NOT_EQUAL,
428 [Audit_bitmask] = AUDIT_BIT_MASK,
429 [Audit_bittest] = AUDIT_BIT_TEST,
430 [Audit_lt] = AUDIT_LESS_THAN,
431 [Audit_gt] = AUDIT_GREATER_THAN,
432 [Audit_le] = AUDIT_LESS_THAN_OR_EQUAL,
433 [Audit_ge] = AUDIT_GREATER_THAN_OR_EQUAL,
434};
435
436static u32 audit_to_op(u32 op)
437{
438 u32 n;
439 for (n = Audit_equal; n < Audit_bad && audit_ops[n] != op; n++)
440 ;
441 return n;
442}
443
444
415/* Translate struct audit_rule to kernel's rule respresentation. 445/* Translate struct audit_rule to kernel's rule respresentation.
416 * Exists for backward compatibility with userspace. */ 446 * Exists for backward compatibility with userspace. */
417static struct audit_entry *audit_rule_to_entry(struct audit_rule *rule) 447static struct audit_entry *audit_rule_to_entry(struct audit_rule *rule)
418{ 448{
419 struct audit_entry *entry; 449 struct audit_entry *entry;
420 struct audit_field *ino_f;
421 int err = 0; 450 int err = 0;
422 int i; 451 int i;
423 452
@@ -427,12 +456,28 @@ static struct audit_entry *audit_rule_to_entry(struct audit_rule *rule)
427 456
428 for (i = 0; i < rule->field_count; i++) { 457 for (i = 0; i < rule->field_count; i++) {
429 struct audit_field *f = &entry->rule.fields[i]; 458 struct audit_field *f = &entry->rule.fields[i];
459 u32 n;
460
461 n = rule->fields[i] & (AUDIT_NEGATE|AUDIT_OPERATORS);
462
463 /* Support for legacy operators where
464 * AUDIT_NEGATE bit signifies != and otherwise assumes == */
465 if (n & AUDIT_NEGATE)
466 f->op = Audit_not_equal;
467 else if (!n)
468 f->op = Audit_equal;
469 else
470 f->op = audit_to_op(n);
471
472 entry->rule.vers_ops = (n & AUDIT_OPERATORS) ? 2 : 1;
430 473
431 f->op = rule->fields[i] & (AUDIT_NEGATE|AUDIT_OPERATORS);
432 f->type = rule->fields[i] & ~(AUDIT_NEGATE|AUDIT_OPERATORS); 474 f->type = rule->fields[i] & ~(AUDIT_NEGATE|AUDIT_OPERATORS);
433 f->val = rule->values[i]; 475 f->val = rule->values[i];
434 476
435 err = -EINVAL; 477 err = -EINVAL;
478 if (f->op == Audit_bad)
479 goto exit_free;
480
436 switch(f->type) { 481 switch(f->type) {
437 default: 482 default:
438 goto exit_free; 483 goto exit_free;
@@ -454,11 +499,8 @@ static struct audit_entry *audit_rule_to_entry(struct audit_rule *rule)
454 case AUDIT_EXIT: 499 case AUDIT_EXIT:
455 case AUDIT_SUCCESS: 500 case AUDIT_SUCCESS:
456 /* bit ops are only useful on syscall args */ 501 /* bit ops are only useful on syscall args */
457 if (f->op == AUDIT_BIT_MASK || 502 if (f->op == Audit_bitmask || f->op == Audit_bittest)
458 f->op == AUDIT_BIT_TEST) {
459 err = -EINVAL;
460 goto exit_free; 503 goto exit_free;
461 }
462 break; 504 break;
463 case AUDIT_ARG0: 505 case AUDIT_ARG0:
464 case AUDIT_ARG1: 506 case AUDIT_ARG1:
@@ -467,11 +509,8 @@ static struct audit_entry *audit_rule_to_entry(struct audit_rule *rule)
467 break; 509 break;
468 /* arch is only allowed to be = or != */ 510 /* arch is only allowed to be = or != */
469 case AUDIT_ARCH: 511 case AUDIT_ARCH:
470 if ((f->op != AUDIT_NOT_EQUAL) && (f->op != AUDIT_EQUAL) 512 if (f->op != Audit_not_equal && f->op != Audit_equal)
471 && (f->op != AUDIT_NEGATE) && (f->op)) {
472 err = -EINVAL;
473 goto exit_free; 513 goto exit_free;
474 }
475 entry->rule.arch_f = f; 514 entry->rule.arch_f = f;
476 break; 515 break;
477 case AUDIT_PERM: 516 case AUDIT_PERM:
@@ -488,33 +527,10 @@ static struct audit_entry *audit_rule_to_entry(struct audit_rule *rule)
488 goto exit_free; 527 goto exit_free;
489 break; 528 break;
490 } 529 }
491
492 entry->rule.vers_ops = (f->op & AUDIT_OPERATORS) ? 2 : 1;
493
494 /* Support for legacy operators where
495 * AUDIT_NEGATE bit signifies != and otherwise assumes == */
496 if (f->op & AUDIT_NEGATE)
497 f->op = AUDIT_NOT_EQUAL;
498 else if (!f->op)
499 f->op = AUDIT_EQUAL;
500 else if (f->op == AUDIT_OPERATORS) {
501 err = -EINVAL;
502 goto exit_free;
503 }
504 } 530 }
505 531
506 ino_f = entry->rule.inode_f; 532 if (entry->rule.inode_f && entry->rule.inode_f->op == Audit_not_equal)
507 if (ino_f) { 533 entry->rule.inode_f = NULL;
508 switch(ino_f->op) {
509 case AUDIT_NOT_EQUAL:
510 entry->rule.inode_f = NULL;
511 case AUDIT_EQUAL:
512 break;
513 default:
514 err = -EINVAL;
515 goto exit_free;
516 }
517 }
518 534
519exit_nofree: 535exit_nofree:
520 return entry; 536 return entry;
@@ -530,7 +546,6 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
530{ 546{
531 int err = 0; 547 int err = 0;
532 struct audit_entry *entry; 548 struct audit_entry *entry;
533 struct audit_field *ino_f;
534 void *bufp; 549 void *bufp;
535 size_t remain = datasz - sizeof(struct audit_rule_data); 550 size_t remain = datasz - sizeof(struct audit_rule_data);
536 int i; 551 int i;
@@ -546,11 +561,11 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
546 struct audit_field *f = &entry->rule.fields[i]; 561 struct audit_field *f = &entry->rule.fields[i];
547 562
548 err = -EINVAL; 563 err = -EINVAL;
549 if (!(data->fieldflags[i] & AUDIT_OPERATORS) || 564
550 data->fieldflags[i] & ~AUDIT_OPERATORS) 565 f->op = audit_to_op(data->fieldflags[i]);
566 if (f->op == Audit_bad)
551 goto exit_free; 567 goto exit_free;
552 568
553 f->op = data->fieldflags[i] & AUDIT_OPERATORS;
554 f->type = data->fields[i]; 569 f->type = data->fields[i];
555 f->val = data->values[i]; 570 f->val = data->values[i];
556 f->lsm_str = NULL; 571 f->lsm_str = NULL;
@@ -662,18 +677,8 @@ static struct audit_entry *audit_data_to_entry(struct audit_rule_data *data,
662 } 677 }
663 } 678 }
664 679
665 ino_f = entry->rule.inode_f; 680 if (entry->rule.inode_f && entry->rule.inode_f->op == Audit_not_equal)
666 if (ino_f) { 681 entry->rule.inode_f = NULL;
667 switch(ino_f->op) {
668 case AUDIT_NOT_EQUAL:
669 entry->rule.inode_f = NULL;
670 case AUDIT_EQUAL:
671 break;
672 default:
673 err = -EINVAL;
674 goto exit_free;
675 }
676 }
677 682
678exit_nofree: 683exit_nofree:
679 return entry; 684 return entry;
@@ -713,10 +718,10 @@ static struct audit_rule *audit_krule_to_rule(struct audit_krule *krule)
713 rule->fields[i] = krule->fields[i].type; 718 rule->fields[i] = krule->fields[i].type;
714 719
715 if (krule->vers_ops == 1) { 720 if (krule->vers_ops == 1) {
716 if (krule->fields[i].op & AUDIT_NOT_EQUAL) 721 if (krule->fields[i].op == Audit_not_equal)
717 rule->fields[i] |= AUDIT_NEGATE; 722 rule->fields[i] |= AUDIT_NEGATE;
718 } else { 723 } else {
719 rule->fields[i] |= krule->fields[i].op; 724 rule->fields[i] |= audit_ops[krule->fields[i].op];
720 } 725 }
721 } 726 }
722 for (i = 0; i < AUDIT_BITMASK_SIZE; i++) rule->mask[i] = krule->mask[i]; 727 for (i = 0; i < AUDIT_BITMASK_SIZE; i++) rule->mask[i] = krule->mask[i];
@@ -744,7 +749,7 @@ static struct audit_rule_data *audit_krule_to_data(struct audit_krule *krule)
744 struct audit_field *f = &krule->fields[i]; 749 struct audit_field *f = &krule->fields[i];
745 750
746 data->fields[i] = f->type; 751 data->fields[i] = f->type;
747 data->fieldflags[i] = f->op; 752 data->fieldflags[i] = audit_ops[f->op];
748 switch(f->type) { 753 switch(f->type) {
749 case AUDIT_SUBJ_USER: 754 case AUDIT_SUBJ_USER:
750 case AUDIT_SUBJ_ROLE: 755 case AUDIT_SUBJ_ROLE:
@@ -919,6 +924,7 @@ static struct audit_entry *audit_dupe_rule(struct audit_krule *old,
919 new->action = old->action; 924 new->action = old->action;
920 for (i = 0; i < AUDIT_BITMASK_SIZE; i++) 925 for (i = 0; i < AUDIT_BITMASK_SIZE; i++)
921 new->mask[i] = old->mask[i]; 926 new->mask[i] = old->mask[i];
927 new->prio = old->prio;
922 new->buflen = old->buflen; 928 new->buflen = old->buflen;
923 new->inode_f = old->inode_f; 929 new->inode_f = old->inode_f;
924 new->watch = NULL; 930 new->watch = NULL;
@@ -987,9 +993,8 @@ static void audit_update_watch(struct audit_parent *parent,
987 993
988 /* If the update involves invalidating rules, do the inode-based 994 /* If the update involves invalidating rules, do the inode-based
989 * filtering now, so we don't omit records. */ 995 * filtering now, so we don't omit records. */
990 if (invalidating && current->audit_context && 996 if (invalidating && current->audit_context)
991 audit_filter_inodes(current, current->audit_context) == AUDIT_RECORD_CONTEXT) 997 audit_filter_inodes(current, current->audit_context);
992 audit_set_auditable(current->audit_context);
993 998
994 nwatch = audit_dupe_watch(owatch); 999 nwatch = audit_dupe_watch(owatch);
995 if (IS_ERR(nwatch)) { 1000 if (IS_ERR(nwatch)) {
@@ -1007,12 +1012,15 @@ static void audit_update_watch(struct audit_parent *parent,
1007 list_del_rcu(&oentry->list); 1012 list_del_rcu(&oentry->list);
1008 1013
1009 nentry = audit_dupe_rule(&oentry->rule, nwatch); 1014 nentry = audit_dupe_rule(&oentry->rule, nwatch);
1010 if (IS_ERR(nentry)) 1015 if (IS_ERR(nentry)) {
1016 list_del(&oentry->rule.list);
1011 audit_panic("error updating watch, removing"); 1017 audit_panic("error updating watch, removing");
1012 else { 1018 } else {
1013 int h = audit_hash_ino((u32)ino); 1019 int h = audit_hash_ino((u32)ino);
1014 list_add(&nentry->rule.rlist, &nwatch->rules); 1020 list_add(&nentry->rule.rlist, &nwatch->rules);
1015 list_add_rcu(&nentry->list, &audit_inode_hash[h]); 1021 list_add_rcu(&nentry->list, &audit_inode_hash[h]);
1022 list_replace(&oentry->rule.list,
1023 &nentry->rule.list);
1016 } 1024 }
1017 1025
1018 call_rcu(&oentry->rcu, audit_free_rule_rcu); 1026 call_rcu(&oentry->rcu, audit_free_rule_rcu);
@@ -1077,6 +1085,7 @@ static void audit_remove_parent_watches(struct audit_parent *parent)
1077 audit_log_end(ab); 1085 audit_log_end(ab);
1078 } 1086 }
1079 list_del(&r->rlist); 1087 list_del(&r->rlist);
1088 list_del(&r->list);
1080 list_del_rcu(&e->list); 1089 list_del_rcu(&e->list);
1081 call_rcu(&e->rcu, audit_free_rule_rcu); 1090 call_rcu(&e->rcu, audit_free_rule_rcu);
1082 } 1091 }
@@ -1102,12 +1111,16 @@ static void audit_inotify_unregister(struct list_head *in_list)
1102/* Find an existing audit rule. 1111/* Find an existing audit rule.
1103 * Caller must hold audit_filter_mutex to prevent stale rule data. */ 1112 * Caller must hold audit_filter_mutex to prevent stale rule data. */
1104static struct audit_entry *audit_find_rule(struct audit_entry *entry, 1113static struct audit_entry *audit_find_rule(struct audit_entry *entry,
1105 struct list_head *list) 1114 struct list_head **p)
1106{ 1115{
1107 struct audit_entry *e, *found = NULL; 1116 struct audit_entry *e, *found = NULL;
1117 struct list_head *list;
1108 int h; 1118 int h;
1109 1119
1110 if (entry->rule.watch) { 1120 if (entry->rule.inode_f) {
1121 h = audit_hash_ino(entry->rule.inode_f->val);
1122 *p = list = &audit_inode_hash[h];
1123 } else if (entry->rule.watch) {
1111 /* we don't know the inode number, so must walk entire hash */ 1124 /* we don't know the inode number, so must walk entire hash */
1112 for (h = 0; h < AUDIT_INODE_BUCKETS; h++) { 1125 for (h = 0; h < AUDIT_INODE_BUCKETS; h++) {
1113 list = &audit_inode_hash[h]; 1126 list = &audit_inode_hash[h];
@@ -1118,6 +1131,8 @@ static struct audit_entry *audit_find_rule(struct audit_entry *entry,
1118 } 1131 }
1119 } 1132 }
1120 goto out; 1133 goto out;
1134 } else {
1135 *p = list = &audit_filter_list[entry->rule.listnr];
1121 } 1136 }
1122 1137
1123 list_for_each_entry(e, list, list) 1138 list_for_each_entry(e, list, list)
@@ -1258,15 +1273,17 @@ static int audit_add_watch(struct audit_krule *krule, struct nameidata *ndp,
1258 return ret; 1273 return ret;
1259} 1274}
1260 1275
1276static u64 prio_low = ~0ULL/2;
1277static u64 prio_high = ~0ULL/2 - 1;
1278
1261/* Add rule to given filterlist if not a duplicate. */ 1279/* Add rule to given filterlist if not a duplicate. */
1262static inline int audit_add_rule(struct audit_entry *entry, 1280static inline int audit_add_rule(struct audit_entry *entry)
1263 struct list_head *list)
1264{ 1281{
1265 struct audit_entry *e; 1282 struct audit_entry *e;
1266 struct audit_field *inode_f = entry->rule.inode_f;
1267 struct audit_watch *watch = entry->rule.watch; 1283 struct audit_watch *watch = entry->rule.watch;
1268 struct audit_tree *tree = entry->rule.tree; 1284 struct audit_tree *tree = entry->rule.tree;
1269 struct nameidata *ndp = NULL, *ndw = NULL; 1285 struct nameidata *ndp = NULL, *ndw = NULL;
1286 struct list_head *list;
1270 int h, err; 1287 int h, err;
1271#ifdef CONFIG_AUDITSYSCALL 1288#ifdef CONFIG_AUDITSYSCALL
1272 int dont_count = 0; 1289 int dont_count = 0;
@@ -1277,13 +1294,8 @@ static inline int audit_add_rule(struct audit_entry *entry,
1277 dont_count = 1; 1294 dont_count = 1;
1278#endif 1295#endif
1279 1296
1280 if (inode_f) {
1281 h = audit_hash_ino(inode_f->val);
1282 list = &audit_inode_hash[h];
1283 }
1284
1285 mutex_lock(&audit_filter_mutex); 1297 mutex_lock(&audit_filter_mutex);
1286 e = audit_find_rule(entry, list); 1298 e = audit_find_rule(entry, &list);
1287 mutex_unlock(&audit_filter_mutex); 1299 mutex_unlock(&audit_filter_mutex);
1288 if (e) { 1300 if (e) {
1289 err = -EEXIST; 1301 err = -EEXIST;
@@ -1319,10 +1331,22 @@ static inline int audit_add_rule(struct audit_entry *entry,
1319 } 1331 }
1320 } 1332 }
1321 1333
1334 entry->rule.prio = ~0ULL;
1335 if (entry->rule.listnr == AUDIT_FILTER_EXIT) {
1336 if (entry->rule.flags & AUDIT_FILTER_PREPEND)
1337 entry->rule.prio = ++prio_high;
1338 else
1339 entry->rule.prio = --prio_low;
1340 }
1341
1322 if (entry->rule.flags & AUDIT_FILTER_PREPEND) { 1342 if (entry->rule.flags & AUDIT_FILTER_PREPEND) {
1343 list_add(&entry->rule.list,
1344 &audit_rules_list[entry->rule.listnr]);
1323 list_add_rcu(&entry->list, list); 1345 list_add_rcu(&entry->list, list);
1324 entry->rule.flags &= ~AUDIT_FILTER_PREPEND; 1346 entry->rule.flags &= ~AUDIT_FILTER_PREPEND;
1325 } else { 1347 } else {
1348 list_add_tail(&entry->rule.list,
1349 &audit_rules_list[entry->rule.listnr]);
1326 list_add_tail_rcu(&entry->list, list); 1350 list_add_tail_rcu(&entry->list, list);
1327 } 1351 }
1328#ifdef CONFIG_AUDITSYSCALL 1352#ifdef CONFIG_AUDITSYSCALL
@@ -1345,15 +1369,14 @@ error:
1345} 1369}
1346 1370
1347/* Remove an existing rule from filterlist. */ 1371/* Remove an existing rule from filterlist. */
1348static inline int audit_del_rule(struct audit_entry *entry, 1372static inline int audit_del_rule(struct audit_entry *entry)
1349 struct list_head *list)
1350{ 1373{
1351 struct audit_entry *e; 1374 struct audit_entry *e;
1352 struct audit_field *inode_f = entry->rule.inode_f;
1353 struct audit_watch *watch, *tmp_watch = entry->rule.watch; 1375 struct audit_watch *watch, *tmp_watch = entry->rule.watch;
1354 struct audit_tree *tree = entry->rule.tree; 1376 struct audit_tree *tree = entry->rule.tree;
1377 struct list_head *list;
1355 LIST_HEAD(inotify_list); 1378 LIST_HEAD(inotify_list);
1356 int h, ret = 0; 1379 int ret = 0;
1357#ifdef CONFIG_AUDITSYSCALL 1380#ifdef CONFIG_AUDITSYSCALL
1358 int dont_count = 0; 1381 int dont_count = 0;
1359 1382
@@ -1363,13 +1386,8 @@ static inline int audit_del_rule(struct audit_entry *entry,
1363 dont_count = 1; 1386 dont_count = 1;
1364#endif 1387#endif
1365 1388
1366 if (inode_f) {
1367 h = audit_hash_ino(inode_f->val);
1368 list = &audit_inode_hash[h];
1369 }
1370
1371 mutex_lock(&audit_filter_mutex); 1389 mutex_lock(&audit_filter_mutex);
1372 e = audit_find_rule(entry, list); 1390 e = audit_find_rule(entry, &list);
1373 if (!e) { 1391 if (!e) {
1374 mutex_unlock(&audit_filter_mutex); 1392 mutex_unlock(&audit_filter_mutex);
1375 ret = -ENOENT; 1393 ret = -ENOENT;
@@ -1404,6 +1422,7 @@ static inline int audit_del_rule(struct audit_entry *entry,
1404 audit_remove_tree_rule(&e->rule); 1422 audit_remove_tree_rule(&e->rule);
1405 1423
1406 list_del_rcu(&e->list); 1424 list_del_rcu(&e->list);
1425 list_del(&e->rule.list);
1407 call_rcu(&e->rcu, audit_free_rule_rcu); 1426 call_rcu(&e->rcu, audit_free_rule_rcu);
1408 1427
1409#ifdef CONFIG_AUDITSYSCALL 1428#ifdef CONFIG_AUDITSYSCALL
@@ -1432,30 +1451,16 @@ out:
1432static void audit_list(int pid, int seq, struct sk_buff_head *q) 1451static void audit_list(int pid, int seq, struct sk_buff_head *q)
1433{ 1452{
1434 struct sk_buff *skb; 1453 struct sk_buff *skb;
1435 struct audit_entry *entry; 1454 struct audit_krule *r;
1436 int i; 1455 int i;
1437 1456
1438 /* This is a blocking read, so use audit_filter_mutex instead of rcu 1457 /* This is a blocking read, so use audit_filter_mutex instead of rcu
1439 * iterator to sync with list writers. */ 1458 * iterator to sync with list writers. */
1440 for (i=0; i<AUDIT_NR_FILTERS; i++) { 1459 for (i=0; i<AUDIT_NR_FILTERS; i++) {
1441 list_for_each_entry(entry, &audit_filter_list[i], list) { 1460 list_for_each_entry(r, &audit_rules_list[i], list) {
1442 struct audit_rule *rule;
1443
1444 rule = audit_krule_to_rule(&entry->rule);
1445 if (unlikely(!rule))
1446 break;
1447 skb = audit_make_reply(pid, seq, AUDIT_LIST, 0, 1,
1448 rule, sizeof(*rule));
1449 if (skb)
1450 skb_queue_tail(q, skb);
1451 kfree(rule);
1452 }
1453 }
1454 for (i = 0; i < AUDIT_INODE_BUCKETS; i++) {
1455 list_for_each_entry(entry, &audit_inode_hash[i], list) {
1456 struct audit_rule *rule; 1461 struct audit_rule *rule;
1457 1462
1458 rule = audit_krule_to_rule(&entry->rule); 1463 rule = audit_krule_to_rule(r);
1459 if (unlikely(!rule)) 1464 if (unlikely(!rule))
1460 break; 1465 break;
1461 skb = audit_make_reply(pid, seq, AUDIT_LIST, 0, 1, 1466 skb = audit_make_reply(pid, seq, AUDIT_LIST, 0, 1,
@@ -1474,30 +1479,16 @@ static void audit_list(int pid, int seq, struct sk_buff_head *q)
1474static void audit_list_rules(int pid, int seq, struct sk_buff_head *q) 1479static void audit_list_rules(int pid, int seq, struct sk_buff_head *q)
1475{ 1480{
1476 struct sk_buff *skb; 1481 struct sk_buff *skb;
1477 struct audit_entry *e; 1482 struct audit_krule *r;
1478 int i; 1483 int i;
1479 1484
1480 /* This is a blocking read, so use audit_filter_mutex instead of rcu 1485 /* This is a blocking read, so use audit_filter_mutex instead of rcu
1481 * iterator to sync with list writers. */ 1486 * iterator to sync with list writers. */
1482 for (i=0; i<AUDIT_NR_FILTERS; i++) { 1487 for (i=0; i<AUDIT_NR_FILTERS; i++) {
1483 list_for_each_entry(e, &audit_filter_list[i], list) { 1488 list_for_each_entry(r, &audit_rules_list[i], list) {
1484 struct audit_rule_data *data;
1485
1486 data = audit_krule_to_data(&e->rule);
1487 if (unlikely(!data))
1488 break;
1489 skb = audit_make_reply(pid, seq, AUDIT_LIST_RULES, 0, 1,
1490 data, sizeof(*data) + data->buflen);
1491 if (skb)
1492 skb_queue_tail(q, skb);
1493 kfree(data);
1494 }
1495 }
1496 for (i=0; i< AUDIT_INODE_BUCKETS; i++) {
1497 list_for_each_entry(e, &audit_inode_hash[i], list) {
1498 struct audit_rule_data *data; 1489 struct audit_rule_data *data;
1499 1490
1500 data = audit_krule_to_data(&e->rule); 1491 data = audit_krule_to_data(r);
1501 if (unlikely(!data)) 1492 if (unlikely(!data))
1502 break; 1493 break;
1503 skb = audit_make_reply(pid, seq, AUDIT_LIST_RULES, 0, 1, 1494 skb = audit_make_reply(pid, seq, AUDIT_LIST_RULES, 0, 1,
@@ -1603,8 +1594,7 @@ int audit_receive_filter(int type, int pid, int uid, int seq, void *data,
1603 if (IS_ERR(entry)) 1594 if (IS_ERR(entry))
1604 return PTR_ERR(entry); 1595 return PTR_ERR(entry);
1605 1596
1606 err = audit_add_rule(entry, 1597 err = audit_add_rule(entry);
1607 &audit_filter_list[entry->rule.listnr]);
1608 audit_log_rule_change(loginuid, sessionid, sid, "add", 1598 audit_log_rule_change(loginuid, sessionid, sid, "add",
1609 &entry->rule, !err); 1599 &entry->rule, !err);
1610 1600
@@ -1620,8 +1610,7 @@ int audit_receive_filter(int type, int pid, int uid, int seq, void *data,
1620 if (IS_ERR(entry)) 1610 if (IS_ERR(entry))
1621 return PTR_ERR(entry); 1611 return PTR_ERR(entry);
1622 1612
1623 err = audit_del_rule(entry, 1613 err = audit_del_rule(entry);
1624 &audit_filter_list[entry->rule.listnr]);
1625 audit_log_rule_change(loginuid, sessionid, sid, "remove", 1614 audit_log_rule_change(loginuid, sessionid, sid, "remove",
1626 &entry->rule, !err); 1615 &entry->rule, !err);
1627 1616
@@ -1634,28 +1623,29 @@ int audit_receive_filter(int type, int pid, int uid, int seq, void *data,
1634 return err; 1623 return err;
1635} 1624}
1636 1625
1637int audit_comparator(const u32 left, const u32 op, const u32 right) 1626int audit_comparator(u32 left, u32 op, u32 right)
1638{ 1627{
1639 switch (op) { 1628 switch (op) {
1640 case AUDIT_EQUAL: 1629 case Audit_equal:
1641 return (left == right); 1630 return (left == right);
1642 case AUDIT_NOT_EQUAL: 1631 case Audit_not_equal:
1643 return (left != right); 1632 return (left != right);
1644 case AUDIT_LESS_THAN: 1633 case Audit_lt:
1645 return (left < right); 1634 return (left < right);
1646 case AUDIT_LESS_THAN_OR_EQUAL: 1635 case Audit_le:
1647 return (left <= right); 1636 return (left <= right);
1648 case AUDIT_GREATER_THAN: 1637 case Audit_gt:
1649 return (left > right); 1638 return (left > right);
1650 case AUDIT_GREATER_THAN_OR_EQUAL: 1639 case Audit_ge:
1651 return (left >= right); 1640 return (left >= right);
1652 case AUDIT_BIT_MASK: 1641 case Audit_bitmask:
1653 return (left & right); 1642 return (left & right);
1654 case AUDIT_BIT_TEST: 1643 case Audit_bittest:
1655 return ((left & right) == right); 1644 return ((left & right) == right);
1645 default:
1646 BUG();
1647 return 0;
1656 } 1648 }
1657 BUG();
1658 return 0;
1659} 1649}
1660 1650
1661/* Compare given dentry name with last component in given path, 1651/* Compare given dentry name with last component in given path,
@@ -1778,6 +1768,43 @@ unlock_and_return:
1778 return result; 1768 return result;
1779} 1769}
1780 1770
1771static int update_lsm_rule(struct audit_krule *r)
1772{
1773 struct audit_entry *entry = container_of(r, struct audit_entry, rule);
1774 struct audit_entry *nentry;
1775 struct audit_watch *watch;
1776 struct audit_tree *tree;
1777 int err = 0;
1778
1779 if (!security_audit_rule_known(r))
1780 return 0;
1781
1782 watch = r->watch;
1783 tree = r->tree;
1784 nentry = audit_dupe_rule(r, watch);
1785 if (IS_ERR(nentry)) {
1786 /* save the first error encountered for the
1787 * return value */
1788 err = PTR_ERR(nentry);
1789 audit_panic("error updating LSM filters");
1790 if (watch)
1791 list_del(&r->rlist);
1792 list_del_rcu(&entry->list);
1793 list_del(&r->list);
1794 } else {
1795 if (watch) {
1796 list_add(&nentry->rule.rlist, &watch->rules);
1797 list_del(&r->rlist);
1798 } else if (tree)
1799 list_replace_init(&r->rlist, &nentry->rule.rlist);
1800 list_replace_rcu(&entry->list, &nentry->list);
1801 list_replace(&r->list, &nentry->rule.list);
1802 }
1803 call_rcu(&entry->rcu, audit_free_rule_rcu);
1804
1805 return err;
1806}
1807
1781/* This function will re-initialize the lsm_rule field of all applicable rules. 1808/* This function will re-initialize the lsm_rule field of all applicable rules.
1782 * It will traverse the filter lists serarching for rules that contain LSM 1809 * It will traverse the filter lists serarching for rules that contain LSM
1783 * specific filter fields. When such a rule is found, it is copied, the 1810 * specific filter fields. When such a rule is found, it is copied, the
@@ -1785,45 +1812,19 @@ unlock_and_return:
1785 * updated rule. */ 1812 * updated rule. */
1786int audit_update_lsm_rules(void) 1813int audit_update_lsm_rules(void)
1787{ 1814{
1788 struct audit_entry *entry, *n, *nentry; 1815 struct audit_krule *r, *n;
1789 struct audit_watch *watch;
1790 struct audit_tree *tree;
1791 int i, err = 0; 1816 int i, err = 0;
1792 1817
1793 /* audit_filter_mutex synchronizes the writers */ 1818 /* audit_filter_mutex synchronizes the writers */
1794 mutex_lock(&audit_filter_mutex); 1819 mutex_lock(&audit_filter_mutex);
1795 1820
1796 for (i = 0; i < AUDIT_NR_FILTERS; i++) { 1821 for (i = 0; i < AUDIT_NR_FILTERS; i++) {
1797 list_for_each_entry_safe(entry, n, &audit_filter_list[i], list) { 1822 list_for_each_entry_safe(r, n, &audit_rules_list[i], list) {
1798 if (!security_audit_rule_known(&entry->rule)) 1823 int res = update_lsm_rule(r);
1799 continue; 1824 if (!err)
1800 1825 err = res;
1801 watch = entry->rule.watch;
1802 tree = entry->rule.tree;
1803 nentry = audit_dupe_rule(&entry->rule, watch);
1804 if (IS_ERR(nentry)) {
1805 /* save the first error encountered for the
1806 * return value */
1807 if (!err)
1808 err = PTR_ERR(nentry);
1809 audit_panic("error updating LSM filters");
1810 if (watch)
1811 list_del(&entry->rule.rlist);
1812 list_del_rcu(&entry->list);
1813 } else {
1814 if (watch) {
1815 list_add(&nentry->rule.rlist,
1816 &watch->rules);
1817 list_del(&entry->rule.rlist);
1818 } else if (tree)
1819 list_replace_init(&entry->rule.rlist,
1820 &nentry->rule.rlist);
1821 list_replace_rcu(&entry->list, &nentry->list);
1822 }
1823 call_rcu(&entry->rcu, audit_free_rule_rcu);
1824 } 1826 }
1825 } 1827 }
1826
1827 mutex_unlock(&audit_filter_mutex); 1828 mutex_unlock(&audit_filter_mutex);
1828 1829
1829 return err; 1830 return err;
diff --git a/kernel/auditsc.c b/kernel/auditsc.c
index cf5bc2f5f9c3..8cbddff6c283 100644
--- a/kernel/auditsc.c
+++ b/kernel/auditsc.c
@@ -65,6 +65,7 @@
65#include <linux/highmem.h> 65#include <linux/highmem.h>
66#include <linux/syscalls.h> 66#include <linux/syscalls.h>
67#include <linux/inotify.h> 67#include <linux/inotify.h>
68#include <linux/capability.h>
68 69
69#include "audit.h" 70#include "audit.h"
70 71
@@ -84,6 +85,15 @@ int audit_n_rules;
84/* determines whether we collect data for signals sent */ 85/* determines whether we collect data for signals sent */
85int audit_signals; 86int audit_signals;
86 87
88struct audit_cap_data {
89 kernel_cap_t permitted;
90 kernel_cap_t inheritable;
91 union {
92 unsigned int fE; /* effective bit of a file capability */
93 kernel_cap_t effective; /* effective set of a process */
94 };
95};
96
87/* When fs/namei.c:getname() is called, we store the pointer in name and 97/* When fs/namei.c:getname() is called, we store the pointer in name and
88 * we don't let putname() free it (instead we free all of the saved 98 * we don't let putname() free it (instead we free all of the saved
89 * pointers at syscall exit time). 99 * pointers at syscall exit time).
@@ -100,6 +110,8 @@ struct audit_names {
100 gid_t gid; 110 gid_t gid;
101 dev_t rdev; 111 dev_t rdev;
102 u32 osid; 112 u32 osid;
113 struct audit_cap_data fcap;
114 unsigned int fcap_ver;
103}; 115};
104 116
105struct audit_aux_data { 117struct audit_aux_data {
@@ -112,43 +124,6 @@ struct audit_aux_data {
112/* Number of target pids per aux struct. */ 124/* Number of target pids per aux struct. */
113#define AUDIT_AUX_PIDS 16 125#define AUDIT_AUX_PIDS 16
114 126
115struct audit_aux_data_mq_open {
116 struct audit_aux_data d;
117 int oflag;
118 mode_t mode;
119 struct mq_attr attr;
120};
121
122struct audit_aux_data_mq_sendrecv {
123 struct audit_aux_data d;
124 mqd_t mqdes;
125 size_t msg_len;
126 unsigned int msg_prio;
127 struct timespec abs_timeout;
128};
129
130struct audit_aux_data_mq_notify {
131 struct audit_aux_data d;
132 mqd_t mqdes;
133 struct sigevent notification;
134};
135
136struct audit_aux_data_mq_getsetattr {
137 struct audit_aux_data d;
138 mqd_t mqdes;
139 struct mq_attr mqstat;
140};
141
142struct audit_aux_data_ipcctl {
143 struct audit_aux_data d;
144 struct ipc_perm p;
145 unsigned long qbytes;
146 uid_t uid;
147 gid_t gid;
148 mode_t mode;
149 u32 osid;
150};
151
152struct audit_aux_data_execve { 127struct audit_aux_data_execve {
153 struct audit_aux_data d; 128 struct audit_aux_data d;
154 int argc; 129 int argc;
@@ -156,23 +131,6 @@ struct audit_aux_data_execve {
156 struct mm_struct *mm; 131 struct mm_struct *mm;
157}; 132};
158 133
159struct audit_aux_data_socketcall {
160 struct audit_aux_data d;
161 int nargs;
162 unsigned long args[0];
163};
164
165struct audit_aux_data_sockaddr {
166 struct audit_aux_data d;
167 int len;
168 char a[0];
169};
170
171struct audit_aux_data_fd_pair {
172 struct audit_aux_data d;
173 int fd[2];
174};
175
176struct audit_aux_data_pids { 134struct audit_aux_data_pids {
177 struct audit_aux_data d; 135 struct audit_aux_data d;
178 pid_t target_pid[AUDIT_AUX_PIDS]; 136 pid_t target_pid[AUDIT_AUX_PIDS];
@@ -184,6 +142,20 @@ struct audit_aux_data_pids {
184 int pid_count; 142 int pid_count;
185}; 143};
186 144
145struct audit_aux_data_bprm_fcaps {
146 struct audit_aux_data d;
147 struct audit_cap_data fcap;
148 unsigned int fcap_ver;
149 struct audit_cap_data old_pcap;
150 struct audit_cap_data new_pcap;
151};
152
153struct audit_aux_data_capset {
154 struct audit_aux_data d;
155 pid_t pid;
156 struct audit_cap_data cap;
157};
158
187struct audit_tree_refs { 159struct audit_tree_refs {
188 struct audit_tree_refs *next; 160 struct audit_tree_refs *next;
189 struct audit_chunk *c[31]; 161 struct audit_chunk *c[31];
@@ -193,14 +165,14 @@ struct audit_tree_refs {
193struct audit_context { 165struct audit_context {
194 int dummy; /* must be the first element */ 166 int dummy; /* must be the first element */
195 int in_syscall; /* 1 if task is in a syscall */ 167 int in_syscall; /* 1 if task is in a syscall */
196 enum audit_state state; 168 enum audit_state state, current_state;
197 unsigned int serial; /* serial number for record */ 169 unsigned int serial; /* serial number for record */
198 struct timespec ctime; /* time of syscall entry */ 170 struct timespec ctime; /* time of syscall entry */
199 int major; /* syscall number */ 171 int major; /* syscall number */
200 unsigned long argv[4]; /* syscall arguments */ 172 unsigned long argv[4]; /* syscall arguments */
201 int return_valid; /* return code is valid */ 173 int return_valid; /* return code is valid */
202 long return_code;/* syscall return code */ 174 long return_code;/* syscall return code */
203 int auditable; /* 1 if record should be written */ 175 u64 prio;
204 int name_count; 176 int name_count;
205 struct audit_names names[AUDIT_NAMES]; 177 struct audit_names names[AUDIT_NAMES];
206 char * filterkey; /* key for rule that triggered record */ 178 char * filterkey; /* key for rule that triggered record */
@@ -208,7 +180,8 @@ struct audit_context {
208 struct audit_context *previous; /* For nested syscalls */ 180 struct audit_context *previous; /* For nested syscalls */
209 struct audit_aux_data *aux; 181 struct audit_aux_data *aux;
210 struct audit_aux_data *aux_pids; 182 struct audit_aux_data *aux_pids;
211 183 struct sockaddr_storage *sockaddr;
184 size_t sockaddr_len;
212 /* Save things to print about task_struct */ 185 /* Save things to print about task_struct */
213 pid_t pid, ppid; 186 pid_t pid, ppid;
214 uid_t uid, euid, suid, fsuid; 187 uid_t uid, euid, suid, fsuid;
@@ -226,6 +199,49 @@ struct audit_context {
226 struct audit_tree_refs *trees, *first_trees; 199 struct audit_tree_refs *trees, *first_trees;
227 int tree_count; 200 int tree_count;
228 201
202 int type;
203 union {
204 struct {
205 int nargs;
206 long args[6];
207 } socketcall;
208 struct {
209 uid_t uid;
210 gid_t gid;
211 mode_t mode;
212 u32 osid;
213 int has_perm;
214 uid_t perm_uid;
215 gid_t perm_gid;
216 mode_t perm_mode;
217 unsigned long qbytes;
218 } ipc;
219 struct {
220 mqd_t mqdes;
221 struct mq_attr mqstat;
222 } mq_getsetattr;
223 struct {
224 mqd_t mqdes;
225 int sigev_signo;
226 } mq_notify;
227 struct {
228 mqd_t mqdes;
229 size_t msg_len;
230 unsigned int msg_prio;
231 struct timespec abs_timeout;
232 } mq_sendrecv;
233 struct {
234 int oflag;
235 mode_t mode;
236 struct mq_attr attr;
237 } mq_open;
238 struct {
239 pid_t pid;
240 struct audit_cap_data cap;
241 } capset;
242 };
243 int fds[2];
244
229#if AUDIT_DEBUG 245#if AUDIT_DEBUG
230 int put_count; 246 int put_count;
231 int ino_count; 247 int ino_count;
@@ -421,6 +437,7 @@ static int audit_filter_rules(struct task_struct *tsk,
421 struct audit_names *name, 437 struct audit_names *name,
422 enum audit_state *state) 438 enum audit_state *state)
423{ 439{
440 const struct cred *cred = get_task_cred(tsk);
424 int i, j, need_sid = 1; 441 int i, j, need_sid = 1;
425 u32 sid; 442 u32 sid;
426 443
@@ -440,28 +457,28 @@ static int audit_filter_rules(struct task_struct *tsk,
440 } 457 }
441 break; 458 break;
442 case AUDIT_UID: 459 case AUDIT_UID:
443 result = audit_comparator(tsk->uid, f->op, f->val); 460 result = audit_comparator(cred->uid, f->op, f->val);
444 break; 461 break;
445 case AUDIT_EUID: 462 case AUDIT_EUID:
446 result = audit_comparator(tsk->euid, f->op, f->val); 463 result = audit_comparator(cred->euid, f->op, f->val);
447 break; 464 break;
448 case AUDIT_SUID: 465 case AUDIT_SUID:
449 result = audit_comparator(tsk->suid, f->op, f->val); 466 result = audit_comparator(cred->suid, f->op, f->val);
450 break; 467 break;
451 case AUDIT_FSUID: 468 case AUDIT_FSUID:
452 result = audit_comparator(tsk->fsuid, f->op, f->val); 469 result = audit_comparator(cred->fsuid, f->op, f->val);
453 break; 470 break;
454 case AUDIT_GID: 471 case AUDIT_GID:
455 result = audit_comparator(tsk->gid, f->op, f->val); 472 result = audit_comparator(cred->gid, f->op, f->val);
456 break; 473 break;
457 case AUDIT_EGID: 474 case AUDIT_EGID:
458 result = audit_comparator(tsk->egid, f->op, f->val); 475 result = audit_comparator(cred->egid, f->op, f->val);
459 break; 476 break;
460 case AUDIT_SGID: 477 case AUDIT_SGID:
461 result = audit_comparator(tsk->sgid, f->op, f->val); 478 result = audit_comparator(cred->sgid, f->op, f->val);
462 break; 479 break;
463 case AUDIT_FSGID: 480 case AUDIT_FSGID:
464 result = audit_comparator(tsk->fsgid, f->op, f->val); 481 result = audit_comparator(cred->fsgid, f->op, f->val);
465 break; 482 break;
466 case AUDIT_PERS: 483 case AUDIT_PERS:
467 result = audit_comparator(tsk->personality, f->op, f->val); 484 result = audit_comparator(tsk->personality, f->op, f->val);
@@ -581,19 +598,12 @@ static int audit_filter_rules(struct task_struct *tsk,
581 } 598 }
582 } 599 }
583 /* Find ipc objects that match */ 600 /* Find ipc objects that match */
584 if (ctx) { 601 if (!ctx || ctx->type != AUDIT_IPC)
585 struct audit_aux_data *aux; 602 break;
586 for (aux = ctx->aux; aux; 603 if (security_audit_rule_match(ctx->ipc.osid,
587 aux = aux->next) { 604 f->type, f->op,
588 if (aux->type == AUDIT_IPC) { 605 f->lsm_rule, ctx))
589 struct audit_aux_data_ipcctl *axi = (void *)aux; 606 ++result;
590 if (security_audit_rule_match(axi->osid, f->type, f->op, f->lsm_rule, ctx)) {
591 ++result;
592 break;
593 }
594 }
595 }
596 }
597 } 607 }
598 break; 608 break;
599 case AUDIT_ARG0: 609 case AUDIT_ARG0:
@@ -615,15 +625,26 @@ static int audit_filter_rules(struct task_struct *tsk,
615 break; 625 break;
616 } 626 }
617 627
618 if (!result) 628 if (!result) {
629 put_cred(cred);
619 return 0; 630 return 0;
631 }
632 }
633
634 if (ctx) {
635 if (rule->prio <= ctx->prio)
636 return 0;
637 if (rule->filterkey) {
638 kfree(ctx->filterkey);
639 ctx->filterkey = kstrdup(rule->filterkey, GFP_ATOMIC);
640 }
641 ctx->prio = rule->prio;
620 } 642 }
621 if (rule->filterkey && ctx)
622 ctx->filterkey = kstrdup(rule->filterkey, GFP_ATOMIC);
623 switch (rule->action) { 643 switch (rule->action) {
624 case AUDIT_NEVER: *state = AUDIT_DISABLED; break; 644 case AUDIT_NEVER: *state = AUDIT_DISABLED; break;
625 case AUDIT_ALWAYS: *state = AUDIT_RECORD_CONTEXT; break; 645 case AUDIT_ALWAYS: *state = AUDIT_RECORD_CONTEXT; break;
626 } 646 }
647 put_cred(cred);
627 return 1; 648 return 1;
628} 649}
629 650
@@ -631,7 +652,7 @@ static int audit_filter_rules(struct task_struct *tsk,
631 * completely disabled for this task. Since we only have the task 652 * completely disabled for this task. Since we only have the task
632 * structure at this point, we can only check uid and gid. 653 * structure at this point, we can only check uid and gid.
633 */ 654 */
634static enum audit_state audit_filter_task(struct task_struct *tsk) 655static enum audit_state audit_filter_task(struct task_struct *tsk, char **key)
635{ 656{
636 struct audit_entry *e; 657 struct audit_entry *e;
637 enum audit_state state; 658 enum audit_state state;
@@ -639,6 +660,8 @@ static enum audit_state audit_filter_task(struct task_struct *tsk)
639 rcu_read_lock(); 660 rcu_read_lock();
640 list_for_each_entry_rcu(e, &audit_filter_list[AUDIT_FILTER_TASK], list) { 661 list_for_each_entry_rcu(e, &audit_filter_list[AUDIT_FILTER_TASK], list) {
641 if (audit_filter_rules(tsk, &e->rule, NULL, NULL, &state)) { 662 if (audit_filter_rules(tsk, &e->rule, NULL, NULL, &state)) {
663 if (state == AUDIT_RECORD_CONTEXT)
664 *key = kstrdup(e->rule.filterkey, GFP_ATOMIC);
642 rcu_read_unlock(); 665 rcu_read_unlock();
643 return state; 666 return state;
644 } 667 }
@@ -672,6 +695,7 @@ static enum audit_state audit_filter_syscall(struct task_struct *tsk,
672 audit_filter_rules(tsk, &e->rule, ctx, NULL, 695 audit_filter_rules(tsk, &e->rule, ctx, NULL,
673 &state)) { 696 &state)) {
674 rcu_read_unlock(); 697 rcu_read_unlock();
698 ctx->current_state = state;
675 return state; 699 return state;
676 } 700 }
677 } 701 }
@@ -685,15 +709,14 @@ static enum audit_state audit_filter_syscall(struct task_struct *tsk,
685 * buckets applicable to the inode numbers in audit_names[]. 709 * buckets applicable to the inode numbers in audit_names[].
686 * Regarding audit_state, same rules apply as for audit_filter_syscall(). 710 * Regarding audit_state, same rules apply as for audit_filter_syscall().
687 */ 711 */
688enum audit_state audit_filter_inodes(struct task_struct *tsk, 712void audit_filter_inodes(struct task_struct *tsk, struct audit_context *ctx)
689 struct audit_context *ctx)
690{ 713{
691 int i; 714 int i;
692 struct audit_entry *e; 715 struct audit_entry *e;
693 enum audit_state state; 716 enum audit_state state;
694 717
695 if (audit_pid && tsk->tgid == audit_pid) 718 if (audit_pid && tsk->tgid == audit_pid)
696 return AUDIT_DISABLED; 719 return;
697 720
698 rcu_read_lock(); 721 rcu_read_lock();
699 for (i = 0; i < ctx->name_count; i++) { 722 for (i = 0; i < ctx->name_count; i++) {
@@ -710,17 +733,20 @@ enum audit_state audit_filter_inodes(struct task_struct *tsk,
710 if ((e->rule.mask[word] & bit) == bit && 733 if ((e->rule.mask[word] & bit) == bit &&
711 audit_filter_rules(tsk, &e->rule, ctx, n, &state)) { 734 audit_filter_rules(tsk, &e->rule, ctx, n, &state)) {
712 rcu_read_unlock(); 735 rcu_read_unlock();
713 return state; 736 ctx->current_state = state;
737 return;
714 } 738 }
715 } 739 }
716 } 740 }
717 rcu_read_unlock(); 741 rcu_read_unlock();
718 return AUDIT_BUILD_CONTEXT;
719} 742}
720 743
721void audit_set_auditable(struct audit_context *ctx) 744static void audit_set_auditable(struct audit_context *ctx)
722{ 745{
723 ctx->auditable = 1; 746 if (!ctx->prio) {
747 ctx->prio = 1;
748 ctx->current_state = AUDIT_RECORD_CONTEXT;
749 }
724} 750}
725 751
726static inline struct audit_context *audit_get_context(struct task_struct *tsk, 752static inline struct audit_context *audit_get_context(struct task_struct *tsk,
@@ -751,23 +777,11 @@ static inline struct audit_context *audit_get_context(struct task_struct *tsk,
751 else 777 else
752 context->return_code = return_code; 778 context->return_code = return_code;
753 779
754 if (context->in_syscall && !context->dummy && !context->auditable) { 780 if (context->in_syscall && !context->dummy) {
755 enum audit_state state; 781 audit_filter_syscall(tsk, context, &audit_filter_list[AUDIT_FILTER_EXIT]);
756 782 audit_filter_inodes(tsk, context);
757 state = audit_filter_syscall(tsk, context, &audit_filter_list[AUDIT_FILTER_EXIT]);
758 if (state == AUDIT_RECORD_CONTEXT) {
759 context->auditable = 1;
760 goto get_context;
761 }
762
763 state = audit_filter_inodes(tsk, context);
764 if (state == AUDIT_RECORD_CONTEXT)
765 context->auditable = 1;
766
767 } 783 }
768 784
769get_context:
770
771 tsk->audit_context = NULL; 785 tsk->audit_context = NULL;
772 return context; 786 return context;
773} 787}
@@ -777,8 +791,7 @@ static inline void audit_free_names(struct audit_context *context)
777 int i; 791 int i;
778 792
779#if AUDIT_DEBUG == 2 793#if AUDIT_DEBUG == 2
780 if (context->auditable 794 if (context->put_count + context->ino_count != context->name_count) {
781 ||context->put_count + context->ino_count != context->name_count) {
782 printk(KERN_ERR "%s:%d(:%d): major=%d in_syscall=%d" 795 printk(KERN_ERR "%s:%d(:%d): major=%d in_syscall=%d"
783 " name_count=%d put_count=%d" 796 " name_count=%d put_count=%d"
784 " ino_count=%d [NOT freeing]\n", 797 " ino_count=%d [NOT freeing]\n",
@@ -829,6 +842,7 @@ static inline void audit_zero_context(struct audit_context *context,
829{ 842{
830 memset(context, 0, sizeof(*context)); 843 memset(context, 0, sizeof(*context));
831 context->state = state; 844 context->state = state;
845 context->prio = state == AUDIT_RECORD_CONTEXT ? ~0ULL : 0;
832} 846}
833 847
834static inline struct audit_context *audit_alloc_context(enum audit_state state) 848static inline struct audit_context *audit_alloc_context(enum audit_state state)
@@ -854,18 +868,21 @@ int audit_alloc(struct task_struct *tsk)
854{ 868{
855 struct audit_context *context; 869 struct audit_context *context;
856 enum audit_state state; 870 enum audit_state state;
871 char *key = NULL;
857 872
858 if (likely(!audit_ever_enabled)) 873 if (likely(!audit_ever_enabled))
859 return 0; /* Return if not auditing. */ 874 return 0; /* Return if not auditing. */
860 875
861 state = audit_filter_task(tsk); 876 state = audit_filter_task(tsk, &key);
862 if (likely(state == AUDIT_DISABLED)) 877 if (likely(state == AUDIT_DISABLED))
863 return 0; 878 return 0;
864 879
865 if (!(context = audit_alloc_context(state))) { 880 if (!(context = audit_alloc_context(state))) {
881 kfree(key);
866 audit_log_lost("out of memory in audit_alloc"); 882 audit_log_lost("out of memory in audit_alloc");
867 return -ENOMEM; 883 return -ENOMEM;
868 } 884 }
885 context->filterkey = key;
869 886
870 tsk->audit_context = context; 887 tsk->audit_context = context;
871 set_tsk_thread_flag(tsk, TIF_SYSCALL_AUDIT); 888 set_tsk_thread_flag(tsk, TIF_SYSCALL_AUDIT);
@@ -891,6 +908,7 @@ static inline void audit_free_context(struct audit_context *context)
891 free_tree_refs(context); 908 free_tree_refs(context);
892 audit_free_aux(context); 909 audit_free_aux(context);
893 kfree(context->filterkey); 910 kfree(context->filterkey);
911 kfree(context->sockaddr);
894 kfree(context); 912 kfree(context);
895 context = previous; 913 context = previous;
896 } while (context); 914 } while (context);
@@ -1171,8 +1189,129 @@ static void audit_log_execve_info(struct audit_context *context,
1171 kfree(buf); 1189 kfree(buf);
1172} 1190}
1173 1191
1192static void audit_log_cap(struct audit_buffer *ab, char *prefix, kernel_cap_t *cap)
1193{
1194 int i;
1195
1196 audit_log_format(ab, " %s=", prefix);
1197 CAP_FOR_EACH_U32(i) {
1198 audit_log_format(ab, "%08x", cap->cap[(_KERNEL_CAPABILITY_U32S-1) - i]);
1199 }
1200}
1201
1202static void audit_log_fcaps(struct audit_buffer *ab, struct audit_names *name)
1203{
1204 kernel_cap_t *perm = &name->fcap.permitted;
1205 kernel_cap_t *inh = &name->fcap.inheritable;
1206 int log = 0;
1207
1208 if (!cap_isclear(*perm)) {
1209 audit_log_cap(ab, "cap_fp", perm);
1210 log = 1;
1211 }
1212 if (!cap_isclear(*inh)) {
1213 audit_log_cap(ab, "cap_fi", inh);
1214 log = 1;
1215 }
1216
1217 if (log)
1218 audit_log_format(ab, " cap_fe=%d cap_fver=%x", name->fcap.fE, name->fcap_ver);
1219}
1220
1221static void show_special(struct audit_context *context, int *call_panic)
1222{
1223 struct audit_buffer *ab;
1224 int i;
1225
1226 ab = audit_log_start(context, GFP_KERNEL, context->type);
1227 if (!ab)
1228 return;
1229
1230 switch (context->type) {
1231 case AUDIT_SOCKETCALL: {
1232 int nargs = context->socketcall.nargs;
1233 audit_log_format(ab, "nargs=%d", nargs);
1234 for (i = 0; i < nargs; i++)
1235 audit_log_format(ab, " a%d=%lx", i,
1236 context->socketcall.args[i]);
1237 break; }
1238 case AUDIT_IPC: {
1239 u32 osid = context->ipc.osid;
1240
1241 audit_log_format(ab, "ouid=%u ogid=%u mode=%#o",
1242 context->ipc.uid, context->ipc.gid, context->ipc.mode);
1243 if (osid) {
1244 char *ctx = NULL;
1245 u32 len;
1246 if (security_secid_to_secctx(osid, &ctx, &len)) {
1247 audit_log_format(ab, " osid=%u", osid);
1248 *call_panic = 1;
1249 } else {
1250 audit_log_format(ab, " obj=%s", ctx);
1251 security_release_secctx(ctx, len);
1252 }
1253 }
1254 if (context->ipc.has_perm) {
1255 audit_log_end(ab);
1256 ab = audit_log_start(context, GFP_KERNEL,
1257 AUDIT_IPC_SET_PERM);
1258 audit_log_format(ab,
1259 "qbytes=%lx ouid=%u ogid=%u mode=%#o",
1260 context->ipc.qbytes,
1261 context->ipc.perm_uid,
1262 context->ipc.perm_gid,
1263 context->ipc.perm_mode);
1264 if (!ab)
1265 return;
1266 }
1267 break; }
1268 case AUDIT_MQ_OPEN: {
1269 audit_log_format(ab,
1270 "oflag=0x%x mode=%#o mq_flags=0x%lx mq_maxmsg=%ld "
1271 "mq_msgsize=%ld mq_curmsgs=%ld",
1272 context->mq_open.oflag, context->mq_open.mode,
1273 context->mq_open.attr.mq_flags,
1274 context->mq_open.attr.mq_maxmsg,
1275 context->mq_open.attr.mq_msgsize,
1276 context->mq_open.attr.mq_curmsgs);
1277 break; }
1278 case AUDIT_MQ_SENDRECV: {
1279 audit_log_format(ab,
1280 "mqdes=%d msg_len=%zd msg_prio=%u "
1281 "abs_timeout_sec=%ld abs_timeout_nsec=%ld",
1282 context->mq_sendrecv.mqdes,
1283 context->mq_sendrecv.msg_len,
1284 context->mq_sendrecv.msg_prio,
1285 context->mq_sendrecv.abs_timeout.tv_sec,
1286 context->mq_sendrecv.abs_timeout.tv_nsec);
1287 break; }
1288 case AUDIT_MQ_NOTIFY: {
1289 audit_log_format(ab, "mqdes=%d sigev_signo=%d",
1290 context->mq_notify.mqdes,
1291 context->mq_notify.sigev_signo);
1292 break; }
1293 case AUDIT_MQ_GETSETATTR: {
1294 struct mq_attr *attr = &context->mq_getsetattr.mqstat;
1295 audit_log_format(ab,
1296 "mqdes=%d mq_flags=0x%lx mq_maxmsg=%ld mq_msgsize=%ld "
1297 "mq_curmsgs=%ld ",
1298 context->mq_getsetattr.mqdes,
1299 attr->mq_flags, attr->mq_maxmsg,
1300 attr->mq_msgsize, attr->mq_curmsgs);
1301 break; }
1302 case AUDIT_CAPSET: {
1303 audit_log_format(ab, "pid=%d", context->capset.pid);
1304 audit_log_cap(ab, "cap_pi", &context->capset.cap.inheritable);
1305 audit_log_cap(ab, "cap_pp", &context->capset.cap.permitted);
1306 audit_log_cap(ab, "cap_pe", &context->capset.cap.effective);
1307 break; }
1308 }
1309 audit_log_end(ab);
1310}
1311
1174static void audit_log_exit(struct audit_context *context, struct task_struct *tsk) 1312static void audit_log_exit(struct audit_context *context, struct task_struct *tsk)
1175{ 1313{
1314 const struct cred *cred;
1176 int i, call_panic = 0; 1315 int i, call_panic = 0;
1177 struct audit_buffer *ab; 1316 struct audit_buffer *ab;
1178 struct audit_aux_data *aux; 1317 struct audit_aux_data *aux;
@@ -1182,14 +1321,15 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts
1182 context->pid = tsk->pid; 1321 context->pid = tsk->pid;
1183 if (!context->ppid) 1322 if (!context->ppid)
1184 context->ppid = sys_getppid(); 1323 context->ppid = sys_getppid();
1185 context->uid = tsk->uid; 1324 cred = current_cred();
1186 context->gid = tsk->gid; 1325 context->uid = cred->uid;
1187 context->euid = tsk->euid; 1326 context->gid = cred->gid;
1188 context->suid = tsk->suid; 1327 context->euid = cred->euid;
1189 context->fsuid = tsk->fsuid; 1328 context->suid = cred->suid;
1190 context->egid = tsk->egid; 1329 context->fsuid = cred->fsuid;
1191 context->sgid = tsk->sgid; 1330 context->egid = cred->egid;
1192 context->fsgid = tsk->fsgid; 1331 context->sgid = cred->sgid;
1332 context->fsgid = cred->fsgid;
1193 context->personality = tsk->personality; 1333 context->personality = tsk->personality;
1194 1334
1195 ab = audit_log_start(context, GFP_KERNEL, AUDIT_SYSCALL); 1335 ab = audit_log_start(context, GFP_KERNEL, AUDIT_SYSCALL);
@@ -1246,96 +1386,50 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts
1246 continue; /* audit_panic has been called */ 1386 continue; /* audit_panic has been called */
1247 1387
1248 switch (aux->type) { 1388 switch (aux->type) {
1249 case AUDIT_MQ_OPEN: {
1250 struct audit_aux_data_mq_open *axi = (void *)aux;
1251 audit_log_format(ab,
1252 "oflag=0x%x mode=%#o mq_flags=0x%lx mq_maxmsg=%ld "
1253 "mq_msgsize=%ld mq_curmsgs=%ld",
1254 axi->oflag, axi->mode, axi->attr.mq_flags,
1255 axi->attr.mq_maxmsg, axi->attr.mq_msgsize,
1256 axi->attr.mq_curmsgs);
1257 break; }
1258
1259 case AUDIT_MQ_SENDRECV: {
1260 struct audit_aux_data_mq_sendrecv *axi = (void *)aux;
1261 audit_log_format(ab,
1262 "mqdes=%d msg_len=%zd msg_prio=%u "
1263 "abs_timeout_sec=%ld abs_timeout_nsec=%ld",
1264 axi->mqdes, axi->msg_len, axi->msg_prio,
1265 axi->abs_timeout.tv_sec, axi->abs_timeout.tv_nsec);
1266 break; }
1267
1268 case AUDIT_MQ_NOTIFY: {
1269 struct audit_aux_data_mq_notify *axi = (void *)aux;
1270 audit_log_format(ab,
1271 "mqdes=%d sigev_signo=%d",
1272 axi->mqdes,
1273 axi->notification.sigev_signo);
1274 break; }
1275
1276 case AUDIT_MQ_GETSETATTR: {
1277 struct audit_aux_data_mq_getsetattr *axi = (void *)aux;
1278 audit_log_format(ab,
1279 "mqdes=%d mq_flags=0x%lx mq_maxmsg=%ld mq_msgsize=%ld "
1280 "mq_curmsgs=%ld ",
1281 axi->mqdes,
1282 axi->mqstat.mq_flags, axi->mqstat.mq_maxmsg,
1283 axi->mqstat.mq_msgsize, axi->mqstat.mq_curmsgs);
1284 break; }
1285
1286 case AUDIT_IPC: {
1287 struct audit_aux_data_ipcctl *axi = (void *)aux;
1288 audit_log_format(ab,
1289 "ouid=%u ogid=%u mode=%#o",
1290 axi->uid, axi->gid, axi->mode);
1291 if (axi->osid != 0) {
1292 char *ctx = NULL;
1293 u32 len;
1294 if (security_secid_to_secctx(
1295 axi->osid, &ctx, &len)) {
1296 audit_log_format(ab, " osid=%u",
1297 axi->osid);
1298 call_panic = 1;
1299 } else {
1300 audit_log_format(ab, " obj=%s", ctx);
1301 security_release_secctx(ctx, len);
1302 }
1303 }
1304 break; }
1305
1306 case AUDIT_IPC_SET_PERM: {
1307 struct audit_aux_data_ipcctl *axi = (void *)aux;
1308 audit_log_format(ab,
1309 "qbytes=%lx ouid=%u ogid=%u mode=%#o",
1310 axi->qbytes, axi->uid, axi->gid, axi->mode);
1311 break; }
1312 1389
1313 case AUDIT_EXECVE: { 1390 case AUDIT_EXECVE: {
1314 struct audit_aux_data_execve *axi = (void *)aux; 1391 struct audit_aux_data_execve *axi = (void *)aux;
1315 audit_log_execve_info(context, &ab, axi); 1392 audit_log_execve_info(context, &ab, axi);
1316 break; } 1393 break; }
1317 1394
1318 case AUDIT_SOCKETCALL: { 1395 case AUDIT_BPRM_FCAPS: {
1319 struct audit_aux_data_socketcall *axs = (void *)aux; 1396 struct audit_aux_data_bprm_fcaps *axs = (void *)aux;
1320 audit_log_format(ab, "nargs=%d", axs->nargs); 1397 audit_log_format(ab, "fver=%x", axs->fcap_ver);
1321 for (i=0; i<axs->nargs; i++) 1398 audit_log_cap(ab, "fp", &axs->fcap.permitted);
1322 audit_log_format(ab, " a%d=%lx", i, axs->args[i]); 1399 audit_log_cap(ab, "fi", &axs->fcap.inheritable);
1400 audit_log_format(ab, " fe=%d", axs->fcap.fE);
1401 audit_log_cap(ab, "old_pp", &axs->old_pcap.permitted);
1402 audit_log_cap(ab, "old_pi", &axs->old_pcap.inheritable);
1403 audit_log_cap(ab, "old_pe", &axs->old_pcap.effective);
1404 audit_log_cap(ab, "new_pp", &axs->new_pcap.permitted);
1405 audit_log_cap(ab, "new_pi", &axs->new_pcap.inheritable);
1406 audit_log_cap(ab, "new_pe", &axs->new_pcap.effective);
1323 break; } 1407 break; }
1324 1408
1325 case AUDIT_SOCKADDR: { 1409 }
1326 struct audit_aux_data_sockaddr *axs = (void *)aux; 1410 audit_log_end(ab);
1411 }
1327 1412
1328 audit_log_format(ab, "saddr="); 1413 if (context->type)
1329 audit_log_n_hex(ab, axs->a, axs->len); 1414 show_special(context, &call_panic);
1330 break; }
1331 1415
1332 case AUDIT_FD_PAIR: { 1416 if (context->fds[0] >= 0) {
1333 struct audit_aux_data_fd_pair *axs = (void *)aux; 1417 ab = audit_log_start(context, GFP_KERNEL, AUDIT_FD_PAIR);
1334 audit_log_format(ab, "fd0=%d fd1=%d", axs->fd[0], axs->fd[1]); 1418 if (ab) {
1335 break; } 1419 audit_log_format(ab, "fd0=%d fd1=%d",
1420 context->fds[0], context->fds[1]);
1421 audit_log_end(ab);
1422 }
1423 }
1336 1424
1425 if (context->sockaddr_len) {
1426 ab = audit_log_start(context, GFP_KERNEL, AUDIT_SOCKADDR);
1427 if (ab) {
1428 audit_log_format(ab, "saddr=");
1429 audit_log_n_hex(ab, (void *)context->sockaddr,
1430 context->sockaddr_len);
1431 audit_log_end(ab);
1337 } 1432 }
1338 audit_log_end(ab);
1339 } 1433 }
1340 1434
1341 for (aux = context->aux_pids; aux; aux = aux->next) { 1435 for (aux = context->aux_pids; aux; aux = aux->next) {
@@ -1421,6 +1515,8 @@ static void audit_log_exit(struct audit_context *context, struct task_struct *ts
1421 } 1515 }
1422 } 1516 }
1423 1517
1518 audit_log_fcaps(ab, n);
1519
1424 audit_log_end(ab); 1520 audit_log_end(ab);
1425 } 1521 }
1426 1522
@@ -1451,7 +1547,7 @@ void audit_free(struct task_struct *tsk)
1451 * We use GFP_ATOMIC here because we might be doing this 1547 * We use GFP_ATOMIC here because we might be doing this
1452 * in the context of the idle thread */ 1548 * in the context of the idle thread */
1453 /* that can happen only if we are called from do_exit() */ 1549 /* that can happen only if we are called from do_exit() */
1454 if (context->in_syscall && context->auditable) 1550 if (context->in_syscall && context->current_state == AUDIT_RECORD_CONTEXT)
1455 audit_log_exit(context, tsk); 1551 audit_log_exit(context, tsk);
1456 1552
1457 audit_free_context(context); 1553 audit_free_context(context);
@@ -1459,7 +1555,6 @@ void audit_free(struct task_struct *tsk)
1459 1555
1460/** 1556/**
1461 * audit_syscall_entry - fill in an audit record at syscall entry 1557 * audit_syscall_entry - fill in an audit record at syscall entry
1462 * @tsk: task being audited
1463 * @arch: architecture type 1558 * @arch: architecture type
1464 * @major: major syscall type (function) 1559 * @major: major syscall type (function)
1465 * @a1: additional syscall register 1 1560 * @a1: additional syscall register 1
@@ -1536,21 +1631,42 @@ void audit_syscall_entry(int arch, int major,
1536 1631
1537 state = context->state; 1632 state = context->state;
1538 context->dummy = !audit_n_rules; 1633 context->dummy = !audit_n_rules;
1539 if (!context->dummy && (state == AUDIT_SETUP_CONTEXT || state == AUDIT_BUILD_CONTEXT)) 1634 if (!context->dummy && state == AUDIT_BUILD_CONTEXT) {
1635 context->prio = 0;
1540 state = audit_filter_syscall(tsk, context, &audit_filter_list[AUDIT_FILTER_ENTRY]); 1636 state = audit_filter_syscall(tsk, context, &audit_filter_list[AUDIT_FILTER_ENTRY]);
1637 }
1541 if (likely(state == AUDIT_DISABLED)) 1638 if (likely(state == AUDIT_DISABLED))
1542 return; 1639 return;
1543 1640
1544 context->serial = 0; 1641 context->serial = 0;
1545 context->ctime = CURRENT_TIME; 1642 context->ctime = CURRENT_TIME;
1546 context->in_syscall = 1; 1643 context->in_syscall = 1;
1547 context->auditable = !!(state == AUDIT_RECORD_CONTEXT); 1644 context->current_state = state;
1548 context->ppid = 0; 1645 context->ppid = 0;
1549} 1646}
1550 1647
1648void audit_finish_fork(struct task_struct *child)
1649{
1650 struct audit_context *ctx = current->audit_context;
1651 struct audit_context *p = child->audit_context;
1652 if (!p || !ctx)
1653 return;
1654 if (!ctx->in_syscall || ctx->current_state != AUDIT_RECORD_CONTEXT)
1655 return;
1656 p->arch = ctx->arch;
1657 p->major = ctx->major;
1658 memcpy(p->argv, ctx->argv, sizeof(ctx->argv));
1659 p->ctime = ctx->ctime;
1660 p->dummy = ctx->dummy;
1661 p->in_syscall = ctx->in_syscall;
1662 p->filterkey = kstrdup(ctx->filterkey, GFP_KERNEL);
1663 p->ppid = current->pid;
1664 p->prio = ctx->prio;
1665 p->current_state = ctx->current_state;
1666}
1667
1551/** 1668/**
1552 * audit_syscall_exit - deallocate audit context after a system call 1669 * audit_syscall_exit - deallocate audit context after a system call
1553 * @tsk: task being audited
1554 * @valid: success/failure flag 1670 * @valid: success/failure flag
1555 * @return_code: syscall return value 1671 * @return_code: syscall return value
1556 * 1672 *
@@ -1570,11 +1686,11 @@ void audit_syscall_exit(int valid, long return_code)
1570 if (likely(!context)) 1686 if (likely(!context))
1571 return; 1687 return;
1572 1688
1573 if (context->in_syscall && context->auditable) 1689 if (context->in_syscall && context->current_state == AUDIT_RECORD_CONTEXT)
1574 audit_log_exit(context, tsk); 1690 audit_log_exit(context, tsk);
1575 1691
1576 context->in_syscall = 0; 1692 context->in_syscall = 0;
1577 context->auditable = 0; 1693 context->prio = context->state == AUDIT_RECORD_CONTEXT ? ~0ULL : 0;
1578 1694
1579 if (context->previous) { 1695 if (context->previous) {
1580 struct audit_context *new_context = context->previous; 1696 struct audit_context *new_context = context->previous;
@@ -1589,8 +1705,13 @@ void audit_syscall_exit(int valid, long return_code)
1589 context->aux_pids = NULL; 1705 context->aux_pids = NULL;
1590 context->target_pid = 0; 1706 context->target_pid = 0;
1591 context->target_sid = 0; 1707 context->target_sid = 0;
1592 kfree(context->filterkey); 1708 context->sockaddr_len = 0;
1593 context->filterkey = NULL; 1709 context->type = 0;
1710 context->fds[0] = -1;
1711 if (context->state != AUDIT_RECORD_CONTEXT) {
1712 kfree(context->filterkey);
1713 context->filterkey = NULL;
1714 }
1594 tsk->audit_context = context; 1715 tsk->audit_context = context;
1595 } 1716 }
1596} 1717}
@@ -1787,8 +1908,36 @@ static int audit_inc_name_count(struct audit_context *context,
1787 return 0; 1908 return 0;
1788} 1909}
1789 1910
1911
1912static inline int audit_copy_fcaps(struct audit_names *name, const struct dentry *dentry)
1913{
1914 struct cpu_vfs_cap_data caps;
1915 int rc;
1916
1917 memset(&name->fcap.permitted, 0, sizeof(kernel_cap_t));
1918 memset(&name->fcap.inheritable, 0, sizeof(kernel_cap_t));
1919 name->fcap.fE = 0;
1920 name->fcap_ver = 0;
1921
1922 if (!dentry)
1923 return 0;
1924
1925 rc = get_vfs_caps_from_disk(dentry, &caps);
1926 if (rc)
1927 return rc;
1928
1929 name->fcap.permitted = caps.permitted;
1930 name->fcap.inheritable = caps.inheritable;
1931 name->fcap.fE = !!(caps.magic_etc & VFS_CAP_FLAGS_EFFECTIVE);
1932 name->fcap_ver = (caps.magic_etc & VFS_CAP_REVISION_MASK) >> VFS_CAP_REVISION_SHIFT;
1933
1934 return 0;
1935}
1936
1937
1790/* Copy inode data into an audit_names. */ 1938/* Copy inode data into an audit_names. */
1791static void audit_copy_inode(struct audit_names *name, const struct inode *inode) 1939static void audit_copy_inode(struct audit_names *name, const struct dentry *dentry,
1940 const struct inode *inode)
1792{ 1941{
1793 name->ino = inode->i_ino; 1942 name->ino = inode->i_ino;
1794 name->dev = inode->i_sb->s_dev; 1943 name->dev = inode->i_sb->s_dev;
@@ -1797,6 +1946,7 @@ static void audit_copy_inode(struct audit_names *name, const struct inode *inode
1797 name->gid = inode->i_gid; 1946 name->gid = inode->i_gid;
1798 name->rdev = inode->i_rdev; 1947 name->rdev = inode->i_rdev;
1799 security_inode_getsecid(inode, &name->osid); 1948 security_inode_getsecid(inode, &name->osid);
1949 audit_copy_fcaps(name, dentry);
1800} 1950}
1801 1951
1802/** 1952/**
@@ -1831,7 +1981,7 @@ void __audit_inode(const char *name, const struct dentry *dentry)
1831 context->names[idx].name = NULL; 1981 context->names[idx].name = NULL;
1832 } 1982 }
1833 handle_path(dentry); 1983 handle_path(dentry);
1834 audit_copy_inode(&context->names[idx], inode); 1984 audit_copy_inode(&context->names[idx], dentry, inode);
1835} 1985}
1836 1986
1837/** 1987/**
@@ -1892,7 +2042,7 @@ void __audit_inode_child(const char *dname, const struct dentry *dentry,
1892 if (!strcmp(dname, n->name) || 2042 if (!strcmp(dname, n->name) ||
1893 !audit_compare_dname_path(dname, n->name, &dirlen)) { 2043 !audit_compare_dname_path(dname, n->name, &dirlen)) {
1894 if (inode) 2044 if (inode)
1895 audit_copy_inode(n, inode); 2045 audit_copy_inode(n, NULL, inode);
1896 else 2046 else
1897 n->ino = (unsigned long)-1; 2047 n->ino = (unsigned long)-1;
1898 found_child = n->name; 2048 found_child = n->name;
@@ -1906,7 +2056,7 @@ add_names:
1906 return; 2056 return;
1907 idx = context->name_count - 1; 2057 idx = context->name_count - 1;
1908 context->names[idx].name = NULL; 2058 context->names[idx].name = NULL;
1909 audit_copy_inode(&context->names[idx], parent); 2059 audit_copy_inode(&context->names[idx], NULL, parent);
1910 } 2060 }
1911 2061
1912 if (!found_child) { 2062 if (!found_child) {
@@ -1927,7 +2077,7 @@ add_names:
1927 } 2077 }
1928 2078
1929 if (inode) 2079 if (inode)
1930 audit_copy_inode(&context->names[idx], inode); 2080 audit_copy_inode(&context->names[idx], NULL, inode);
1931 else 2081 else
1932 context->names[idx].ino = (unsigned long)-1; 2082 context->names[idx].ino = (unsigned long)-1;
1933 } 2083 }
@@ -1942,15 +2092,21 @@ EXPORT_SYMBOL_GPL(__audit_inode_child);
1942 * 2092 *
1943 * Also sets the context as auditable. 2093 * Also sets the context as auditable.
1944 */ 2094 */
1945void auditsc_get_stamp(struct audit_context *ctx, 2095int auditsc_get_stamp(struct audit_context *ctx,
1946 struct timespec *t, unsigned int *serial) 2096 struct timespec *t, unsigned int *serial)
1947{ 2097{
2098 if (!ctx->in_syscall)
2099 return 0;
1948 if (!ctx->serial) 2100 if (!ctx->serial)
1949 ctx->serial = audit_serial(); 2101 ctx->serial = audit_serial();
1950 t->tv_sec = ctx->ctime.tv_sec; 2102 t->tv_sec = ctx->ctime.tv_sec;
1951 t->tv_nsec = ctx->ctime.tv_nsec; 2103 t->tv_nsec = ctx->ctime.tv_nsec;
1952 *serial = ctx->serial; 2104 *serial = ctx->serial;
1953 ctx->auditable = 1; 2105 if (!ctx->prio) {
2106 ctx->prio = 1;
2107 ctx->current_state = AUDIT_RECORD_CONTEXT;
2108 }
2109 return 1;
1954} 2110}
1955 2111
1956/* global counter which is incremented every time something logs in */ 2112/* global counter which is incremented every time something logs in */
@@ -1978,7 +2134,7 @@ int audit_set_loginuid(struct task_struct *task, uid_t loginuid)
1978 audit_log_format(ab, "login pid=%d uid=%u " 2134 audit_log_format(ab, "login pid=%d uid=%u "
1979 "old auid=%u new auid=%u" 2135 "old auid=%u new auid=%u"
1980 " old ses=%u new ses=%u", 2136 " old ses=%u new ses=%u",
1981 task->pid, task->uid, 2137 task->pid, task_uid(task),
1982 task->loginuid, loginuid, 2138 task->loginuid, loginuid,
1983 task->sessionid, sessionid); 2139 task->sessionid, sessionid);
1984 audit_log_end(ab); 2140 audit_log_end(ab);
@@ -1995,132 +2151,46 @@ int audit_set_loginuid(struct task_struct *task, uid_t loginuid)
1995 * @mode: mode bits 2151 * @mode: mode bits
1996 * @u_attr: queue attributes 2152 * @u_attr: queue attributes
1997 * 2153 *
1998 * Returns 0 for success or NULL context or < 0 on error.
1999 */ 2154 */
2000int __audit_mq_open(int oflag, mode_t mode, struct mq_attr __user *u_attr) 2155void __audit_mq_open(int oflag, mode_t mode, struct mq_attr *attr)
2001{ 2156{
2002 struct audit_aux_data_mq_open *ax;
2003 struct audit_context *context = current->audit_context; 2157 struct audit_context *context = current->audit_context;
2004 2158
2005 if (!audit_enabled) 2159 if (attr)
2006 return 0; 2160 memcpy(&context->mq_open.attr, attr, sizeof(struct mq_attr));
2007 2161 else
2008 if (likely(!context)) 2162 memset(&context->mq_open.attr, 0, sizeof(struct mq_attr));
2009 return 0;
2010
2011 ax = kmalloc(sizeof(*ax), GFP_ATOMIC);
2012 if (!ax)
2013 return -ENOMEM;
2014
2015 if (u_attr != NULL) {
2016 if (copy_from_user(&ax->attr, u_attr, sizeof(ax->attr))) {
2017 kfree(ax);
2018 return -EFAULT;
2019 }
2020 } else
2021 memset(&ax->attr, 0, sizeof(ax->attr));
2022 2163
2023 ax->oflag = oflag; 2164 context->mq_open.oflag = oflag;
2024 ax->mode = mode; 2165 context->mq_open.mode = mode;
2025 2166
2026 ax->d.type = AUDIT_MQ_OPEN; 2167 context->type = AUDIT_MQ_OPEN;
2027 ax->d.next = context->aux;
2028 context->aux = (void *)ax;
2029 return 0;
2030} 2168}
2031 2169
2032/** 2170/**
2033 * __audit_mq_timedsend - record audit data for a POSIX MQ timed send 2171 * __audit_mq_sendrecv - record audit data for a POSIX MQ timed send/receive
2034 * @mqdes: MQ descriptor 2172 * @mqdes: MQ descriptor
2035 * @msg_len: Message length 2173 * @msg_len: Message length
2036 * @msg_prio: Message priority 2174 * @msg_prio: Message priority
2037 * @u_abs_timeout: Message timeout in absolute time 2175 * @abs_timeout: Message timeout in absolute time
2038 * 2176 *
2039 * Returns 0 for success or NULL context or < 0 on error.
2040 */ 2177 */
2041int __audit_mq_timedsend(mqd_t mqdes, size_t msg_len, unsigned int msg_prio, 2178void __audit_mq_sendrecv(mqd_t mqdes, size_t msg_len, unsigned int msg_prio,
2042 const struct timespec __user *u_abs_timeout) 2179 const struct timespec *abs_timeout)
2043{ 2180{
2044 struct audit_aux_data_mq_sendrecv *ax;
2045 struct audit_context *context = current->audit_context; 2181 struct audit_context *context = current->audit_context;
2182 struct timespec *p = &context->mq_sendrecv.abs_timeout;
2046 2183
2047 if (!audit_enabled) 2184 if (abs_timeout)
2048 return 0; 2185 memcpy(p, abs_timeout, sizeof(struct timespec));
2049 2186 else
2050 if (likely(!context)) 2187 memset(p, 0, sizeof(struct timespec));
2051 return 0;
2052
2053 ax = kmalloc(sizeof(*ax), GFP_ATOMIC);
2054 if (!ax)
2055 return -ENOMEM;
2056
2057 if (u_abs_timeout != NULL) {
2058 if (copy_from_user(&ax->abs_timeout, u_abs_timeout, sizeof(ax->abs_timeout))) {
2059 kfree(ax);
2060 return -EFAULT;
2061 }
2062 } else
2063 memset(&ax->abs_timeout, 0, sizeof(ax->abs_timeout));
2064
2065 ax->mqdes = mqdes;
2066 ax->msg_len = msg_len;
2067 ax->msg_prio = msg_prio;
2068
2069 ax->d.type = AUDIT_MQ_SENDRECV;
2070 ax->d.next = context->aux;
2071 context->aux = (void *)ax;
2072 return 0;
2073}
2074
2075/**
2076 * __audit_mq_timedreceive - record audit data for a POSIX MQ timed receive
2077 * @mqdes: MQ descriptor
2078 * @msg_len: Message length
2079 * @u_msg_prio: Message priority
2080 * @u_abs_timeout: Message timeout in absolute time
2081 *
2082 * Returns 0 for success or NULL context or < 0 on error.
2083 */
2084int __audit_mq_timedreceive(mqd_t mqdes, size_t msg_len,
2085 unsigned int __user *u_msg_prio,
2086 const struct timespec __user *u_abs_timeout)
2087{
2088 struct audit_aux_data_mq_sendrecv *ax;
2089 struct audit_context *context = current->audit_context;
2090
2091 if (!audit_enabled)
2092 return 0;
2093
2094 if (likely(!context))
2095 return 0;
2096
2097 ax = kmalloc(sizeof(*ax), GFP_ATOMIC);
2098 if (!ax)
2099 return -ENOMEM;
2100
2101 if (u_msg_prio != NULL) {
2102 if (get_user(ax->msg_prio, u_msg_prio)) {
2103 kfree(ax);
2104 return -EFAULT;
2105 }
2106 } else
2107 ax->msg_prio = 0;
2108
2109 if (u_abs_timeout != NULL) {
2110 if (copy_from_user(&ax->abs_timeout, u_abs_timeout, sizeof(ax->abs_timeout))) {
2111 kfree(ax);
2112 return -EFAULT;
2113 }
2114 } else
2115 memset(&ax->abs_timeout, 0, sizeof(ax->abs_timeout));
2116 2188
2117 ax->mqdes = mqdes; 2189 context->mq_sendrecv.mqdes = mqdes;
2118 ax->msg_len = msg_len; 2190 context->mq_sendrecv.msg_len = msg_len;
2191 context->mq_sendrecv.msg_prio = msg_prio;
2119 2192
2120 ax->d.type = AUDIT_MQ_SENDRECV; 2193 context->type = AUDIT_MQ_SENDRECV;
2121 ax->d.next = context->aux;
2122 context->aux = (void *)ax;
2123 return 0;
2124} 2194}
2125 2195
2126/** 2196/**
@@ -2128,38 +2198,19 @@ int __audit_mq_timedreceive(mqd_t mqdes, size_t msg_len,
2128 * @mqdes: MQ descriptor 2198 * @mqdes: MQ descriptor
2129 * @u_notification: Notification event 2199 * @u_notification: Notification event
2130 * 2200 *
2131 * Returns 0 for success or NULL context or < 0 on error.
2132 */ 2201 */
2133 2202
2134int __audit_mq_notify(mqd_t mqdes, const struct sigevent __user *u_notification) 2203void __audit_mq_notify(mqd_t mqdes, const struct sigevent *notification)
2135{ 2204{
2136 struct audit_aux_data_mq_notify *ax;
2137 struct audit_context *context = current->audit_context; 2205 struct audit_context *context = current->audit_context;
2138 2206
2139 if (!audit_enabled) 2207 if (notification)
2140 return 0; 2208 context->mq_notify.sigev_signo = notification->sigev_signo;
2141 2209 else
2142 if (likely(!context)) 2210 context->mq_notify.sigev_signo = 0;
2143 return 0;
2144
2145 ax = kmalloc(sizeof(*ax), GFP_ATOMIC);
2146 if (!ax)
2147 return -ENOMEM;
2148
2149 if (u_notification != NULL) {
2150 if (copy_from_user(&ax->notification, u_notification, sizeof(ax->notification))) {
2151 kfree(ax);
2152 return -EFAULT;
2153 }
2154 } else
2155 memset(&ax->notification, 0, sizeof(ax->notification));
2156
2157 ax->mqdes = mqdes;
2158 2211
2159 ax->d.type = AUDIT_MQ_NOTIFY; 2212 context->mq_notify.mqdes = mqdes;
2160 ax->d.next = context->aux; 2213 context->type = AUDIT_MQ_NOTIFY;
2161 context->aux = (void *)ax;
2162 return 0;
2163} 2214}
2164 2215
2165/** 2216/**
@@ -2167,55 +2218,29 @@ int __audit_mq_notify(mqd_t mqdes, const struct sigevent __user *u_notification)
2167 * @mqdes: MQ descriptor 2218 * @mqdes: MQ descriptor
2168 * @mqstat: MQ flags 2219 * @mqstat: MQ flags
2169 * 2220 *
2170 * Returns 0 for success or NULL context or < 0 on error.
2171 */ 2221 */
2172int __audit_mq_getsetattr(mqd_t mqdes, struct mq_attr *mqstat) 2222void __audit_mq_getsetattr(mqd_t mqdes, struct mq_attr *mqstat)
2173{ 2223{
2174 struct audit_aux_data_mq_getsetattr *ax;
2175 struct audit_context *context = current->audit_context; 2224 struct audit_context *context = current->audit_context;
2176 2225 context->mq_getsetattr.mqdes = mqdes;
2177 if (!audit_enabled) 2226 context->mq_getsetattr.mqstat = *mqstat;
2178 return 0; 2227 context->type = AUDIT_MQ_GETSETATTR;
2179
2180 if (likely(!context))
2181 return 0;
2182
2183 ax = kmalloc(sizeof(*ax), GFP_ATOMIC);
2184 if (!ax)
2185 return -ENOMEM;
2186
2187 ax->mqdes = mqdes;
2188 ax->mqstat = *mqstat;
2189
2190 ax->d.type = AUDIT_MQ_GETSETATTR;
2191 ax->d.next = context->aux;
2192 context->aux = (void *)ax;
2193 return 0;
2194} 2228}
2195 2229
2196/** 2230/**
2197 * audit_ipc_obj - record audit data for ipc object 2231 * audit_ipc_obj - record audit data for ipc object
2198 * @ipcp: ipc permissions 2232 * @ipcp: ipc permissions
2199 * 2233 *
2200 * Returns 0 for success or NULL context or < 0 on error.
2201 */ 2234 */
2202int __audit_ipc_obj(struct kern_ipc_perm *ipcp) 2235void __audit_ipc_obj(struct kern_ipc_perm *ipcp)
2203{ 2236{
2204 struct audit_aux_data_ipcctl *ax;
2205 struct audit_context *context = current->audit_context; 2237 struct audit_context *context = current->audit_context;
2206 2238 context->ipc.uid = ipcp->uid;
2207 ax = kmalloc(sizeof(*ax), GFP_ATOMIC); 2239 context->ipc.gid = ipcp->gid;
2208 if (!ax) 2240 context->ipc.mode = ipcp->mode;
2209 return -ENOMEM; 2241 context->ipc.has_perm = 0;
2210 2242 security_ipc_getsecid(ipcp, &context->ipc.osid);
2211 ax->uid = ipcp->uid; 2243 context->type = AUDIT_IPC;
2212 ax->gid = ipcp->gid;
2213 ax->mode = ipcp->mode;
2214 security_ipc_getsecid(ipcp, &ax->osid);
2215 ax->d.type = AUDIT_IPC;
2216 ax->d.next = context->aux;
2217 context->aux = (void *)ax;
2218 return 0;
2219} 2244}
2220 2245
2221/** 2246/**
@@ -2225,26 +2250,17 @@ int __audit_ipc_obj(struct kern_ipc_perm *ipcp)
2225 * @gid: msgq group id 2250 * @gid: msgq group id
2226 * @mode: msgq mode (permissions) 2251 * @mode: msgq mode (permissions)
2227 * 2252 *
2228 * Returns 0 for success or NULL context or < 0 on error. 2253 * Called only after audit_ipc_obj().
2229 */ 2254 */
2230int __audit_ipc_set_perm(unsigned long qbytes, uid_t uid, gid_t gid, mode_t mode) 2255void __audit_ipc_set_perm(unsigned long qbytes, uid_t uid, gid_t gid, mode_t mode)
2231{ 2256{
2232 struct audit_aux_data_ipcctl *ax;
2233 struct audit_context *context = current->audit_context; 2257 struct audit_context *context = current->audit_context;
2234 2258
2235 ax = kmalloc(sizeof(*ax), GFP_ATOMIC); 2259 context->ipc.qbytes = qbytes;
2236 if (!ax) 2260 context->ipc.perm_uid = uid;
2237 return -ENOMEM; 2261 context->ipc.perm_gid = gid;
2238 2262 context->ipc.perm_mode = mode;
2239 ax->qbytes = qbytes; 2263 context->ipc.has_perm = 1;
2240 ax->uid = uid;
2241 ax->gid = gid;
2242 ax->mode = mode;
2243
2244 ax->d.type = AUDIT_IPC_SET_PERM;
2245 ax->d.next = context->aux;
2246 context->aux = (void *)ax;
2247 return 0;
2248} 2264}
2249 2265
2250int audit_bprm(struct linux_binprm *bprm) 2266int audit_bprm(struct linux_binprm *bprm)
@@ -2274,27 +2290,17 @@ int audit_bprm(struct linux_binprm *bprm)
2274 * @nargs: number of args 2290 * @nargs: number of args
2275 * @args: args array 2291 * @args: args array
2276 * 2292 *
2277 * Returns 0 for success or NULL context or < 0 on error.
2278 */ 2293 */
2279int audit_socketcall(int nargs, unsigned long *args) 2294void audit_socketcall(int nargs, unsigned long *args)
2280{ 2295{
2281 struct audit_aux_data_socketcall *ax;
2282 struct audit_context *context = current->audit_context; 2296 struct audit_context *context = current->audit_context;
2283 2297
2284 if (likely(!context || context->dummy)) 2298 if (likely(!context || context->dummy))
2285 return 0; 2299 return;
2286
2287 ax = kmalloc(sizeof(*ax) + nargs * sizeof(unsigned long), GFP_KERNEL);
2288 if (!ax)
2289 return -ENOMEM;
2290
2291 ax->nargs = nargs;
2292 memcpy(ax->args, args, nargs * sizeof(unsigned long));
2293 2300
2294 ax->d.type = AUDIT_SOCKETCALL; 2301 context->type = AUDIT_SOCKETCALL;
2295 ax->d.next = context->aux; 2302 context->socketcall.nargs = nargs;
2296 context->aux = (void *)ax; 2303 memcpy(context->socketcall.args, args, nargs * sizeof(unsigned long));
2297 return 0;
2298} 2304}
2299 2305
2300/** 2306/**
@@ -2302,29 +2308,12 @@ int audit_socketcall(int nargs, unsigned long *args)
2302 * @fd1: the first file descriptor 2308 * @fd1: the first file descriptor
2303 * @fd2: the second file descriptor 2309 * @fd2: the second file descriptor
2304 * 2310 *
2305 * Returns 0 for success or NULL context or < 0 on error.
2306 */ 2311 */
2307int __audit_fd_pair(int fd1, int fd2) 2312void __audit_fd_pair(int fd1, int fd2)
2308{ 2313{
2309 struct audit_context *context = current->audit_context; 2314 struct audit_context *context = current->audit_context;
2310 struct audit_aux_data_fd_pair *ax; 2315 context->fds[0] = fd1;
2311 2316 context->fds[1] = fd2;
2312 if (likely(!context)) {
2313 return 0;
2314 }
2315
2316 ax = kmalloc(sizeof(*ax), GFP_KERNEL);
2317 if (!ax) {
2318 return -ENOMEM;
2319 }
2320
2321 ax->fd[0] = fd1;
2322 ax->fd[1] = fd2;
2323
2324 ax->d.type = AUDIT_FD_PAIR;
2325 ax->d.next = context->aux;
2326 context->aux = (void *)ax;
2327 return 0;
2328} 2317}
2329 2318
2330/** 2319/**
@@ -2336,22 +2325,20 @@ int __audit_fd_pair(int fd1, int fd2)
2336 */ 2325 */
2337int audit_sockaddr(int len, void *a) 2326int audit_sockaddr(int len, void *a)
2338{ 2327{
2339 struct audit_aux_data_sockaddr *ax;
2340 struct audit_context *context = current->audit_context; 2328 struct audit_context *context = current->audit_context;
2341 2329
2342 if (likely(!context || context->dummy)) 2330 if (likely(!context || context->dummy))
2343 return 0; 2331 return 0;
2344 2332
2345 ax = kmalloc(sizeof(*ax) + len, GFP_KERNEL); 2333 if (!context->sockaddr) {
2346 if (!ax) 2334 void *p = kmalloc(sizeof(struct sockaddr_storage), GFP_KERNEL);
2347 return -ENOMEM; 2335 if (!p)
2348 2336 return -ENOMEM;
2349 ax->len = len; 2337 context->sockaddr = p;
2350 memcpy(ax->a, a, len); 2338 }
2351 2339
2352 ax->d.type = AUDIT_SOCKADDR; 2340 context->sockaddr_len = len;
2353 ax->d.next = context->aux; 2341 memcpy(context->sockaddr, a, len);
2354 context->aux = (void *)ax;
2355 return 0; 2342 return 0;
2356} 2343}
2357 2344
@@ -2361,7 +2348,7 @@ void __audit_ptrace(struct task_struct *t)
2361 2348
2362 context->target_pid = t->pid; 2349 context->target_pid = t->pid;
2363 context->target_auid = audit_get_loginuid(t); 2350 context->target_auid = audit_get_loginuid(t);
2364 context->target_uid = t->uid; 2351 context->target_uid = task_uid(t);
2365 context->target_sessionid = audit_get_sessionid(t); 2352 context->target_sessionid = audit_get_sessionid(t);
2366 security_task_getsecid(t, &context->target_sid); 2353 security_task_getsecid(t, &context->target_sid);
2367 memcpy(context->target_comm, t->comm, TASK_COMM_LEN); 2354 memcpy(context->target_comm, t->comm, TASK_COMM_LEN);
@@ -2380,6 +2367,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
2380 struct audit_aux_data_pids *axp; 2367 struct audit_aux_data_pids *axp;
2381 struct task_struct *tsk = current; 2368 struct task_struct *tsk = current;
2382 struct audit_context *ctx = tsk->audit_context; 2369 struct audit_context *ctx = tsk->audit_context;
2370 uid_t uid = current_uid(), t_uid = task_uid(t);
2383 2371
2384 if (audit_pid && t->tgid == audit_pid) { 2372 if (audit_pid && t->tgid == audit_pid) {
2385 if (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1 || sig == SIGUSR2) { 2373 if (sig == SIGTERM || sig == SIGHUP || sig == SIGUSR1 || sig == SIGUSR2) {
@@ -2387,7 +2375,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
2387 if (tsk->loginuid != -1) 2375 if (tsk->loginuid != -1)
2388 audit_sig_uid = tsk->loginuid; 2376 audit_sig_uid = tsk->loginuid;
2389 else 2377 else
2390 audit_sig_uid = tsk->uid; 2378 audit_sig_uid = uid;
2391 security_task_getsecid(tsk, &audit_sig_sid); 2379 security_task_getsecid(tsk, &audit_sig_sid);
2392 } 2380 }
2393 if (!audit_signals || audit_dummy_context()) 2381 if (!audit_signals || audit_dummy_context())
@@ -2399,7 +2387,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
2399 if (!ctx->target_pid) { 2387 if (!ctx->target_pid) {
2400 ctx->target_pid = t->tgid; 2388 ctx->target_pid = t->tgid;
2401 ctx->target_auid = audit_get_loginuid(t); 2389 ctx->target_auid = audit_get_loginuid(t);
2402 ctx->target_uid = t->uid; 2390 ctx->target_uid = t_uid;
2403 ctx->target_sessionid = audit_get_sessionid(t); 2391 ctx->target_sessionid = audit_get_sessionid(t);
2404 security_task_getsecid(t, &ctx->target_sid); 2392 security_task_getsecid(t, &ctx->target_sid);
2405 memcpy(ctx->target_comm, t->comm, TASK_COMM_LEN); 2393 memcpy(ctx->target_comm, t->comm, TASK_COMM_LEN);
@@ -2420,7 +2408,7 @@ int __audit_signal_info(int sig, struct task_struct *t)
2420 2408
2421 axp->target_pid[axp->pid_count] = t->tgid; 2409 axp->target_pid[axp->pid_count] = t->tgid;
2422 axp->target_auid[axp->pid_count] = audit_get_loginuid(t); 2410 axp->target_auid[axp->pid_count] = audit_get_loginuid(t);
2423 axp->target_uid[axp->pid_count] = t->uid; 2411 axp->target_uid[axp->pid_count] = t_uid;
2424 axp->target_sessionid[axp->pid_count] = audit_get_sessionid(t); 2412 axp->target_sessionid[axp->pid_count] = audit_get_sessionid(t);
2425 security_task_getsecid(t, &axp->target_sid[axp->pid_count]); 2413 security_task_getsecid(t, &axp->target_sid[axp->pid_count]);
2426 memcpy(axp->target_comm[axp->pid_count], t->comm, TASK_COMM_LEN); 2414 memcpy(axp->target_comm[axp->pid_count], t->comm, TASK_COMM_LEN);
@@ -2430,6 +2418,72 @@ int __audit_signal_info(int sig, struct task_struct *t)
2430} 2418}
2431 2419
2432/** 2420/**
2421 * __audit_log_bprm_fcaps - store information about a loading bprm and relevant fcaps
2422 * @bprm: pointer to the bprm being processed
2423 * @new: the proposed new credentials
2424 * @old: the old credentials
2425 *
2426 * Simply check if the proc already has the caps given by the file and if not
2427 * store the priv escalation info for later auditing at the end of the syscall
2428 *
2429 * -Eric
2430 */
2431int __audit_log_bprm_fcaps(struct linux_binprm *bprm,
2432 const struct cred *new, const struct cred *old)
2433{
2434 struct audit_aux_data_bprm_fcaps *ax;
2435 struct audit_context *context = current->audit_context;
2436 struct cpu_vfs_cap_data vcaps;
2437 struct dentry *dentry;
2438
2439 ax = kmalloc(sizeof(*ax), GFP_KERNEL);
2440 if (!ax)
2441 return -ENOMEM;
2442
2443 ax->d.type = AUDIT_BPRM_FCAPS;
2444 ax->d.next = context->aux;
2445 context->aux = (void *)ax;
2446
2447 dentry = dget(bprm->file->f_dentry);
2448 get_vfs_caps_from_disk(dentry, &vcaps);
2449 dput(dentry);
2450
2451 ax->fcap.permitted = vcaps.permitted;
2452 ax->fcap.inheritable = vcaps.inheritable;
2453 ax->fcap.fE = !!(vcaps.magic_etc & VFS_CAP_FLAGS_EFFECTIVE);
2454 ax->fcap_ver = (vcaps.magic_etc & VFS_CAP_REVISION_MASK) >> VFS_CAP_REVISION_SHIFT;
2455
2456 ax->old_pcap.permitted = old->cap_permitted;
2457 ax->old_pcap.inheritable = old->cap_inheritable;
2458 ax->old_pcap.effective = old->cap_effective;
2459
2460 ax->new_pcap.permitted = new->cap_permitted;
2461 ax->new_pcap.inheritable = new->cap_inheritable;
2462 ax->new_pcap.effective = new->cap_effective;
2463 return 0;
2464}
2465
2466/**
2467 * __audit_log_capset - store information about the arguments to the capset syscall
2468 * @pid: target pid of the capset call
2469 * @new: the new credentials
2470 * @old: the old (current) credentials
2471 *
2472 * Record the aguments userspace sent to sys_capset for later printing by the
2473 * audit system if applicable
2474 */
2475void __audit_log_capset(pid_t pid,
2476 const struct cred *new, const struct cred *old)
2477{
2478 struct audit_context *context = current->audit_context;
2479 context->capset.pid = pid;
2480 context->capset.cap.effective = new->cap_effective;
2481 context->capset.cap.inheritable = new->cap_effective;
2482 context->capset.cap.permitted = new->cap_permitted;
2483 context->type = AUDIT_CAPSET;
2484}
2485
2486/**
2433 * audit_core_dumps - record information about processes that end abnormally 2487 * audit_core_dumps - record information about processes that end abnormally
2434 * @signr: signal value 2488 * @signr: signal value
2435 * 2489 *
@@ -2440,7 +2494,8 @@ void audit_core_dumps(long signr)
2440{ 2494{
2441 struct audit_buffer *ab; 2495 struct audit_buffer *ab;
2442 u32 sid; 2496 u32 sid;
2443 uid_t auid = audit_get_loginuid(current); 2497 uid_t auid = audit_get_loginuid(current), uid;
2498 gid_t gid;
2444 unsigned int sessionid = audit_get_sessionid(current); 2499 unsigned int sessionid = audit_get_sessionid(current);
2445 2500
2446 if (!audit_enabled) 2501 if (!audit_enabled)
@@ -2450,8 +2505,9 @@ void audit_core_dumps(long signr)
2450 return; 2505 return;
2451 2506
2452 ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_ANOM_ABEND); 2507 ab = audit_log_start(NULL, GFP_KERNEL, AUDIT_ANOM_ABEND);
2508 current_uid_gid(&uid, &gid);
2453 audit_log_format(ab, "auid=%u uid=%u gid=%u ses=%u", 2509 audit_log_format(ab, "auid=%u uid=%u gid=%u ses=%u",
2454 auid, current->uid, current->gid, sessionid); 2510 auid, uid, gid, sessionid);
2455 security_task_getsecid(current, &sid); 2511 security_task_getsecid(current, &sid);
2456 if (sid) { 2512 if (sid) {
2457 char *ctx = NULL; 2513 char *ctx = NULL;
diff --git a/kernel/capability.c b/kernel/capability.c
index 33e51e78c2d8..c598d9d5be4f 100644
--- a/kernel/capability.c
+++ b/kernel/capability.c
@@ -7,6 +7,7 @@
7 * 30 May 2002: Cleanup, Robert M. Love <rml@tech9.net> 7 * 30 May 2002: Cleanup, Robert M. Love <rml@tech9.net>
8 */ 8 */
9 9
10#include <linux/audit.h>
10#include <linux/capability.h> 11#include <linux/capability.h>
11#include <linux/mm.h> 12#include <linux/mm.h>
12#include <linux/module.h> 13#include <linux/module.h>
@@ -14,12 +15,7 @@
14#include <linux/syscalls.h> 15#include <linux/syscalls.h>
15#include <linux/pid_namespace.h> 16#include <linux/pid_namespace.h>
16#include <asm/uaccess.h> 17#include <asm/uaccess.h>
17 18#include "cred-internals.h"
18/*
19 * This lock protects task->cap_* for all tasks including current.
20 * Locking rule: acquire this prior to tasklist_lock.
21 */
22static DEFINE_SPINLOCK(task_capability_lock);
23 19
24/* 20/*
25 * Leveraged for setting/resetting capabilities 21 * Leveraged for setting/resetting capabilities
@@ -33,6 +29,17 @@ EXPORT_SYMBOL(__cap_empty_set);
33EXPORT_SYMBOL(__cap_full_set); 29EXPORT_SYMBOL(__cap_full_set);
34EXPORT_SYMBOL(__cap_init_eff_set); 30EXPORT_SYMBOL(__cap_init_eff_set);
35 31
32#ifdef CONFIG_SECURITY_FILE_CAPABILITIES
33int file_caps_enabled = 1;
34
35static int __init file_caps_disable(char *str)
36{
37 file_caps_enabled = 0;
38 return 1;
39}
40__setup("no_file_caps", file_caps_disable);
41#endif
42
36/* 43/*
37 * More recent versions of libcap are available from: 44 * More recent versions of libcap are available from:
38 * 45 *
@@ -115,167 +122,12 @@ static int cap_validate_magic(cap_user_header_t header, unsigned *tocopy)
115 return 0; 122 return 0;
116} 123}
117 124
118#ifndef CONFIG_SECURITY_FILE_CAPABILITIES
119
120/* 125/*
121 * Without filesystem capability support, we nominally support one process 126 * The only thing that can change the capabilities of the current
122 * setting the capabilities of another 127 * process is the current process. As such, we can't be in this code
123 */ 128 * at the same time as we are in the process of setting capabilities
124static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp, 129 * in this process. The net result is that we can limit our use of
125 kernel_cap_t *pIp, kernel_cap_t *pPp) 130 * locks to when we are reading the caps of another process.
126{
127 struct task_struct *target;
128 int ret;
129
130 spin_lock(&task_capability_lock);
131 read_lock(&tasklist_lock);
132
133 if (pid && pid != task_pid_vnr(current)) {
134 target = find_task_by_vpid(pid);
135 if (!target) {
136 ret = -ESRCH;
137 goto out;
138 }
139 } else
140 target = current;
141
142 ret = security_capget(target, pEp, pIp, pPp);
143
144out:
145 read_unlock(&tasklist_lock);
146 spin_unlock(&task_capability_lock);
147
148 return ret;
149}
150
151/*
152 * cap_set_pg - set capabilities for all processes in a given process
153 * group. We call this holding task_capability_lock and tasklist_lock.
154 */
155static inline int cap_set_pg(int pgrp_nr, kernel_cap_t *effective,
156 kernel_cap_t *inheritable,
157 kernel_cap_t *permitted)
158{
159 struct task_struct *g, *target;
160 int ret = -EPERM;
161 int found = 0;
162 struct pid *pgrp;
163
164 spin_lock(&task_capability_lock);
165 read_lock(&tasklist_lock);
166
167 pgrp = find_vpid(pgrp_nr);
168 do_each_pid_task(pgrp, PIDTYPE_PGID, g) {
169 target = g;
170 while_each_thread(g, target) {
171 if (!security_capset_check(target, effective,
172 inheritable, permitted)) {
173 security_capset_set(target, effective,
174 inheritable, permitted);
175 ret = 0;
176 }
177 found = 1;
178 }
179 } while_each_pid_task(pgrp, PIDTYPE_PGID, g);
180
181 read_unlock(&tasklist_lock);
182 spin_unlock(&task_capability_lock);
183
184 if (!found)
185 ret = 0;
186 return ret;
187}
188
189/*
190 * cap_set_all - set capabilities for all processes other than init
191 * and self. We call this holding task_capability_lock and tasklist_lock.
192 */
193static inline int cap_set_all(kernel_cap_t *effective,
194 kernel_cap_t *inheritable,
195 kernel_cap_t *permitted)
196{
197 struct task_struct *g, *target;
198 int ret = -EPERM;
199 int found = 0;
200
201 spin_lock(&task_capability_lock);
202 read_lock(&tasklist_lock);
203
204 do_each_thread(g, target) {
205 if (target == current
206 || is_container_init(target->group_leader))
207 continue;
208 found = 1;
209 if (security_capset_check(target, effective, inheritable,
210 permitted))
211 continue;
212 ret = 0;
213 security_capset_set(target, effective, inheritable, permitted);
214 } while_each_thread(g, target);
215
216 read_unlock(&tasklist_lock);
217 spin_unlock(&task_capability_lock);
218
219 if (!found)
220 ret = 0;
221
222 return ret;
223}
224
225/*
226 * Given the target pid does not refer to the current process we
227 * need more elaborate support... (This support is not present when
228 * filesystem capabilities are configured.)
229 */
230static inline int do_sys_capset_other_tasks(pid_t pid, kernel_cap_t *effective,
231 kernel_cap_t *inheritable,
232 kernel_cap_t *permitted)
233{
234 struct task_struct *target;
235 int ret;
236
237 if (!capable(CAP_SETPCAP))
238 return -EPERM;
239
240 if (pid == -1) /* all procs other than current and init */
241 return cap_set_all(effective, inheritable, permitted);
242
243 else if (pid < 0) /* all procs in process group */
244 return cap_set_pg(-pid, effective, inheritable, permitted);
245
246 /* target != current */
247 spin_lock(&task_capability_lock);
248 read_lock(&tasklist_lock);
249
250 target = find_task_by_vpid(pid);
251 if (!target)
252 ret = -ESRCH;
253 else {
254 ret = security_capset_check(target, effective, inheritable,
255 permitted);
256
257 /* having verified that the proposed changes are legal,
258 we now put them into effect. */
259 if (!ret)
260 security_capset_set(target, effective, inheritable,
261 permitted);
262 }
263
264 read_unlock(&tasklist_lock);
265 spin_unlock(&task_capability_lock);
266
267 return ret;
268}
269
270#else /* ie., def CONFIG_SECURITY_FILE_CAPABILITIES */
271
272/*
273 * If we have configured with filesystem capability support, then the
274 * only thing that can change the capabilities of the current process
275 * is the current process. As such, we can't be in this code at the
276 * same time as we are in the process of setting capabilities in this
277 * process. The net result is that we can limit our use of locks to
278 * when we are reading the caps of another process.
279 */ 131 */
280static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp, 132static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp,
281 kernel_cap_t *pIp, kernel_cap_t *pPp) 133 kernel_cap_t *pIp, kernel_cap_t *pPp)
@@ -285,7 +137,6 @@ static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp,
285 if (pid && (pid != task_pid_vnr(current))) { 137 if (pid && (pid != task_pid_vnr(current))) {
286 struct task_struct *target; 138 struct task_struct *target;
287 139
288 spin_lock(&task_capability_lock);
289 read_lock(&tasklist_lock); 140 read_lock(&tasklist_lock);
290 141
291 target = find_task_by_vpid(pid); 142 target = find_task_by_vpid(pid);
@@ -295,50 +146,12 @@ static inline int cap_get_target_pid(pid_t pid, kernel_cap_t *pEp,
295 ret = security_capget(target, pEp, pIp, pPp); 146 ret = security_capget(target, pEp, pIp, pPp);
296 147
297 read_unlock(&tasklist_lock); 148 read_unlock(&tasklist_lock);
298 spin_unlock(&task_capability_lock);
299 } else 149 } else
300 ret = security_capget(current, pEp, pIp, pPp); 150 ret = security_capget(current, pEp, pIp, pPp);
301 151
302 return ret; 152 return ret;
303} 153}
304 154
305/*
306 * With filesystem capability support configured, the kernel does not
307 * permit the changing of capabilities in one process by another
308 * process. (CAP_SETPCAP has much less broad semantics when configured
309 * this way.)
310 */
311static inline int do_sys_capset_other_tasks(pid_t pid,
312 kernel_cap_t *effective,
313 kernel_cap_t *inheritable,
314 kernel_cap_t *permitted)
315{
316 return -EPERM;
317}
318
319#endif /* ie., ndef CONFIG_SECURITY_FILE_CAPABILITIES */
320
321/*
322 * Atomically modify the effective capabilities returning the original
323 * value. No permission check is performed here - it is assumed that the
324 * caller is permitted to set the desired effective capabilities.
325 */
326kernel_cap_t cap_set_effective(const kernel_cap_t pE_new)
327{
328 kernel_cap_t pE_old;
329
330 spin_lock(&task_capability_lock);
331
332 pE_old = current->cap_effective;
333 current->cap_effective = pE_new;
334
335 spin_unlock(&task_capability_lock);
336
337 return pE_old;
338}
339
340EXPORT_SYMBOL(cap_set_effective);
341
342/** 155/**
343 * sys_capget - get the capabilities of a given process. 156 * sys_capget - get the capabilities of a given process.
344 * @header: pointer to struct that contains capability version and 157 * @header: pointer to struct that contains capability version and
@@ -366,7 +179,6 @@ asmlinkage long sys_capget(cap_user_header_t header, cap_user_data_t dataptr)
366 return -EINVAL; 179 return -EINVAL;
367 180
368 ret = cap_get_target_pid(pid, &pE, &pI, &pP); 181 ret = cap_get_target_pid(pid, &pE, &pI, &pP);
369
370 if (!ret) { 182 if (!ret) {
371 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S]; 183 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S];
372 unsigned i; 184 unsigned i;
@@ -412,16 +224,14 @@ asmlinkage long sys_capget(cap_user_header_t header, cap_user_data_t dataptr)
412 * @data: pointer to struct that contains the effective, permitted, 224 * @data: pointer to struct that contains the effective, permitted,
413 * and inheritable capabilities 225 * and inheritable capabilities
414 * 226 *
415 * Set capabilities for a given process, all processes, or all 227 * Set capabilities for the current process only. The ability to any other
416 * processes in a given process group. 228 * process(es) has been deprecated and removed.
417 * 229 *
418 * The restrictions on setting capabilities are specified as: 230 * The restrictions on setting capabilities are specified as:
419 * 231 *
420 * [pid is for the 'target' task. 'current' is the calling task.] 232 * I: any raised capabilities must be a subset of the old permitted
421 * 233 * P: any raised capabilities must be a subset of the old permitted
422 * I: any raised capabilities must be a subset of the (old current) permitted 234 * E: must be set to a subset of new permitted
423 * P: any raised capabilities must be a subset of the (old current) permitted
424 * E: must be set to a subset of (new target) permitted
425 * 235 *
426 * Returns 0 on success and < 0 on error. 236 * Returns 0 on success and < 0 on error.
427 */ 237 */
@@ -430,6 +240,7 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
430 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S]; 240 struct __user_cap_data_struct kdata[_KERNEL_CAPABILITY_U32S];
431 unsigned i, tocopy; 241 unsigned i, tocopy;
432 kernel_cap_t inheritable, permitted, effective; 242 kernel_cap_t inheritable, permitted, effective;
243 struct cred *new;
433 int ret; 244 int ret;
434 pid_t pid; 245 pid_t pid;
435 246
@@ -440,10 +251,13 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
440 if (get_user(pid, &header->pid)) 251 if (get_user(pid, &header->pid))
441 return -EFAULT; 252 return -EFAULT;
442 253
443 if (copy_from_user(&kdata, data, tocopy 254 /* may only affect current now */
444 * sizeof(struct __user_cap_data_struct))) { 255 if (pid != 0 && pid != task_pid_vnr(current))
256 return -EPERM;
257
258 if (copy_from_user(&kdata, data,
259 tocopy * sizeof(struct __user_cap_data_struct)))
445 return -EFAULT; 260 return -EFAULT;
446 }
447 261
448 for (i = 0; i < tocopy; i++) { 262 for (i = 0; i < tocopy; i++) {
449 effective.cap[i] = kdata[i].effective; 263 effective.cap[i] = kdata[i].effective;
@@ -457,32 +271,21 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
457 i++; 271 i++;
458 } 272 }
459 273
460 if (pid && (pid != task_pid_vnr(current))) 274 new = prepare_creds();
461 ret = do_sys_capset_other_tasks(pid, &effective, &inheritable, 275 if (!new)
462 &permitted); 276 return -ENOMEM;
463 else {
464 /*
465 * This lock is required even when filesystem
466 * capability support is configured - it protects the
467 * sys_capget() call from returning incorrect data in
468 * the case that the targeted process is not the
469 * current one.
470 */
471 spin_lock(&task_capability_lock);
472 277
473 ret = security_capset_check(current, &effective, &inheritable, 278 ret = security_capset(new, current_cred(),
474 &permitted); 279 &effective, &inheritable, &permitted);
475 /* 280 if (ret < 0)
476 * Having verified that the proposed changes are 281 goto error;
477 * legal, we now put them into effect. 282
478 */ 283 audit_log_capset(pid, new, current_cred());
479 if (!ret)
480 security_capset_set(current, &effective, &inheritable,
481 &permitted);
482 spin_unlock(&task_capability_lock);
483 }
484 284
285 return commit_creds(new);
485 286
287error:
288 abort_creds(new);
486 return ret; 289 return ret;
487} 290}
488 291
@@ -498,6 +301,11 @@ asmlinkage long sys_capset(cap_user_header_t header, const cap_user_data_t data)
498 */ 301 */
499int capable(int cap) 302int capable(int cap)
500{ 303{
304 if (unlikely(!cap_valid(cap))) {
305 printk(KERN_CRIT "capable() called with invalid cap=%u\n", cap);
306 BUG();
307 }
308
501 if (has_capability(current, cap)) { 309 if (has_capability(current, cap)) {
502 current->flags |= PF_SUPERPRIV; 310 current->flags |= PF_SUPERPRIV;
503 return 1; 311 return 1;
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index fe00b3b983a8..87bb0258fd27 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -571,9 +571,8 @@ static struct inode *cgroup_new_inode(mode_t mode, struct super_block *sb)
571 571
572 if (inode) { 572 if (inode) {
573 inode->i_mode = mode; 573 inode->i_mode = mode;
574 inode->i_uid = current->fsuid; 574 inode->i_uid = current_fsuid();
575 inode->i_gid = current->fsgid; 575 inode->i_gid = current_fsgid();
576 inode->i_blocks = 0;
577 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME; 576 inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
578 inode->i_mapping->backing_dev_info = &cgroup_backing_dev_info; 577 inode->i_mapping->backing_dev_info = &cgroup_backing_dev_info;
579 } 578 }
@@ -702,7 +701,7 @@ static int rebind_subsystems(struct cgroupfs_root *root,
702 * any child cgroups exist. This is theoretically supportable 701 * any child cgroups exist. This is theoretically supportable
703 * but involves complex error handling, so it's being left until 702 * but involves complex error handling, so it's being left until
704 * later */ 703 * later */
705 if (!list_empty(&cgrp->children)) 704 if (root->number_of_cgroups > 1)
706 return -EBUSY; 705 return -EBUSY;
707 706
708 /* Process each subsystem */ 707 /* Process each subsystem */
@@ -1024,7 +1023,7 @@ static int cgroup_get_sb(struct file_system_type *fs_type,
1024 if (ret == -EBUSY) { 1023 if (ret == -EBUSY) {
1025 mutex_unlock(&cgroup_mutex); 1024 mutex_unlock(&cgroup_mutex);
1026 mutex_unlock(&inode->i_mutex); 1025 mutex_unlock(&inode->i_mutex);
1027 goto drop_new_super; 1026 goto free_cg_links;
1028 } 1027 }
1029 1028
1030 /* EBUSY should be the only error here */ 1029 /* EBUSY should be the only error here */
@@ -1073,10 +1072,11 @@ static int cgroup_get_sb(struct file_system_type *fs_type,
1073 1072
1074 return simple_set_mnt(mnt, sb); 1073 return simple_set_mnt(mnt, sb);
1075 1074
1075 free_cg_links:
1076 free_cg_links(&tmp_cg_links);
1076 drop_new_super: 1077 drop_new_super:
1077 up_write(&sb->s_umount); 1078 up_write(&sb->s_umount);
1078 deactivate_super(sb); 1079 deactivate_super(sb);
1079 free_cg_links(&tmp_cg_links);
1080 return ret; 1080 return ret;
1081} 1081}
1082 1082
@@ -1279,6 +1279,7 @@ int cgroup_attach_task(struct cgroup *cgrp, struct task_struct *tsk)
1279static int attach_task_by_pid(struct cgroup *cgrp, u64 pid) 1279static int attach_task_by_pid(struct cgroup *cgrp, u64 pid)
1280{ 1280{
1281 struct task_struct *tsk; 1281 struct task_struct *tsk;
1282 const struct cred *cred = current_cred(), *tcred;
1282 int ret; 1283 int ret;
1283 1284
1284 if (pid) { 1285 if (pid) {
@@ -1288,14 +1289,16 @@ static int attach_task_by_pid(struct cgroup *cgrp, u64 pid)
1288 rcu_read_unlock(); 1289 rcu_read_unlock();
1289 return -ESRCH; 1290 return -ESRCH;
1290 } 1291 }
1291 get_task_struct(tsk);
1292 rcu_read_unlock();
1293 1292
1294 if ((current->euid) && (current->euid != tsk->uid) 1293 tcred = __task_cred(tsk);
1295 && (current->euid != tsk->suid)) { 1294 if (cred->euid &&
1296 put_task_struct(tsk); 1295 cred->euid != tcred->uid &&
1296 cred->euid != tcred->suid) {
1297 rcu_read_unlock();
1297 return -EACCES; 1298 return -EACCES;
1298 } 1299 }
1300 get_task_struct(tsk);
1301 rcu_read_unlock();
1299 } else { 1302 } else {
1300 tsk = current; 1303 tsk = current;
1301 get_task_struct(tsk); 1304 get_task_struct(tsk);
@@ -2934,9 +2937,6 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys,
2934 again: 2937 again:
2935 root = subsys->root; 2938 root = subsys->root;
2936 if (root == &rootnode) { 2939 if (root == &rootnode) {
2937 printk(KERN_INFO
2938 "Not cloning cgroup for unused subsystem %s\n",
2939 subsys->name);
2940 mutex_unlock(&cgroup_mutex); 2940 mutex_unlock(&cgroup_mutex);
2941 return 0; 2941 return 0;
2942 } 2942 }
@@ -2944,7 +2944,11 @@ int cgroup_clone(struct task_struct *tsk, struct cgroup_subsys *subsys,
2944 parent = task_cgroup(tsk, subsys->subsys_id); 2944 parent = task_cgroup(tsk, subsys->subsys_id);
2945 2945
2946 /* Pin the hierarchy */ 2946 /* Pin the hierarchy */
2947 atomic_inc(&parent->root->sb->s_active); 2947 if (!atomic_inc_not_zero(&parent->root->sb->s_active)) {
2948 /* We race with the final deactivate_super() */
2949 mutex_unlock(&cgroup_mutex);
2950 return 0;
2951 }
2948 2952
2949 /* Keep the cgroup alive */ 2953 /* Keep the cgroup alive */
2950 get_css_set(cg); 2954 get_css_set(cg);
diff --git a/kernel/compat.c b/kernel/compat.c
index 8eafe3eb50d9..d52e2ec1deb5 100644
--- a/kernel/compat.c
+++ b/kernel/compat.c
@@ -454,16 +454,16 @@ asmlinkage long compat_sys_waitid(int which, compat_pid_t pid,
454} 454}
455 455
456static int compat_get_user_cpu_mask(compat_ulong_t __user *user_mask_ptr, 456static int compat_get_user_cpu_mask(compat_ulong_t __user *user_mask_ptr,
457 unsigned len, cpumask_t *new_mask) 457 unsigned len, struct cpumask *new_mask)
458{ 458{
459 unsigned long *k; 459 unsigned long *k;
460 460
461 if (len < sizeof(cpumask_t)) 461 if (len < cpumask_size())
462 memset(new_mask, 0, sizeof(cpumask_t)); 462 memset(new_mask, 0, cpumask_size());
463 else if (len > sizeof(cpumask_t)) 463 else if (len > cpumask_size())
464 len = sizeof(cpumask_t); 464 len = cpumask_size();
465 465
466 k = cpus_addr(*new_mask); 466 k = cpumask_bits(new_mask);
467 return compat_get_bitmap(k, user_mask_ptr, len * 8); 467 return compat_get_bitmap(k, user_mask_ptr, len * 8);
468} 468}
469 469
@@ -471,40 +471,51 @@ asmlinkage long compat_sys_sched_setaffinity(compat_pid_t pid,
471 unsigned int len, 471 unsigned int len,
472 compat_ulong_t __user *user_mask_ptr) 472 compat_ulong_t __user *user_mask_ptr)
473{ 473{
474 cpumask_t new_mask; 474 cpumask_var_t new_mask;
475 int retval; 475 int retval;
476 476
477 retval = compat_get_user_cpu_mask(user_mask_ptr, len, &new_mask); 477 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
478 return -ENOMEM;
479
480 retval = compat_get_user_cpu_mask(user_mask_ptr, len, new_mask);
478 if (retval) 481 if (retval)
479 return retval; 482 goto out;
480 483
481 return sched_setaffinity(pid, &new_mask); 484 retval = sched_setaffinity(pid, new_mask);
485out:
486 free_cpumask_var(new_mask);
487 return retval;
482} 488}
483 489
484asmlinkage long compat_sys_sched_getaffinity(compat_pid_t pid, unsigned int len, 490asmlinkage long compat_sys_sched_getaffinity(compat_pid_t pid, unsigned int len,
485 compat_ulong_t __user *user_mask_ptr) 491 compat_ulong_t __user *user_mask_ptr)
486{ 492{
487 int ret; 493 int ret;
488 cpumask_t mask; 494 cpumask_var_t mask;
489 unsigned long *k; 495 unsigned long *k;
490 unsigned int min_length = sizeof(cpumask_t); 496 unsigned int min_length = cpumask_size();
491 497
492 if (NR_CPUS <= BITS_PER_COMPAT_LONG) 498 if (nr_cpu_ids <= BITS_PER_COMPAT_LONG)
493 min_length = sizeof(compat_ulong_t); 499 min_length = sizeof(compat_ulong_t);
494 500
495 if (len < min_length) 501 if (len < min_length)
496 return -EINVAL; 502 return -EINVAL;
497 503
498 ret = sched_getaffinity(pid, &mask); 504 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
505 return -ENOMEM;
506
507 ret = sched_getaffinity(pid, mask);
499 if (ret < 0) 508 if (ret < 0)
500 return ret; 509 goto out;
501 510
502 k = cpus_addr(mask); 511 k = cpumask_bits(mask);
503 ret = compat_put_bitmap(user_mask_ptr, k, min_length * 8); 512 ret = compat_put_bitmap(user_mask_ptr, k, min_length * 8);
504 if (ret) 513 if (ret == 0)
505 return ret; 514 ret = min_length;
506 515
507 return min_length; 516out:
517 free_cpumask_var(mask);
518 return ret;
508} 519}
509 520
510int get_compat_itimerspec(struct itimerspec *dst, 521int get_compat_itimerspec(struct itimerspec *dst,
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 5a732c5ef08b..30e74dd6d01b 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -15,29 +15,8 @@
15#include <linux/stop_machine.h> 15#include <linux/stop_machine.h>
16#include <linux/mutex.h> 16#include <linux/mutex.h>
17 17
18/* 18#ifdef CONFIG_SMP
19 * Represents all cpu's present in the system 19/* Serializes the updates to cpu_online_mask, cpu_present_mask */
20 * In systems capable of hotplug, this map could dynamically grow
21 * as new cpu's are detected in the system via any platform specific
22 * method, such as ACPI for e.g.
23 */
24cpumask_t cpu_present_map __read_mostly;
25EXPORT_SYMBOL(cpu_present_map);
26
27#ifndef CONFIG_SMP
28
29/*
30 * Represents all cpu's that are currently online.
31 */
32cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL;
33EXPORT_SYMBOL(cpu_online_map);
34
35cpumask_t cpu_possible_map __read_mostly = CPU_MASK_ALL;
36EXPORT_SYMBOL(cpu_possible_map);
37
38#else /* CONFIG_SMP */
39
40/* Serializes the updates to cpu_online_map, cpu_present_map */
41static DEFINE_MUTEX(cpu_add_remove_lock); 20static DEFINE_MUTEX(cpu_add_remove_lock);
42 21
43static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain); 22static __cpuinitdata RAW_NOTIFIER_HEAD(cpu_chain);
@@ -64,8 +43,6 @@ void __init cpu_hotplug_init(void)
64 cpu_hotplug.refcount = 0; 43 cpu_hotplug.refcount = 0;
65} 44}
66 45
67cpumask_t cpu_active_map;
68
69#ifdef CONFIG_HOTPLUG_CPU 46#ifdef CONFIG_HOTPLUG_CPU
70 47
71void get_online_cpus(void) 48void get_online_cpus(void)
@@ -96,7 +73,7 @@ EXPORT_SYMBOL_GPL(put_online_cpus);
96 73
97/* 74/*
98 * The following two API's must be used when attempting 75 * The following two API's must be used when attempting
99 * to serialize the updates to cpu_online_map, cpu_present_map. 76 * to serialize the updates to cpu_online_mask, cpu_present_mask.
100 */ 77 */
101void cpu_maps_update_begin(void) 78void cpu_maps_update_begin(void)
102{ 79{
@@ -217,7 +194,7 @@ static int __ref take_cpu_down(void *_param)
217static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) 194static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
218{ 195{
219 int err, nr_calls = 0; 196 int err, nr_calls = 0;
220 cpumask_t old_allowed, tmp; 197 cpumask_var_t old_allowed;
221 void *hcpu = (void *)(long)cpu; 198 void *hcpu = (void *)(long)cpu;
222 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; 199 unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0;
223 struct take_cpu_down_param tcd_param = { 200 struct take_cpu_down_param tcd_param = {
@@ -231,6 +208,9 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
231 if (!cpu_online(cpu)) 208 if (!cpu_online(cpu))
232 return -EINVAL; 209 return -EINVAL;
233 210
211 if (!alloc_cpumask_var(&old_allowed, GFP_KERNEL))
212 return -ENOMEM;
213
234 cpu_hotplug_begin(); 214 cpu_hotplug_begin();
235 err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod, 215 err = __raw_notifier_call_chain(&cpu_chain, CPU_DOWN_PREPARE | mod,
236 hcpu, -1, &nr_calls); 216 hcpu, -1, &nr_calls);
@@ -245,13 +225,11 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
245 } 225 }
246 226
247 /* Ensure that we are not runnable on dying cpu */ 227 /* Ensure that we are not runnable on dying cpu */
248 old_allowed = current->cpus_allowed; 228 cpumask_copy(old_allowed, &current->cpus_allowed);
249 cpus_setall(tmp); 229 set_cpus_allowed_ptr(current,
250 cpu_clear(cpu, tmp); 230 cpumask_of(cpumask_any_but(cpu_online_mask, cpu)));
251 set_cpus_allowed_ptr(current, &tmp);
252 tmp = cpumask_of_cpu(cpu);
253 231
254 err = __stop_machine(take_cpu_down, &tcd_param, &tmp); 232 err = __stop_machine(take_cpu_down, &tcd_param, cpumask_of(cpu));
255 if (err) { 233 if (err) {
256 /* CPU didn't die: tell everyone. Can't complain. */ 234 /* CPU didn't die: tell everyone. Can't complain. */
257 if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod, 235 if (raw_notifier_call_chain(&cpu_chain, CPU_DOWN_FAILED | mod,
@@ -277,7 +255,7 @@ static int __ref _cpu_down(unsigned int cpu, int tasks_frozen)
277 check_for_tasks(cpu); 255 check_for_tasks(cpu);
278 256
279out_allowed: 257out_allowed:
280 set_cpus_allowed_ptr(current, &old_allowed); 258 set_cpus_allowed_ptr(current, old_allowed);
281out_release: 259out_release:
282 cpu_hotplug_done(); 260 cpu_hotplug_done();
283 if (!err) { 261 if (!err) {
@@ -285,13 +263,17 @@ out_release:
285 hcpu) == NOTIFY_BAD) 263 hcpu) == NOTIFY_BAD)
286 BUG(); 264 BUG();
287 } 265 }
266 free_cpumask_var(old_allowed);
288 return err; 267 return err;
289} 268}
290 269
291int __ref cpu_down(unsigned int cpu) 270int __ref cpu_down(unsigned int cpu)
292{ 271{
293 int err = 0; 272 int err;
294 273
274 err = stop_machine_create();
275 if (err)
276 return err;
295 cpu_maps_update_begin(); 277 cpu_maps_update_begin();
296 278
297 if (cpu_hotplug_disabled) { 279 if (cpu_hotplug_disabled) {
@@ -303,7 +285,7 @@ int __ref cpu_down(unsigned int cpu)
303 285
304 /* 286 /*
305 * Make sure the all cpus did the reschedule and are not 287 * Make sure the all cpus did the reschedule and are not
306 * using stale version of the cpu_active_map. 288 * using stale version of the cpu_active_mask.
307 * This is not strictly necessary becuase stop_machine() 289 * This is not strictly necessary becuase stop_machine()
308 * that we run down the line already provides the required 290 * that we run down the line already provides the required
309 * synchronization. But it's really a side effect and we do not 291 * synchronization. But it's really a side effect and we do not
@@ -318,6 +300,7 @@ int __ref cpu_down(unsigned int cpu)
318 300
319out: 301out:
320 cpu_maps_update_done(); 302 cpu_maps_update_done();
303 stop_machine_destroy();
321 return err; 304 return err;
322} 305}
323EXPORT_SYMBOL(cpu_down); 306EXPORT_SYMBOL(cpu_down);
@@ -367,7 +350,7 @@ out_notify:
367int __cpuinit cpu_up(unsigned int cpu) 350int __cpuinit cpu_up(unsigned int cpu)
368{ 351{
369 int err = 0; 352 int err = 0;
370 if (!cpu_isset(cpu, cpu_possible_map)) { 353 if (!cpu_possible(cpu)) {
371 printk(KERN_ERR "can't online cpu %d because it is not " 354 printk(KERN_ERR "can't online cpu %d because it is not "
372 "configured as may-hotadd at boot time\n", cpu); 355 "configured as may-hotadd at boot time\n", cpu);
373#if defined(CONFIG_IA64) || defined(CONFIG_X86_64) 356#if defined(CONFIG_IA64) || defined(CONFIG_X86_64)
@@ -392,25 +375,25 @@ out:
392} 375}
393 376
394#ifdef CONFIG_PM_SLEEP_SMP 377#ifdef CONFIG_PM_SLEEP_SMP
395static cpumask_t frozen_cpus; 378static cpumask_var_t frozen_cpus;
396 379
397int disable_nonboot_cpus(void) 380int disable_nonboot_cpus(void)
398{ 381{
399 int cpu, first_cpu, error = 0; 382 int cpu, first_cpu, error = 0;
400 383
401 cpu_maps_update_begin(); 384 cpu_maps_update_begin();
402 first_cpu = first_cpu(cpu_online_map); 385 first_cpu = cpumask_first(cpu_online_mask);
403 /* We take down all of the non-boot CPUs in one shot to avoid races 386 /* We take down all of the non-boot CPUs in one shot to avoid races
404 * with the userspace trying to use the CPU hotplug at the same time 387 * with the userspace trying to use the CPU hotplug at the same time
405 */ 388 */
406 cpus_clear(frozen_cpus); 389 cpumask_clear(frozen_cpus);
407 printk("Disabling non-boot CPUs ...\n"); 390 printk("Disabling non-boot CPUs ...\n");
408 for_each_online_cpu(cpu) { 391 for_each_online_cpu(cpu) {
409 if (cpu == first_cpu) 392 if (cpu == first_cpu)
410 continue; 393 continue;
411 error = _cpu_down(cpu, 1); 394 error = _cpu_down(cpu, 1);
412 if (!error) { 395 if (!error) {
413 cpu_set(cpu, frozen_cpus); 396 cpumask_set_cpu(cpu, frozen_cpus);
414 printk("CPU%d is down\n", cpu); 397 printk("CPU%d is down\n", cpu);
415 } else { 398 } else {
416 printk(KERN_ERR "Error taking CPU%d down: %d\n", 399 printk(KERN_ERR "Error taking CPU%d down: %d\n",
@@ -436,11 +419,11 @@ void __ref enable_nonboot_cpus(void)
436 /* Allow everyone to use the CPU hotplug again */ 419 /* Allow everyone to use the CPU hotplug again */
437 cpu_maps_update_begin(); 420 cpu_maps_update_begin();
438 cpu_hotplug_disabled = 0; 421 cpu_hotplug_disabled = 0;
439 if (cpus_empty(frozen_cpus)) 422 if (cpumask_empty(frozen_cpus))
440 goto out; 423 goto out;
441 424
442 printk("Enabling non-boot CPUs ...\n"); 425 printk("Enabling non-boot CPUs ...\n");
443 for_each_cpu_mask_nr(cpu, frozen_cpus) { 426 for_each_cpu(cpu, frozen_cpus) {
444 error = _cpu_up(cpu, 1); 427 error = _cpu_up(cpu, 1);
445 if (!error) { 428 if (!error) {
446 printk("CPU%d is up\n", cpu); 429 printk("CPU%d is up\n", cpu);
@@ -448,10 +431,18 @@ void __ref enable_nonboot_cpus(void)
448 } 431 }
449 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error); 432 printk(KERN_WARNING "Error taking CPU%d up: %d\n", cpu, error);
450 } 433 }
451 cpus_clear(frozen_cpus); 434 cpumask_clear(frozen_cpus);
452out: 435out:
453 cpu_maps_update_done(); 436 cpu_maps_update_done();
454} 437}
438
439static int alloc_frozen_cpus(void)
440{
441 if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
442 return -ENOMEM;
443 return 0;
444}
445core_initcall(alloc_frozen_cpus);
455#endif /* CONFIG_PM_SLEEP_SMP */ 446#endif /* CONFIG_PM_SLEEP_SMP */
456 447
457/** 448/**
@@ -462,12 +453,12 @@ out:
462 * It must be called by the arch code on the new cpu, before the new cpu 453 * It must be called by the arch code on the new cpu, before the new cpu
463 * enables interrupts and before the "boot" cpu returns from __cpu_up(). 454 * enables interrupts and before the "boot" cpu returns from __cpu_up().
464 */ 455 */
465void notify_cpu_starting(unsigned int cpu) 456void __cpuinit notify_cpu_starting(unsigned int cpu)
466{ 457{
467 unsigned long val = CPU_STARTING; 458 unsigned long val = CPU_STARTING;
468 459
469#ifdef CONFIG_PM_SLEEP_SMP 460#ifdef CONFIG_PM_SLEEP_SMP
470 if (cpu_isset(cpu, frozen_cpus)) 461 if (frozen_cpus != NULL && cpumask_test_cpu(cpu, frozen_cpus))
471 val = CPU_STARTING_FROZEN; 462 val = CPU_STARTING_FROZEN;
472#endif /* CONFIG_PM_SLEEP_SMP */ 463#endif /* CONFIG_PM_SLEEP_SMP */
473 raw_notifier_call_chain(&cpu_chain, val, (void *)(long)cpu); 464 raw_notifier_call_chain(&cpu_chain, val, (void *)(long)cpu);
@@ -479,7 +470,7 @@ void notify_cpu_starting(unsigned int cpu)
479 * cpu_bit_bitmap[] is a special, "compressed" data structure that 470 * cpu_bit_bitmap[] is a special, "compressed" data structure that
480 * represents all NR_CPUS bits binary values of 1<<nr. 471 * represents all NR_CPUS bits binary values of 1<<nr.
481 * 472 *
482 * It is used by cpumask_of_cpu() to get a constant address to a CPU 473 * It is used by cpumask_of() to get a constant address to a CPU
483 * mask value that has a single bit set only. 474 * mask value that has a single bit set only.
484 */ 475 */
485 476
@@ -502,3 +493,71 @@ EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
502 493
503const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL; 494const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
504EXPORT_SYMBOL(cpu_all_bits); 495EXPORT_SYMBOL(cpu_all_bits);
496
497#ifdef CONFIG_INIT_ALL_POSSIBLE
498static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly
499 = CPU_BITS_ALL;
500#else
501static DECLARE_BITMAP(cpu_possible_bits, CONFIG_NR_CPUS) __read_mostly;
502#endif
503const struct cpumask *const cpu_possible_mask = to_cpumask(cpu_possible_bits);
504EXPORT_SYMBOL(cpu_possible_mask);
505
506static DECLARE_BITMAP(cpu_online_bits, CONFIG_NR_CPUS) __read_mostly;
507const struct cpumask *const cpu_online_mask = to_cpumask(cpu_online_bits);
508EXPORT_SYMBOL(cpu_online_mask);
509
510static DECLARE_BITMAP(cpu_present_bits, CONFIG_NR_CPUS) __read_mostly;
511const struct cpumask *const cpu_present_mask = to_cpumask(cpu_present_bits);
512EXPORT_SYMBOL(cpu_present_mask);
513
514static DECLARE_BITMAP(cpu_active_bits, CONFIG_NR_CPUS) __read_mostly;
515const struct cpumask *const cpu_active_mask = to_cpumask(cpu_active_bits);
516EXPORT_SYMBOL(cpu_active_mask);
517
518void set_cpu_possible(unsigned int cpu, bool possible)
519{
520 if (possible)
521 cpumask_set_cpu(cpu, to_cpumask(cpu_possible_bits));
522 else
523 cpumask_clear_cpu(cpu, to_cpumask(cpu_possible_bits));
524}
525
526void set_cpu_present(unsigned int cpu, bool present)
527{
528 if (present)
529 cpumask_set_cpu(cpu, to_cpumask(cpu_present_bits));
530 else
531 cpumask_clear_cpu(cpu, to_cpumask(cpu_present_bits));
532}
533
534void set_cpu_online(unsigned int cpu, bool online)
535{
536 if (online)
537 cpumask_set_cpu(cpu, to_cpumask(cpu_online_bits));
538 else
539 cpumask_clear_cpu(cpu, to_cpumask(cpu_online_bits));
540}
541
542void set_cpu_active(unsigned int cpu, bool active)
543{
544 if (active)
545 cpumask_set_cpu(cpu, to_cpumask(cpu_active_bits));
546 else
547 cpumask_clear_cpu(cpu, to_cpumask(cpu_active_bits));
548}
549
550void init_cpu_present(const struct cpumask *src)
551{
552 cpumask_copy(to_cpumask(cpu_present_bits), src);
553}
554
555void init_cpu_possible(const struct cpumask *src)
556{
557 cpumask_copy(to_cpumask(cpu_possible_bits), src);
558}
559
560void init_cpu_online(const struct cpumask *src)
561{
562 cpumask_copy(to_cpumask(cpu_online_bits), src);
563}
diff --git a/kernel/cpuset.c b/kernel/cpuset.c
index da7ff6137f37..39c1a4c1c5a9 100644
--- a/kernel/cpuset.c
+++ b/kernel/cpuset.c
@@ -585,7 +585,7 @@ static int generate_sched_domains(cpumask_t **domains,
585 int i, j, k; /* indices for partition finding loops */ 585 int i, j, k; /* indices for partition finding loops */
586 cpumask_t *doms; /* resulting partition; i.e. sched domains */ 586 cpumask_t *doms; /* resulting partition; i.e. sched domains */
587 struct sched_domain_attr *dattr; /* attributes for custom domains */ 587 struct sched_domain_attr *dattr; /* attributes for custom domains */
588 int ndoms; /* number of sched domains in result */ 588 int ndoms = 0; /* number of sched domains in result */
589 int nslot; /* next empty doms[] cpumask_t slot */ 589 int nslot; /* next empty doms[] cpumask_t slot */
590 590
591 doms = NULL; 591 doms = NULL;
@@ -896,7 +896,7 @@ static int update_cpumask(struct cpuset *cs, const char *buf)
896 if (!*buf) { 896 if (!*buf) {
897 cpus_clear(trialcs.cpus_allowed); 897 cpus_clear(trialcs.cpus_allowed);
898 } else { 898 } else {
899 retval = cpulist_parse(buf, trialcs.cpus_allowed); 899 retval = cpulist_parse(buf, &trialcs.cpus_allowed);
900 if (retval < 0) 900 if (retval < 0)
901 return retval; 901 return retval;
902 902
@@ -1482,7 +1482,7 @@ static int cpuset_sprintf_cpulist(char *page, struct cpuset *cs)
1482 mask = cs->cpus_allowed; 1482 mask = cs->cpus_allowed;
1483 mutex_unlock(&callback_mutex); 1483 mutex_unlock(&callback_mutex);
1484 1484
1485 return cpulist_scnprintf(page, PAGE_SIZE, mask); 1485 return cpulist_scnprintf(page, PAGE_SIZE, &mask);
1486} 1486}
1487 1487
1488static int cpuset_sprintf_memlist(char *page, struct cpuset *cs) 1488static int cpuset_sprintf_memlist(char *page, struct cpuset *cs)
diff --git a/kernel/cred-internals.h b/kernel/cred-internals.h
new file mode 100644
index 000000000000..2dc4fc2d0bf1
--- /dev/null
+++ b/kernel/cred-internals.h
@@ -0,0 +1,21 @@
1/* Internal credentials stuff
2 *
3 * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public Licence
8 * as published by the Free Software Foundation; either version
9 * 2 of the Licence, or (at your option) any later version.
10 */
11
12/*
13 * user.c
14 */
15static inline void sched_switch_user(struct task_struct *p)
16{
17#ifdef CONFIG_USER_SCHED
18 sched_move_task(p);
19#endif /* CONFIG_USER_SCHED */
20}
21
diff --git a/kernel/cred.c b/kernel/cred.c
new file mode 100644
index 000000000000..ff7bc071991c
--- /dev/null
+++ b/kernel/cred.c
@@ -0,0 +1,588 @@
1/* Task credentials management - see Documentation/credentials.txt
2 *
3 * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public Licence
8 * as published by the Free Software Foundation; either version
9 * 2 of the Licence, or (at your option) any later version.
10 */
11#include <linux/module.h>
12#include <linux/cred.h>
13#include <linux/sched.h>
14#include <linux/key.h>
15#include <linux/keyctl.h>
16#include <linux/init_task.h>
17#include <linux/security.h>
18#include <linux/cn_proc.h>
19#include "cred-internals.h"
20
21static struct kmem_cache *cred_jar;
22
23/*
24 * The common credentials for the initial task's thread group
25 */
26#ifdef CONFIG_KEYS
27static struct thread_group_cred init_tgcred = {
28 .usage = ATOMIC_INIT(2),
29 .tgid = 0,
30 .lock = SPIN_LOCK_UNLOCKED,
31};
32#endif
33
34/*
35 * The initial credentials for the initial task
36 */
37struct cred init_cred = {
38 .usage = ATOMIC_INIT(4),
39 .securebits = SECUREBITS_DEFAULT,
40 .cap_inheritable = CAP_INIT_INH_SET,
41 .cap_permitted = CAP_FULL_SET,
42 .cap_effective = CAP_INIT_EFF_SET,
43 .cap_bset = CAP_INIT_BSET,
44 .user = INIT_USER,
45 .group_info = &init_groups,
46#ifdef CONFIG_KEYS
47 .tgcred = &init_tgcred,
48#endif
49};
50
51/*
52 * Dispose of the shared task group credentials
53 */
54#ifdef CONFIG_KEYS
55static void release_tgcred_rcu(struct rcu_head *rcu)
56{
57 struct thread_group_cred *tgcred =
58 container_of(rcu, struct thread_group_cred, rcu);
59
60 BUG_ON(atomic_read(&tgcred->usage) != 0);
61
62 key_put(tgcred->session_keyring);
63 key_put(tgcred->process_keyring);
64 kfree(tgcred);
65}
66#endif
67
68/*
69 * Release a set of thread group credentials.
70 */
71static void release_tgcred(struct cred *cred)
72{
73#ifdef CONFIG_KEYS
74 struct thread_group_cred *tgcred = cred->tgcred;
75
76 if (atomic_dec_and_test(&tgcred->usage))
77 call_rcu(&tgcred->rcu, release_tgcred_rcu);
78#endif
79}
80
81/*
82 * The RCU callback to actually dispose of a set of credentials
83 */
84static void put_cred_rcu(struct rcu_head *rcu)
85{
86 struct cred *cred = container_of(rcu, struct cred, rcu);
87
88 if (atomic_read(&cred->usage) != 0)
89 panic("CRED: put_cred_rcu() sees %p with usage %d\n",
90 cred, atomic_read(&cred->usage));
91
92 security_cred_free(cred);
93 key_put(cred->thread_keyring);
94 key_put(cred->request_key_auth);
95 release_tgcred(cred);
96 put_group_info(cred->group_info);
97 free_uid(cred->user);
98 kmem_cache_free(cred_jar, cred);
99}
100
101/**
102 * __put_cred - Destroy a set of credentials
103 * @cred: The record to release
104 *
105 * Destroy a set of credentials on which no references remain.
106 */
107void __put_cred(struct cred *cred)
108{
109 BUG_ON(atomic_read(&cred->usage) != 0);
110
111 call_rcu(&cred->rcu, put_cred_rcu);
112}
113EXPORT_SYMBOL(__put_cred);
114
115/**
116 * prepare_creds - Prepare a new set of credentials for modification
117 *
118 * Prepare a new set of task credentials for modification. A task's creds
119 * shouldn't generally be modified directly, therefore this function is used to
120 * prepare a new copy, which the caller then modifies and then commits by
121 * calling commit_creds().
122 *
123 * Preparation involves making a copy of the objective creds for modification.
124 *
125 * Returns a pointer to the new creds-to-be if successful, NULL otherwise.
126 *
127 * Call commit_creds() or abort_creds() to clean up.
128 */
129struct cred *prepare_creds(void)
130{
131 struct task_struct *task = current;
132 const struct cred *old;
133 struct cred *new;
134
135 BUG_ON(atomic_read(&task->real_cred->usage) < 1);
136
137 new = kmem_cache_alloc(cred_jar, GFP_KERNEL);
138 if (!new)
139 return NULL;
140
141 old = task->cred;
142 memcpy(new, old, sizeof(struct cred));
143
144 atomic_set(&new->usage, 1);
145 get_group_info(new->group_info);
146 get_uid(new->user);
147
148#ifdef CONFIG_KEYS
149 key_get(new->thread_keyring);
150 key_get(new->request_key_auth);
151 atomic_inc(&new->tgcred->usage);
152#endif
153
154#ifdef CONFIG_SECURITY
155 new->security = NULL;
156#endif
157
158 if (security_prepare_creds(new, old, GFP_KERNEL) < 0)
159 goto error;
160 return new;
161
162error:
163 abort_creds(new);
164 return NULL;
165}
166EXPORT_SYMBOL(prepare_creds);
167
168/*
169 * Prepare credentials for current to perform an execve()
170 * - The caller must hold current->cred_exec_mutex
171 */
172struct cred *prepare_exec_creds(void)
173{
174 struct thread_group_cred *tgcred = NULL;
175 struct cred *new;
176
177#ifdef CONFIG_KEYS
178 tgcred = kmalloc(sizeof(*tgcred), GFP_KERNEL);
179 if (!tgcred)
180 return NULL;
181#endif
182
183 new = prepare_creds();
184 if (!new) {
185 kfree(tgcred);
186 return new;
187 }
188
189#ifdef CONFIG_KEYS
190 /* newly exec'd tasks don't get a thread keyring */
191 key_put(new->thread_keyring);
192 new->thread_keyring = NULL;
193
194 /* create a new per-thread-group creds for all this set of threads to
195 * share */
196 memcpy(tgcred, new->tgcred, sizeof(struct thread_group_cred));
197
198 atomic_set(&tgcred->usage, 1);
199 spin_lock_init(&tgcred->lock);
200
201 /* inherit the session keyring; new process keyring */
202 key_get(tgcred->session_keyring);
203 tgcred->process_keyring = NULL;
204
205 release_tgcred(new);
206 new->tgcred = tgcred;
207#endif
208
209 return new;
210}
211
212/*
213 * prepare new credentials for the usermode helper dispatcher
214 */
215struct cred *prepare_usermodehelper_creds(void)
216{
217#ifdef CONFIG_KEYS
218 struct thread_group_cred *tgcred = NULL;
219#endif
220 struct cred *new;
221
222#ifdef CONFIG_KEYS
223 tgcred = kzalloc(sizeof(*new->tgcred), GFP_ATOMIC);
224 if (!tgcred)
225 return NULL;
226#endif
227
228 new = kmem_cache_alloc(cred_jar, GFP_ATOMIC);
229 if (!new)
230 return NULL;
231
232 memcpy(new, &init_cred, sizeof(struct cred));
233
234 atomic_set(&new->usage, 1);
235 get_group_info(new->group_info);
236 get_uid(new->user);
237
238#ifdef CONFIG_KEYS
239 new->thread_keyring = NULL;
240 new->request_key_auth = NULL;
241 new->jit_keyring = KEY_REQKEY_DEFL_DEFAULT;
242
243 atomic_set(&tgcred->usage, 1);
244 spin_lock_init(&tgcred->lock);
245 new->tgcred = tgcred;
246#endif
247
248#ifdef CONFIG_SECURITY
249 new->security = NULL;
250#endif
251 if (security_prepare_creds(new, &init_cred, GFP_ATOMIC) < 0)
252 goto error;
253
254 BUG_ON(atomic_read(&new->usage) != 1);
255 return new;
256
257error:
258 put_cred(new);
259 return NULL;
260}
261
262/*
263 * Copy credentials for the new process created by fork()
264 *
265 * We share if we can, but under some circumstances we have to generate a new
266 * set.
267 *
268 * The new process gets the current process's subjective credentials as its
269 * objective and subjective credentials
270 */
271int copy_creds(struct task_struct *p, unsigned long clone_flags)
272{
273#ifdef CONFIG_KEYS
274 struct thread_group_cred *tgcred;
275#endif
276 struct cred *new;
277 int ret;
278
279 mutex_init(&p->cred_exec_mutex);
280
281 if (
282#ifdef CONFIG_KEYS
283 !p->cred->thread_keyring &&
284#endif
285 clone_flags & CLONE_THREAD
286 ) {
287 p->real_cred = get_cred(p->cred);
288 get_cred(p->cred);
289 atomic_inc(&p->cred->user->processes);
290 return 0;
291 }
292
293 new = prepare_creds();
294 if (!new)
295 return -ENOMEM;
296
297 if (clone_flags & CLONE_NEWUSER) {
298 ret = create_user_ns(new);
299 if (ret < 0)
300 goto error_put;
301 }
302
303#ifdef CONFIG_KEYS
304 /* new threads get their own thread keyrings if their parent already
305 * had one */
306 if (new->thread_keyring) {
307 key_put(new->thread_keyring);
308 new->thread_keyring = NULL;
309 if (clone_flags & CLONE_THREAD)
310 install_thread_keyring_to_cred(new);
311 }
312
313 /* we share the process and session keyrings between all the threads in
314 * a process - this is slightly icky as we violate COW credentials a
315 * bit */
316 if (!(clone_flags & CLONE_THREAD)) {
317 tgcred = kmalloc(sizeof(*tgcred), GFP_KERNEL);
318 if (!tgcred) {
319 ret = -ENOMEM;
320 goto error_put;
321 }
322 atomic_set(&tgcred->usage, 1);
323 spin_lock_init(&tgcred->lock);
324 tgcred->process_keyring = NULL;
325 tgcred->session_keyring = key_get(new->tgcred->session_keyring);
326
327 release_tgcred(new);
328 new->tgcred = tgcred;
329 }
330#endif
331
332 atomic_inc(&new->user->processes);
333 p->cred = p->real_cred = get_cred(new);
334 return 0;
335
336error_put:
337 put_cred(new);
338 return ret;
339}
340
341/**
342 * commit_creds - Install new credentials upon the current task
343 * @new: The credentials to be assigned
344 *
345 * Install a new set of credentials to the current task, using RCU to replace
346 * the old set. Both the objective and the subjective credentials pointers are
347 * updated. This function may not be called if the subjective credentials are
348 * in an overridden state.
349 *
350 * This function eats the caller's reference to the new credentials.
351 *
352 * Always returns 0 thus allowing this function to be tail-called at the end
353 * of, say, sys_setgid().
354 */
355int commit_creds(struct cred *new)
356{
357 struct task_struct *task = current;
358 const struct cred *old;
359
360 BUG_ON(task->cred != task->real_cred);
361 BUG_ON(atomic_read(&task->real_cred->usage) < 2);
362 BUG_ON(atomic_read(&new->usage) < 1);
363
364 old = task->real_cred;
365 security_commit_creds(new, old);
366
367 get_cred(new); /* we will require a ref for the subj creds too */
368
369 /* dumpability changes */
370 if (old->euid != new->euid ||
371 old->egid != new->egid ||
372 old->fsuid != new->fsuid ||
373 old->fsgid != new->fsgid ||
374 !cap_issubset(new->cap_permitted, old->cap_permitted)) {
375 set_dumpable(task->mm, suid_dumpable);
376 task->pdeath_signal = 0;
377 smp_wmb();
378 }
379
380 /* alter the thread keyring */
381 if (new->fsuid != old->fsuid)
382 key_fsuid_changed(task);
383 if (new->fsgid != old->fsgid)
384 key_fsgid_changed(task);
385
386 /* do it
387 * - What if a process setreuid()'s and this brings the
388 * new uid over his NPROC rlimit? We can check this now
389 * cheaply with the new uid cache, so if it matters
390 * we should be checking for it. -DaveM
391 */
392 if (new->user != old->user)
393 atomic_inc(&new->user->processes);
394 rcu_assign_pointer(task->real_cred, new);
395 rcu_assign_pointer(task->cred, new);
396 if (new->user != old->user)
397 atomic_dec(&old->user->processes);
398
399 sched_switch_user(task);
400
401 /* send notifications */
402 if (new->uid != old->uid ||
403 new->euid != old->euid ||
404 new->suid != old->suid ||
405 new->fsuid != old->fsuid)
406 proc_id_connector(task, PROC_EVENT_UID);
407
408 if (new->gid != old->gid ||
409 new->egid != old->egid ||
410 new->sgid != old->sgid ||
411 new->fsgid != old->fsgid)
412 proc_id_connector(task, PROC_EVENT_GID);
413
414 /* release the old obj and subj refs both */
415 put_cred(old);
416 put_cred(old);
417 return 0;
418}
419EXPORT_SYMBOL(commit_creds);
420
421/**
422 * abort_creds - Discard a set of credentials and unlock the current task
423 * @new: The credentials that were going to be applied
424 *
425 * Discard a set of credentials that were under construction and unlock the
426 * current task.
427 */
428void abort_creds(struct cred *new)
429{
430 BUG_ON(atomic_read(&new->usage) < 1);
431 put_cred(new);
432}
433EXPORT_SYMBOL(abort_creds);
434
435/**
436 * override_creds - Override the current process's subjective credentials
437 * @new: The credentials to be assigned
438 *
439 * Install a set of temporary override subjective credentials on the current
440 * process, returning the old set for later reversion.
441 */
442const struct cred *override_creds(const struct cred *new)
443{
444 const struct cred *old = current->cred;
445
446 rcu_assign_pointer(current->cred, get_cred(new));
447 return old;
448}
449EXPORT_SYMBOL(override_creds);
450
451/**
452 * revert_creds - Revert a temporary subjective credentials override
453 * @old: The credentials to be restored
454 *
455 * Revert a temporary set of override subjective credentials to an old set,
456 * discarding the override set.
457 */
458void revert_creds(const struct cred *old)
459{
460 const struct cred *override = current->cred;
461
462 rcu_assign_pointer(current->cred, old);
463 put_cred(override);
464}
465EXPORT_SYMBOL(revert_creds);
466
467/*
468 * initialise the credentials stuff
469 */
470void __init cred_init(void)
471{
472 /* allocate a slab in which we can store credentials */
473 cred_jar = kmem_cache_create("cred_jar", sizeof(struct cred),
474 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
475}
476
477/**
478 * prepare_kernel_cred - Prepare a set of credentials for a kernel service
479 * @daemon: A userspace daemon to be used as a reference
480 *
481 * Prepare a set of credentials for a kernel service. This can then be used to
482 * override a task's own credentials so that work can be done on behalf of that
483 * task that requires a different subjective context.
484 *
485 * @daemon is used to provide a base for the security record, but can be NULL.
486 * If @daemon is supplied, then the security data will be derived from that;
487 * otherwise they'll be set to 0 and no groups, full capabilities and no keys.
488 *
489 * The caller may change these controls afterwards if desired.
490 *
491 * Returns the new credentials or NULL if out of memory.
492 *
493 * Does not take, and does not return holding current->cred_replace_mutex.
494 */
495struct cred *prepare_kernel_cred(struct task_struct *daemon)
496{
497 const struct cred *old;
498 struct cred *new;
499
500 new = kmem_cache_alloc(cred_jar, GFP_KERNEL);
501 if (!new)
502 return NULL;
503
504 if (daemon)
505 old = get_task_cred(daemon);
506 else
507 old = get_cred(&init_cred);
508
509 get_uid(new->user);
510 get_group_info(new->group_info);
511
512#ifdef CONFIG_KEYS
513 atomic_inc(&init_tgcred.usage);
514 new->tgcred = &init_tgcred;
515 new->request_key_auth = NULL;
516 new->thread_keyring = NULL;
517 new->jit_keyring = KEY_REQKEY_DEFL_THREAD_KEYRING;
518#endif
519
520#ifdef CONFIG_SECURITY
521 new->security = NULL;
522#endif
523 if (security_prepare_creds(new, old, GFP_KERNEL) < 0)
524 goto error;
525
526 atomic_set(&new->usage, 1);
527 put_cred(old);
528 return new;
529
530error:
531 put_cred(new);
532 return NULL;
533}
534EXPORT_SYMBOL(prepare_kernel_cred);
535
536/**
537 * set_security_override - Set the security ID in a set of credentials
538 * @new: The credentials to alter
539 * @secid: The LSM security ID to set
540 *
541 * Set the LSM security ID in a set of credentials so that the subjective
542 * security is overridden when an alternative set of credentials is used.
543 */
544int set_security_override(struct cred *new, u32 secid)
545{
546 return security_kernel_act_as(new, secid);
547}
548EXPORT_SYMBOL(set_security_override);
549
550/**
551 * set_security_override_from_ctx - Set the security ID in a set of credentials
552 * @new: The credentials to alter
553 * @secctx: The LSM security context to generate the security ID from.
554 *
555 * Set the LSM security ID in a set of credentials so that the subjective
556 * security is overridden when an alternative set of credentials is used. The
557 * security ID is specified in string form as a security context to be
558 * interpreted by the LSM.
559 */
560int set_security_override_from_ctx(struct cred *new, const char *secctx)
561{
562 u32 secid;
563 int ret;
564
565 ret = security_secctx_to_secid(secctx, strlen(secctx), &secid);
566 if (ret < 0)
567 return ret;
568
569 return set_security_override(new, secid);
570}
571EXPORT_SYMBOL(set_security_override_from_ctx);
572
573/**
574 * set_create_files_as - Set the LSM file create context in a set of credentials
575 * @new: The credentials to alter
576 * @inode: The inode to take the context from
577 *
578 * Change the LSM file creation context in a set of credentials to be the same
579 * as the object context of the specified inode, so that the new inodes have
580 * the same MAC context as that inode.
581 */
582int set_create_files_as(struct cred *new, struct inode *inode)
583{
584 new->fsuid = inode->i_uid;
585 new->fsgid = inode->i_gid;
586 return security_kernel_create_files_as(new, inode);
587}
588EXPORT_SYMBOL(set_create_files_as);
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index b3179dad71be..abb6e17505e2 100644
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -127,7 +127,7 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
127 */ 127 */
128 t1 = tsk->sched_info.pcount; 128 t1 = tsk->sched_info.pcount;
129 t2 = tsk->sched_info.run_delay; 129 t2 = tsk->sched_info.run_delay;
130 t3 = tsk->sched_info.cpu_time; 130 t3 = tsk->se.sum_exec_runtime;
131 131
132 d->cpu_count += t1; 132 d->cpu_count += t1;
133 133
diff --git a/kernel/exit.c b/kernel/exit.c
index 2d8be7ebb0f7..c9e5a1c14e08 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -46,12 +46,18 @@
46#include <linux/blkdev.h> 46#include <linux/blkdev.h>
47#include <linux/task_io_accounting_ops.h> 47#include <linux/task_io_accounting_ops.h>
48#include <linux/tracehook.h> 48#include <linux/tracehook.h>
49#include <linux/init_task.h>
49#include <trace/sched.h> 50#include <trace/sched.h>
50 51
51#include <asm/uaccess.h> 52#include <asm/uaccess.h>
52#include <asm/unistd.h> 53#include <asm/unistd.h>
53#include <asm/pgtable.h> 54#include <asm/pgtable.h>
54#include <asm/mmu_context.h> 55#include <asm/mmu_context.h>
56#include "cred-internals.h"
57
58DEFINE_TRACE(sched_process_free);
59DEFINE_TRACE(sched_process_exit);
60DEFINE_TRACE(sched_process_wait);
55 61
56static void exit_mm(struct task_struct * tsk); 62static void exit_mm(struct task_struct * tsk);
57 63
@@ -164,7 +170,10 @@ void release_task(struct task_struct * p)
164 int zap_leader; 170 int zap_leader;
165repeat: 171repeat:
166 tracehook_prepare_release_task(p); 172 tracehook_prepare_release_task(p);
167 atomic_dec(&p->user->processes); 173 /* don't need to get the RCU readlock here - the process is dead and
174 * can't be modifying its own credentials */
175 atomic_dec(&__task_cred(p)->user->processes);
176
168 proc_flush_task(p); 177 proc_flush_task(p);
169 write_lock_irq(&tasklist_lock); 178 write_lock_irq(&tasklist_lock);
170 tracehook_finish_release_task(p); 179 tracehook_finish_release_task(p);
@@ -339,12 +348,12 @@ static void reparent_to_kthreadd(void)
339 /* cpus_allowed? */ 348 /* cpus_allowed? */
340 /* rt_priority? */ 349 /* rt_priority? */
341 /* signals? */ 350 /* signals? */
342 security_task_reparent_to_init(current);
343 memcpy(current->signal->rlim, init_task.signal->rlim, 351 memcpy(current->signal->rlim, init_task.signal->rlim,
344 sizeof(current->signal->rlim)); 352 sizeof(current->signal->rlim));
345 atomic_inc(&(INIT_USER->__count)); 353
354 atomic_inc(&init_cred.usage);
355 commit_creds(&init_cred);
346 write_unlock_irq(&tasklist_lock); 356 write_unlock_irq(&tasklist_lock);
347 switch_uid(INIT_USER);
348} 357}
349 358
350void __set_special_pids(struct pid *pid) 359void __set_special_pids(struct pid *pid)
@@ -1028,8 +1037,6 @@ NORET_TYPE void do_exit(long code)
1028 * task into the wait for ever nirwana as well. 1037 * task into the wait for ever nirwana as well.
1029 */ 1038 */
1030 tsk->flags |= PF_EXITPIDONE; 1039 tsk->flags |= PF_EXITPIDONE;
1031 if (tsk->io_context)
1032 exit_io_context();
1033 set_current_state(TASK_UNINTERRUPTIBLE); 1040 set_current_state(TASK_UNINTERRUPTIBLE);
1034 schedule(); 1041 schedule();
1035 } 1042 }
@@ -1078,7 +1085,6 @@ NORET_TYPE void do_exit(long code)
1078 check_stack_usage(); 1085 check_stack_usage();
1079 exit_thread(); 1086 exit_thread();
1080 cgroup_exit(tsk, 1); 1087 cgroup_exit(tsk, 1);
1081 exit_keys(tsk);
1082 1088
1083 if (group_dead && tsk->signal->leader) 1089 if (group_dead && tsk->signal->leader)
1084 disassociate_ctty(1); 1090 disassociate_ctty(1);
@@ -1123,7 +1129,6 @@ NORET_TYPE void do_exit(long code)
1123 preempt_disable(); 1129 preempt_disable();
1124 /* causes final put_task_struct in finish_task_switch(). */ 1130 /* causes final put_task_struct in finish_task_switch(). */
1125 tsk->state = TASK_DEAD; 1131 tsk->state = TASK_DEAD;
1126
1127 schedule(); 1132 schedule();
1128 BUG(); 1133 BUG();
1129 /* Avoid "noreturn function does return". */ 1134 /* Avoid "noreturn function does return". */
@@ -1263,12 +1268,12 @@ static int wait_task_zombie(struct task_struct *p, int options,
1263 unsigned long state; 1268 unsigned long state;
1264 int retval, status, traced; 1269 int retval, status, traced;
1265 pid_t pid = task_pid_vnr(p); 1270 pid_t pid = task_pid_vnr(p);
1271 uid_t uid = __task_cred(p)->uid;
1266 1272
1267 if (!likely(options & WEXITED)) 1273 if (!likely(options & WEXITED))
1268 return 0; 1274 return 0;
1269 1275
1270 if (unlikely(options & WNOWAIT)) { 1276 if (unlikely(options & WNOWAIT)) {
1271 uid_t uid = p->uid;
1272 int exit_code = p->exit_code; 1277 int exit_code = p->exit_code;
1273 int why, status; 1278 int why, status;
1274 1279
@@ -1321,10 +1326,10 @@ static int wait_task_zombie(struct task_struct *p, int options,
1321 * group, which consolidates times for all threads in the 1326 * group, which consolidates times for all threads in the
1322 * group including the group leader. 1327 * group including the group leader.
1323 */ 1328 */
1329 thread_group_cputime(p, &cputime);
1324 spin_lock_irq(&p->parent->sighand->siglock); 1330 spin_lock_irq(&p->parent->sighand->siglock);
1325 psig = p->parent->signal; 1331 psig = p->parent->signal;
1326 sig = p->signal; 1332 sig = p->signal;
1327 thread_group_cputime(p, &cputime);
1328 psig->cutime = 1333 psig->cutime =
1329 cputime_add(psig->cutime, 1334 cputime_add(psig->cutime,
1330 cputime_add(cputime.utime, 1335 cputime_add(cputime.utime,
@@ -1389,7 +1394,7 @@ static int wait_task_zombie(struct task_struct *p, int options,
1389 if (!retval && infop) 1394 if (!retval && infop)
1390 retval = put_user(pid, &infop->si_pid); 1395 retval = put_user(pid, &infop->si_pid);
1391 if (!retval && infop) 1396 if (!retval && infop)
1392 retval = put_user(p->uid, &infop->si_uid); 1397 retval = put_user(uid, &infop->si_uid);
1393 if (!retval) 1398 if (!retval)
1394 retval = pid; 1399 retval = pid;
1395 1400
@@ -1454,7 +1459,8 @@ static int wait_task_stopped(int ptrace, struct task_struct *p,
1454 if (!unlikely(options & WNOWAIT)) 1459 if (!unlikely(options & WNOWAIT))
1455 p->exit_code = 0; 1460 p->exit_code = 0;
1456 1461
1457 uid = p->uid; 1462 /* don't need the RCU readlock here as we're holding a spinlock */
1463 uid = __task_cred(p)->uid;
1458unlock_sig: 1464unlock_sig:
1459 spin_unlock_irq(&p->sighand->siglock); 1465 spin_unlock_irq(&p->sighand->siglock);
1460 if (!exit_code) 1466 if (!exit_code)
@@ -1528,10 +1534,10 @@ static int wait_task_continued(struct task_struct *p, int options,
1528 } 1534 }
1529 if (!unlikely(options & WNOWAIT)) 1535 if (!unlikely(options & WNOWAIT))
1530 p->signal->flags &= ~SIGNAL_STOP_CONTINUED; 1536 p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
1537 uid = __task_cred(p)->uid;
1531 spin_unlock_irq(&p->sighand->siglock); 1538 spin_unlock_irq(&p->sighand->siglock);
1532 1539
1533 pid = task_pid_vnr(p); 1540 pid = task_pid_vnr(p);
1534 uid = p->uid;
1535 get_task_struct(p); 1541 get_task_struct(p);
1536 read_unlock(&tasklist_lock); 1542 read_unlock(&tasklist_lock);
1537 1543
diff --git a/kernel/extable.c b/kernel/extable.c
index a26cb2e17023..e136ed8d82ba 100644
--- a/kernel/extable.c
+++ b/kernel/extable.c
@@ -17,6 +17,7 @@
17*/ 17*/
18#include <linux/module.h> 18#include <linux/module.h>
19#include <linux/init.h> 19#include <linux/init.h>
20#include <linux/ftrace.h>
20#include <asm/uaccess.h> 21#include <asm/uaccess.h>
21#include <asm/sections.h> 22#include <asm/sections.h>
22 23
@@ -40,7 +41,7 @@ const struct exception_table_entry *search_exception_tables(unsigned long addr)
40 return e; 41 return e;
41} 42}
42 43
43int core_kernel_text(unsigned long addr) 44__notrace_funcgraph int core_kernel_text(unsigned long addr)
44{ 45{
45 if (addr >= (unsigned long)_stext && 46 if (addr >= (unsigned long)_stext &&
46 addr <= (unsigned long)_etext) 47 addr <= (unsigned long)_etext)
@@ -53,7 +54,7 @@ int core_kernel_text(unsigned long addr)
53 return 0; 54 return 0;
54} 55}
55 56
56int __kernel_text_address(unsigned long addr) 57__notrace_funcgraph int __kernel_text_address(unsigned long addr)
57{ 58{
58 if (core_kernel_text(addr)) 59 if (core_kernel_text(addr))
59 return 1; 60 return 1;
@@ -66,3 +67,19 @@ int kernel_text_address(unsigned long addr)
66 return 1; 67 return 1;
67 return module_text_address(addr) != NULL; 68 return module_text_address(addr) != NULL;
68} 69}
70
71/*
72 * On some architectures (PPC64, IA64) function pointers
73 * are actually only tokens to some data that then holds the
74 * real function address. As a result, to find if a function
75 * pointer is part of the kernel text, we need to do some
76 * special dereferencing first.
77 */
78int func_ptr_is_kernel_text(void *ptr)
79{
80 unsigned long addr;
81 addr = (unsigned long) dereference_function_descriptor(ptr);
82 if (core_kernel_text(addr))
83 return 1;
84 return module_text_address(addr) != NULL;
85}
diff --git a/kernel/fork.c b/kernel/fork.c
index 2a372a0e206f..43cbf30669e6 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -47,6 +47,7 @@
47#include <linux/mount.h> 47#include <linux/mount.h>
48#include <linux/audit.h> 48#include <linux/audit.h>
49#include <linux/memcontrol.h> 49#include <linux/memcontrol.h>
50#include <linux/ftrace.h>
50#include <linux/profile.h> 51#include <linux/profile.h>
51#include <linux/rmap.h> 52#include <linux/rmap.h>
52#include <linux/acct.h> 53#include <linux/acct.h>
@@ -80,6 +81,8 @@ DEFINE_PER_CPU(unsigned long, process_counts) = 0;
80 81
81__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ 82__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */
82 83
84DEFINE_TRACE(sched_process_fork);
85
83int nr_processes(void) 86int nr_processes(void)
84{ 87{
85 int cpu; 88 int cpu;
@@ -137,6 +140,7 @@ void free_task(struct task_struct *tsk)
137 prop_local_destroy_single(&tsk->dirties); 140 prop_local_destroy_single(&tsk->dirties);
138 free_thread_info(tsk->stack); 141 free_thread_info(tsk->stack);
139 rt_mutex_debug_task_free(tsk); 142 rt_mutex_debug_task_free(tsk);
143 ftrace_graph_exit_task(tsk);
140 free_task_struct(tsk); 144 free_task_struct(tsk);
141} 145}
142EXPORT_SYMBOL(free_task); 146EXPORT_SYMBOL(free_task);
@@ -147,9 +151,8 @@ void __put_task_struct(struct task_struct *tsk)
147 WARN_ON(atomic_read(&tsk->usage)); 151 WARN_ON(atomic_read(&tsk->usage));
148 WARN_ON(tsk == current); 152 WARN_ON(tsk == current);
149 153
150 security_task_free(tsk); 154 put_cred(tsk->real_cred);
151 free_uid(tsk->user); 155 put_cred(tsk->cred);
152 put_group_info(tsk->group_info);
153 delayacct_tsk_free(tsk); 156 delayacct_tsk_free(tsk);
154 157
155 if (!profile_handoff_task(tsk)) 158 if (!profile_handoff_task(tsk))
@@ -315,17 +318,20 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
315 file = tmp->vm_file; 318 file = tmp->vm_file;
316 if (file) { 319 if (file) {
317 struct inode *inode = file->f_path.dentry->d_inode; 320 struct inode *inode = file->f_path.dentry->d_inode;
321 struct address_space *mapping = file->f_mapping;
322
318 get_file(file); 323 get_file(file);
319 if (tmp->vm_flags & VM_DENYWRITE) 324 if (tmp->vm_flags & VM_DENYWRITE)
320 atomic_dec(&inode->i_writecount); 325 atomic_dec(&inode->i_writecount);
321 326 spin_lock(&mapping->i_mmap_lock);
322 /* insert tmp into the share list, just after mpnt */ 327 if (tmp->vm_flags & VM_SHARED)
323 spin_lock(&file->f_mapping->i_mmap_lock); 328 mapping->i_mmap_writable++;
324 tmp->vm_truncate_count = mpnt->vm_truncate_count; 329 tmp->vm_truncate_count = mpnt->vm_truncate_count;
325 flush_dcache_mmap_lock(file->f_mapping); 330 flush_dcache_mmap_lock(mapping);
331 /* insert tmp into the share list, just after mpnt */
326 vma_prio_tree_add(tmp, mpnt); 332 vma_prio_tree_add(tmp, mpnt);
327 flush_dcache_mmap_unlock(file->f_mapping); 333 flush_dcache_mmap_unlock(mapping);
328 spin_unlock(&file->f_mapping->i_mmap_lock); 334 spin_unlock(&mapping->i_mmap_lock);
329 } 335 }
330 336
331 /* 337 /*
@@ -409,8 +415,8 @@ static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
409 set_mm_counter(mm, file_rss, 0); 415 set_mm_counter(mm, file_rss, 0);
410 set_mm_counter(mm, anon_rss, 0); 416 set_mm_counter(mm, anon_rss, 0);
411 spin_lock_init(&mm->page_table_lock); 417 spin_lock_init(&mm->page_table_lock);
412 rwlock_init(&mm->ioctx_list_lock); 418 spin_lock_init(&mm->ioctx_lock);
413 mm->ioctx_list = NULL; 419 INIT_HLIST_HEAD(&mm->ioctx_list);
414 mm->free_area_cache = TASK_UNMAPPED_BASE; 420 mm->free_area_cache = TASK_UNMAPPED_BASE;
415 mm->cached_hole_size = ~0UL; 421 mm->cached_hole_size = ~0UL;
416 mm_init_owner(mm, p); 422 mm_init_owner(mm, p);
@@ -815,12 +821,6 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
815 if (!sig) 821 if (!sig)
816 return -ENOMEM; 822 return -ENOMEM;
817 823
818 ret = copy_thread_group_keys(tsk);
819 if (ret < 0) {
820 kmem_cache_free(signal_cachep, sig);
821 return ret;
822 }
823
824 atomic_set(&sig->count, 1); 824 atomic_set(&sig->count, 1);
825 atomic_set(&sig->live, 1); 825 atomic_set(&sig->live, 1);
826 init_waitqueue_head(&sig->wait_chldexit); 826 init_waitqueue_head(&sig->wait_chldexit);
@@ -865,7 +865,6 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
865void __cleanup_signal(struct signal_struct *sig) 865void __cleanup_signal(struct signal_struct *sig)
866{ 866{
867 thread_group_cputime_free(sig); 867 thread_group_cputime_free(sig);
868 exit_thread_group_keys(sig);
869 tty_kref_put(sig->tty); 868 tty_kref_put(sig->tty);
870 kmem_cache_free(signal_cachep, sig); 869 kmem_cache_free(signal_cachep, sig);
871} 870}
@@ -981,16 +980,16 @@ static struct task_struct *copy_process(unsigned long clone_flags,
981 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled); 980 DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
982#endif 981#endif
983 retval = -EAGAIN; 982 retval = -EAGAIN;
984 if (atomic_read(&p->user->processes) >= 983 if (atomic_read(&p->real_cred->user->processes) >=
985 p->signal->rlim[RLIMIT_NPROC].rlim_cur) { 984 p->signal->rlim[RLIMIT_NPROC].rlim_cur) {
986 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) && 985 if (!capable(CAP_SYS_ADMIN) && !capable(CAP_SYS_RESOURCE) &&
987 p->user != current->nsproxy->user_ns->root_user) 986 p->real_cred->user != INIT_USER)
988 goto bad_fork_free; 987 goto bad_fork_free;
989 } 988 }
990 989
991 atomic_inc(&p->user->__count); 990 retval = copy_creds(p, clone_flags);
992 atomic_inc(&p->user->processes); 991 if (retval < 0)
993 get_group_info(p->group_info); 992 goto bad_fork_free;
994 993
995 /* 994 /*
996 * If multiple threads are within copy_process(), then this check 995 * If multiple threads are within copy_process(), then this check
@@ -1045,10 +1044,6 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1045 do_posix_clock_monotonic_gettime(&p->start_time); 1044 do_posix_clock_monotonic_gettime(&p->start_time);
1046 p->real_start_time = p->start_time; 1045 p->real_start_time = p->start_time;
1047 monotonic_to_bootbased(&p->real_start_time); 1046 monotonic_to_bootbased(&p->real_start_time);
1048#ifdef CONFIG_SECURITY
1049 p->security = NULL;
1050#endif
1051 p->cap_bset = current->cap_bset;
1052 p->io_context = NULL; 1047 p->io_context = NULL;
1053 p->audit_context = NULL; 1048 p->audit_context = NULL;
1054 cgroup_fork(p); 1049 cgroup_fork(p);
@@ -1089,14 +1084,14 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1089#ifdef CONFIG_DEBUG_MUTEXES 1084#ifdef CONFIG_DEBUG_MUTEXES
1090 p->blocked_on = NULL; /* not blocked yet */ 1085 p->blocked_on = NULL; /* not blocked yet */
1091#endif 1086#endif
1087 if (unlikely(ptrace_reparented(current)))
1088 ptrace_fork(p, clone_flags);
1092 1089
1093 /* Perform scheduler related setup. Assign this task to a CPU. */ 1090 /* Perform scheduler related setup. Assign this task to a CPU. */
1094 sched_fork(p, clone_flags); 1091 sched_fork(p, clone_flags);
1095 1092
1096 if ((retval = security_task_alloc(p)))
1097 goto bad_fork_cleanup_policy;
1098 if ((retval = audit_alloc(p))) 1093 if ((retval = audit_alloc(p)))
1099 goto bad_fork_cleanup_security; 1094 goto bad_fork_cleanup_policy;
1100 /* copy all the process information */ 1095 /* copy all the process information */
1101 if ((retval = copy_semundo(clone_flags, p))) 1096 if ((retval = copy_semundo(clone_flags, p)))
1102 goto bad_fork_cleanup_audit; 1097 goto bad_fork_cleanup_audit;
@@ -1110,10 +1105,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1110 goto bad_fork_cleanup_sighand; 1105 goto bad_fork_cleanup_sighand;
1111 if ((retval = copy_mm(clone_flags, p))) 1106 if ((retval = copy_mm(clone_flags, p)))
1112 goto bad_fork_cleanup_signal; 1107 goto bad_fork_cleanup_signal;
1113 if ((retval = copy_keys(clone_flags, p)))
1114 goto bad_fork_cleanup_mm;
1115 if ((retval = copy_namespaces(clone_flags, p))) 1108 if ((retval = copy_namespaces(clone_flags, p)))
1116 goto bad_fork_cleanup_keys; 1109 goto bad_fork_cleanup_mm;
1117 if ((retval = copy_io(clone_flags, p))) 1110 if ((retval = copy_io(clone_flags, p)))
1118 goto bad_fork_cleanup_namespaces; 1111 goto bad_fork_cleanup_namespaces;
1119 retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs); 1112 retval = copy_thread(0, clone_flags, stack_start, stack_size, p, regs);
@@ -1133,6 +1126,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1133 } 1126 }
1134 } 1127 }
1135 1128
1129 ftrace_graph_init_task(p);
1130
1136 p->pid = pid_nr(pid); 1131 p->pid = pid_nr(pid);
1137 p->tgid = p->pid; 1132 p->tgid = p->pid;
1138 if (clone_flags & CLONE_THREAD) 1133 if (clone_flags & CLONE_THREAD)
@@ -1141,7 +1136,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1141 if (current->nsproxy != p->nsproxy) { 1136 if (current->nsproxy != p->nsproxy) {
1142 retval = ns_cgroup_clone(p, pid); 1137 retval = ns_cgroup_clone(p, pid);
1143 if (retval) 1138 if (retval)
1144 goto bad_fork_free_pid; 1139 goto bad_fork_free_graph;
1145 } 1140 }
1146 1141
1147 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL; 1142 p->set_child_tid = (clone_flags & CLONE_CHILD_SETTID) ? child_tidptr : NULL;
@@ -1234,7 +1229,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1234 spin_unlock(&current->sighand->siglock); 1229 spin_unlock(&current->sighand->siglock);
1235 write_unlock_irq(&tasklist_lock); 1230 write_unlock_irq(&tasklist_lock);
1236 retval = -ERESTARTNOINTR; 1231 retval = -ERESTARTNOINTR;
1237 goto bad_fork_free_pid; 1232 goto bad_fork_free_graph;
1238 } 1233 }
1239 1234
1240 if (clone_flags & CLONE_THREAD) { 1235 if (clone_flags & CLONE_THREAD) {
@@ -1271,6 +1266,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1271 cgroup_post_fork(p); 1266 cgroup_post_fork(p);
1272 return p; 1267 return p;
1273 1268
1269bad_fork_free_graph:
1270 ftrace_graph_exit_task(p);
1274bad_fork_free_pid: 1271bad_fork_free_pid:
1275 if (pid != &init_struct_pid) 1272 if (pid != &init_struct_pid)
1276 free_pid(pid); 1273 free_pid(pid);
@@ -1278,8 +1275,6 @@ bad_fork_cleanup_io:
1278 put_io_context(p->io_context); 1275 put_io_context(p->io_context);
1279bad_fork_cleanup_namespaces: 1276bad_fork_cleanup_namespaces:
1280 exit_task_namespaces(p); 1277 exit_task_namespaces(p);
1281bad_fork_cleanup_keys:
1282 exit_keys(p);
1283bad_fork_cleanup_mm: 1278bad_fork_cleanup_mm:
1284 if (p->mm) 1279 if (p->mm)
1285 mmput(p->mm); 1280 mmput(p->mm);
@@ -1295,8 +1290,6 @@ bad_fork_cleanup_semundo:
1295 exit_sem(p); 1290 exit_sem(p);
1296bad_fork_cleanup_audit: 1291bad_fork_cleanup_audit:
1297 audit_free(p); 1292 audit_free(p);
1298bad_fork_cleanup_security:
1299 security_task_free(p);
1300bad_fork_cleanup_policy: 1293bad_fork_cleanup_policy:
1301#ifdef CONFIG_NUMA 1294#ifdef CONFIG_NUMA
1302 mpol_put(p->mempolicy); 1295 mpol_put(p->mempolicy);
@@ -1309,9 +1302,9 @@ bad_fork_cleanup_cgroup:
1309bad_fork_cleanup_put_domain: 1302bad_fork_cleanup_put_domain:
1310 module_put(task_thread_info(p)->exec_domain->module); 1303 module_put(task_thread_info(p)->exec_domain->module);
1311bad_fork_cleanup_count: 1304bad_fork_cleanup_count:
1312 put_group_info(p->group_info); 1305 atomic_dec(&p->cred->user->processes);
1313 atomic_dec(&p->user->processes); 1306 put_cred(p->real_cred);
1314 free_uid(p->user); 1307 put_cred(p->cred);
1315bad_fork_free: 1308bad_fork_free:
1316 free_task(p); 1309 free_task(p);
1317fork_out: 1310fork_out:
@@ -1355,6 +1348,21 @@ long do_fork(unsigned long clone_flags,
1355 long nr; 1348 long nr;
1356 1349
1357 /* 1350 /*
1351 * Do some preliminary argument and permissions checking before we
1352 * actually start allocating stuff
1353 */
1354 if (clone_flags & CLONE_NEWUSER) {
1355 if (clone_flags & CLONE_THREAD)
1356 return -EINVAL;
1357 /* hopefully this check will go away when userns support is
1358 * complete
1359 */
1360 if (!capable(CAP_SYS_ADMIN) || !capable(CAP_SETUID) ||
1361 !capable(CAP_SETGID))
1362 return -EPERM;
1363 }
1364
1365 /*
1358 * We hope to recycle these flags after 2.6.26 1366 * We hope to recycle these flags after 2.6.26
1359 */ 1367 */
1360 if (unlikely(clone_flags & CLONE_STOPPED)) { 1368 if (unlikely(clone_flags & CLONE_STOPPED)) {
@@ -1398,6 +1406,7 @@ long do_fork(unsigned long clone_flags,
1398 init_completion(&vfork); 1406 init_completion(&vfork);
1399 } 1407 }
1400 1408
1409 audit_finish_fork(p);
1401 tracehook_report_clone(trace, regs, clone_flags, nr, p); 1410 tracehook_report_clone(trace, regs, clone_flags, nr, p);
1402 1411
1403 /* 1412 /*
@@ -1601,8 +1610,7 @@ asmlinkage long sys_unshare(unsigned long unshare_flags)
1601 err = -EINVAL; 1610 err = -EINVAL;
1602 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND| 1611 if (unshare_flags & ~(CLONE_THREAD|CLONE_FS|CLONE_NEWNS|CLONE_SIGHAND|
1603 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM| 1612 CLONE_VM|CLONE_FILES|CLONE_SYSVSEM|
1604 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWUSER| 1613 CLONE_NEWUTS|CLONE_NEWIPC|CLONE_NEWNET))
1605 CLONE_NEWNET))
1606 goto bad_unshare_out; 1614 goto bad_unshare_out;
1607 1615
1608 /* 1616 /*
diff --git a/kernel/futex.c b/kernel/futex.c
index 206d4c906885..002aa189eb09 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -409,13 +409,20 @@ static void free_pi_state(struct futex_pi_state *pi_state)
409static struct task_struct * futex_find_get_task(pid_t pid) 409static struct task_struct * futex_find_get_task(pid_t pid)
410{ 410{
411 struct task_struct *p; 411 struct task_struct *p;
412 const struct cred *cred = current_cred(), *pcred;
412 413
413 rcu_read_lock(); 414 rcu_read_lock();
414 p = find_task_by_vpid(pid); 415 p = find_task_by_vpid(pid);
415 if (!p || ((current->euid != p->euid) && (current->euid != p->uid))) 416 if (!p) {
416 p = ERR_PTR(-ESRCH); 417 p = ERR_PTR(-ESRCH);
417 else 418 } else {
418 get_task_struct(p); 419 pcred = __task_cred(p);
420 if (cred->euid != pcred->euid &&
421 cred->euid != pcred->uid)
422 p = ERR_PTR(-ESRCH);
423 else
424 get_task_struct(p);
425 }
419 426
420 rcu_read_unlock(); 427 rcu_read_unlock();
421 428
@@ -1755,6 +1762,7 @@ sys_get_robust_list(int pid, struct robust_list_head __user * __user *head_ptr,
1755{ 1762{
1756 struct robust_list_head __user *head; 1763 struct robust_list_head __user *head;
1757 unsigned long ret; 1764 unsigned long ret;
1765 const struct cred *cred = current_cred(), *pcred;
1758 1766
1759 if (!futex_cmpxchg_enabled) 1767 if (!futex_cmpxchg_enabled)
1760 return -ENOSYS; 1768 return -ENOSYS;
@@ -1770,8 +1778,10 @@ sys_get_robust_list(int pid, struct robust_list_head __user * __user *head_ptr,
1770 if (!p) 1778 if (!p)
1771 goto err_unlock; 1779 goto err_unlock;
1772 ret = -EPERM; 1780 ret = -EPERM;
1773 if ((current->euid != p->euid) && (current->euid != p->uid) && 1781 pcred = __task_cred(p);
1774 !capable(CAP_SYS_PTRACE)) 1782 if (cred->euid != pcred->euid &&
1783 cred->euid != pcred->uid &&
1784 !capable(CAP_SYS_PTRACE))
1775 goto err_unlock; 1785 goto err_unlock;
1776 head = p->robust_list; 1786 head = p->robust_list;
1777 rcu_read_unlock(); 1787 rcu_read_unlock();
diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c
index 04ac3a9e42cf..d607a5b9ee29 100644
--- a/kernel/futex_compat.c
+++ b/kernel/futex_compat.c
@@ -135,6 +135,7 @@ compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr,
135{ 135{
136 struct compat_robust_list_head __user *head; 136 struct compat_robust_list_head __user *head;
137 unsigned long ret; 137 unsigned long ret;
138 const struct cred *cred = current_cred(), *pcred;
138 139
139 if (!futex_cmpxchg_enabled) 140 if (!futex_cmpxchg_enabled)
140 return -ENOSYS; 141 return -ENOSYS;
@@ -150,8 +151,10 @@ compat_sys_get_robust_list(int pid, compat_uptr_t __user *head_ptr,
150 if (!p) 151 if (!p)
151 goto err_unlock; 152 goto err_unlock;
152 ret = -EPERM; 153 ret = -EPERM;
153 if ((current->euid != p->euid) && (current->euid != p->uid) && 154 pcred = __task_cred(p);
154 !capable(CAP_SYS_PTRACE)) 155 if (cred->euid != pcred->euid &&
156 cred->euid != pcred->uid &&
157 !capable(CAP_SYS_PTRACE))
155 goto err_unlock; 158 goto err_unlock;
156 head = p->compat_robust_list; 159 head = p->compat_robust_list;
157 read_unlock(&tasklist_lock); 160 read_unlock(&tasklist_lock);
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index 47e63349d1b2..eb2bfefa6dcc 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -32,7 +32,6 @@
32 */ 32 */
33 33
34#include <linux/cpu.h> 34#include <linux/cpu.h>
35#include <linux/irq.h>
36#include <linux/module.h> 35#include <linux/module.h>
37#include <linux/percpu.h> 36#include <linux/percpu.h>
38#include <linux/hrtimer.h> 37#include <linux/hrtimer.h>
@@ -442,22 +441,6 @@ static inline void debug_hrtimer_activate(struct hrtimer *timer) { }
442static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } 441static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
443#endif 442#endif
444 443
445/*
446 * Check, whether the timer is on the callback pending list
447 */
448static inline int hrtimer_cb_pending(const struct hrtimer *timer)
449{
450 return timer->state & HRTIMER_STATE_PENDING;
451}
452
453/*
454 * Remove a timer from the callback pending list
455 */
456static inline void hrtimer_remove_cb_pending(struct hrtimer *timer)
457{
458 list_del_init(&timer->cb_entry);
459}
460
461/* High resolution timer related functions */ 444/* High resolution timer related functions */
462#ifdef CONFIG_HIGH_RES_TIMERS 445#ifdef CONFIG_HIGH_RES_TIMERS
463 446
@@ -651,6 +634,8 @@ static inline void hrtimer_init_timer_hres(struct hrtimer *timer)
651{ 634{
652} 635}
653 636
637static void __run_hrtimer(struct hrtimer *timer);
638
654/* 639/*
655 * When High resolution timers are active, try to reprogram. Note, that in case 640 * When High resolution timers are active, try to reprogram. Note, that in case
656 * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry 641 * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry
@@ -661,31 +646,14 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
661 struct hrtimer_clock_base *base) 646 struct hrtimer_clock_base *base)
662{ 647{
663 if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) { 648 if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {
664 649 /*
665 /* Timer is expired, act upon the callback mode */ 650 * XXX: recursion check?
666 switch(timer->cb_mode) { 651 * hrtimer_forward() should round up with timer granularity
667 case HRTIMER_CB_IRQSAFE_PERCPU: 652 * so that we never get into inf recursion here,
668 case HRTIMER_CB_IRQSAFE_UNLOCKED: 653 * it doesn't do that though
669 /* 654 */
670 * This is solely for the sched tick emulation with 655 __run_hrtimer(timer);
671 * dynamic tick support to ensure that we do not 656 return 1;
672 * restart the tick right on the edge and end up with
673 * the tick timer in the softirq ! The calling site
674 * takes care of this. Also used for hrtimer sleeper !
675 */
676 debug_hrtimer_deactivate(timer);
677 return 1;
678 case HRTIMER_CB_SOFTIRQ:
679 /*
680 * Move everything else into the softirq pending list !
681 */
682 list_add_tail(&timer->cb_entry,
683 &base->cpu_base->cb_pending);
684 timer->state = HRTIMER_STATE_PENDING;
685 return 1;
686 default:
687 BUG();
688 }
689 } 657 }
690 return 0; 658 return 0;
691} 659}
@@ -724,11 +692,6 @@ static int hrtimer_switch_to_hres(void)
724 return 1; 692 return 1;
725} 693}
726 694
727static inline void hrtimer_raise_softirq(void)
728{
729 raise_softirq(HRTIMER_SOFTIRQ);
730}
731
732#else 695#else
733 696
734static inline int hrtimer_hres_active(void) { return 0; } 697static inline int hrtimer_hres_active(void) { return 0; }
@@ -747,7 +710,6 @@ static inline int hrtimer_reprogram(struct hrtimer *timer,
747{ 710{
748 return 0; 711 return 0;
749} 712}
750static inline void hrtimer_raise_softirq(void) { }
751 713
752#endif /* CONFIG_HIGH_RES_TIMERS */ 714#endif /* CONFIG_HIGH_RES_TIMERS */
753 715
@@ -890,10 +852,7 @@ static void __remove_hrtimer(struct hrtimer *timer,
890 struct hrtimer_clock_base *base, 852 struct hrtimer_clock_base *base,
891 unsigned long newstate, int reprogram) 853 unsigned long newstate, int reprogram)
892{ 854{
893 /* High res. callback list. NOP for !HIGHRES */ 855 if (timer->state & HRTIMER_STATE_ENQUEUED) {
894 if (hrtimer_cb_pending(timer))
895 hrtimer_remove_cb_pending(timer);
896 else {
897 /* 856 /*
898 * Remove the timer from the rbtree and replace the 857 * Remove the timer from the rbtree and replace the
899 * first entry pointer if necessary. 858 * first entry pointer if necessary.
@@ -953,7 +912,7 @@ hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_n
953{ 912{
954 struct hrtimer_clock_base *base, *new_base; 913 struct hrtimer_clock_base *base, *new_base;
955 unsigned long flags; 914 unsigned long flags;
956 int ret, raise; 915 int ret;
957 916
958 base = lock_hrtimer_base(timer, &flags); 917 base = lock_hrtimer_base(timer, &flags);
959 918
@@ -988,26 +947,8 @@ hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_n
988 enqueue_hrtimer(timer, new_base, 947 enqueue_hrtimer(timer, new_base,
989 new_base->cpu_base == &__get_cpu_var(hrtimer_bases)); 948 new_base->cpu_base == &__get_cpu_var(hrtimer_bases));
990 949
991 /*
992 * The timer may be expired and moved to the cb_pending
993 * list. We can not raise the softirq with base lock held due
994 * to a possible deadlock with runqueue lock.
995 */
996 raise = timer->state == HRTIMER_STATE_PENDING;
997
998 /*
999 * We use preempt_disable to prevent this task from migrating after
1000 * setting up the softirq and raising it. Otherwise, if me migrate
1001 * we will raise the softirq on the wrong CPU.
1002 */
1003 preempt_disable();
1004
1005 unlock_hrtimer_base(timer, &flags); 950 unlock_hrtimer_base(timer, &flags);
1006 951
1007 if (raise)
1008 hrtimer_raise_softirq();
1009 preempt_enable();
1010
1011 return ret; 952 return ret;
1012} 953}
1013EXPORT_SYMBOL_GPL(hrtimer_start_range_ns); 954EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
@@ -1192,75 +1133,6 @@ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp)
1192} 1133}
1193EXPORT_SYMBOL_GPL(hrtimer_get_res); 1134EXPORT_SYMBOL_GPL(hrtimer_get_res);
1194 1135
1195static void run_hrtimer_pending(struct hrtimer_cpu_base *cpu_base)
1196{
1197 spin_lock_irq(&cpu_base->lock);
1198
1199 while (!list_empty(&cpu_base->cb_pending)) {
1200 enum hrtimer_restart (*fn)(struct hrtimer *);
1201 struct hrtimer *timer;
1202 int restart;
1203 int emulate_hardirq_ctx = 0;
1204
1205 timer = list_entry(cpu_base->cb_pending.next,
1206 struct hrtimer, cb_entry);
1207
1208 debug_hrtimer_deactivate(timer);
1209 timer_stats_account_hrtimer(timer);
1210
1211 fn = timer->function;
1212 /*
1213 * A timer might have been added to the cb_pending list
1214 * when it was migrated during a cpu-offline operation.
1215 * Emulate hardirq context for such timers.
1216 */
1217 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU ||
1218 timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED)
1219 emulate_hardirq_ctx = 1;
1220
1221 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_CALLBACK, 0);
1222 spin_unlock_irq(&cpu_base->lock);
1223
1224 if (unlikely(emulate_hardirq_ctx)) {
1225 local_irq_disable();
1226 restart = fn(timer);
1227 local_irq_enable();
1228 } else
1229 restart = fn(timer);
1230
1231 spin_lock_irq(&cpu_base->lock);
1232
1233 timer->state &= ~HRTIMER_STATE_CALLBACK;
1234 if (restart == HRTIMER_RESTART) {
1235 BUG_ON(hrtimer_active(timer));
1236 /*
1237 * Enqueue the timer, allow reprogramming of the event
1238 * device
1239 */
1240 enqueue_hrtimer(timer, timer->base, 1);
1241 } else if (hrtimer_active(timer)) {
1242 /*
1243 * If the timer was rearmed on another CPU, reprogram
1244 * the event device.
1245 */
1246 struct hrtimer_clock_base *base = timer->base;
1247
1248 if (base->first == &timer->node &&
1249 hrtimer_reprogram(timer, base)) {
1250 /*
1251 * Timer is expired. Thus move it from tree to
1252 * pending list again.
1253 */
1254 __remove_hrtimer(timer, base,
1255 HRTIMER_STATE_PENDING, 0);
1256 list_add_tail(&timer->cb_entry,
1257 &base->cpu_base->cb_pending);
1258 }
1259 }
1260 }
1261 spin_unlock_irq(&cpu_base->lock);
1262}
1263
1264static void __run_hrtimer(struct hrtimer *timer) 1136static void __run_hrtimer(struct hrtimer *timer)
1265{ 1137{
1266 struct hrtimer_clock_base *base = timer->base; 1138 struct hrtimer_clock_base *base = timer->base;
@@ -1268,25 +1140,21 @@ static void __run_hrtimer(struct hrtimer *timer)
1268 enum hrtimer_restart (*fn)(struct hrtimer *); 1140 enum hrtimer_restart (*fn)(struct hrtimer *);
1269 int restart; 1141 int restart;
1270 1142
1143 WARN_ON(!irqs_disabled());
1144
1271 debug_hrtimer_deactivate(timer); 1145 debug_hrtimer_deactivate(timer);
1272 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); 1146 __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0);
1273 timer_stats_account_hrtimer(timer); 1147 timer_stats_account_hrtimer(timer);
1274
1275 fn = timer->function; 1148 fn = timer->function;
1276 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU || 1149
1277 timer->cb_mode == HRTIMER_CB_IRQSAFE_UNLOCKED) { 1150 /*
1278 /* 1151 * Because we run timers from hardirq context, there is no chance
1279 * Used for scheduler timers, avoid lock inversion with 1152 * they get migrated to another cpu, therefore its safe to unlock
1280 * rq->lock and tasklist_lock. 1153 * the timer base.
1281 * 1154 */
1282 * These timers are required to deal with enqueue expiry 1155 spin_unlock(&cpu_base->lock);
1283 * themselves and are not allowed to migrate. 1156 restart = fn(timer);
1284 */ 1157 spin_lock(&cpu_base->lock);
1285 spin_unlock(&cpu_base->lock);
1286 restart = fn(timer);
1287 spin_lock(&cpu_base->lock);
1288 } else
1289 restart = fn(timer);
1290 1158
1291 /* 1159 /*
1292 * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid 1160 * Note: We clear the CALLBACK bit after enqueue_hrtimer to avoid
@@ -1311,7 +1179,7 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1311 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases); 1179 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1312 struct hrtimer_clock_base *base; 1180 struct hrtimer_clock_base *base;
1313 ktime_t expires_next, now; 1181 ktime_t expires_next, now;
1314 int i, raise = 0; 1182 int i;
1315 1183
1316 BUG_ON(!cpu_base->hres_active); 1184 BUG_ON(!cpu_base->hres_active);
1317 cpu_base->nr_events++; 1185 cpu_base->nr_events++;
@@ -1360,16 +1228,6 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1360 break; 1228 break;
1361 } 1229 }
1362 1230
1363 /* Move softirq callbacks to the pending list */
1364 if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
1365 __remove_hrtimer(timer, base,
1366 HRTIMER_STATE_PENDING, 0);
1367 list_add_tail(&timer->cb_entry,
1368 &base->cpu_base->cb_pending);
1369 raise = 1;
1370 continue;
1371 }
1372
1373 __run_hrtimer(timer); 1231 __run_hrtimer(timer);
1374 } 1232 }
1375 spin_unlock(&cpu_base->lock); 1233 spin_unlock(&cpu_base->lock);
@@ -1383,10 +1241,6 @@ void hrtimer_interrupt(struct clock_event_device *dev)
1383 if (tick_program_event(expires_next, 0)) 1241 if (tick_program_event(expires_next, 0))
1384 goto retry; 1242 goto retry;
1385 } 1243 }
1386
1387 /* Raise softirq ? */
1388 if (raise)
1389 raise_softirq(HRTIMER_SOFTIRQ);
1390} 1244}
1391 1245
1392/** 1246/**
@@ -1413,11 +1267,6 @@ void hrtimer_peek_ahead_timers(void)
1413 local_irq_restore(flags); 1267 local_irq_restore(flags);
1414} 1268}
1415 1269
1416static void run_hrtimer_softirq(struct softirq_action *h)
1417{
1418 run_hrtimer_pending(&__get_cpu_var(hrtimer_bases));
1419}
1420
1421#endif /* CONFIG_HIGH_RES_TIMERS */ 1270#endif /* CONFIG_HIGH_RES_TIMERS */
1422 1271
1423/* 1272/*
@@ -1429,8 +1278,6 @@ static void run_hrtimer_softirq(struct softirq_action *h)
1429 */ 1278 */
1430void hrtimer_run_pending(void) 1279void hrtimer_run_pending(void)
1431{ 1280{
1432 struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
1433
1434 if (hrtimer_hres_active()) 1281 if (hrtimer_hres_active())
1435 return; 1282 return;
1436 1283
@@ -1444,8 +1291,6 @@ void hrtimer_run_pending(void)
1444 */ 1291 */
1445 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) 1292 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
1446 hrtimer_switch_to_hres(); 1293 hrtimer_switch_to_hres();
1447
1448 run_hrtimer_pending(cpu_base);
1449} 1294}
1450 1295
1451/* 1296/*
@@ -1482,14 +1327,6 @@ void hrtimer_run_queues(void)
1482 hrtimer_get_expires_tv64(timer)) 1327 hrtimer_get_expires_tv64(timer))
1483 break; 1328 break;
1484 1329
1485 if (timer->cb_mode == HRTIMER_CB_SOFTIRQ) {
1486 __remove_hrtimer(timer, base,
1487 HRTIMER_STATE_PENDING, 0);
1488 list_add_tail(&timer->cb_entry,
1489 &base->cpu_base->cb_pending);
1490 continue;
1491 }
1492
1493 __run_hrtimer(timer); 1330 __run_hrtimer(timer);
1494 } 1331 }
1495 spin_unlock(&cpu_base->lock); 1332 spin_unlock(&cpu_base->lock);
@@ -1516,9 +1353,6 @@ void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1516{ 1353{
1517 sl->timer.function = hrtimer_wakeup; 1354 sl->timer.function = hrtimer_wakeup;
1518 sl->task = task; 1355 sl->task = task;
1519#ifdef CONFIG_HIGH_RES_TIMERS
1520 sl->timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
1521#endif
1522} 1356}
1523 1357
1524static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) 1358static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
@@ -1655,18 +1489,16 @@ static void __cpuinit init_hrtimers_cpu(int cpu)
1655 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) 1489 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)
1656 cpu_base->clock_base[i].cpu_base = cpu_base; 1490 cpu_base->clock_base[i].cpu_base = cpu_base;
1657 1491
1658 INIT_LIST_HEAD(&cpu_base->cb_pending);
1659 hrtimer_init_hres(cpu_base); 1492 hrtimer_init_hres(cpu_base);
1660} 1493}
1661 1494
1662#ifdef CONFIG_HOTPLUG_CPU 1495#ifdef CONFIG_HOTPLUG_CPU
1663 1496
1664static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base, 1497static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1665 struct hrtimer_clock_base *new_base, int dcpu) 1498 struct hrtimer_clock_base *new_base)
1666{ 1499{
1667 struct hrtimer *timer; 1500 struct hrtimer *timer;
1668 struct rb_node *node; 1501 struct rb_node *node;
1669 int raise = 0;
1670 1502
1671 while ((node = rb_first(&old_base->active))) { 1503 while ((node = rb_first(&old_base->active))) {
1672 timer = rb_entry(node, struct hrtimer, node); 1504 timer = rb_entry(node, struct hrtimer, node);
@@ -1674,18 +1506,6 @@ static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1674 debug_hrtimer_deactivate(timer); 1506 debug_hrtimer_deactivate(timer);
1675 1507
1676 /* 1508 /*
1677 * Should not happen. Per CPU timers should be
1678 * canceled _before_ the migration code is called
1679 */
1680 if (timer->cb_mode == HRTIMER_CB_IRQSAFE_PERCPU) {
1681 __remove_hrtimer(timer, old_base,
1682 HRTIMER_STATE_INACTIVE, 0);
1683 WARN(1, "hrtimer (%p %p)active but cpu %d dead\n",
1684 timer, timer->function, dcpu);
1685 continue;
1686 }
1687
1688 /*
1689 * Mark it as STATE_MIGRATE not INACTIVE otherwise the 1509 * Mark it as STATE_MIGRATE not INACTIVE otherwise the
1690 * timer could be seen as !active and just vanish away 1510 * timer could be seen as !active and just vanish away
1691 * under us on another CPU 1511 * under us on another CPU
@@ -1693,69 +1513,34 @@ static int migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1693 __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0); 1513 __remove_hrtimer(timer, old_base, HRTIMER_STATE_MIGRATE, 0);
1694 timer->base = new_base; 1514 timer->base = new_base;
1695 /* 1515 /*
1696 * Enqueue the timer. Allow reprogramming of the event device 1516 * Enqueue the timers on the new cpu, but do not reprogram
1517 * the timer as that would enable a deadlock between
1518 * hrtimer_enqueue_reprogramm() running the timer and us still
1519 * holding a nested base lock.
1520 *
1521 * Instead we tickle the hrtimer interrupt after the migration
1522 * is done, which will run all expired timers and re-programm
1523 * the timer device.
1697 */ 1524 */
1698 enqueue_hrtimer(timer, new_base, 1); 1525 enqueue_hrtimer(timer, new_base, 0);
1699 1526
1700#ifdef CONFIG_HIGH_RES_TIMERS
1701 /*
1702 * Happens with high res enabled when the timer was
1703 * already expired and the callback mode is
1704 * HRTIMER_CB_IRQSAFE_UNLOCKED (hrtimer_sleeper). The
1705 * enqueue code does not move them to the soft irq
1706 * pending list for performance/latency reasons, but
1707 * in the migration state, we need to do that
1708 * otherwise we end up with a stale timer.
1709 */
1710 if (timer->state == HRTIMER_STATE_MIGRATE) {
1711 timer->state = HRTIMER_STATE_PENDING;
1712 list_add_tail(&timer->cb_entry,
1713 &new_base->cpu_base->cb_pending);
1714 raise = 1;
1715 }
1716#endif
1717 /* Clear the migration state bit */ 1527 /* Clear the migration state bit */
1718 timer->state &= ~HRTIMER_STATE_MIGRATE; 1528 timer->state &= ~HRTIMER_STATE_MIGRATE;
1719 } 1529 }
1720 return raise;
1721}
1722
1723#ifdef CONFIG_HIGH_RES_TIMERS
1724static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
1725 struct hrtimer_cpu_base *new_base)
1726{
1727 struct hrtimer *timer;
1728 int raise = 0;
1729
1730 while (!list_empty(&old_base->cb_pending)) {
1731 timer = list_entry(old_base->cb_pending.next,
1732 struct hrtimer, cb_entry);
1733
1734 __remove_hrtimer(timer, timer->base, HRTIMER_STATE_PENDING, 0);
1735 timer->base = &new_base->clock_base[timer->base->index];
1736 list_add_tail(&timer->cb_entry, &new_base->cb_pending);
1737 raise = 1;
1738 }
1739 return raise;
1740}
1741#else
1742static int migrate_hrtimer_pending(struct hrtimer_cpu_base *old_base,
1743 struct hrtimer_cpu_base *new_base)
1744{
1745 return 0;
1746} 1530}
1747#endif
1748 1531
1749static void migrate_hrtimers(int cpu) 1532static int migrate_hrtimers(int scpu)
1750{ 1533{
1751 struct hrtimer_cpu_base *old_base, *new_base; 1534 struct hrtimer_cpu_base *old_base, *new_base;
1752 int i, raise = 0; 1535 int dcpu, i;
1753 1536
1754 BUG_ON(cpu_online(cpu)); 1537 BUG_ON(cpu_online(scpu));
1755 old_base = &per_cpu(hrtimer_bases, cpu); 1538 old_base = &per_cpu(hrtimer_bases, scpu);
1756 new_base = &get_cpu_var(hrtimer_bases); 1539 new_base = &get_cpu_var(hrtimer_bases);
1757 1540
1758 tick_cancel_sched_timer(cpu); 1541 dcpu = smp_processor_id();
1542
1543 tick_cancel_sched_timer(scpu);
1759 /* 1544 /*
1760 * The caller is globally serialized and nobody else 1545 * The caller is globally serialized and nobody else
1761 * takes two locks at once, deadlock is not possible. 1546 * takes two locks at once, deadlock is not possible.
@@ -1764,41 +1549,47 @@ static void migrate_hrtimers(int cpu)
1764 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); 1549 spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1765 1550
1766 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { 1551 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1767 if (migrate_hrtimer_list(&old_base->clock_base[i], 1552 migrate_hrtimer_list(&old_base->clock_base[i],
1768 &new_base->clock_base[i], cpu)) 1553 &new_base->clock_base[i]);
1769 raise = 1;
1770 } 1554 }
1771 1555
1772 if (migrate_hrtimer_pending(old_base, new_base))
1773 raise = 1;
1774
1775 spin_unlock(&old_base->lock); 1556 spin_unlock(&old_base->lock);
1776 spin_unlock_irq(&new_base->lock); 1557 spin_unlock_irq(&new_base->lock);
1777 put_cpu_var(hrtimer_bases); 1558 put_cpu_var(hrtimer_bases);
1778 1559
1779 if (raise) 1560 return dcpu;
1780 hrtimer_raise_softirq(); 1561}
1562
1563static void tickle_timers(void *arg)
1564{
1565 hrtimer_peek_ahead_timers();
1781} 1566}
1567
1782#endif /* CONFIG_HOTPLUG_CPU */ 1568#endif /* CONFIG_HOTPLUG_CPU */
1783 1569
1784static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self, 1570static int __cpuinit hrtimer_cpu_notify(struct notifier_block *self,
1785 unsigned long action, void *hcpu) 1571 unsigned long action, void *hcpu)
1786{ 1572{
1787 unsigned int cpu = (long)hcpu; 1573 int scpu = (long)hcpu;
1788 1574
1789 switch (action) { 1575 switch (action) {
1790 1576
1791 case CPU_UP_PREPARE: 1577 case CPU_UP_PREPARE:
1792 case CPU_UP_PREPARE_FROZEN: 1578 case CPU_UP_PREPARE_FROZEN:
1793 init_hrtimers_cpu(cpu); 1579 init_hrtimers_cpu(scpu);
1794 break; 1580 break;
1795 1581
1796#ifdef CONFIG_HOTPLUG_CPU 1582#ifdef CONFIG_HOTPLUG_CPU
1797 case CPU_DEAD: 1583 case CPU_DEAD:
1798 case CPU_DEAD_FROZEN: 1584 case CPU_DEAD_FROZEN:
1799 clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &cpu); 1585 {
1800 migrate_hrtimers(cpu); 1586 int dcpu;
1587
1588 clockevents_notify(CLOCK_EVT_NOTIFY_CPU_DEAD, &scpu);
1589 dcpu = migrate_hrtimers(scpu);
1590 smp_call_function_single(dcpu, tickle_timers, NULL, 0);
1801 break; 1591 break;
1592 }
1802#endif 1593#endif
1803 1594
1804 default: 1595 default:
@@ -1817,9 +1608,6 @@ void __init hrtimers_init(void)
1817 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, 1608 hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE,
1818 (void *)(long)smp_processor_id()); 1609 (void *)(long)smp_processor_id());
1819 register_cpu_notifier(&hrtimers_nb); 1610 register_cpu_notifier(&hrtimers_nb);
1820#ifdef CONFIG_HIGH_RES_TIMERS
1821 open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq);
1822#endif
1823} 1611}
1824 1612
1825/** 1613/**
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 681c52dbfe22..4dd5b1edac98 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -3,3 +3,4 @@ obj-y := handle.o manage.o spurious.o resend.o chip.o devres.o
3obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o 3obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o
4obj-$(CONFIG_PROC_FS) += proc.o 4obj-$(CONFIG_PROC_FS) += proc.o
5obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o 5obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o
6obj-$(CONFIG_NUMA_MIGRATE_IRQ_DESC) += numa_migrate.o
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index 10b5092e9bfe..f63c706d25e1 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -24,9 +24,10 @@
24 */ 24 */
25void dynamic_irq_init(unsigned int irq) 25void dynamic_irq_init(unsigned int irq)
26{ 26{
27 struct irq_desc *desc = irq_to_desc(irq); 27 struct irq_desc *desc;
28 unsigned long flags; 28 unsigned long flags;
29 29
30 desc = irq_to_desc(irq);
30 if (!desc) { 31 if (!desc) {
31 WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq); 32 WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq);
32 return; 33 return;
@@ -45,7 +46,7 @@ void dynamic_irq_init(unsigned int irq)
45 desc->irq_count = 0; 46 desc->irq_count = 0;
46 desc->irqs_unhandled = 0; 47 desc->irqs_unhandled = 0;
47#ifdef CONFIG_SMP 48#ifdef CONFIG_SMP
48 cpus_setall(desc->affinity); 49 cpumask_setall(&desc->affinity);
49#endif 50#endif
50 spin_unlock_irqrestore(&desc->lock, flags); 51 spin_unlock_irqrestore(&desc->lock, flags);
51} 52}
@@ -124,6 +125,7 @@ int set_irq_type(unsigned int irq, unsigned int type)
124 return -ENODEV; 125 return -ENODEV;
125 } 126 }
126 127
128 type &= IRQ_TYPE_SENSE_MASK;
127 if (type == IRQ_TYPE_NONE) 129 if (type == IRQ_TYPE_NONE)
128 return 0; 130 return 0;
129 131
@@ -352,6 +354,7 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
352 354
353 spin_lock(&desc->lock); 355 spin_lock(&desc->lock);
354 mask_ack_irq(desc, irq); 356 mask_ack_irq(desc, irq);
357 desc = irq_remap_to_desc(irq, desc);
355 358
356 if (unlikely(desc->status & IRQ_INPROGRESS)) 359 if (unlikely(desc->status & IRQ_INPROGRESS))
357 goto out_unlock; 360 goto out_unlock;
@@ -429,6 +432,7 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
429 desc->status &= ~IRQ_INPROGRESS; 432 desc->status &= ~IRQ_INPROGRESS;
430out: 433out:
431 desc->chip->eoi(irq); 434 desc->chip->eoi(irq);
435 desc = irq_remap_to_desc(irq, desc);
432 436
433 spin_unlock(&desc->lock); 437 spin_unlock(&desc->lock);
434} 438}
@@ -465,12 +469,14 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
465 !desc->action)) { 469 !desc->action)) {
466 desc->status |= (IRQ_PENDING | IRQ_MASKED); 470 desc->status |= (IRQ_PENDING | IRQ_MASKED);
467 mask_ack_irq(desc, irq); 471 mask_ack_irq(desc, irq);
472 desc = irq_remap_to_desc(irq, desc);
468 goto out_unlock; 473 goto out_unlock;
469 } 474 }
470 kstat_incr_irqs_this_cpu(irq, desc); 475 kstat_incr_irqs_this_cpu(irq, desc);
471 476
472 /* Start handling the irq */ 477 /* Start handling the irq */
473 desc->chip->ack(irq); 478 desc->chip->ack(irq);
479 desc = irq_remap_to_desc(irq, desc);
474 480
475 /* Mark the IRQ currently in progress.*/ 481 /* Mark the IRQ currently in progress.*/
476 desc->status |= IRQ_INPROGRESS; 482 desc->status |= IRQ_INPROGRESS;
@@ -531,8 +537,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
531 if (!noirqdebug) 537 if (!noirqdebug)
532 note_interrupt(irq, desc, action_ret); 538 note_interrupt(irq, desc, action_ret);
533 539
534 if (desc->chip->eoi) 540 if (desc->chip->eoi) {
535 desc->chip->eoi(irq); 541 desc->chip->eoi(irq);
542 desc = irq_remap_to_desc(irq, desc);
543 }
536} 544}
537 545
538void 546void
@@ -567,8 +575,10 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
567 575
568 /* Uninstall? */ 576 /* Uninstall? */
569 if (handle == handle_bad_irq) { 577 if (handle == handle_bad_irq) {
570 if (desc->chip != &no_irq_chip) 578 if (desc->chip != &no_irq_chip) {
571 mask_ack_irq(desc, irq); 579 mask_ack_irq(desc, irq);
580 desc = irq_remap_to_desc(irq, desc);
581 }
572 desc->status |= IRQ_DISABLED; 582 desc->status |= IRQ_DISABLED;
573 desc->depth = 1; 583 desc->depth = 1;
574 } 584 }
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index c815b42d0f5b..c20db0be9173 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -15,9 +15,16 @@
15#include <linux/random.h> 15#include <linux/random.h>
16#include <linux/interrupt.h> 16#include <linux/interrupt.h>
17#include <linux/kernel_stat.h> 17#include <linux/kernel_stat.h>
18#include <linux/rculist.h>
19#include <linux/hash.h>
18 20
19#include "internals.h" 21#include "internals.h"
20 22
23/*
24 * lockdep: we want to handle all irq_desc locks as a single lock-class:
25 */
26struct lock_class_key irq_desc_lock_class;
27
21/** 28/**
22 * handle_bad_irq - handle spurious and unhandled irqs 29 * handle_bad_irq - handle spurious and unhandled irqs
23 * @irq: the interrupt number 30 * @irq: the interrupt number
@@ -49,6 +56,150 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc)
49int nr_irqs = NR_IRQS; 56int nr_irqs = NR_IRQS;
50EXPORT_SYMBOL_GPL(nr_irqs); 57EXPORT_SYMBOL_GPL(nr_irqs);
51 58
59#ifdef CONFIG_SPARSE_IRQ
60static struct irq_desc irq_desc_init = {
61 .irq = -1,
62 .status = IRQ_DISABLED,
63 .chip = &no_irq_chip,
64 .handle_irq = handle_bad_irq,
65 .depth = 1,
66 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock),
67#ifdef CONFIG_SMP
68 .affinity = CPU_MASK_ALL
69#endif
70};
71
72void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr)
73{
74 unsigned long bytes;
75 char *ptr;
76 int node;
77
78 /* Compute how many bytes we need per irq and allocate them */
79 bytes = nr * sizeof(unsigned int);
80
81 node = cpu_to_node(cpu);
82 ptr = kzalloc_node(bytes, GFP_ATOMIC, node);
83 printk(KERN_DEBUG " alloc kstat_irqs on cpu %d node %d\n", cpu, node);
84
85 if (ptr)
86 desc->kstat_irqs = (unsigned int *)ptr;
87}
88
89static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu)
90{
91 memcpy(desc, &irq_desc_init, sizeof(struct irq_desc));
92
93 spin_lock_init(&desc->lock);
94 desc->irq = irq;
95#ifdef CONFIG_SMP
96 desc->cpu = cpu;
97#endif
98 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
99 init_kstat_irqs(desc, cpu, nr_cpu_ids);
100 if (!desc->kstat_irqs) {
101 printk(KERN_ERR "can not alloc kstat_irqs\n");
102 BUG_ON(1);
103 }
104 arch_init_chip_data(desc, cpu);
105}
106
107/*
108 * Protect the sparse_irqs:
109 */
110DEFINE_SPINLOCK(sparse_irq_lock);
111
112struct irq_desc *irq_desc_ptrs[NR_IRQS] __read_mostly;
113
114static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = {
115 [0 ... NR_IRQS_LEGACY-1] = {
116 .irq = -1,
117 .status = IRQ_DISABLED,
118 .chip = &no_irq_chip,
119 .handle_irq = handle_bad_irq,
120 .depth = 1,
121 .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock),
122#ifdef CONFIG_SMP
123 .affinity = CPU_MASK_ALL
124#endif
125 }
126};
127
128/* FIXME: use bootmem alloc ...*/
129static unsigned int kstat_irqs_legacy[NR_IRQS_LEGACY][NR_CPUS];
130
131int __init early_irq_init(void)
132{
133 struct irq_desc *desc;
134 int legacy_count;
135 int i;
136
137 desc = irq_desc_legacy;
138 legacy_count = ARRAY_SIZE(irq_desc_legacy);
139
140 for (i = 0; i < legacy_count; i++) {
141 desc[i].irq = i;
142 desc[i].kstat_irqs = kstat_irqs_legacy[i];
143 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
144
145 irq_desc_ptrs[i] = desc + i;
146 }
147
148 for (i = legacy_count; i < NR_IRQS; i++)
149 irq_desc_ptrs[i] = NULL;
150
151 return arch_early_irq_init();
152}
153
154struct irq_desc *irq_to_desc(unsigned int irq)
155{
156 return (irq < NR_IRQS) ? irq_desc_ptrs[irq] : NULL;
157}
158
159struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu)
160{
161 struct irq_desc *desc;
162 unsigned long flags;
163 int node;
164
165 if (irq >= NR_IRQS) {
166 printk(KERN_WARNING "irq >= NR_IRQS in irq_to_desc_alloc: %d %d\n",
167 irq, NR_IRQS);
168 WARN_ON(1);
169 return NULL;
170 }
171
172 desc = irq_desc_ptrs[irq];
173 if (desc)
174 return desc;
175
176 spin_lock_irqsave(&sparse_irq_lock, flags);
177
178 /* We have to check it to avoid races with another CPU */
179 desc = irq_desc_ptrs[irq];
180 if (desc)
181 goto out_unlock;
182
183 node = cpu_to_node(cpu);
184 desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node);
185 printk(KERN_DEBUG " alloc irq_desc for %d on cpu %d node %d\n",
186 irq, cpu, node);
187 if (!desc) {
188 printk(KERN_ERR "can not alloc irq_desc\n");
189 BUG_ON(1);
190 }
191 init_one_irq_desc(irq, desc, cpu);
192
193 irq_desc_ptrs[irq] = desc;
194
195out_unlock:
196 spin_unlock_irqrestore(&sparse_irq_lock, flags);
197
198 return desc;
199}
200
201#else /* !CONFIG_SPARSE_IRQ */
202
52struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { 203struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
53 [0 ... NR_IRQS-1] = { 204 [0 ... NR_IRQS-1] = {
54 .status = IRQ_DISABLED, 205 .status = IRQ_DISABLED,
@@ -62,6 +213,32 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = {
62 } 213 }
63}; 214};
64 215
216int __init early_irq_init(void)
217{
218 struct irq_desc *desc;
219 int count;
220 int i;
221
222 desc = irq_desc;
223 count = ARRAY_SIZE(irq_desc);
224
225 for (i = 0; i < count; i++)
226 desc[i].irq = i;
227
228 return arch_early_irq_init();
229}
230
231struct irq_desc *irq_to_desc(unsigned int irq)
232{
233 return (irq < NR_IRQS) ? irq_desc + irq : NULL;
234}
235
236struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu)
237{
238 return irq_to_desc(irq);
239}
240#endif /* !CONFIG_SPARSE_IRQ */
241
65/* 242/*
66 * What should we do if we get a hw irq event on an illegal vector? 243 * What should we do if we get a hw irq event on an illegal vector?
67 * Each architecture has to answer this themself. 244 * Each architecture has to answer this themself.
@@ -179,8 +356,11 @@ unsigned int __do_IRQ(unsigned int irq)
179 /* 356 /*
180 * No locking required for CPU-local interrupts: 357 * No locking required for CPU-local interrupts:
181 */ 358 */
182 if (desc->chip->ack) 359 if (desc->chip->ack) {
183 desc->chip->ack(irq); 360 desc->chip->ack(irq);
361 /* get new one */
362 desc = irq_remap_to_desc(irq, desc);
363 }
184 if (likely(!(desc->status & IRQ_DISABLED))) { 364 if (likely(!(desc->status & IRQ_DISABLED))) {
185 action_ret = handle_IRQ_event(irq, desc->action); 365 action_ret = handle_IRQ_event(irq, desc->action);
186 if (!noirqdebug) 366 if (!noirqdebug)
@@ -191,8 +371,10 @@ unsigned int __do_IRQ(unsigned int irq)
191 } 371 }
192 372
193 spin_lock(&desc->lock); 373 spin_lock(&desc->lock);
194 if (desc->chip->ack) 374 if (desc->chip->ack) {
195 desc->chip->ack(irq); 375 desc->chip->ack(irq);
376 desc = irq_remap_to_desc(irq, desc);
377 }
196 /* 378 /*
197 * REPLAY is when Linux resends an IRQ that was dropped earlier 379 * REPLAY is when Linux resends an IRQ that was dropped earlier
198 * WAITING is used by probe to mark irqs that are being tested 380 * WAITING is used by probe to mark irqs that are being tested
@@ -259,19 +441,22 @@ out:
259} 441}
260#endif 442#endif
261 443
262
263#ifdef CONFIG_TRACE_IRQFLAGS
264/*
265 * lockdep: we want to handle all irq_desc locks as a single lock-class:
266 */
267static struct lock_class_key irq_desc_lock_class;
268
269void early_init_irq_lock_class(void) 444void early_init_irq_lock_class(void)
270{ 445{
271 struct irq_desc *desc; 446 struct irq_desc *desc;
272 int i; 447 int i;
273 448
274 for_each_irq_desc(i, desc) 449 for_each_irq_desc(i, desc) {
275 lockdep_set_class(&desc->lock, &irq_desc_lock_class); 450 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
451 }
452}
453
454#ifdef CONFIG_SPARSE_IRQ
455unsigned int kstat_irqs_cpu(unsigned int irq, int cpu)
456{
457 struct irq_desc *desc = irq_to_desc(irq);
458 return desc ? desc->kstat_irqs[cpu] : 0;
276} 459}
277#endif 460#endif
461EXPORT_SYMBOL(kstat_irqs_cpu);
462
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index c9767e641980..e6d0a43cc125 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -13,6 +13,11 @@ extern void compat_irq_chip_set_default_handler(struct irq_desc *desc);
13extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, 13extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
14 unsigned long flags); 14 unsigned long flags);
15 15
16extern struct lock_class_key irq_desc_lock_class;
17extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr);
18extern spinlock_t sparse_irq_lock;
19extern struct irq_desc *irq_desc_ptrs[NR_IRQS];
20
16#ifdef CONFIG_PROC_FS 21#ifdef CONFIG_PROC_FS
17extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); 22extern void register_irq_proc(unsigned int irq, struct irq_desc *desc);
18extern void register_handler_proc(unsigned int irq, struct irqaction *action); 23extern void register_handler_proc(unsigned int irq, struct irqaction *action);
@@ -25,6 +30,8 @@ static inline void unregister_handler_proc(unsigned int irq,
25 struct irqaction *action) { } 30 struct irqaction *action) { }
26#endif 31#endif
27 32
33extern int irq_select_affinity_usr(unsigned int irq);
34
28/* 35/*
29 * Debugging printout: 36 * Debugging printout:
30 */ 37 */
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index c498a1b8c621..cd0cd8dcb345 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -16,8 +16,15 @@
16#include "internals.h" 16#include "internals.h"
17 17
18#ifdef CONFIG_SMP 18#ifdef CONFIG_SMP
19cpumask_var_t irq_default_affinity;
19 20
20cpumask_t irq_default_affinity = CPU_MASK_ALL; 21static int init_irq_default_affinity(void)
22{
23 alloc_cpumask_var(&irq_default_affinity, GFP_KERNEL);
24 cpumask_setall(irq_default_affinity);
25 return 0;
26}
27core_initcall(init_irq_default_affinity);
21 28
22/** 29/**
23 * synchronize_irq - wait for pending IRQ handlers (on other CPUs) 30 * synchronize_irq - wait for pending IRQ handlers (on other CPUs)
@@ -79,27 +86,30 @@ int irq_can_set_affinity(unsigned int irq)
79 * @cpumask: cpumask 86 * @cpumask: cpumask
80 * 87 *
81 */ 88 */
82int irq_set_affinity(unsigned int irq, cpumask_t cpumask) 89int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
83{ 90{
84 struct irq_desc *desc = irq_to_desc(irq); 91 struct irq_desc *desc = irq_to_desc(irq);
92 unsigned long flags;
85 93
86 if (!desc->chip->set_affinity) 94 if (!desc->chip->set_affinity)
87 return -EINVAL; 95 return -EINVAL;
88 96
97 spin_lock_irqsave(&desc->lock, flags);
98
89#ifdef CONFIG_GENERIC_PENDING_IRQ 99#ifdef CONFIG_GENERIC_PENDING_IRQ
90 if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) { 100 if (desc->status & IRQ_MOVE_PCNTXT || desc->status & IRQ_DISABLED) {
91 unsigned long flags; 101 cpumask_copy(&desc->affinity, cpumask);
92
93 spin_lock_irqsave(&desc->lock, flags);
94 desc->affinity = cpumask;
95 desc->chip->set_affinity(irq, cpumask); 102 desc->chip->set_affinity(irq, cpumask);
96 spin_unlock_irqrestore(&desc->lock, flags); 103 } else {
97 } else 104 desc->status |= IRQ_MOVE_PENDING;
98 set_pending_irq(irq, cpumask); 105 cpumask_copy(&desc->pending_mask, cpumask);
106 }
99#else 107#else
100 desc->affinity = cpumask; 108 cpumask_copy(&desc->affinity, cpumask);
101 desc->chip->set_affinity(irq, cpumask); 109 desc->chip->set_affinity(irq, cpumask);
102#endif 110#endif
111 desc->status |= IRQ_AFFINITY_SET;
112 spin_unlock_irqrestore(&desc->lock, flags);
103 return 0; 113 return 0;
104} 114}
105 115
@@ -107,24 +117,57 @@ int irq_set_affinity(unsigned int irq, cpumask_t cpumask)
107/* 117/*
108 * Generic version of the affinity autoselector. 118 * Generic version of the affinity autoselector.
109 */ 119 */
110int irq_select_affinity(unsigned int irq) 120int do_irq_select_affinity(unsigned int irq, struct irq_desc *desc)
111{ 121{
112 cpumask_t mask;
113 struct irq_desc *desc;
114
115 if (!irq_can_set_affinity(irq)) 122 if (!irq_can_set_affinity(irq))
116 return 0; 123 return 0;
117 124
118 cpus_and(mask, cpu_online_map, irq_default_affinity); 125 /*
126 * Preserve an userspace affinity setup, but make sure that
127 * one of the targets is online.
128 */
129 if (desc->status & (IRQ_AFFINITY_SET | IRQ_NO_BALANCING)) {
130 if (cpumask_any_and(&desc->affinity, cpu_online_mask)
131 < nr_cpu_ids)
132 goto set_affinity;
133 else
134 desc->status &= ~IRQ_AFFINITY_SET;
135 }
119 136
120 desc = irq_to_desc(irq); 137 cpumask_and(&desc->affinity, cpu_online_mask, irq_default_affinity);
121 desc->affinity = mask; 138set_affinity:
122 desc->chip->set_affinity(irq, mask); 139 desc->chip->set_affinity(irq, &desc->affinity);
123 140
124 return 0; 141 return 0;
125} 142}
143#else
144static inline int do_irq_select_affinity(unsigned int irq, struct irq_desc *d)
145{
146 return irq_select_affinity(irq);
147}
126#endif 148#endif
127 149
150/*
151 * Called when affinity is set via /proc/irq
152 */
153int irq_select_affinity_usr(unsigned int irq)
154{
155 struct irq_desc *desc = irq_to_desc(irq);
156 unsigned long flags;
157 int ret;
158
159 spin_lock_irqsave(&desc->lock, flags);
160 ret = do_irq_select_affinity(irq, desc);
161 spin_unlock_irqrestore(&desc->lock, flags);
162
163 return ret;
164}
165
166#else
167static inline int do_irq_select_affinity(int irq, struct irq_desc *desc)
168{
169 return 0;
170}
128#endif 171#endif
129 172
130/** 173/**
@@ -327,21 +370,23 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq,
327 * IRQF_TRIGGER_* but the PIC does not support multiple 370 * IRQF_TRIGGER_* but the PIC does not support multiple
328 * flow-types? 371 * flow-types?
329 */ 372 */
330 pr_warning("No set_type function for IRQ %d (%s)\n", irq, 373 pr_debug("No set_type function for IRQ %d (%s)\n", irq,
331 chip ? (chip->name ? : "unknown") : "unknown"); 374 chip ? (chip->name ? : "unknown") : "unknown");
332 return 0; 375 return 0;
333 } 376 }
334 377
335 ret = chip->set_type(irq, flags & IRQF_TRIGGER_MASK); 378 /* caller masked out all except trigger mode flags */
379 ret = chip->set_type(irq, flags);
336 380
337 if (ret) 381 if (ret)
338 pr_err("setting trigger mode %d for irq %u failed (%pF)\n", 382 pr_err("setting trigger mode %d for irq %u failed (%pF)\n",
339 (int)(flags & IRQF_TRIGGER_MASK), 383 (int)flags, irq, chip->set_type);
340 irq, chip->set_type);
341 else { 384 else {
385 if (flags & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH))
386 flags |= IRQ_LEVEL;
342 /* note that IRQF_TRIGGER_MASK == IRQ_TYPE_SENSE_MASK */ 387 /* note that IRQF_TRIGGER_MASK == IRQ_TYPE_SENSE_MASK */
343 desc->status &= ~IRQ_TYPE_SENSE_MASK; 388 desc->status &= ~(IRQ_LEVEL | IRQ_TYPE_SENSE_MASK);
344 desc->status |= flags & IRQ_TYPE_SENSE_MASK; 389 desc->status |= flags;
345 } 390 }
346 391
347 return ret; 392 return ret;
@@ -421,7 +466,8 @@ __setup_irq(unsigned int irq, struct irq_desc * desc, struct irqaction *new)
421 466
422 /* Setup the type (level, edge polarity) if configured: */ 467 /* Setup the type (level, edge polarity) if configured: */
423 if (new->flags & IRQF_TRIGGER_MASK) { 468 if (new->flags & IRQF_TRIGGER_MASK) {
424 ret = __irq_set_trigger(desc, irq, new->flags); 469 ret = __irq_set_trigger(desc, irq,
470 new->flags & IRQF_TRIGGER_MASK);
425 471
426 if (ret) { 472 if (ret) {
427 spin_unlock_irqrestore(&desc->lock, flags); 473 spin_unlock_irqrestore(&desc->lock, flags);
@@ -445,8 +491,12 @@ __setup_irq(unsigned int irq, struct irq_desc * desc, struct irqaction *new)
445 /* Undo nested disables: */ 491 /* Undo nested disables: */
446 desc->depth = 1; 492 desc->depth = 1;
447 493
494 /* Exclude IRQ from balancing if requested */
495 if (new->flags & IRQF_NOBALANCING)
496 desc->status |= IRQ_NO_BALANCING;
497
448 /* Set default affinity mask once everything is setup */ 498 /* Set default affinity mask once everything is setup */
449 irq_select_affinity(irq); 499 do_irq_select_affinity(irq, desc);
450 500
451 } else if ((new->flags & IRQF_TRIGGER_MASK) 501 } else if ((new->flags & IRQF_TRIGGER_MASK)
452 && (new->flags & IRQF_TRIGGER_MASK) 502 && (new->flags & IRQF_TRIGGER_MASK)
@@ -459,10 +509,6 @@ __setup_irq(unsigned int irq, struct irq_desc * desc, struct irqaction *new)
459 509
460 *p = new; 510 *p = new;
461 511
462 /* Exclude IRQ from balancing */
463 if (new->flags & IRQF_NOBALANCING)
464 desc->status |= IRQ_NO_BALANCING;
465
466 /* Reset broken irq detection when installing new handler */ 512 /* Reset broken irq detection when installing new handler */
467 desc->irq_count = 0; 513 desc->irq_count = 0;
468 desc->irqs_unhandled = 0; 514 desc->irqs_unhandled = 0;
@@ -635,6 +681,18 @@ int request_irq(unsigned int irq, irq_handler_t handler,
635 struct irq_desc *desc; 681 struct irq_desc *desc;
636 int retval; 682 int retval;
637 683
684 /*
685 * handle_IRQ_event() always ignores IRQF_DISABLED except for
686 * the _first_ irqaction (sigh). That can cause oopsing, but
687 * the behavior is classified as "will not fix" so we need to
688 * start nudging drivers away from using that idiom.
689 */
690 if ((irqflags & (IRQF_SHARED|IRQF_DISABLED))
691 == (IRQF_SHARED|IRQF_DISABLED))
692 pr_warning("IRQ %d/%s: IRQF_DISABLED is not "
693 "guaranteed on shared IRQs\n",
694 irq, devname);
695
638#ifdef CONFIG_LOCKDEP 696#ifdef CONFIG_LOCKDEP
639 /* 697 /*
640 * Lockdep wants atomic interrupt handlers: 698 * Lockdep wants atomic interrupt handlers:
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index 90b920d3f52b..bd72329e630c 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -1,21 +1,9 @@
1 1
2#include <linux/irq.h> 2#include <linux/irq.h>
3 3
4void set_pending_irq(unsigned int irq, cpumask_t mask)
5{
6 struct irq_desc *desc = irq_to_desc(irq);
7 unsigned long flags;
8
9 spin_lock_irqsave(&desc->lock, flags);
10 desc->status |= IRQ_MOVE_PENDING;
11 desc->pending_mask = mask;
12 spin_unlock_irqrestore(&desc->lock, flags);
13}
14
15void move_masked_irq(int irq) 4void move_masked_irq(int irq)
16{ 5{
17 struct irq_desc *desc = irq_to_desc(irq); 6 struct irq_desc *desc = irq_to_desc(irq);
18 cpumask_t tmp;
19 7
20 if (likely(!(desc->status & IRQ_MOVE_PENDING))) 8 if (likely(!(desc->status & IRQ_MOVE_PENDING)))
21 return; 9 return;
@@ -30,7 +18,7 @@ void move_masked_irq(int irq)
30 18
31 desc->status &= ~IRQ_MOVE_PENDING; 19 desc->status &= ~IRQ_MOVE_PENDING;
32 20
33 if (unlikely(cpus_empty(desc->pending_mask))) 21 if (unlikely(cpumask_empty(&desc->pending_mask)))
34 return; 22 return;
35 23
36 if (!desc->chip->set_affinity) 24 if (!desc->chip->set_affinity)
@@ -38,8 +26,6 @@ void move_masked_irq(int irq)
38 26
39 assert_spin_locked(&desc->lock); 27 assert_spin_locked(&desc->lock);
40 28
41 cpus_and(tmp, desc->pending_mask, cpu_online_map);
42
43 /* 29 /*
44 * If there was a valid mask to work with, please 30 * If there was a valid mask to work with, please
45 * do the disable, re-program, enable sequence. 31 * do the disable, re-program, enable sequence.
@@ -52,10 +38,13 @@ void move_masked_irq(int irq)
52 * For correct operation this depends on the caller 38 * For correct operation this depends on the caller
53 * masking the irqs. 39 * masking the irqs.
54 */ 40 */
55 if (likely(!cpus_empty(tmp))) { 41 if (likely(cpumask_any_and(&desc->pending_mask, cpu_online_mask)
56 desc->chip->set_affinity(irq,tmp); 42 < nr_cpu_ids)) {
43 cpumask_and(&desc->affinity,
44 &desc->pending_mask, cpu_online_mask);
45 desc->chip->set_affinity(irq, &desc->affinity);
57 } 46 }
58 cpus_clear(desc->pending_mask); 47 cpumask_clear(&desc->pending_mask);
59} 48}
60 49
61void move_native_irq(int irq) 50void move_native_irq(int irq)
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c
new file mode 100644
index 000000000000..ecf765c6a77a
--- /dev/null
+++ b/kernel/irq/numa_migrate.c
@@ -0,0 +1,119 @@
1/*
2 * NUMA irq-desc migration code
3 *
4 * Migrate IRQ data structures (irq_desc, chip_data, etc.) over to
5 * the new "home node" of the IRQ.
6 */
7
8#include <linux/irq.h>
9#include <linux/module.h>
10#include <linux/random.h>
11#include <linux/interrupt.h>
12#include <linux/kernel_stat.h>
13
14#include "internals.h"
15
16static void init_copy_kstat_irqs(struct irq_desc *old_desc,
17 struct irq_desc *desc,
18 int cpu, int nr)
19{
20 unsigned long bytes;
21
22 init_kstat_irqs(desc, cpu, nr);
23
24 if (desc->kstat_irqs != old_desc->kstat_irqs) {
25 /* Compute how many bytes we need per irq and allocate them */
26 bytes = nr * sizeof(unsigned int);
27
28 memcpy(desc->kstat_irqs, old_desc->kstat_irqs, bytes);
29 }
30}
31
32static void free_kstat_irqs(struct irq_desc *old_desc, struct irq_desc *desc)
33{
34 if (old_desc->kstat_irqs == desc->kstat_irqs)
35 return;
36
37 kfree(old_desc->kstat_irqs);
38 old_desc->kstat_irqs = NULL;
39}
40
41static void init_copy_one_irq_desc(int irq, struct irq_desc *old_desc,
42 struct irq_desc *desc, int cpu)
43{
44 memcpy(desc, old_desc, sizeof(struct irq_desc));
45 spin_lock_init(&desc->lock);
46 desc->cpu = cpu;
47 lockdep_set_class(&desc->lock, &irq_desc_lock_class);
48 init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids);
49 arch_init_copy_chip_data(old_desc, desc, cpu);
50}
51
52static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc)
53{
54 free_kstat_irqs(old_desc, desc);
55 arch_free_chip_data(old_desc, desc);
56}
57
58static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc,
59 int cpu)
60{
61 struct irq_desc *desc;
62 unsigned int irq;
63 unsigned long flags;
64 int node;
65
66 irq = old_desc->irq;
67
68 spin_lock_irqsave(&sparse_irq_lock, flags);
69
70 /* We have to check it to avoid races with another CPU */
71 desc = irq_desc_ptrs[irq];
72
73 if (desc && old_desc != desc)
74 goto out_unlock;
75
76 node = cpu_to_node(cpu);
77 desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node);
78 if (!desc) {
79 printk(KERN_ERR "irq %d: can not get new irq_desc for migration.\n", irq);
80 /* still use old one */
81 desc = old_desc;
82 goto out_unlock;
83 }
84 init_copy_one_irq_desc(irq, old_desc, desc, cpu);
85
86 irq_desc_ptrs[irq] = desc;
87
88 /* free the old one */
89 free_one_irq_desc(old_desc, desc);
90 kfree(old_desc);
91
92out_unlock:
93 spin_unlock_irqrestore(&sparse_irq_lock, flags);
94
95 return desc;
96}
97
98struct irq_desc *move_irq_desc(struct irq_desc *desc, int cpu)
99{
100 int old_cpu;
101 int node, old_node;
102
103 /* those all static, do move them */
104 if (desc->irq < NR_IRQS_LEGACY)
105 return desc;
106
107 old_cpu = desc->cpu;
108 if (old_cpu != cpu) {
109 node = cpu_to_node(cpu);
110 old_node = cpu_to_node(old_cpu);
111 if (old_node != node)
112 desc = __real_move_irq_desc(desc, cpu);
113 else
114 desc->cpu = cpu;
115 }
116
117 return desc;
118}
119
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index 4d161c70ba55..aae3f742bcec 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -20,7 +20,7 @@ static struct proc_dir_entry *root_irq_dir;
20static int irq_affinity_proc_show(struct seq_file *m, void *v) 20static int irq_affinity_proc_show(struct seq_file *m, void *v)
21{ 21{
22 struct irq_desc *desc = irq_to_desc((long)m->private); 22 struct irq_desc *desc = irq_to_desc((long)m->private);
23 cpumask_t *mask = &desc->affinity; 23 const struct cpumask *mask = &desc->affinity;
24 24
25#ifdef CONFIG_GENERIC_PENDING_IRQ 25#ifdef CONFIG_GENERIC_PENDING_IRQ
26 if (desc->status & IRQ_MOVE_PENDING) 26 if (desc->status & IRQ_MOVE_PENDING)
@@ -40,33 +40,42 @@ static ssize_t irq_affinity_proc_write(struct file *file,
40 const char __user *buffer, size_t count, loff_t *pos) 40 const char __user *buffer, size_t count, loff_t *pos)
41{ 41{
42 unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data; 42 unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data;
43 cpumask_t new_value; 43 cpumask_var_t new_value;
44 int err; 44 int err;
45 45
46 if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity || 46 if (!irq_to_desc(irq)->chip->set_affinity || no_irq_affinity ||
47 irq_balancing_disabled(irq)) 47 irq_balancing_disabled(irq))
48 return -EIO; 48 return -EIO;
49 49
50 if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
51 return -ENOMEM;
52
50 err = cpumask_parse_user(buffer, count, new_value); 53 err = cpumask_parse_user(buffer, count, new_value);
51 if (err) 54 if (err)
52 return err; 55 goto free_cpumask;
53 56
54 if (!is_affinity_mask_valid(new_value)) 57 if (!is_affinity_mask_valid(new_value)) {
55 return -EINVAL; 58 err = -EINVAL;
59 goto free_cpumask;
60 }
56 61
57 /* 62 /*
58 * Do not allow disabling IRQs completely - it's a too easy 63 * Do not allow disabling IRQs completely - it's a too easy
59 * way to make the system unusable accidentally :-) At least 64 * way to make the system unusable accidentally :-) At least
60 * one online CPU still has to be targeted. 65 * one online CPU still has to be targeted.
61 */ 66 */
62 if (!cpus_intersects(new_value, cpu_online_map)) 67 if (!cpumask_intersects(new_value, cpu_online_mask)) {
63 /* Special case for empty set - allow the architecture 68 /* Special case for empty set - allow the architecture
64 code to set default SMP affinity. */ 69 code to set default SMP affinity. */
65 return irq_select_affinity(irq) ? -EINVAL : count; 70 err = irq_select_affinity_usr(irq) ? -EINVAL : count;
66 71 } else {
67 irq_set_affinity(irq, new_value); 72 irq_set_affinity(irq, new_value);
73 err = count;
74 }
68 75
69 return count; 76free_cpumask:
77 free_cpumask_var(new_value);
78 return err;
70} 79}
71 80
72static int irq_affinity_proc_open(struct inode *inode, struct file *file) 81static int irq_affinity_proc_open(struct inode *inode, struct file *file)
@@ -84,7 +93,7 @@ static const struct file_operations irq_affinity_proc_fops = {
84 93
85static int default_affinity_show(struct seq_file *m, void *v) 94static int default_affinity_show(struct seq_file *m, void *v)
86{ 95{
87 seq_cpumask(m, &irq_default_affinity); 96 seq_cpumask(m, irq_default_affinity);
88 seq_putc(m, '\n'); 97 seq_putc(m, '\n');
89 return 0; 98 return 0;
90} 99}
@@ -92,27 +101,37 @@ static int default_affinity_show(struct seq_file *m, void *v)
92static ssize_t default_affinity_write(struct file *file, 101static ssize_t default_affinity_write(struct file *file,
93 const char __user *buffer, size_t count, loff_t *ppos) 102 const char __user *buffer, size_t count, loff_t *ppos)
94{ 103{
95 cpumask_t new_value; 104 cpumask_var_t new_value;
96 int err; 105 int err;
97 106
107 if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
108 return -ENOMEM;
109
98 err = cpumask_parse_user(buffer, count, new_value); 110 err = cpumask_parse_user(buffer, count, new_value);
99 if (err) 111 if (err)
100 return err; 112 goto out;
101 113
102 if (!is_affinity_mask_valid(new_value)) 114 if (!is_affinity_mask_valid(new_value)) {
103 return -EINVAL; 115 err = -EINVAL;
116 goto out;
117 }
104 118
105 /* 119 /*
106 * Do not allow disabling IRQs completely - it's a too easy 120 * Do not allow disabling IRQs completely - it's a too easy
107 * way to make the system unusable accidentally :-) At least 121 * way to make the system unusable accidentally :-) At least
108 * one online CPU still has to be targeted. 122 * one online CPU still has to be targeted.
109 */ 123 */
110 if (!cpus_intersects(new_value, cpu_online_map)) 124 if (!cpumask_intersects(new_value, cpu_online_mask)) {
111 return -EINVAL; 125 err = -EINVAL;
126 goto out;
127 }
112 128
113 irq_default_affinity = new_value; 129 cpumask_copy(irq_default_affinity, new_value);
130 err = count;
114 131
115 return count; 132out:
133 free_cpumask_var(new_value);
134 return err;
116} 135}
117 136
118static int default_affinity_open(struct inode *inode, struct file *file) 137static int default_affinity_open(struct inode *inode, struct file *file)
@@ -243,7 +262,11 @@ void init_irq_proc(void)
243 /* 262 /*
244 * Create entries for all existing IRQs. 263 * Create entries for all existing IRQs.
245 */ 264 */
246 for_each_irq_desc(irq, desc) 265 for_each_irq_desc(irq, desc) {
266 if (!desc)
267 continue;
268
247 register_irq_proc(irq, desc); 269 register_irq_proc(irq, desc);
270 }
248} 271}
249 272
diff --git a/kernel/kallsyms.c b/kernel/kallsyms.c
index 7b8b0f21a5b1..e694afa0eb8c 100644
--- a/kernel/kallsyms.c
+++ b/kernel/kallsyms.c
@@ -30,20 +30,19 @@
30#define all_var 0 30#define all_var 0
31#endif 31#endif
32 32
33/* These will be re-linked against their real values during the second link stage */ 33extern const unsigned long kallsyms_addresses[];
34extern const unsigned long kallsyms_addresses[] __attribute__((weak)); 34extern const u8 kallsyms_names[];
35extern const u8 kallsyms_names[] __attribute__((weak));
36 35
37/* tell the compiler that the count isn't in the small data section if the arch 36/* tell the compiler that the count isn't in the small data section if the arch
38 * has one (eg: FRV) 37 * has one (eg: FRV)
39 */ 38 */
40extern const unsigned long kallsyms_num_syms 39extern const unsigned long kallsyms_num_syms
41__attribute__((weak, section(".rodata"))); 40 __attribute__((__section__(".rodata")));
42 41
43extern const u8 kallsyms_token_table[] __attribute__((weak)); 42extern const u8 kallsyms_token_table[];
44extern const u16 kallsyms_token_index[] __attribute__((weak)); 43extern const u16 kallsyms_token_index[];
45 44
46extern const unsigned long kallsyms_markers[] __attribute__((weak)); 45extern const unsigned long kallsyms_markers[];
47 46
48static inline int is_kernel_inittext(unsigned long addr) 47static inline int is_kernel_inittext(unsigned long addr)
49{ 48{
@@ -168,9 +167,6 @@ static unsigned long get_symbol_pos(unsigned long addr,
168 unsigned long symbol_start = 0, symbol_end = 0; 167 unsigned long symbol_start = 0, symbol_end = 0;
169 unsigned long i, low, high, mid; 168 unsigned long i, low, high, mid;
170 169
171 /* This kernel should never had been booted. */
172 BUG_ON(!kallsyms_addresses);
173
174 /* do a binary search on the sorted kallsyms_addresses array */ 170 /* do a binary search on the sorted kallsyms_addresses array */
175 low = 0; 171 low = 0;
176 high = kallsyms_num_syms; 172 high = kallsyms_num_syms;
diff --git a/kernel/kexec.c b/kernel/kexec.c
index ac0fde7b54d0..3fb855ad6aa0 100644
--- a/kernel/kexec.c
+++ b/kernel/kexec.c
@@ -1116,7 +1116,7 @@ void crash_save_cpu(struct pt_regs *regs, int cpu)
1116 struct elf_prstatus prstatus; 1116 struct elf_prstatus prstatus;
1117 u32 *buf; 1117 u32 *buf;
1118 1118
1119 if ((cpu < 0) || (cpu >= NR_CPUS)) 1119 if ((cpu < 0) || (cpu >= nr_cpu_ids))
1120 return; 1120 return;
1121 1121
1122 /* Using ELF notes here is opportunistic. 1122 /* Using ELF notes here is opportunistic.
diff --git a/kernel/kmod.c b/kernel/kmod.c
index 3d3c3ea3a023..b46dbb908669 100644
--- a/kernel/kmod.c
+++ b/kernel/kmod.c
@@ -118,10 +118,10 @@ EXPORT_SYMBOL(request_module);
118struct subprocess_info { 118struct subprocess_info {
119 struct work_struct work; 119 struct work_struct work;
120 struct completion *complete; 120 struct completion *complete;
121 struct cred *cred;
121 char *path; 122 char *path;
122 char **argv; 123 char **argv;
123 char **envp; 124 char **envp;
124 struct key *ring;
125 enum umh_wait wait; 125 enum umh_wait wait;
126 int retval; 126 int retval;
127 struct file *stdin; 127 struct file *stdin;
@@ -134,19 +134,20 @@ struct subprocess_info {
134static int ____call_usermodehelper(void *data) 134static int ____call_usermodehelper(void *data)
135{ 135{
136 struct subprocess_info *sub_info = data; 136 struct subprocess_info *sub_info = data;
137 struct key *new_session, *old_session;
138 int retval; 137 int retval;
139 138
140 /* Unblock all signals and set the session keyring. */ 139 BUG_ON(atomic_read(&sub_info->cred->usage) != 1);
141 new_session = key_get(sub_info->ring); 140
141 /* Unblock all signals */
142 spin_lock_irq(&current->sighand->siglock); 142 spin_lock_irq(&current->sighand->siglock);
143 old_session = __install_session_keyring(current, new_session);
144 flush_signal_handlers(current, 1); 143 flush_signal_handlers(current, 1);
145 sigemptyset(&current->blocked); 144 sigemptyset(&current->blocked);
146 recalc_sigpending(); 145 recalc_sigpending();
147 spin_unlock_irq(&current->sighand->siglock); 146 spin_unlock_irq(&current->sighand->siglock);
148 147
149 key_put(old_session); 148 /* Install the credentials */
149 commit_creds(sub_info->cred);
150 sub_info->cred = NULL;
150 151
151 /* Install input pipe when needed */ 152 /* Install input pipe when needed */
152 if (sub_info->stdin) { 153 if (sub_info->stdin) {
@@ -185,6 +186,8 @@ void call_usermodehelper_freeinfo(struct subprocess_info *info)
185{ 186{
186 if (info->cleanup) 187 if (info->cleanup)
187 (*info->cleanup)(info->argv, info->envp); 188 (*info->cleanup)(info->argv, info->envp);
189 if (info->cred)
190 put_cred(info->cred);
188 kfree(info); 191 kfree(info);
189} 192}
190EXPORT_SYMBOL(call_usermodehelper_freeinfo); 193EXPORT_SYMBOL(call_usermodehelper_freeinfo);
@@ -240,6 +243,8 @@ static void __call_usermodehelper(struct work_struct *work)
240 pid_t pid; 243 pid_t pid;
241 enum umh_wait wait = sub_info->wait; 244 enum umh_wait wait = sub_info->wait;
242 245
246 BUG_ON(atomic_read(&sub_info->cred->usage) != 1);
247
243 /* CLONE_VFORK: wait until the usermode helper has execve'd 248 /* CLONE_VFORK: wait until the usermode helper has execve'd
244 * successfully We need the data structures to stay around 249 * successfully We need the data structures to stay around
245 * until that is done. */ 250 * until that is done. */
@@ -362,6 +367,9 @@ struct subprocess_info *call_usermodehelper_setup(char *path, char **argv,
362 sub_info->path = path; 367 sub_info->path = path;
363 sub_info->argv = argv; 368 sub_info->argv = argv;
364 sub_info->envp = envp; 369 sub_info->envp = envp;
370 sub_info->cred = prepare_usermodehelper_creds();
371 if (!sub_info->cred)
372 return NULL;
365 373
366 out: 374 out:
367 return sub_info; 375 return sub_info;
@@ -376,7 +384,13 @@ EXPORT_SYMBOL(call_usermodehelper_setup);
376void call_usermodehelper_setkeys(struct subprocess_info *info, 384void call_usermodehelper_setkeys(struct subprocess_info *info,
377 struct key *session_keyring) 385 struct key *session_keyring)
378{ 386{
379 info->ring = session_keyring; 387#ifdef CONFIG_KEYS
388 struct thread_group_cred *tgcred = info->cred->tgcred;
389 key_put(tgcred->session_keyring);
390 tgcred->session_keyring = key_get(session_keyring);
391#else
392 BUG();
393#endif
380} 394}
381EXPORT_SYMBOL(call_usermodehelper_setkeys); 395EXPORT_SYMBOL(call_usermodehelper_setkeys);
382 396
@@ -444,6 +458,8 @@ int call_usermodehelper_exec(struct subprocess_info *sub_info,
444 DECLARE_COMPLETION_ONSTACK(done); 458 DECLARE_COMPLETION_ONSTACK(done);
445 int retval = 0; 459 int retval = 0;
446 460
461 BUG_ON(atomic_read(&sub_info->cred->usage) != 1);
462
447 helper_lock(); 463 helper_lock();
448 if (sub_info->path[0] == '\0') 464 if (sub_info->path[0] == '\0')
449 goto out; 465 goto out;
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 8e7a7ce3ed0a..4fbc456f393d 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -21,6 +21,9 @@ static DEFINE_SPINLOCK(kthread_create_lock);
21static LIST_HEAD(kthread_create_list); 21static LIST_HEAD(kthread_create_list);
22struct task_struct *kthreadd_task; 22struct task_struct *kthreadd_task;
23 23
24DEFINE_TRACE(sched_kthread_stop);
25DEFINE_TRACE(sched_kthread_stop_ret);
26
24struct kthread_create_info 27struct kthread_create_info
25{ 28{
26 /* Information passed to kthread() from kthreadd. */ 29 /* Information passed to kthread() from kthreadd. */
diff --git a/kernel/latencytop.c b/kernel/latencytop.c
index 5e7b45c56923..449db466bdbc 100644
--- a/kernel/latencytop.c
+++ b/kernel/latencytop.c
@@ -191,7 +191,7 @@ static int lstats_show(struct seq_file *m, void *v)
191 latency_record[i].time, 191 latency_record[i].time,
192 latency_record[i].max); 192 latency_record[i].max);
193 for (q = 0; q < LT_BACKTRACEDEPTH; q++) { 193 for (q = 0; q < LT_BACKTRACEDEPTH; q++) {
194 char sym[KSYM_NAME_LEN]; 194 char sym[KSYM_SYMBOL_LEN];
195 char *c; 195 char *c;
196 if (!latency_record[i].backtrace[q]) 196 if (!latency_record[i].backtrace[q])
197 break; 197 break;
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
index 06e157119d2b..06b0c3568f0b 100644
--- a/kernel/lockdep.c
+++ b/kernel/lockdep.c
@@ -25,6 +25,7 @@
25 * Thanks to Arjan van de Ven for coming up with the initial idea of 25 * Thanks to Arjan van de Ven for coming up with the initial idea of
26 * mapping lock dependencies runtime. 26 * mapping lock dependencies runtime.
27 */ 27 */
28#define DISABLE_BRANCH_PROFILING
28#include <linux/mutex.h> 29#include <linux/mutex.h>
29#include <linux/sched.h> 30#include <linux/sched.h>
30#include <linux/delay.h> 31#include <linux/delay.h>
@@ -136,16 +137,16 @@ static inline struct lock_class *hlock_class(struct held_lock *hlock)
136#ifdef CONFIG_LOCK_STAT 137#ifdef CONFIG_LOCK_STAT
137static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats); 138static DEFINE_PER_CPU(struct lock_class_stats[MAX_LOCKDEP_KEYS], lock_stats);
138 139
139static int lock_contention_point(struct lock_class *class, unsigned long ip) 140static int lock_point(unsigned long points[], unsigned long ip)
140{ 141{
141 int i; 142 int i;
142 143
143 for (i = 0; i < ARRAY_SIZE(class->contention_point); i++) { 144 for (i = 0; i < LOCKSTAT_POINTS; i++) {
144 if (class->contention_point[i] == 0) { 145 if (points[i] == 0) {
145 class->contention_point[i] = ip; 146 points[i] = ip;
146 break; 147 break;
147 } 148 }
148 if (class->contention_point[i] == ip) 149 if (points[i] == ip)
149 break; 150 break;
150 } 151 }
151 152
@@ -185,6 +186,9 @@ struct lock_class_stats lock_stats(struct lock_class *class)
185 for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++) 186 for (i = 0; i < ARRAY_SIZE(stats.contention_point); i++)
186 stats.contention_point[i] += pcs->contention_point[i]; 187 stats.contention_point[i] += pcs->contention_point[i];
187 188
189 for (i = 0; i < ARRAY_SIZE(stats.contending_point); i++)
190 stats.contending_point[i] += pcs->contending_point[i];
191
188 lock_time_add(&pcs->read_waittime, &stats.read_waittime); 192 lock_time_add(&pcs->read_waittime, &stats.read_waittime);
189 lock_time_add(&pcs->write_waittime, &stats.write_waittime); 193 lock_time_add(&pcs->write_waittime, &stats.write_waittime);
190 194
@@ -209,6 +213,7 @@ void clear_lock_stats(struct lock_class *class)
209 memset(cpu_stats, 0, sizeof(struct lock_class_stats)); 213 memset(cpu_stats, 0, sizeof(struct lock_class_stats));
210 } 214 }
211 memset(class->contention_point, 0, sizeof(class->contention_point)); 215 memset(class->contention_point, 0, sizeof(class->contention_point));
216 memset(class->contending_point, 0, sizeof(class->contending_point));
212} 217}
213 218
214static struct lock_class_stats *get_lock_stats(struct lock_class *class) 219static struct lock_class_stats *get_lock_stats(struct lock_class *class)
@@ -287,14 +292,12 @@ void lockdep_off(void)
287{ 292{
288 current->lockdep_recursion++; 293 current->lockdep_recursion++;
289} 294}
290
291EXPORT_SYMBOL(lockdep_off); 295EXPORT_SYMBOL(lockdep_off);
292 296
293void lockdep_on(void) 297void lockdep_on(void)
294{ 298{
295 current->lockdep_recursion--; 299 current->lockdep_recursion--;
296} 300}
297
298EXPORT_SYMBOL(lockdep_on); 301EXPORT_SYMBOL(lockdep_on);
299 302
300/* 303/*
@@ -576,7 +579,8 @@ static void print_lock_class_header(struct lock_class *class, int depth)
576/* 579/*
577 * printk all lock dependencies starting at <entry>: 580 * printk all lock dependencies starting at <entry>:
578 */ 581 */
579static void print_lock_dependencies(struct lock_class *class, int depth) 582static void __used
583print_lock_dependencies(struct lock_class *class, int depth)
580{ 584{
581 struct lock_list *entry; 585 struct lock_list *entry;
582 586
@@ -2508,7 +2512,6 @@ void lockdep_init_map(struct lockdep_map *lock, const char *name,
2508 if (subclass) 2512 if (subclass)
2509 register_lock_class(lock, subclass, 1); 2513 register_lock_class(lock, subclass, 1);
2510} 2514}
2511
2512EXPORT_SYMBOL_GPL(lockdep_init_map); 2515EXPORT_SYMBOL_GPL(lockdep_init_map);
2513 2516
2514/* 2517/*
@@ -2689,8 +2692,9 @@ static int check_unlock(struct task_struct *curr, struct lockdep_map *lock,
2689} 2692}
2690 2693
2691static int 2694static int
2692__lock_set_subclass(struct lockdep_map *lock, 2695__lock_set_class(struct lockdep_map *lock, const char *name,
2693 unsigned int subclass, unsigned long ip) 2696 struct lock_class_key *key, unsigned int subclass,
2697 unsigned long ip)
2694{ 2698{
2695 struct task_struct *curr = current; 2699 struct task_struct *curr = current;
2696 struct held_lock *hlock, *prev_hlock; 2700 struct held_lock *hlock, *prev_hlock;
@@ -2717,6 +2721,7 @@ __lock_set_subclass(struct lockdep_map *lock,
2717 return print_unlock_inbalance_bug(curr, lock, ip); 2721 return print_unlock_inbalance_bug(curr, lock, ip);
2718 2722
2719found_it: 2723found_it:
2724 lockdep_init_map(lock, name, key, 0);
2720 class = register_lock_class(lock, subclass, 0); 2725 class = register_lock_class(lock, subclass, 0);
2721 hlock->class_idx = class - lock_classes + 1; 2726 hlock->class_idx = class - lock_classes + 1;
2722 2727
@@ -2901,9 +2906,9 @@ static void check_flags(unsigned long flags)
2901#endif 2906#endif
2902} 2907}
2903 2908
2904void 2909void lock_set_class(struct lockdep_map *lock, const char *name,
2905lock_set_subclass(struct lockdep_map *lock, 2910 struct lock_class_key *key, unsigned int subclass,
2906 unsigned int subclass, unsigned long ip) 2911 unsigned long ip)
2907{ 2912{
2908 unsigned long flags; 2913 unsigned long flags;
2909 2914
@@ -2913,13 +2918,12 @@ lock_set_subclass(struct lockdep_map *lock,
2913 raw_local_irq_save(flags); 2918 raw_local_irq_save(flags);
2914 current->lockdep_recursion = 1; 2919 current->lockdep_recursion = 1;
2915 check_flags(flags); 2920 check_flags(flags);
2916 if (__lock_set_subclass(lock, subclass, ip)) 2921 if (__lock_set_class(lock, name, key, subclass, ip))
2917 check_chain_key(current); 2922 check_chain_key(current);
2918 current->lockdep_recursion = 0; 2923 current->lockdep_recursion = 0;
2919 raw_local_irq_restore(flags); 2924 raw_local_irq_restore(flags);
2920} 2925}
2921 2926EXPORT_SYMBOL_GPL(lock_set_class);
2922EXPORT_SYMBOL_GPL(lock_set_subclass);
2923 2927
2924/* 2928/*
2925 * We are not always called with irqs disabled - do that here, 2929 * We are not always called with irqs disabled - do that here,
@@ -2943,7 +2947,6 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
2943 current->lockdep_recursion = 0; 2947 current->lockdep_recursion = 0;
2944 raw_local_irq_restore(flags); 2948 raw_local_irq_restore(flags);
2945} 2949}
2946
2947EXPORT_SYMBOL_GPL(lock_acquire); 2950EXPORT_SYMBOL_GPL(lock_acquire);
2948 2951
2949void lock_release(struct lockdep_map *lock, int nested, 2952void lock_release(struct lockdep_map *lock, int nested,
@@ -2961,7 +2964,6 @@ void lock_release(struct lockdep_map *lock, int nested,
2961 current->lockdep_recursion = 0; 2964 current->lockdep_recursion = 0;
2962 raw_local_irq_restore(flags); 2965 raw_local_irq_restore(flags);
2963} 2966}
2964
2965EXPORT_SYMBOL_GPL(lock_release); 2967EXPORT_SYMBOL_GPL(lock_release);
2966 2968
2967#ifdef CONFIG_LOCK_STAT 2969#ifdef CONFIG_LOCK_STAT
@@ -2999,7 +3001,7 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
2999 struct held_lock *hlock, *prev_hlock; 3001 struct held_lock *hlock, *prev_hlock;
3000 struct lock_class_stats *stats; 3002 struct lock_class_stats *stats;
3001 unsigned int depth; 3003 unsigned int depth;
3002 int i, point; 3004 int i, contention_point, contending_point;
3003 3005
3004 depth = curr->lockdep_depth; 3006 depth = curr->lockdep_depth;
3005 if (DEBUG_LOCKS_WARN_ON(!depth)) 3007 if (DEBUG_LOCKS_WARN_ON(!depth))
@@ -3023,18 +3025,22 @@ __lock_contended(struct lockdep_map *lock, unsigned long ip)
3023found_it: 3025found_it:
3024 hlock->waittime_stamp = sched_clock(); 3026 hlock->waittime_stamp = sched_clock();
3025 3027
3026 point = lock_contention_point(hlock_class(hlock), ip); 3028 contention_point = lock_point(hlock_class(hlock)->contention_point, ip);
3029 contending_point = lock_point(hlock_class(hlock)->contending_point,
3030 lock->ip);
3027 3031
3028 stats = get_lock_stats(hlock_class(hlock)); 3032 stats = get_lock_stats(hlock_class(hlock));
3029 if (point < ARRAY_SIZE(stats->contention_point)) 3033 if (contention_point < LOCKSTAT_POINTS)
3030 stats->contention_point[point]++; 3034 stats->contention_point[contention_point]++;
3035 if (contending_point < LOCKSTAT_POINTS)
3036 stats->contending_point[contending_point]++;
3031 if (lock->cpu != smp_processor_id()) 3037 if (lock->cpu != smp_processor_id())
3032 stats->bounces[bounce_contended + !!hlock->read]++; 3038 stats->bounces[bounce_contended + !!hlock->read]++;
3033 put_lock_stats(stats); 3039 put_lock_stats(stats);
3034} 3040}
3035 3041
3036static void 3042static void
3037__lock_acquired(struct lockdep_map *lock) 3043__lock_acquired(struct lockdep_map *lock, unsigned long ip)
3038{ 3044{
3039 struct task_struct *curr = current; 3045 struct task_struct *curr = current;
3040 struct held_lock *hlock, *prev_hlock; 3046 struct held_lock *hlock, *prev_hlock;
@@ -3083,6 +3089,7 @@ found_it:
3083 put_lock_stats(stats); 3089 put_lock_stats(stats);
3084 3090
3085 lock->cpu = cpu; 3091 lock->cpu = cpu;
3092 lock->ip = ip;
3086} 3093}
3087 3094
3088void lock_contended(struct lockdep_map *lock, unsigned long ip) 3095void lock_contended(struct lockdep_map *lock, unsigned long ip)
@@ -3104,7 +3111,7 @@ void lock_contended(struct lockdep_map *lock, unsigned long ip)
3104} 3111}
3105EXPORT_SYMBOL_GPL(lock_contended); 3112EXPORT_SYMBOL_GPL(lock_contended);
3106 3113
3107void lock_acquired(struct lockdep_map *lock) 3114void lock_acquired(struct lockdep_map *lock, unsigned long ip)
3108{ 3115{
3109 unsigned long flags; 3116 unsigned long flags;
3110 3117
@@ -3117,7 +3124,7 @@ void lock_acquired(struct lockdep_map *lock)
3117 raw_local_irq_save(flags); 3124 raw_local_irq_save(flags);
3118 check_flags(flags); 3125 check_flags(flags);
3119 current->lockdep_recursion = 1; 3126 current->lockdep_recursion = 1;
3120 __lock_acquired(lock); 3127 __lock_acquired(lock, ip);
3121 current->lockdep_recursion = 0; 3128 current->lockdep_recursion = 0;
3122 raw_local_irq_restore(flags); 3129 raw_local_irq_restore(flags);
3123} 3130}
@@ -3276,10 +3283,10 @@ void __init lockdep_info(void)
3276{ 3283{
3277 printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n"); 3284 printk("Lock dependency validator: Copyright (c) 2006 Red Hat, Inc., Ingo Molnar\n");
3278 3285
3279 printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES); 3286 printk("... MAX_LOCKDEP_SUBCLASSES: %lu\n", MAX_LOCKDEP_SUBCLASSES);
3280 printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH); 3287 printk("... MAX_LOCK_DEPTH: %lu\n", MAX_LOCK_DEPTH);
3281 printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS); 3288 printk("... MAX_LOCKDEP_KEYS: %lu\n", MAX_LOCKDEP_KEYS);
3282 printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE); 3289 printk("... CLASSHASH_SIZE: %lu\n", CLASSHASH_SIZE);
3283 printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES); 3290 printk("... MAX_LOCKDEP_ENTRIES: %lu\n", MAX_LOCKDEP_ENTRIES);
3284 printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS); 3291 printk("... MAX_LOCKDEP_CHAINS: %lu\n", MAX_LOCKDEP_CHAINS);
3285 printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE); 3292 printk("... CHAINHASH_SIZE: %lu\n", CHAINHASH_SIZE);
@@ -3441,7 +3448,6 @@ retry:
3441 if (unlock) 3448 if (unlock)
3442 read_unlock(&tasklist_lock); 3449 read_unlock(&tasklist_lock);
3443} 3450}
3444
3445EXPORT_SYMBOL_GPL(debug_show_all_locks); 3451EXPORT_SYMBOL_GPL(debug_show_all_locks);
3446 3452
3447/* 3453/*
@@ -3462,7 +3468,6 @@ void debug_show_held_locks(struct task_struct *task)
3462{ 3468{
3463 __debug_show_held_locks(task); 3469 __debug_show_held_locks(task);
3464} 3470}
3465
3466EXPORT_SYMBOL_GPL(debug_show_held_locks); 3471EXPORT_SYMBOL_GPL(debug_show_held_locks);
3467 3472
3468void lockdep_sys_exit(void) 3473void lockdep_sys_exit(void)
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index 20dbcbf9c7dd..13716b813896 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -470,11 +470,12 @@ static void seq_line(struct seq_file *m, char c, int offset, int length)
470 470
471static void snprint_time(char *buf, size_t bufsiz, s64 nr) 471static void snprint_time(char *buf, size_t bufsiz, s64 nr)
472{ 472{
473 unsigned long rem; 473 s64 div;
474 s32 rem;
474 475
475 nr += 5; /* for display rounding */ 476 nr += 5; /* for display rounding */
476 rem = do_div(nr, 1000); /* XXX: do_div_signed */ 477 div = div_s64_rem(nr, 1000, &rem);
477 snprintf(buf, bufsiz, "%lld.%02d", (long long)nr, (int)rem/10); 478 snprintf(buf, bufsiz, "%lld.%02d", (long long)div, (int)rem/10);
478} 479}
479 480
480static void seq_time(struct seq_file *m, s64 time) 481static void seq_time(struct seq_file *m, s64 time)
@@ -556,7 +557,7 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
556 if (stats->read_holdtime.nr) 557 if (stats->read_holdtime.nr)
557 namelen += 2; 558 namelen += 2;
558 559
559 for (i = 0; i < ARRAY_SIZE(class->contention_point); i++) { 560 for (i = 0; i < LOCKSTAT_POINTS; i++) {
560 char sym[KSYM_SYMBOL_LEN]; 561 char sym[KSYM_SYMBOL_LEN];
561 char ip[32]; 562 char ip[32];
562 563
@@ -573,6 +574,23 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
573 stats->contention_point[i], 574 stats->contention_point[i],
574 ip, sym); 575 ip, sym);
575 } 576 }
577 for (i = 0; i < LOCKSTAT_POINTS; i++) {
578 char sym[KSYM_SYMBOL_LEN];
579 char ip[32];
580
581 if (class->contending_point[i] == 0)
582 break;
583
584 if (!i)
585 seq_line(m, '-', 40-namelen, namelen);
586
587 sprint_symbol(sym, class->contending_point[i]);
588 snprintf(ip, sizeof(ip), "[<%p>]",
589 (void *)class->contending_point[i]);
590 seq_printf(m, "%40s %14lu %29s %s\n", name,
591 stats->contending_point[i],
592 ip, sym);
593 }
576 if (i) { 594 if (i) {
577 seq_puts(m, "\n"); 595 seq_puts(m, "\n");
578 seq_line(m, '.', 0, 40 + 1 + 10 * (14 + 1)); 596 seq_line(m, '.', 0, 40 + 1 + 10 * (14 + 1));
@@ -582,7 +600,7 @@ static void seq_stats(struct seq_file *m, struct lock_stat_data *data)
582 600
583static void seq_header(struct seq_file *m) 601static void seq_header(struct seq_file *m)
584{ 602{
585 seq_printf(m, "lock_stat version 0.2\n"); 603 seq_printf(m, "lock_stat version 0.3\n");
586 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1)); 604 seq_line(m, '-', 0, 40 + 1 + 10 * (14 + 1));
587 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s " 605 seq_printf(m, "%40s %14s %14s %14s %14s %14s %14s %14s %14s "
588 "%14s %14s\n", 606 "%14s %14s\n",
diff --git a/kernel/marker.c b/kernel/marker.c
index e9c6b2bc9400..ea54f2647868 100644
--- a/kernel/marker.c
+++ b/kernel/marker.c
@@ -43,6 +43,7 @@ static DEFINE_MUTEX(markers_mutex);
43 */ 43 */
44#define MARKER_HASH_BITS 6 44#define MARKER_HASH_BITS 6
45#define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS) 45#define MARKER_TABLE_SIZE (1 << MARKER_HASH_BITS)
46static struct hlist_head marker_table[MARKER_TABLE_SIZE];
46 47
47/* 48/*
48 * Note about RCU : 49 * Note about RCU :
@@ -64,11 +65,10 @@ struct marker_entry {
64 void *oldptr; 65 void *oldptr;
65 int rcu_pending; 66 int rcu_pending;
66 unsigned char ptype:1; 67 unsigned char ptype:1;
68 unsigned char format_allocated:1;
67 char name[0]; /* Contains name'\0'format'\0' */ 69 char name[0]; /* Contains name'\0'format'\0' */
68}; 70};
69 71
70static struct hlist_head marker_table[MARKER_TABLE_SIZE];
71
72/** 72/**
73 * __mark_empty_function - Empty probe callback 73 * __mark_empty_function - Empty probe callback
74 * @probe_private: probe private data 74 * @probe_private: probe private data
@@ -81,7 +81,7 @@ static struct hlist_head marker_table[MARKER_TABLE_SIZE];
81 * though the function pointer change and the marker enabling are two distinct 81 * though the function pointer change and the marker enabling are two distinct
82 * operations that modifies the execution flow of preemptible code. 82 * operations that modifies the execution flow of preemptible code.
83 */ 83 */
84void __mark_empty_function(void *probe_private, void *call_private, 84notrace void __mark_empty_function(void *probe_private, void *call_private,
85 const char *fmt, va_list *args) 85 const char *fmt, va_list *args)
86{ 86{
87} 87}
@@ -97,7 +97,8 @@ EXPORT_SYMBOL_GPL(__mark_empty_function);
97 * need to put a full smp_rmb() in this branch. This is why we do not use 97 * need to put a full smp_rmb() in this branch. This is why we do not use
98 * rcu_dereference() for the pointer read. 98 * rcu_dereference() for the pointer read.
99 */ 99 */
100void marker_probe_cb(const struct marker *mdata, void *call_private, ...) 100notrace void marker_probe_cb(const struct marker *mdata,
101 void *call_private, ...)
101{ 102{
102 va_list args; 103 va_list args;
103 char ptype; 104 char ptype;
@@ -107,7 +108,7 @@ void marker_probe_cb(const struct marker *mdata, void *call_private, ...)
107 * sure the teardown of the callbacks can be done correctly when they 108 * sure the teardown of the callbacks can be done correctly when they
108 * are in modules and they insure RCU read coherency. 109 * are in modules and they insure RCU read coherency.
109 */ 110 */
110 rcu_read_lock_sched(); 111 rcu_read_lock_sched_notrace();
111 ptype = mdata->ptype; 112 ptype = mdata->ptype;
112 if (likely(!ptype)) { 113 if (likely(!ptype)) {
113 marker_probe_func *func; 114 marker_probe_func *func;
@@ -145,7 +146,7 @@ void marker_probe_cb(const struct marker *mdata, void *call_private, ...)
145 va_end(args); 146 va_end(args);
146 } 147 }
147 } 148 }
148 rcu_read_unlock_sched(); 149 rcu_read_unlock_sched_notrace();
149} 150}
150EXPORT_SYMBOL_GPL(marker_probe_cb); 151EXPORT_SYMBOL_GPL(marker_probe_cb);
151 152
@@ -157,12 +158,13 @@ EXPORT_SYMBOL_GPL(marker_probe_cb);
157 * 158 *
158 * Should be connected to markers "MARK_NOARGS". 159 * Should be connected to markers "MARK_NOARGS".
159 */ 160 */
160void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...) 161static notrace void marker_probe_cb_noarg(const struct marker *mdata,
162 void *call_private, ...)
161{ 163{
162 va_list args; /* not initialized */ 164 va_list args; /* not initialized */
163 char ptype; 165 char ptype;
164 166
165 rcu_read_lock_sched(); 167 rcu_read_lock_sched_notrace();
166 ptype = mdata->ptype; 168 ptype = mdata->ptype;
167 if (likely(!ptype)) { 169 if (likely(!ptype)) {
168 marker_probe_func *func; 170 marker_probe_func *func;
@@ -195,9 +197,8 @@ void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...)
195 multi[i].func(multi[i].probe_private, call_private, 197 multi[i].func(multi[i].probe_private, call_private,
196 mdata->format, &args); 198 mdata->format, &args);
197 } 199 }
198 rcu_read_unlock_sched(); 200 rcu_read_unlock_sched_notrace();
199} 201}
200EXPORT_SYMBOL_GPL(marker_probe_cb_noarg);
201 202
202static void free_old_closure(struct rcu_head *head) 203static void free_old_closure(struct rcu_head *head)
203{ 204{
@@ -416,6 +417,7 @@ static struct marker_entry *add_marker(const char *name, const char *format)
416 e->single.probe_private = NULL; 417 e->single.probe_private = NULL;
417 e->multi = NULL; 418 e->multi = NULL;
418 e->ptype = 0; 419 e->ptype = 0;
420 e->format_allocated = 0;
419 e->refcount = 0; 421 e->refcount = 0;
420 e->rcu_pending = 0; 422 e->rcu_pending = 0;
421 hlist_add_head(&e->hlist, head); 423 hlist_add_head(&e->hlist, head);
@@ -447,6 +449,8 @@ static int remove_marker(const char *name)
447 if (e->single.func != __mark_empty_function) 449 if (e->single.func != __mark_empty_function)
448 return -EBUSY; 450 return -EBUSY;
449 hlist_del(&e->hlist); 451 hlist_del(&e->hlist);
452 if (e->format_allocated)
453 kfree(e->format);
450 /* Make sure the call_rcu has been executed */ 454 /* Make sure the call_rcu has been executed */
451 if (e->rcu_pending) 455 if (e->rcu_pending)
452 rcu_barrier_sched(); 456 rcu_barrier_sched();
@@ -457,57 +461,34 @@ static int remove_marker(const char *name)
457/* 461/*
458 * Set the mark_entry format to the format found in the element. 462 * Set the mark_entry format to the format found in the element.
459 */ 463 */
460static int marker_set_format(struct marker_entry **entry, const char *format) 464static int marker_set_format(struct marker_entry *entry, const char *format)
461{ 465{
462 struct marker_entry *e; 466 entry->format = kstrdup(format, GFP_KERNEL);
463 size_t name_len = strlen((*entry)->name) + 1; 467 if (!entry->format)
464 size_t format_len = strlen(format) + 1;
465
466
467 e = kmalloc(sizeof(struct marker_entry) + name_len + format_len,
468 GFP_KERNEL);
469 if (!e)
470 return -ENOMEM; 468 return -ENOMEM;
471 memcpy(&e->name[0], (*entry)->name, name_len); 469 entry->format_allocated = 1;
472 e->format = &e->name[name_len]; 470
473 memcpy(e->format, format, format_len);
474 if (strcmp(e->format, MARK_NOARGS) == 0)
475 e->call = marker_probe_cb_noarg;
476 else
477 e->call = marker_probe_cb;
478 e->single = (*entry)->single;
479 e->multi = (*entry)->multi;
480 e->ptype = (*entry)->ptype;
481 e->refcount = (*entry)->refcount;
482 e->rcu_pending = 0;
483 hlist_add_before(&e->hlist, &(*entry)->hlist);
484 hlist_del(&(*entry)->hlist);
485 /* Make sure the call_rcu has been executed */
486 if ((*entry)->rcu_pending)
487 rcu_barrier_sched();
488 kfree(*entry);
489 *entry = e;
490 trace_mark(core_marker_format, "name %s format %s", 471 trace_mark(core_marker_format, "name %s format %s",
491 e->name, e->format); 472 entry->name, entry->format);
492 return 0; 473 return 0;
493} 474}
494 475
495/* 476/*
496 * Sets the probe callback corresponding to one marker. 477 * Sets the probe callback corresponding to one marker.
497 */ 478 */
498static int set_marker(struct marker_entry **entry, struct marker *elem, 479static int set_marker(struct marker_entry *entry, struct marker *elem,
499 int active) 480 int active)
500{ 481{
501 int ret; 482 int ret = 0;
502 WARN_ON(strcmp((*entry)->name, elem->name) != 0); 483 WARN_ON(strcmp(entry->name, elem->name) != 0);
503 484
504 if ((*entry)->format) { 485 if (entry->format) {
505 if (strcmp((*entry)->format, elem->format) != 0) { 486 if (strcmp(entry->format, elem->format) != 0) {
506 printk(KERN_NOTICE 487 printk(KERN_NOTICE
507 "Format mismatch for probe %s " 488 "Format mismatch for probe %s "
508 "(%s), marker (%s)\n", 489 "(%s), marker (%s)\n",
509 (*entry)->name, 490 entry->name,
510 (*entry)->format, 491 entry->format,
511 elem->format); 492 elem->format);
512 return -EPERM; 493 return -EPERM;
513 } 494 }
@@ -523,37 +504,67 @@ static int set_marker(struct marker_entry **entry, struct marker *elem,
523 * pass from a "safe" callback (with argument) to an "unsafe" 504 * pass from a "safe" callback (with argument) to an "unsafe"
524 * callback (does not set arguments). 505 * callback (does not set arguments).
525 */ 506 */
526 elem->call = (*entry)->call; 507 elem->call = entry->call;
527 /* 508 /*
528 * Sanity check : 509 * Sanity check :
529 * We only update the single probe private data when the ptr is 510 * We only update the single probe private data when the ptr is
530 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1) 511 * set to a _non_ single probe! (0 -> 1 and N -> 1, N != 1)
531 */ 512 */
532 WARN_ON(elem->single.func != __mark_empty_function 513 WARN_ON(elem->single.func != __mark_empty_function
533 && elem->single.probe_private 514 && elem->single.probe_private != entry->single.probe_private
534 != (*entry)->single.probe_private && 515 && !elem->ptype);
535 !elem->ptype); 516 elem->single.probe_private = entry->single.probe_private;
536 elem->single.probe_private = (*entry)->single.probe_private;
537 /* 517 /*
538 * Make sure the private data is valid when we update the 518 * Make sure the private data is valid when we update the
539 * single probe ptr. 519 * single probe ptr.
540 */ 520 */
541 smp_wmb(); 521 smp_wmb();
542 elem->single.func = (*entry)->single.func; 522 elem->single.func = entry->single.func;
543 /* 523 /*
544 * We also make sure that the new probe callbacks array is consistent 524 * We also make sure that the new probe callbacks array is consistent
545 * before setting a pointer to it. 525 * before setting a pointer to it.
546 */ 526 */
547 rcu_assign_pointer(elem->multi, (*entry)->multi); 527 rcu_assign_pointer(elem->multi, entry->multi);
548 /* 528 /*
549 * Update the function or multi probe array pointer before setting the 529 * Update the function or multi probe array pointer before setting the
550 * ptype. 530 * ptype.
551 */ 531 */
552 smp_wmb(); 532 smp_wmb();
553 elem->ptype = (*entry)->ptype; 533 elem->ptype = entry->ptype;
534
535 if (elem->tp_name && (active ^ elem->state)) {
536 WARN_ON(!elem->tp_cb);
537 /*
538 * It is ok to directly call the probe registration because type
539 * checking has been done in the __trace_mark_tp() macro.
540 */
541
542 if (active) {
543 /*
544 * try_module_get should always succeed because we hold
545 * lock_module() to get the tp_cb address.
546 */
547 ret = try_module_get(__module_text_address(
548 (unsigned long)elem->tp_cb));
549 BUG_ON(!ret);
550 ret = tracepoint_probe_register_noupdate(
551 elem->tp_name,
552 elem->tp_cb);
553 } else {
554 ret = tracepoint_probe_unregister_noupdate(
555 elem->tp_name,
556 elem->tp_cb);
557 /*
558 * tracepoint_probe_update_all() must be called
559 * before the module containing tp_cb is unloaded.
560 */
561 module_put(__module_text_address(
562 (unsigned long)elem->tp_cb));
563 }
564 }
554 elem->state = active; 565 elem->state = active;
555 566
556 return 0; 567 return ret;
557} 568}
558 569
559/* 570/*
@@ -564,7 +575,24 @@ static int set_marker(struct marker_entry **entry, struct marker *elem,
564 */ 575 */
565static void disable_marker(struct marker *elem) 576static void disable_marker(struct marker *elem)
566{ 577{
578 int ret;
579
567 /* leave "call" as is. It is known statically. */ 580 /* leave "call" as is. It is known statically. */
581 if (elem->tp_name && elem->state) {
582 WARN_ON(!elem->tp_cb);
583 /*
584 * It is ok to directly call the probe registration because type
585 * checking has been done in the __trace_mark_tp() macro.
586 */
587 ret = tracepoint_probe_unregister_noupdate(elem->tp_name,
588 elem->tp_cb);
589 WARN_ON(ret);
590 /*
591 * tracepoint_probe_update_all() must be called
592 * before the module containing tp_cb is unloaded.
593 */
594 module_put(__module_text_address((unsigned long)elem->tp_cb));
595 }
568 elem->state = 0; 596 elem->state = 0;
569 elem->single.func = __mark_empty_function; 597 elem->single.func = __mark_empty_function;
570 /* Update the function before setting the ptype */ 598 /* Update the function before setting the ptype */
@@ -594,8 +622,7 @@ void marker_update_probe_range(struct marker *begin,
594 for (iter = begin; iter < end; iter++) { 622 for (iter = begin; iter < end; iter++) {
595 mark_entry = get_marker(iter->name); 623 mark_entry = get_marker(iter->name);
596 if (mark_entry) { 624 if (mark_entry) {
597 set_marker(&mark_entry, iter, 625 set_marker(mark_entry, iter, !!mark_entry->refcount);
598 !!mark_entry->refcount);
599 /* 626 /*
600 * ignore error, continue 627 * ignore error, continue
601 */ 628 */
@@ -629,6 +656,7 @@ static void marker_update_probes(void)
629 marker_update_probe_range(__start___markers, __stop___markers); 656 marker_update_probe_range(__start___markers, __stop___markers);
630 /* Markers in modules. */ 657 /* Markers in modules. */
631 module_update_markers(); 658 module_update_markers();
659 tracepoint_probe_update_all();
632} 660}
633 661
634/** 662/**
@@ -657,7 +685,7 @@ int marker_probe_register(const char *name, const char *format,
657 ret = PTR_ERR(entry); 685 ret = PTR_ERR(entry);
658 } else if (format) { 686 } else if (format) {
659 if (!entry->format) 687 if (!entry->format)
660 ret = marker_set_format(&entry, format); 688 ret = marker_set_format(entry, format);
661 else if (strcmp(entry->format, format)) 689 else if (strcmp(entry->format, format))
662 ret = -EPERM; 690 ret = -EPERM;
663 } 691 }
@@ -676,10 +704,11 @@ int marker_probe_register(const char *name, const char *format,
676 goto end; 704 goto end;
677 } 705 }
678 mutex_unlock(&markers_mutex); 706 mutex_unlock(&markers_mutex);
679 marker_update_probes(); /* may update entry */ 707 marker_update_probes();
680 mutex_lock(&markers_mutex); 708 mutex_lock(&markers_mutex);
681 entry = get_marker(name); 709 entry = get_marker(name);
682 WARN_ON(!entry); 710 if (!entry)
711 goto end;
683 if (entry->rcu_pending) 712 if (entry->rcu_pending)
684 rcu_barrier_sched(); 713 rcu_barrier_sched();
685 entry->oldptr = old; 714 entry->oldptr = old;
@@ -720,7 +749,7 @@ int marker_probe_unregister(const char *name,
720 rcu_barrier_sched(); 749 rcu_barrier_sched();
721 old = marker_entry_remove_probe(entry, probe, probe_private); 750 old = marker_entry_remove_probe(entry, probe, probe_private);
722 mutex_unlock(&markers_mutex); 751 mutex_unlock(&markers_mutex);
723 marker_update_probes(); /* may update entry */ 752 marker_update_probes();
724 mutex_lock(&markers_mutex); 753 mutex_lock(&markers_mutex);
725 entry = get_marker(name); 754 entry = get_marker(name);
726 if (!entry) 755 if (!entry)
@@ -801,10 +830,11 @@ int marker_probe_unregister_private_data(marker_probe_func *probe,
801 rcu_barrier_sched(); 830 rcu_barrier_sched();
802 old = marker_entry_remove_probe(entry, NULL, probe_private); 831 old = marker_entry_remove_probe(entry, NULL, probe_private);
803 mutex_unlock(&markers_mutex); 832 mutex_unlock(&markers_mutex);
804 marker_update_probes(); /* may update entry */ 833 marker_update_probes();
805 mutex_lock(&markers_mutex); 834 mutex_lock(&markers_mutex);
806 entry = get_marker_from_private_data(probe, probe_private); 835 entry = get_marker_from_private_data(probe, probe_private);
807 WARN_ON(!entry); 836 if (!entry)
837 goto end;
808 if (entry->rcu_pending) 838 if (entry->rcu_pending)
809 rcu_barrier_sched(); 839 rcu_barrier_sched();
810 entry->oldptr = old; 840 entry->oldptr = old;
@@ -848,8 +878,6 @@ void *marker_get_private_data(const char *name, marker_probe_func *probe,
848 if (!e->ptype) { 878 if (!e->ptype) {
849 if (num == 0 && e->single.func == probe) 879 if (num == 0 && e->single.func == probe)
850 return e->single.probe_private; 880 return e->single.probe_private;
851 else
852 break;
853 } else { 881 } else {
854 struct marker_probe_closure *closure; 882 struct marker_probe_closure *closure;
855 int match = 0; 883 int match = 0;
@@ -861,8 +889,42 @@ void *marker_get_private_data(const char *name, marker_probe_func *probe,
861 return closure[i].probe_private; 889 return closure[i].probe_private;
862 } 890 }
863 } 891 }
892 break;
864 } 893 }
865 } 894 }
866 return ERR_PTR(-ENOENT); 895 return ERR_PTR(-ENOENT);
867} 896}
868EXPORT_SYMBOL_GPL(marker_get_private_data); 897EXPORT_SYMBOL_GPL(marker_get_private_data);
898
899#ifdef CONFIG_MODULES
900
901int marker_module_notify(struct notifier_block *self,
902 unsigned long val, void *data)
903{
904 struct module *mod = data;
905
906 switch (val) {
907 case MODULE_STATE_COMING:
908 marker_update_probe_range(mod->markers,
909 mod->markers + mod->num_markers);
910 break;
911 case MODULE_STATE_GOING:
912 marker_update_probe_range(mod->markers,
913 mod->markers + mod->num_markers);
914 break;
915 }
916 return 0;
917}
918
919struct notifier_block marker_module_nb = {
920 .notifier_call = marker_module_notify,
921 .priority = 0,
922};
923
924static int init_markers(void)
925{
926 return register_module_notifier(&marker_module_nb);
927}
928__initcall(init_markers);
929
930#endif /* CONFIG_MODULES */
diff --git a/kernel/module.c b/kernel/module.c
index 1f4cc00e0c20..f47cce910f25 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -757,8 +757,16 @@ sys_delete_module(const char __user *name_user, unsigned int flags)
757 return -EFAULT; 757 return -EFAULT;
758 name[MODULE_NAME_LEN-1] = '\0'; 758 name[MODULE_NAME_LEN-1] = '\0';
759 759
760 if (mutex_lock_interruptible(&module_mutex) != 0) 760 /* Create stop_machine threads since free_module relies on
761 return -EINTR; 761 * a non-failing stop_machine call. */
762 ret = stop_machine_create();
763 if (ret)
764 return ret;
765
766 if (mutex_lock_interruptible(&module_mutex) != 0) {
767 ret = -EINTR;
768 goto out_stop;
769 }
762 770
763 mod = find_module(name); 771 mod = find_module(name);
764 if (!mod) { 772 if (!mod) {
@@ -817,10 +825,12 @@ sys_delete_module(const char __user *name_user, unsigned int flags)
817 825
818 out: 826 out:
819 mutex_unlock(&module_mutex); 827 mutex_unlock(&module_mutex);
828out_stop:
829 stop_machine_destroy();
820 return ret; 830 return ret;
821} 831}
822 832
823static void print_unload_info(struct seq_file *m, struct module *mod) 833static inline void print_unload_info(struct seq_file *m, struct module *mod)
824{ 834{
825 struct module_use *use; 835 struct module_use *use;
826 int printed_something = 0; 836 int printed_something = 0;
@@ -893,7 +903,7 @@ void module_put(struct module *module)
893EXPORT_SYMBOL(module_put); 903EXPORT_SYMBOL(module_put);
894 904
895#else /* !CONFIG_MODULE_UNLOAD */ 905#else /* !CONFIG_MODULE_UNLOAD */
896static void print_unload_info(struct seq_file *m, struct module *mod) 906static inline void print_unload_info(struct seq_file *m, struct module *mod)
897{ 907{
898 /* We don't know the usage count, or what modules are using. */ 908 /* We don't know the usage count, or what modules are using. */
899 seq_printf(m, " - -"); 909 seq_printf(m, " - -");
@@ -1578,11 +1588,21 @@ static int simplify_symbols(Elf_Shdr *sechdrs,
1578 return ret; 1588 return ret;
1579} 1589}
1580 1590
1591/* Additional bytes needed by arch in front of individual sections */
1592unsigned int __weak arch_mod_section_prepend(struct module *mod,
1593 unsigned int section)
1594{
1595 /* default implementation just returns zero */
1596 return 0;
1597}
1598
1581/* Update size with this section: return offset. */ 1599/* Update size with this section: return offset. */
1582static long get_offset(unsigned int *size, Elf_Shdr *sechdr) 1600static long get_offset(struct module *mod, unsigned int *size,
1601 Elf_Shdr *sechdr, unsigned int section)
1583{ 1602{
1584 long ret; 1603 long ret;
1585 1604
1605 *size += arch_mod_section_prepend(mod, section);
1586 ret = ALIGN(*size, sechdr->sh_addralign ?: 1); 1606 ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
1587 *size = ret + sechdr->sh_size; 1607 *size = ret + sechdr->sh_size;
1588 return ret; 1608 return ret;
@@ -1622,7 +1642,7 @@ static void layout_sections(struct module *mod,
1622 || strncmp(secstrings + s->sh_name, 1642 || strncmp(secstrings + s->sh_name,
1623 ".init", 5) == 0) 1643 ".init", 5) == 0)
1624 continue; 1644 continue;
1625 s->sh_entsize = get_offset(&mod->core_size, s); 1645 s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
1626 DEBUGP("\t%s\n", secstrings + s->sh_name); 1646 DEBUGP("\t%s\n", secstrings + s->sh_name);
1627 } 1647 }
1628 if (m == 0) 1648 if (m == 0)
@@ -1640,7 +1660,7 @@ static void layout_sections(struct module *mod,
1640 || strncmp(secstrings + s->sh_name, 1660 || strncmp(secstrings + s->sh_name,
1641 ".init", 5) != 0) 1661 ".init", 5) != 0)
1642 continue; 1662 continue;
1643 s->sh_entsize = (get_offset(&mod->init_size, s) 1663 s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
1644 | INIT_OFFSET_MASK); 1664 | INIT_OFFSET_MASK);
1645 DEBUGP("\t%s\n", secstrings + s->sh_name); 1665 DEBUGP("\t%s\n", secstrings + s->sh_name);
1646 } 1666 }
@@ -1725,15 +1745,15 @@ static const struct kernel_symbol *lookup_symbol(const char *name,
1725 return NULL; 1745 return NULL;
1726} 1746}
1727 1747
1728static int is_exported(const char *name, const struct module *mod) 1748static int is_exported(const char *name, unsigned long value,
1749 const struct module *mod)
1729{ 1750{
1730 if (!mod && lookup_symbol(name, __start___ksymtab, __stop___ksymtab)) 1751 const struct kernel_symbol *ks;
1731 return 1; 1752 if (!mod)
1753 ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
1732 else 1754 else
1733 if (mod && lookup_symbol(name, mod->syms, mod->syms + mod->num_syms)) 1755 ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
1734 return 1; 1756 return ks != NULL && ks->value == value;
1735 else
1736 return 0;
1737} 1757}
1738 1758
1739/* As per nm */ 1759/* As per nm */
@@ -1865,6 +1885,13 @@ static noinline struct module *load_module(void __user *umod,
1865 /* vmalloc barfs on "unusual" numbers. Check here */ 1885 /* vmalloc barfs on "unusual" numbers. Check here */
1866 if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL) 1886 if (len > 64 * 1024 * 1024 || (hdr = vmalloc(len)) == NULL)
1867 return ERR_PTR(-ENOMEM); 1887 return ERR_PTR(-ENOMEM);
1888
1889 /* Create stop_machine threads since the error path relies on
1890 * a non-failing stop_machine call. */
1891 err = stop_machine_create();
1892 if (err)
1893 goto free_hdr;
1894
1868 if (copy_from_user(hdr, umod, len) != 0) { 1895 if (copy_from_user(hdr, umod, len) != 0) {
1869 err = -EFAULT; 1896 err = -EFAULT;
1870 goto free_hdr; 1897 goto free_hdr;
@@ -2184,24 +2211,15 @@ static noinline struct module *load_module(void __user *umod,
2184 struct mod_debug *debug; 2211 struct mod_debug *debug;
2185 unsigned int num_debug; 2212 unsigned int num_debug;
2186 2213
2187#ifdef CONFIG_MARKERS
2188 marker_update_probe_range(mod->markers,
2189 mod->markers + mod->num_markers);
2190#endif
2191 debug = section_objs(hdr, sechdrs, secstrings, "__verbose", 2214 debug = section_objs(hdr, sechdrs, secstrings, "__verbose",
2192 sizeof(*debug), &num_debug); 2215 sizeof(*debug), &num_debug);
2193 dynamic_printk_setup(debug, num_debug); 2216 dynamic_printk_setup(debug, num_debug);
2194
2195#ifdef CONFIG_TRACEPOINTS
2196 tracepoint_update_probe_range(mod->tracepoints,
2197 mod->tracepoints + mod->num_tracepoints);
2198#endif
2199 } 2217 }
2200 2218
2201 /* sechdrs[0].sh_size is always zero */ 2219 /* sechdrs[0].sh_size is always zero */
2202 mseg = section_objs(hdr, sechdrs, secstrings, "__mcount_loc", 2220 mseg = section_objs(hdr, sechdrs, secstrings, "__mcount_loc",
2203 sizeof(*mseg), &num_mcount); 2221 sizeof(*mseg), &num_mcount);
2204 ftrace_init_module(mseg, mseg + num_mcount); 2222 ftrace_init_module(mod, mseg, mseg + num_mcount);
2205 2223
2206 err = module_finalize(hdr, sechdrs, mod); 2224 err = module_finalize(hdr, sechdrs, mod);
2207 if (err < 0) 2225 if (err < 0)
@@ -2257,6 +2275,7 @@ static noinline struct module *load_module(void __user *umod,
2257 /* Get rid of temporary copy */ 2275 /* Get rid of temporary copy */
2258 vfree(hdr); 2276 vfree(hdr);
2259 2277
2278 stop_machine_destroy();
2260 /* Done! */ 2279 /* Done! */
2261 return mod; 2280 return mod;
2262 2281
@@ -2279,6 +2298,7 @@ static noinline struct module *load_module(void __user *umod,
2279 kfree(args); 2298 kfree(args);
2280 free_hdr: 2299 free_hdr:
2281 vfree(hdr); 2300 vfree(hdr);
2301 stop_machine_destroy();
2282 return ERR_PTR(err); 2302 return ERR_PTR(err);
2283 2303
2284 truncated: 2304 truncated:
@@ -2513,7 +2533,7 @@ int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2513 strlcpy(name, mod->strtab + mod->symtab[symnum].st_name, 2533 strlcpy(name, mod->strtab + mod->symtab[symnum].st_name,
2514 KSYM_NAME_LEN); 2534 KSYM_NAME_LEN);
2515 strlcpy(module_name, mod->name, MODULE_NAME_LEN); 2535 strlcpy(module_name, mod->name, MODULE_NAME_LEN);
2516 *exported = is_exported(name, mod); 2536 *exported = is_exported(name, *value, mod);
2517 preempt_enable(); 2537 preempt_enable();
2518 return 0; 2538 return 0;
2519 } 2539 }
@@ -2713,7 +2733,7 @@ int is_module_address(unsigned long addr)
2713 2733
2714 2734
2715/* Is this a valid kernel address? */ 2735/* Is this a valid kernel address? */
2716struct module *__module_text_address(unsigned long addr) 2736__notrace_funcgraph struct module *__module_text_address(unsigned long addr)
2717{ 2737{
2718 struct module *mod; 2738 struct module *mod;
2719 2739
diff --git a/kernel/mutex.c b/kernel/mutex.c
index 12c779dc65d4..4f45d4b658ef 100644
--- a/kernel/mutex.c
+++ b/kernel/mutex.c
@@ -59,7 +59,7 @@ EXPORT_SYMBOL(__mutex_init);
59 * We also put the fastpath first in the kernel image, to make sure the 59 * We also put the fastpath first in the kernel image, to make sure the
60 * branch is predicted by the CPU as default-untaken. 60 * branch is predicted by the CPU as default-untaken.
61 */ 61 */
62static void noinline __sched 62static __used noinline void __sched
63__mutex_lock_slowpath(atomic_t *lock_count); 63__mutex_lock_slowpath(atomic_t *lock_count);
64 64
65/*** 65/***
@@ -96,7 +96,7 @@ void inline __sched mutex_lock(struct mutex *lock)
96EXPORT_SYMBOL(mutex_lock); 96EXPORT_SYMBOL(mutex_lock);
97#endif 97#endif
98 98
99static noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count); 99static __used noinline void __sched __mutex_unlock_slowpath(atomic_t *lock_count);
100 100
101/*** 101/***
102 * mutex_unlock - release the mutex 102 * mutex_unlock - release the mutex
@@ -184,7 +184,7 @@ __mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
184 } 184 }
185 185
186done: 186done:
187 lock_acquired(&lock->dep_map); 187 lock_acquired(&lock->dep_map, ip);
188 /* got the lock - rejoice! */ 188 /* got the lock - rejoice! */
189 mutex_remove_waiter(lock, &waiter, task_thread_info(task)); 189 mutex_remove_waiter(lock, &waiter, task_thread_info(task));
190 debug_mutex_set_owner(lock, task_thread_info(task)); 190 debug_mutex_set_owner(lock, task_thread_info(task));
@@ -268,7 +268,7 @@ __mutex_unlock_common_slowpath(atomic_t *lock_count, int nested)
268/* 268/*
269 * Release the lock, slowpath: 269 * Release the lock, slowpath:
270 */ 270 */
271static noinline void 271static __used noinline void
272__mutex_unlock_slowpath(atomic_t *lock_count) 272__mutex_unlock_slowpath(atomic_t *lock_count)
273{ 273{
274 __mutex_unlock_common_slowpath(lock_count, 1); 274 __mutex_unlock_common_slowpath(lock_count, 1);
@@ -313,7 +313,7 @@ int __sched mutex_lock_killable(struct mutex *lock)
313} 313}
314EXPORT_SYMBOL(mutex_lock_killable); 314EXPORT_SYMBOL(mutex_lock_killable);
315 315
316static noinline void __sched 316static __used noinline void __sched
317__mutex_lock_slowpath(atomic_t *lock_count) 317__mutex_lock_slowpath(atomic_t *lock_count)
318{ 318{
319 struct mutex *lock = container_of(lock_count, struct mutex, count); 319 struct mutex *lock = container_of(lock_count, struct mutex, count);
diff --git a/kernel/notifier.c b/kernel/notifier.c
index 4282c0a40a57..61d5aa5eced3 100644
--- a/kernel/notifier.c
+++ b/kernel/notifier.c
@@ -82,6 +82,14 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl,
82 82
83 while (nb && nr_to_call) { 83 while (nb && nr_to_call) {
84 next_nb = rcu_dereference(nb->next); 84 next_nb = rcu_dereference(nb->next);
85
86#ifdef CONFIG_DEBUG_NOTIFIERS
87 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
88 WARN(1, "Invalid notifier called!");
89 nb = next_nb;
90 continue;
91 }
92#endif
85 ret = nb->notifier_call(nb, val, v); 93 ret = nb->notifier_call(nb, val, v);
86 94
87 if (nr_calls) 95 if (nr_calls)
diff --git a/kernel/nsproxy.c b/kernel/nsproxy.c
index 1d3ef29a2583..63598dca2d0c 100644
--- a/kernel/nsproxy.c
+++ b/kernel/nsproxy.c
@@ -80,12 +80,6 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
80 goto out_pid; 80 goto out_pid;
81 } 81 }
82 82
83 new_nsp->user_ns = copy_user_ns(flags, tsk->nsproxy->user_ns);
84 if (IS_ERR(new_nsp->user_ns)) {
85 err = PTR_ERR(new_nsp->user_ns);
86 goto out_user;
87 }
88
89 new_nsp->net_ns = copy_net_ns(flags, tsk->nsproxy->net_ns); 83 new_nsp->net_ns = copy_net_ns(flags, tsk->nsproxy->net_ns);
90 if (IS_ERR(new_nsp->net_ns)) { 84 if (IS_ERR(new_nsp->net_ns)) {
91 err = PTR_ERR(new_nsp->net_ns); 85 err = PTR_ERR(new_nsp->net_ns);
@@ -95,9 +89,6 @@ static struct nsproxy *create_new_namespaces(unsigned long flags,
95 return new_nsp; 89 return new_nsp;
96 90
97out_net: 91out_net:
98 if (new_nsp->user_ns)
99 put_user_ns(new_nsp->user_ns);
100out_user:
101 if (new_nsp->pid_ns) 92 if (new_nsp->pid_ns)
102 put_pid_ns(new_nsp->pid_ns); 93 put_pid_ns(new_nsp->pid_ns);
103out_pid: 94out_pid:
@@ -130,7 +121,7 @@ int copy_namespaces(unsigned long flags, struct task_struct *tsk)
130 get_nsproxy(old_ns); 121 get_nsproxy(old_ns);
131 122
132 if (!(flags & (CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC | 123 if (!(flags & (CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC |
133 CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNET))) 124 CLONE_NEWPID | CLONE_NEWNET)))
134 return 0; 125 return 0;
135 126
136 if (!capable(CAP_SYS_ADMIN)) { 127 if (!capable(CAP_SYS_ADMIN)) {
@@ -173,8 +164,6 @@ void free_nsproxy(struct nsproxy *ns)
173 put_ipc_ns(ns->ipc_ns); 164 put_ipc_ns(ns->ipc_ns);
174 if (ns->pid_ns) 165 if (ns->pid_ns)
175 put_pid_ns(ns->pid_ns); 166 put_pid_ns(ns->pid_ns);
176 if (ns->user_ns)
177 put_user_ns(ns->user_ns);
178 put_net(ns->net_ns); 167 put_net(ns->net_ns);
179 kmem_cache_free(nsproxy_cachep, ns); 168 kmem_cache_free(nsproxy_cachep, ns);
180} 169}
@@ -189,7 +178,7 @@ int unshare_nsproxy_namespaces(unsigned long unshare_flags,
189 int err = 0; 178 int err = 0;
190 179
191 if (!(unshare_flags & (CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC | 180 if (!(unshare_flags & (CLONE_NEWNS | CLONE_NEWUTS | CLONE_NEWIPC |
192 CLONE_NEWUSER | CLONE_NEWNET))) 181 CLONE_NEWNET)))
193 return 0; 182 return 0;
194 183
195 if (!capable(CAP_SYS_ADMIN)) 184 if (!capable(CAP_SYS_ADMIN))
diff --git a/kernel/panic.c b/kernel/panic.c
index 6513aac8e992..13f06349a786 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -21,6 +21,7 @@
21#include <linux/debug_locks.h> 21#include <linux/debug_locks.h>
22#include <linux/random.h> 22#include <linux/random.h>
23#include <linux/kallsyms.h> 23#include <linux/kallsyms.h>
24#include <linux/dmi.h>
24 25
25int panic_on_oops; 26int panic_on_oops;
26static unsigned long tainted_mask; 27static unsigned long tainted_mask;
@@ -167,6 +168,7 @@ static const struct tnt tnts[] = {
167 * 'M' - System experienced a machine check exception. 168 * 'M' - System experienced a machine check exception.
168 * 'B' - System has hit bad_page. 169 * 'B' - System has hit bad_page.
169 * 'U' - Userspace-defined naughtiness. 170 * 'U' - Userspace-defined naughtiness.
171 * 'D' - Kernel has oopsed before
170 * 'A' - ACPI table overridden. 172 * 'A' - ACPI table overridden.
171 * 'W' - Taint on warning. 173 * 'W' - Taint on warning.
172 * 'C' - modules from drivers/staging are loaded. 174 * 'C' - modules from drivers/staging are loaded.
@@ -320,36 +322,27 @@ void oops_exit(void)
320} 322}
321 323
322#ifdef WANT_WARN_ON_SLOWPATH 324#ifdef WANT_WARN_ON_SLOWPATH
323void warn_on_slowpath(const char *file, int line)
324{
325 char function[KSYM_SYMBOL_LEN];
326 unsigned long caller = (unsigned long) __builtin_return_address(0);
327 sprint_symbol(function, caller);
328
329 printk(KERN_WARNING "------------[ cut here ]------------\n");
330 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file,
331 line, function);
332 print_modules();
333 dump_stack();
334 print_oops_end_marker();
335 add_taint(TAINT_WARN);
336}
337EXPORT_SYMBOL(warn_on_slowpath);
338
339
340void warn_slowpath(const char *file, int line, const char *fmt, ...) 325void warn_slowpath(const char *file, int line, const char *fmt, ...)
341{ 326{
342 va_list args; 327 va_list args;
343 char function[KSYM_SYMBOL_LEN]; 328 char function[KSYM_SYMBOL_LEN];
344 unsigned long caller = (unsigned long)__builtin_return_address(0); 329 unsigned long caller = (unsigned long)__builtin_return_address(0);
330 const char *board;
331
345 sprint_symbol(function, caller); 332 sprint_symbol(function, caller);
346 333
347 printk(KERN_WARNING "------------[ cut here ]------------\n"); 334 printk(KERN_WARNING "------------[ cut here ]------------\n");
348 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file, 335 printk(KERN_WARNING "WARNING: at %s:%d %s()\n", file,
349 line, function); 336 line, function);
350 va_start(args, fmt); 337 board = dmi_get_system_info(DMI_PRODUCT_NAME);
351 vprintk(fmt, args); 338 if (board)
352 va_end(args); 339 printk(KERN_WARNING "Hardware name: %s\n", board);
340
341 if (fmt) {
342 va_start(args, fmt);
343 vprintk(fmt, args);
344 va_end(args);
345 }
353 346
354 print_modules(); 347 print_modules();
355 dump_stack(); 348 dump_stack();
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index 895337b16a24..157de3a47832 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -58,21 +58,21 @@ void thread_group_cputime(
58 struct task_struct *tsk, 58 struct task_struct *tsk,
59 struct task_cputime *times) 59 struct task_cputime *times)
60{ 60{
61 struct signal_struct *sig; 61 struct task_cputime *totals, *tot;
62 int i; 62 int i;
63 struct task_cputime *tot;
64 63
65 sig = tsk->signal; 64 totals = tsk->signal->cputime.totals;
66 if (unlikely(!sig) || !sig->cputime.totals) { 65 if (!totals) {
67 times->utime = tsk->utime; 66 times->utime = tsk->utime;
68 times->stime = tsk->stime; 67 times->stime = tsk->stime;
69 times->sum_exec_runtime = tsk->se.sum_exec_runtime; 68 times->sum_exec_runtime = tsk->se.sum_exec_runtime;
70 return; 69 return;
71 } 70 }
71
72 times->stime = times->utime = cputime_zero; 72 times->stime = times->utime = cputime_zero;
73 times->sum_exec_runtime = 0; 73 times->sum_exec_runtime = 0;
74 for_each_possible_cpu(i) { 74 for_each_possible_cpu(i) {
75 tot = per_cpu_ptr(tsk->signal->cputime.totals, i); 75 tot = per_cpu_ptr(totals, i);
76 times->utime = cputime_add(times->utime, tot->utime); 76 times->utime = cputime_add(times->utime, tot->utime);
77 times->stime = cputime_add(times->stime, tot->stime); 77 times->stime = cputime_add(times->stime, tot->stime);
78 times->sum_exec_runtime += tot->sum_exec_runtime; 78 times->sum_exec_runtime += tot->sum_exec_runtime;
@@ -311,7 +311,7 @@ static int cpu_clock_sample_group(const clockid_t which_clock,
311 struct task_cputime cputime; 311 struct task_cputime cputime;
312 312
313 thread_group_cputime(p, &cputime); 313 thread_group_cputime(p, &cputime);
314 switch (which_clock) { 314 switch (CPUCLOCK_WHICH(which_clock)) {
315 default: 315 default:
316 return -EINVAL; 316 return -EINVAL;
317 case CPUCLOCK_PROF: 317 case CPUCLOCK_PROF:
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c
index 5e79c662294b..887c63787de6 100644
--- a/kernel/posix-timers.c
+++ b/kernel/posix-timers.c
@@ -116,7 +116,7 @@ static DEFINE_SPINLOCK(idr_lock);
116 * must supply functions here, even if the function just returns 116 * must supply functions here, even if the function just returns
117 * ENOSYS. The standard POSIX timer management code assumes the 117 * ENOSYS. The standard POSIX timer management code assumes the
118 * following: 1.) The k_itimer struct (sched.h) is used for the 118 * following: 1.) The k_itimer struct (sched.h) is used for the
119 * timer. 2.) The list, it_lock, it_clock, it_id and it_process 119 * timer. 2.) The list, it_lock, it_clock, it_id and it_pid
120 * fields are not modified by timer code. 120 * fields are not modified by timer code.
121 * 121 *
122 * At this time all functions EXCEPT clock_nanosleep can be 122 * At this time all functions EXCEPT clock_nanosleep can be
@@ -197,6 +197,11 @@ static int common_timer_create(struct k_itimer *new_timer)
197 return 0; 197 return 0;
198} 198}
199 199
200static int no_timer_create(struct k_itimer *new_timer)
201{
202 return -EOPNOTSUPP;
203}
204
200/* 205/*
201 * Return nonzero if we know a priori this clockid_t value is bogus. 206 * Return nonzero if we know a priori this clockid_t value is bogus.
202 */ 207 */
@@ -248,6 +253,7 @@ static __init int init_posix_timers(void)
248 .clock_getres = hrtimer_get_res, 253 .clock_getres = hrtimer_get_res,
249 .clock_get = posix_get_monotonic_raw, 254 .clock_get = posix_get_monotonic_raw,
250 .clock_set = do_posix_clock_nosettime, 255 .clock_set = do_posix_clock_nosettime,
256 .timer_create = no_timer_create,
251 }; 257 };
252 258
253 register_posix_clock(CLOCK_REALTIME, &clock_realtime); 259 register_posix_clock(CLOCK_REALTIME, &clock_realtime);
@@ -313,7 +319,8 @@ void do_schedule_next_timer(struct siginfo *info)
313 319
314int posix_timer_event(struct k_itimer *timr, int si_private) 320int posix_timer_event(struct k_itimer *timr, int si_private)
315{ 321{
316 int shared, ret; 322 struct task_struct *task;
323 int shared, ret = -1;
317 /* 324 /*
318 * FIXME: if ->sigq is queued we can race with 325 * FIXME: if ->sigq is queued we can race with
319 * dequeue_signal()->do_schedule_next_timer(). 326 * dequeue_signal()->do_schedule_next_timer().
@@ -327,8 +334,13 @@ int posix_timer_event(struct k_itimer *timr, int si_private)
327 */ 334 */
328 timr->sigq->info.si_sys_private = si_private; 335 timr->sigq->info.si_sys_private = si_private;
329 336
330 shared = !(timr->it_sigev_notify & SIGEV_THREAD_ID); 337 rcu_read_lock();
331 ret = send_sigqueue(timr->sigq, timr->it_process, shared); 338 task = pid_task(timr->it_pid, PIDTYPE_PID);
339 if (task) {
340 shared = !(timr->it_sigev_notify & SIGEV_THREAD_ID);
341 ret = send_sigqueue(timr->sigq, task, shared);
342 }
343 rcu_read_unlock();
332 /* If we failed to send the signal the timer stops. */ 344 /* If we failed to send the signal the timer stops. */
333 return ret > 0; 345 return ret > 0;
334} 346}
@@ -405,7 +417,7 @@ static enum hrtimer_restart posix_timer_fn(struct hrtimer *timer)
405 return ret; 417 return ret;
406} 418}
407 419
408static struct task_struct * good_sigevent(sigevent_t * event) 420static struct pid *good_sigevent(sigevent_t * event)
409{ 421{
410 struct task_struct *rtn = current->group_leader; 422 struct task_struct *rtn = current->group_leader;
411 423
@@ -419,7 +431,7 @@ static struct task_struct * good_sigevent(sigevent_t * event)
419 ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) 431 ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX)))
420 return NULL; 432 return NULL;
421 433
422 return rtn; 434 return task_pid(rtn);
423} 435}
424 436
425void register_posix_clock(const clockid_t clock_id, struct k_clock *new_clock) 437void register_posix_clock(const clockid_t clock_id, struct k_clock *new_clock)
@@ -458,6 +470,7 @@ static void release_posix_timer(struct k_itimer *tmr, int it_id_set)
458 idr_remove(&posix_timers_id, tmr->it_id); 470 idr_remove(&posix_timers_id, tmr->it_id);
459 spin_unlock_irqrestore(&idr_lock, flags); 471 spin_unlock_irqrestore(&idr_lock, flags);
460 } 472 }
473 put_pid(tmr->it_pid);
461 sigqueue_free(tmr->sigq); 474 sigqueue_free(tmr->sigq);
462 kmem_cache_free(posix_timers_cache, tmr); 475 kmem_cache_free(posix_timers_cache, tmr);
463} 476}
@@ -471,7 +484,6 @@ sys_timer_create(const clockid_t which_clock,
471{ 484{
472 struct k_itimer *new_timer; 485 struct k_itimer *new_timer;
473 int error, new_timer_id; 486 int error, new_timer_id;
474 struct task_struct *process;
475 sigevent_t event; 487 sigevent_t event;
476 int it_id_set = IT_ID_NOT_SET; 488 int it_id_set = IT_ID_NOT_SET;
477 489
@@ -525,11 +537,9 @@ sys_timer_create(const clockid_t which_clock,
525 goto out; 537 goto out;
526 } 538 }
527 rcu_read_lock(); 539 rcu_read_lock();
528 process = good_sigevent(&event); 540 new_timer->it_pid = get_pid(good_sigevent(&event));
529 if (process)
530 get_task_struct(process);
531 rcu_read_unlock(); 541 rcu_read_unlock();
532 if (!process) { 542 if (!new_timer->it_pid) {
533 error = -EINVAL; 543 error = -EINVAL;
534 goto out; 544 goto out;
535 } 545 }
@@ -537,8 +547,7 @@ sys_timer_create(const clockid_t which_clock,
537 event.sigev_notify = SIGEV_SIGNAL; 547 event.sigev_notify = SIGEV_SIGNAL;
538 event.sigev_signo = SIGALRM; 548 event.sigev_signo = SIGALRM;
539 event.sigev_value.sival_int = new_timer->it_id; 549 event.sigev_value.sival_int = new_timer->it_id;
540 process = current->group_leader; 550 new_timer->it_pid = get_pid(task_tgid(current));
541 get_task_struct(process);
542 } 551 }
543 552
544 new_timer->it_sigev_notify = event.sigev_notify; 553 new_timer->it_sigev_notify = event.sigev_notify;
@@ -548,7 +557,7 @@ sys_timer_create(const clockid_t which_clock,
548 new_timer->sigq->info.si_code = SI_TIMER; 557 new_timer->sigq->info.si_code = SI_TIMER;
549 558
550 spin_lock_irq(&current->sighand->siglock); 559 spin_lock_irq(&current->sighand->siglock);
551 new_timer->it_process = process; 560 new_timer->it_signal = current->signal;
552 list_add(&new_timer->list, &current->signal->posix_timers); 561 list_add(&new_timer->list, &current->signal->posix_timers);
553 spin_unlock_irq(&current->sighand->siglock); 562 spin_unlock_irq(&current->sighand->siglock);
554 563
@@ -583,8 +592,7 @@ static struct k_itimer *lock_timer(timer_t timer_id, unsigned long *flags)
583 timr = idr_find(&posix_timers_id, (int)timer_id); 592 timr = idr_find(&posix_timers_id, (int)timer_id);
584 if (timr) { 593 if (timr) {
585 spin_lock(&timr->it_lock); 594 spin_lock(&timr->it_lock);
586 if (timr->it_process && 595 if (timr->it_signal == current->signal) {
587 same_thread_group(timr->it_process, current)) {
588 spin_unlock(&idr_lock); 596 spin_unlock(&idr_lock);
589 return timr; 597 return timr;
590 } 598 }
@@ -831,8 +839,7 @@ retry_delete:
831 * This keeps any tasks waiting on the spin lock from thinking 839 * This keeps any tasks waiting on the spin lock from thinking
832 * they got something (see the lock code above). 840 * they got something (see the lock code above).
833 */ 841 */
834 put_task_struct(timer->it_process); 842 timer->it_signal = NULL;
835 timer->it_process = NULL;
836 843
837 unlock_timer(timer, flags); 844 unlock_timer(timer, flags);
838 release_posix_timer(timer, IT_ID_SET); 845 release_posix_timer(timer, IT_ID_SET);
@@ -858,8 +865,7 @@ retry_delete:
858 * This keeps any tasks waiting on the spin lock from thinking 865 * This keeps any tasks waiting on the spin lock from thinking
859 * they got something (see the lock code above). 866 * they got something (see the lock code above).
860 */ 867 */
861 put_task_struct(timer->it_process); 868 timer->it_signal = NULL;
862 timer->it_process = NULL;
863 869
864 unlock_timer(timer, flags); 870 unlock_timer(timer, flags);
865 release_posix_timer(timer, IT_ID_SET); 871 release_posix_timer(timer, IT_ID_SET);
diff --git a/kernel/power/disk.c b/kernel/power/disk.c
index c9d74083746f..f77d3819ef57 100644
--- a/kernel/power/disk.c
+++ b/kernel/power/disk.c
@@ -22,7 +22,6 @@
22#include <linux/console.h> 22#include <linux/console.h>
23#include <linux/cpu.h> 23#include <linux/cpu.h>
24#include <linux/freezer.h> 24#include <linux/freezer.h>
25#include <linux/ftrace.h>
26 25
27#include "power.h" 26#include "power.h"
28 27
@@ -257,7 +256,7 @@ static int create_image(int platform_mode)
257 256
258int hibernation_snapshot(int platform_mode) 257int hibernation_snapshot(int platform_mode)
259{ 258{
260 int error, ftrace_save; 259 int error;
261 260
262 /* Free memory before shutting down devices. */ 261 /* Free memory before shutting down devices. */
263 error = swsusp_shrink_memory(); 262 error = swsusp_shrink_memory();
@@ -269,7 +268,6 @@ int hibernation_snapshot(int platform_mode)
269 goto Close; 268 goto Close;
270 269
271 suspend_console(); 270 suspend_console();
272 ftrace_save = __ftrace_enabled_save();
273 error = device_suspend(PMSG_FREEZE); 271 error = device_suspend(PMSG_FREEZE);
274 if (error) 272 if (error)
275 goto Recover_platform; 273 goto Recover_platform;
@@ -299,7 +297,6 @@ int hibernation_snapshot(int platform_mode)
299 Resume_devices: 297 Resume_devices:
300 device_resume(in_suspend ? 298 device_resume(in_suspend ?
301 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE); 299 (error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
302 __ftrace_enabled_restore(ftrace_save);
303 resume_console(); 300 resume_console();
304 Close: 301 Close:
305 platform_end(platform_mode); 302 platform_end(platform_mode);
@@ -370,11 +367,10 @@ static int resume_target_kernel(void)
370 367
371int hibernation_restore(int platform_mode) 368int hibernation_restore(int platform_mode)
372{ 369{
373 int error, ftrace_save; 370 int error;
374 371
375 pm_prepare_console(); 372 pm_prepare_console();
376 suspend_console(); 373 suspend_console();
377 ftrace_save = __ftrace_enabled_save();
378 error = device_suspend(PMSG_QUIESCE); 374 error = device_suspend(PMSG_QUIESCE);
379 if (error) 375 if (error)
380 goto Finish; 376 goto Finish;
@@ -389,7 +385,6 @@ int hibernation_restore(int platform_mode)
389 platform_restore_cleanup(platform_mode); 385 platform_restore_cleanup(platform_mode);
390 device_resume(PMSG_RECOVER); 386 device_resume(PMSG_RECOVER);
391 Finish: 387 Finish:
392 __ftrace_enabled_restore(ftrace_save);
393 resume_console(); 388 resume_console();
394 pm_restore_console(); 389 pm_restore_console();
395 return error; 390 return error;
@@ -402,7 +397,7 @@ int hibernation_restore(int platform_mode)
402 397
403int hibernation_platform_enter(void) 398int hibernation_platform_enter(void)
404{ 399{
405 int error, ftrace_save; 400 int error;
406 401
407 if (!hibernation_ops) 402 if (!hibernation_ops)
408 return -ENOSYS; 403 return -ENOSYS;
@@ -417,7 +412,6 @@ int hibernation_platform_enter(void)
417 goto Close; 412 goto Close;
418 413
419 suspend_console(); 414 suspend_console();
420 ftrace_save = __ftrace_enabled_save();
421 error = device_suspend(PMSG_HIBERNATE); 415 error = device_suspend(PMSG_HIBERNATE);
422 if (error) { 416 if (error) {
423 if (hibernation_ops->recover) 417 if (hibernation_ops->recover)
@@ -452,7 +446,6 @@ int hibernation_platform_enter(void)
452 hibernation_ops->finish(); 446 hibernation_ops->finish();
453 Resume_devices: 447 Resume_devices:
454 device_resume(PMSG_RESTORE); 448 device_resume(PMSG_RESTORE);
455 __ftrace_enabled_restore(ftrace_save);
456 resume_console(); 449 resume_console();
457 Close: 450 Close:
458 hibernation_ops->end(); 451 hibernation_ops->end();
diff --git a/kernel/power/main.c b/kernel/power/main.c
index b8f7ce9473e8..613f16941b85 100644
--- a/kernel/power/main.c
+++ b/kernel/power/main.c
@@ -22,7 +22,6 @@
22#include <linux/freezer.h> 22#include <linux/freezer.h>
23#include <linux/vmstat.h> 23#include <linux/vmstat.h>
24#include <linux/syscalls.h> 24#include <linux/syscalls.h>
25#include <linux/ftrace.h>
26 25
27#include "power.h" 26#include "power.h"
28 27
@@ -317,7 +316,7 @@ static int suspend_enter(suspend_state_t state)
317 */ 316 */
318int suspend_devices_and_enter(suspend_state_t state) 317int suspend_devices_and_enter(suspend_state_t state)
319{ 318{
320 int error, ftrace_save; 319 int error;
321 320
322 if (!suspend_ops) 321 if (!suspend_ops)
323 return -ENOSYS; 322 return -ENOSYS;
@@ -328,7 +327,6 @@ int suspend_devices_and_enter(suspend_state_t state)
328 goto Close; 327 goto Close;
329 } 328 }
330 suspend_console(); 329 suspend_console();
331 ftrace_save = __ftrace_enabled_save();
332 suspend_test_start(); 330 suspend_test_start();
333 error = device_suspend(PMSG_SUSPEND); 331 error = device_suspend(PMSG_SUSPEND);
334 if (error) { 332 if (error) {
@@ -360,7 +358,6 @@ int suspend_devices_and_enter(suspend_state_t state)
360 suspend_test_start(); 358 suspend_test_start();
361 device_resume(PMSG_RESUME); 359 device_resume(PMSG_RESUME);
362 suspend_test_finish("resume devices"); 360 suspend_test_finish("resume devices");
363 __ftrace_enabled_restore(ftrace_save);
364 resume_console(); 361 resume_console();
365 Close: 362 Close:
366 if (suspend_ops->end) 363 if (suspend_ops->end)
diff --git a/kernel/power/poweroff.c b/kernel/power/poweroff.c
index 72016f051477..97890831e1b5 100644
--- a/kernel/power/poweroff.c
+++ b/kernel/power/poweroff.c
@@ -27,7 +27,7 @@ static DECLARE_WORK(poweroff_work, do_poweroff);
27static void handle_poweroff(int key, struct tty_struct *tty) 27static void handle_poweroff(int key, struct tty_struct *tty)
28{ 28{
29 /* run sysrq poweroff on boot cpu */ 29 /* run sysrq poweroff on boot cpu */
30 schedule_work_on(first_cpu(cpu_online_map), &poweroff_work); 30 schedule_work_on(cpumask_first(cpu_online_mask), &poweroff_work);
31} 31}
32 32
33static struct sysrq_key_op sysrq_poweroff_op = { 33static struct sysrq_key_op sysrq_poweroff_op = {
diff --git a/kernel/power/swap.c b/kernel/power/swap.c
index b7713b53d07a..6da14358537c 100644
--- a/kernel/power/swap.c
+++ b/kernel/power/swap.c
@@ -633,7 +633,7 @@ void swsusp_close(fmode_t mode)
633 return; 633 return;
634 } 634 }
635 635
636 blkdev_put(resume_bdev, mode); /* move up */ 636 blkdev_put(resume_bdev, mode);
637} 637}
638 638
639static int swsusp_header_init(void) 639static int swsusp_header_init(void)
diff --git a/kernel/printk.c b/kernel/printk.c
index f492f1583d77..e651ab05655f 100644
--- a/kernel/printk.c
+++ b/kernel/printk.c
@@ -662,7 +662,7 @@ asmlinkage int vprintk(const char *fmt, va_list args)
662 if (recursion_bug) { 662 if (recursion_bug) {
663 recursion_bug = 0; 663 recursion_bug = 0;
664 strcpy(printk_buf, recursion_bug_msg); 664 strcpy(printk_buf, recursion_bug_msg);
665 printed_len = sizeof(recursion_bug_msg); 665 printed_len = strlen(recursion_bug_msg);
666 } 666 }
667 /* Emit the output into the temporary buffer */ 667 /* Emit the output into the temporary buffer */
668 printed_len += vscnprintf(printk_buf + printed_len, 668 printed_len += vscnprintf(printk_buf + printed_len,
diff --git a/kernel/profile.c b/kernel/profile.c
index 5b7d1ac7124c..d18e2d2654f2 100644
--- a/kernel/profile.c
+++ b/kernel/profile.c
@@ -45,7 +45,7 @@ static unsigned long prof_len, prof_shift;
45int prof_on __read_mostly; 45int prof_on __read_mostly;
46EXPORT_SYMBOL_GPL(prof_on); 46EXPORT_SYMBOL_GPL(prof_on);
47 47
48static cpumask_t prof_cpu_mask = CPU_MASK_ALL; 48static cpumask_var_t prof_cpu_mask;
49#ifdef CONFIG_SMP 49#ifdef CONFIG_SMP
50static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits); 50static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
51static DEFINE_PER_CPU(int, cpu_profile_flip); 51static DEFINE_PER_CPU(int, cpu_profile_flip);
@@ -113,9 +113,13 @@ int __ref profile_init(void)
113 buffer_bytes = prof_len*sizeof(atomic_t); 113 buffer_bytes = prof_len*sizeof(atomic_t);
114 if (!slab_is_available()) { 114 if (!slab_is_available()) {
115 prof_buffer = alloc_bootmem(buffer_bytes); 115 prof_buffer = alloc_bootmem(buffer_bytes);
116 alloc_bootmem_cpumask_var(&prof_cpu_mask);
116 return 0; 117 return 0;
117 } 118 }
118 119
120 if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
121 return -ENOMEM;
122
119 prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL); 123 prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL);
120 if (prof_buffer) 124 if (prof_buffer)
121 return 0; 125 return 0;
@@ -128,6 +132,7 @@ int __ref profile_init(void)
128 if (prof_buffer) 132 if (prof_buffer)
129 return 0; 133 return 0;
130 134
135 free_cpumask_var(prof_cpu_mask);
131 return -ENOMEM; 136 return -ENOMEM;
132} 137}
133 138
@@ -351,7 +356,7 @@ out:
351 put_cpu(); 356 put_cpu();
352} 357}
353 358
354static int __devinit profile_cpu_callback(struct notifier_block *info, 359static int __cpuinit profile_cpu_callback(struct notifier_block *info,
355 unsigned long action, void *__cpu) 360 unsigned long action, void *__cpu)
356{ 361{
357 int node, cpu = (unsigned long)__cpu; 362 int node, cpu = (unsigned long)__cpu;
@@ -386,13 +391,15 @@ out_free:
386 return NOTIFY_BAD; 391 return NOTIFY_BAD;
387 case CPU_ONLINE: 392 case CPU_ONLINE:
388 case CPU_ONLINE_FROZEN: 393 case CPU_ONLINE_FROZEN:
389 cpu_set(cpu, prof_cpu_mask); 394 if (prof_cpu_mask != NULL)
395 cpumask_set_cpu(cpu, prof_cpu_mask);
390 break; 396 break;
391 case CPU_UP_CANCELED: 397 case CPU_UP_CANCELED:
392 case CPU_UP_CANCELED_FROZEN: 398 case CPU_UP_CANCELED_FROZEN:
393 case CPU_DEAD: 399 case CPU_DEAD:
394 case CPU_DEAD_FROZEN: 400 case CPU_DEAD_FROZEN:
395 cpu_clear(cpu, prof_cpu_mask); 401 if (prof_cpu_mask != NULL)
402 cpumask_clear_cpu(cpu, prof_cpu_mask);
396 if (per_cpu(cpu_profile_hits, cpu)[0]) { 403 if (per_cpu(cpu_profile_hits, cpu)[0]) {
397 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]); 404 page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[0]);
398 per_cpu(cpu_profile_hits, cpu)[0] = NULL; 405 per_cpu(cpu_profile_hits, cpu)[0] = NULL;
@@ -430,7 +437,8 @@ void profile_tick(int type)
430 437
431 if (type == CPU_PROFILING && timer_hook) 438 if (type == CPU_PROFILING && timer_hook)
432 timer_hook(regs); 439 timer_hook(regs);
433 if (!user_mode(regs) && cpu_isset(smp_processor_id(), prof_cpu_mask)) 440 if (!user_mode(regs) && prof_cpu_mask != NULL &&
441 cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
434 profile_hit(type, (void *)profile_pc(regs)); 442 profile_hit(type, (void *)profile_pc(regs));
435} 443}
436 444
@@ -442,7 +450,7 @@ void profile_tick(int type)
442static int prof_cpu_mask_read_proc(char *page, char **start, off_t off, 450static int prof_cpu_mask_read_proc(char *page, char **start, off_t off,
443 int count, int *eof, void *data) 451 int count, int *eof, void *data)
444{ 452{
445 int len = cpumask_scnprintf(page, count, *(cpumask_t *)data); 453 int len = cpumask_scnprintf(page, count, data);
446 if (count - len < 2) 454 if (count - len < 2)
447 return -EINVAL; 455 return -EINVAL;
448 len += sprintf(page + len, "\n"); 456 len += sprintf(page + len, "\n");
@@ -452,16 +460,20 @@ static int prof_cpu_mask_read_proc(char *page, char **start, off_t off,
452static int prof_cpu_mask_write_proc(struct file *file, 460static int prof_cpu_mask_write_proc(struct file *file,
453 const char __user *buffer, unsigned long count, void *data) 461 const char __user *buffer, unsigned long count, void *data)
454{ 462{
455 cpumask_t *mask = (cpumask_t *)data; 463 struct cpumask *mask = data;
456 unsigned long full_count = count, err; 464 unsigned long full_count = count, err;
457 cpumask_t new_value; 465 cpumask_var_t new_value;
458 466
459 err = cpumask_parse_user(buffer, count, new_value); 467 if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
460 if (err) 468 return -ENOMEM;
461 return err;
462 469
463 *mask = new_value; 470 err = cpumask_parse_user(buffer, count, new_value);
464 return full_count; 471 if (!err) {
472 cpumask_copy(mask, new_value);
473 err = full_count;
474 }
475 free_cpumask_var(new_value);
476 return err;
465} 477}
466 478
467void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir) 479void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir)
@@ -472,7 +484,7 @@ void create_prof_cpu_mask(struct proc_dir_entry *root_irq_dir)
472 entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir); 484 entry = create_proc_entry("prof_cpu_mask", 0600, root_irq_dir);
473 if (!entry) 485 if (!entry)
474 return; 486 return;
475 entry->data = (void *)&prof_cpu_mask; 487 entry->data = prof_cpu_mask;
476 entry->read_proc = prof_cpu_mask_read_proc; 488 entry->read_proc = prof_cpu_mask_read_proc;
477 entry->write_proc = prof_cpu_mask_write_proc; 489 entry->write_proc = prof_cpu_mask_write_proc;
478} 490}
@@ -544,7 +556,7 @@ static const struct file_operations proc_profile_operations = {
544}; 556};
545 557
546#ifdef CONFIG_SMP 558#ifdef CONFIG_SMP
547static inline void profile_nop(void *unused) 559static void profile_nop(void *unused)
548{ 560{
549} 561}
550 562
@@ -596,7 +608,7 @@ out_cleanup:
596#define create_hash_tables() ({ 0; }) 608#define create_hash_tables() ({ 0; })
597#endif 609#endif
598 610
599int create_proc_profile(void) 611int __ref create_proc_profile(void) /* false positive from hotcpu_notifier */
600{ 612{
601 struct proc_dir_entry *entry; 613 struct proc_dir_entry *entry;
602 614
diff --git a/kernel/ptrace.c b/kernel/ptrace.c
index 1e68e4c39e2c..29dc700e198c 100644
--- a/kernel/ptrace.c
+++ b/kernel/ptrace.c
@@ -25,6 +25,17 @@
25#include <asm/pgtable.h> 25#include <asm/pgtable.h>
26#include <asm/uaccess.h> 26#include <asm/uaccess.h>
27 27
28
29/*
30 * Initialize a new task whose father had been ptraced.
31 *
32 * Called from copy_process().
33 */
34void ptrace_fork(struct task_struct *child, unsigned long clone_flags)
35{
36 arch_ptrace_fork(child, clone_flags);
37}
38
28/* 39/*
29 * ptrace a task: make the debugger its new parent and 40 * ptrace a task: make the debugger its new parent and
30 * move it to the ptrace list. 41 * move it to the ptrace list.
@@ -72,6 +83,7 @@ void __ptrace_unlink(struct task_struct *child)
72 child->parent = child->real_parent; 83 child->parent = child->real_parent;
73 list_del_init(&child->ptrace_entry); 84 list_del_init(&child->ptrace_entry);
74 85
86 arch_ptrace_untrace(child);
75 if (task_is_traced(child)) 87 if (task_is_traced(child))
76 ptrace_untrace(child); 88 ptrace_untrace(child);
77} 89}
@@ -115,6 +127,8 @@ int ptrace_check_attach(struct task_struct *child, int kill)
115 127
116int __ptrace_may_access(struct task_struct *task, unsigned int mode) 128int __ptrace_may_access(struct task_struct *task, unsigned int mode)
117{ 129{
130 const struct cred *cred = current_cred(), *tcred;
131
118 /* May we inspect the given task? 132 /* May we inspect the given task?
119 * This check is used both for attaching with ptrace 133 * This check is used both for attaching with ptrace
120 * and for allowing access to sensitive information in /proc. 134 * and for allowing access to sensitive information in /proc.
@@ -127,13 +141,19 @@ int __ptrace_may_access(struct task_struct *task, unsigned int mode)
127 /* Don't let security modules deny introspection */ 141 /* Don't let security modules deny introspection */
128 if (task == current) 142 if (task == current)
129 return 0; 143 return 0;
130 if (((current->uid != task->euid) || 144 rcu_read_lock();
131 (current->uid != task->suid) || 145 tcred = __task_cred(task);
132 (current->uid != task->uid) || 146 if ((cred->uid != tcred->euid ||
133 (current->gid != task->egid) || 147 cred->uid != tcred->suid ||
134 (current->gid != task->sgid) || 148 cred->uid != tcred->uid ||
135 (current->gid != task->gid)) && !capable(CAP_SYS_PTRACE)) 149 cred->gid != tcred->egid ||
150 cred->gid != tcred->sgid ||
151 cred->gid != tcred->gid) &&
152 !capable(CAP_SYS_PTRACE)) {
153 rcu_read_unlock();
136 return -EPERM; 154 return -EPERM;
155 }
156 rcu_read_unlock();
137 smp_rmb(); 157 smp_rmb();
138 if (task->mm) 158 if (task->mm)
139 dumpable = get_dumpable(task->mm); 159 dumpable = get_dumpable(task->mm);
@@ -163,6 +183,14 @@ int ptrace_attach(struct task_struct *task)
163 if (same_thread_group(task, current)) 183 if (same_thread_group(task, current))
164 goto out; 184 goto out;
165 185
186 /* Protect exec's credential calculations against our interference;
187 * SUID, SGID and LSM creds get determined differently under ptrace.
188 */
189 retval = mutex_lock_interruptible(&current->cred_exec_mutex);
190 if (retval < 0)
191 goto out;
192
193 retval = -EPERM;
166repeat: 194repeat:
167 /* 195 /*
168 * Nasty, nasty. 196 * Nasty, nasty.
@@ -202,6 +230,7 @@ repeat:
202bad: 230bad:
203 write_unlock_irqrestore(&tasklist_lock, flags); 231 write_unlock_irqrestore(&tasklist_lock, flags);
204 task_unlock(task); 232 task_unlock(task);
233 mutex_unlock(&current->cred_exec_mutex);
205out: 234out:
206 return retval; 235 return retval;
207} 236}
@@ -612,7 +641,7 @@ int generic_ptrace_pokedata(struct task_struct *tsk, long addr, long data)
612 return (copied == sizeof(data)) ? 0 : -EIO; 641 return (copied == sizeof(data)) ? 0 : -EIO;
613} 642}
614 643
615#if defined CONFIG_COMPAT && defined __ARCH_WANT_COMPAT_SYS_PTRACE 644#if defined CONFIG_COMPAT
616#include <linux/compat.h> 645#include <linux/compat.h>
617 646
618int compat_ptrace_request(struct task_struct *child, compat_long_t request, 647int compat_ptrace_request(struct task_struct *child, compat_long_t request,
@@ -709,4 +738,4 @@ asmlinkage long compat_sys_ptrace(compat_long_t request, compat_long_t pid,
709 unlock_kernel(); 738 unlock_kernel();
710 return ret; 739 return ret;
711} 740}
712#endif /* CONFIG_COMPAT && __ARCH_WANT_COMPAT_SYS_PTRACE */ 741#endif /* CONFIG_COMPAT */
diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c
index 37f72e551542..490934fc7ac3 100644
--- a/kernel/rcuclassic.c
+++ b/kernel/rcuclassic.c
@@ -63,14 +63,14 @@ static struct rcu_ctrlblk rcu_ctrlblk = {
63 .completed = -300, 63 .completed = -300,
64 .pending = -300, 64 .pending = -300,
65 .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock), 65 .lock = __SPIN_LOCK_UNLOCKED(&rcu_ctrlblk.lock),
66 .cpumask = CPU_MASK_NONE, 66 .cpumask = CPU_BITS_NONE,
67}; 67};
68static struct rcu_ctrlblk rcu_bh_ctrlblk = { 68static struct rcu_ctrlblk rcu_bh_ctrlblk = {
69 .cur = -300, 69 .cur = -300,
70 .completed = -300, 70 .completed = -300,
71 .pending = -300, 71 .pending = -300,
72 .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock), 72 .lock = __SPIN_LOCK_UNLOCKED(&rcu_bh_ctrlblk.lock),
73 .cpumask = CPU_MASK_NONE, 73 .cpumask = CPU_BITS_NONE,
74}; 74};
75 75
76DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L }; 76DEFINE_PER_CPU(struct rcu_data, rcu_data) = { 0L };
@@ -85,7 +85,6 @@ static void force_quiescent_state(struct rcu_data *rdp,
85 struct rcu_ctrlblk *rcp) 85 struct rcu_ctrlblk *rcp)
86{ 86{
87 int cpu; 87 int cpu;
88 cpumask_t cpumask;
89 unsigned long flags; 88 unsigned long flags;
90 89
91 set_need_resched(); 90 set_need_resched();
@@ -96,10 +95,10 @@ static void force_quiescent_state(struct rcu_data *rdp,
96 * Don't send IPI to itself. With irqs disabled, 95 * Don't send IPI to itself. With irqs disabled,
97 * rdp->cpu is the current cpu. 96 * rdp->cpu is the current cpu.
98 * 97 *
99 * cpu_online_map is updated by the _cpu_down() 98 * cpu_online_mask is updated by the _cpu_down()
100 * using __stop_machine(). Since we're in irqs disabled 99 * using __stop_machine(). Since we're in irqs disabled
101 * section, __stop_machine() is not exectuting, hence 100 * section, __stop_machine() is not exectuting, hence
102 * the cpu_online_map is stable. 101 * the cpu_online_mask is stable.
103 * 102 *
104 * However, a cpu might have been offlined _just_ before 103 * However, a cpu might have been offlined _just_ before
105 * we disabled irqs while entering here. 104 * we disabled irqs while entering here.
@@ -107,13 +106,14 @@ static void force_quiescent_state(struct rcu_data *rdp,
107 * notification, leading to the offlined cpu's bit 106 * notification, leading to the offlined cpu's bit
108 * being set in the rcp->cpumask. 107 * being set in the rcp->cpumask.
109 * 108 *
110 * Hence cpumask = (rcp->cpumask & cpu_online_map) to prevent 109 * Hence cpumask = (rcp->cpumask & cpu_online_mask) to prevent
111 * sending smp_reschedule() to an offlined CPU. 110 * sending smp_reschedule() to an offlined CPU.
112 */ 111 */
113 cpus_and(cpumask, rcp->cpumask, cpu_online_map); 112 for_each_cpu_and(cpu,
114 cpu_clear(rdp->cpu, cpumask); 113 to_cpumask(rcp->cpumask), cpu_online_mask) {
115 for_each_cpu_mask_nr(cpu, cpumask) 114 if (cpu != rdp->cpu)
116 smp_send_reschedule(cpu); 115 smp_send_reschedule(cpu);
116 }
117 } 117 }
118 spin_unlock_irqrestore(&rcp->lock, flags); 118 spin_unlock_irqrestore(&rcp->lock, flags);
119} 119}
@@ -191,9 +191,9 @@ static void print_other_cpu_stall(struct rcu_ctrlblk *rcp)
191 191
192 /* OK, time to rat on our buddy... */ 192 /* OK, time to rat on our buddy... */
193 193
194 printk(KERN_ERR "RCU detected CPU stalls:"); 194 printk(KERN_ERR "INFO: RCU detected CPU stalls:");
195 for_each_possible_cpu(cpu) { 195 for_each_possible_cpu(cpu) {
196 if (cpu_isset(cpu, rcp->cpumask)) 196 if (cpumask_test_cpu(cpu, to_cpumask(rcp->cpumask)))
197 printk(" %d", cpu); 197 printk(" %d", cpu);
198 } 198 }
199 printk(" (detected by %d, t=%ld jiffies)\n", 199 printk(" (detected by %d, t=%ld jiffies)\n",
@@ -204,7 +204,7 @@ static void print_cpu_stall(struct rcu_ctrlblk *rcp)
204{ 204{
205 unsigned long flags; 205 unsigned long flags;
206 206
207 printk(KERN_ERR "RCU detected CPU %d stall (t=%lu/%lu jiffies)\n", 207 printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu/%lu jiffies)\n",
208 smp_processor_id(), jiffies, 208 smp_processor_id(), jiffies,
209 jiffies - rcp->gp_start); 209 jiffies - rcp->gp_start);
210 dump_stack(); 210 dump_stack();
@@ -221,7 +221,8 @@ static void check_cpu_stall(struct rcu_ctrlblk *rcp)
221 long delta; 221 long delta;
222 222
223 delta = jiffies - rcp->jiffies_stall; 223 delta = jiffies - rcp->jiffies_stall;
224 if (cpu_isset(smp_processor_id(), rcp->cpumask) && delta >= 0) { 224 if (cpumask_test_cpu(smp_processor_id(), to_cpumask(rcp->cpumask)) &&
225 delta >= 0) {
225 226
226 /* We haven't checked in, so go dump stack. */ 227 /* We haven't checked in, so go dump stack. */
227 print_cpu_stall(rcp); 228 print_cpu_stall(rcp);
@@ -393,7 +394,8 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp)
393 * unnecessarily. 394 * unnecessarily.
394 */ 395 */
395 smp_mb(); 396 smp_mb();
396 cpus_andnot(rcp->cpumask, cpu_online_map, nohz_cpu_mask); 397 cpumask_andnot(to_cpumask(rcp->cpumask),
398 cpu_online_mask, nohz_cpu_mask);
397 399
398 rcp->signaled = 0; 400 rcp->signaled = 0;
399 } 401 }
@@ -406,8 +408,8 @@ static void rcu_start_batch(struct rcu_ctrlblk *rcp)
406 */ 408 */
407static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp) 409static void cpu_quiet(int cpu, struct rcu_ctrlblk *rcp)
408{ 410{
409 cpu_clear(cpu, rcp->cpumask); 411 cpumask_clear_cpu(cpu, to_cpumask(rcp->cpumask));
410 if (cpus_empty(rcp->cpumask)) { 412 if (cpumask_empty(to_cpumask(rcp->cpumask))) {
411 /* batch completed ! */ 413 /* batch completed ! */
412 rcp->completed = rcp->cur; 414 rcp->completed = rcp->cur;
413 rcu_start_batch(rcp); 415 rcu_start_batch(rcp);
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index ad63af8b2521..d92a76a881aa 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -77,8 +77,15 @@ void wakeme_after_rcu(struct rcu_head *head)
77 * sections are delimited by rcu_read_lock() and rcu_read_unlock(), 77 * sections are delimited by rcu_read_lock() and rcu_read_unlock(),
78 * and may be nested. 78 * and may be nested.
79 */ 79 */
80void synchronize_rcu(void); /* Makes kernel-doc tools happy */ 80void synchronize_rcu(void)
81synchronize_rcu_xxx(synchronize_rcu, call_rcu) 81{
82 struct rcu_synchronize rcu;
83 init_completion(&rcu.completion);
84 /* Will wake me after RCU finished. */
85 call_rcu(&rcu.head, wakeme_after_rcu);
86 /* Wait for it. */
87 wait_for_completion(&rcu.completion);
88}
82EXPORT_SYMBOL_GPL(synchronize_rcu); 89EXPORT_SYMBOL_GPL(synchronize_rcu);
83 90
84static void rcu_barrier_callback(struct rcu_head *notused) 91static void rcu_barrier_callback(struct rcu_head *notused)
diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c
index 59236e8b9daa..33cfc50781f9 100644
--- a/kernel/rcupreempt.c
+++ b/kernel/rcupreempt.c
@@ -164,7 +164,8 @@ static char *rcu_try_flip_state_names[] =
164 { "idle", "waitack", "waitzero", "waitmb" }; 164 { "idle", "waitack", "waitzero", "waitmb" };
165#endif /* #ifdef CONFIG_RCU_TRACE */ 165#endif /* #ifdef CONFIG_RCU_TRACE */
166 166
167static cpumask_t rcu_cpu_online_map __read_mostly = CPU_MASK_NONE; 167static DECLARE_BITMAP(rcu_cpu_online_map, NR_CPUS) __read_mostly
168 = CPU_BITS_NONE;
168 169
169/* 170/*
170 * Enum and per-CPU flag to determine when each CPU has seen 171 * Enum and per-CPU flag to determine when each CPU has seen
@@ -551,6 +552,16 @@ void rcu_irq_exit(void)
551 } 552 }
552} 553}
553 554
555void rcu_nmi_enter(void)
556{
557 rcu_irq_enter();
558}
559
560void rcu_nmi_exit(void)
561{
562 rcu_irq_exit();
563}
564
554static void dyntick_save_progress_counter(int cpu) 565static void dyntick_save_progress_counter(int cpu)
555{ 566{
556 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); 567 struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu);
@@ -748,7 +759,7 @@ rcu_try_flip_idle(void)
748 759
749 /* Now ask each CPU for acknowledgement of the flip. */ 760 /* Now ask each CPU for acknowledgement of the flip. */
750 761
751 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) { 762 for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
752 per_cpu(rcu_flip_flag, cpu) = rcu_flipped; 763 per_cpu(rcu_flip_flag, cpu) = rcu_flipped;
753 dyntick_save_progress_counter(cpu); 764 dyntick_save_progress_counter(cpu);
754 } 765 }
@@ -766,7 +777,7 @@ rcu_try_flip_waitack(void)
766 int cpu; 777 int cpu;
767 778
768 RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); 779 RCU_TRACE_ME(rcupreempt_trace_try_flip_a1);
769 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) 780 for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
770 if (rcu_try_flip_waitack_needed(cpu) && 781 if (rcu_try_flip_waitack_needed(cpu) &&
771 per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { 782 per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) {
772 RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); 783 RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1);
@@ -798,7 +809,7 @@ rcu_try_flip_waitzero(void)
798 /* Check to see if the sum of the "last" counters is zero. */ 809 /* Check to see if the sum of the "last" counters is zero. */
799 810
800 RCU_TRACE_ME(rcupreempt_trace_try_flip_z1); 811 RCU_TRACE_ME(rcupreempt_trace_try_flip_z1);
801 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) 812 for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
802 sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx]; 813 sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx];
803 if (sum != 0) { 814 if (sum != 0) {
804 RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1); 815 RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1);
@@ -813,7 +824,7 @@ rcu_try_flip_waitzero(void)
813 smp_mb(); /* ^^^^^^^^^^^^ */ 824 smp_mb(); /* ^^^^^^^^^^^^ */
814 825
815 /* Call for a memory barrier from each CPU. */ 826 /* Call for a memory barrier from each CPU. */
816 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) { 827 for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map)) {
817 per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; 828 per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed;
818 dyntick_save_progress_counter(cpu); 829 dyntick_save_progress_counter(cpu);
819 } 830 }
@@ -833,7 +844,7 @@ rcu_try_flip_waitmb(void)
833 int cpu; 844 int cpu;
834 845
835 RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); 846 RCU_TRACE_ME(rcupreempt_trace_try_flip_m1);
836 for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) 847 for_each_cpu(cpu, to_cpumask(rcu_cpu_online_map))
837 if (rcu_try_flip_waitmb_needed(cpu) && 848 if (rcu_try_flip_waitmb_needed(cpu) &&
838 per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { 849 per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) {
839 RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); 850 RCU_TRACE_ME(rcupreempt_trace_try_flip_me1);
@@ -1022,7 +1033,7 @@ void rcu_offline_cpu(int cpu)
1022 RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0; 1033 RCU_DATA_CPU(cpu)->rcu_flipctr[0] = 0;
1023 RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0; 1034 RCU_DATA_CPU(cpu)->rcu_flipctr[1] = 0;
1024 1035
1025 cpu_clear(cpu, rcu_cpu_online_map); 1036 cpumask_clear_cpu(cpu, to_cpumask(rcu_cpu_online_map));
1026 1037
1027 spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); 1038 spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
1028 1039
@@ -1062,7 +1073,7 @@ void __cpuinit rcu_online_cpu(int cpu)
1062 struct rcu_data *rdp; 1073 struct rcu_data *rdp;
1063 1074
1064 spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); 1075 spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags);
1065 cpu_set(cpu, rcu_cpu_online_map); 1076 cpumask_set_cpu(cpu, to_cpumask(rcu_cpu_online_map));
1066 spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); 1077 spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags);
1067 1078
1068 /* 1079 /*
@@ -1166,7 +1177,16 @@ EXPORT_SYMBOL_GPL(call_rcu_sched);
1166 * in -rt this does -not- necessarily result in all currently executing 1177 * in -rt this does -not- necessarily result in all currently executing
1167 * interrupt -handlers- having completed. 1178 * interrupt -handlers- having completed.
1168 */ 1179 */
1169synchronize_rcu_xxx(__synchronize_sched, call_rcu_sched) 1180void __synchronize_sched(void)
1181{
1182 struct rcu_synchronize rcu;
1183
1184 init_completion(&rcu.completion);
1185 /* Will wake me after RCU finished. */
1186 call_rcu_sched(&rcu.head, wakeme_after_rcu);
1187 /* Wait for it. */
1188 wait_for_completion(&rcu.completion);
1189}
1170EXPORT_SYMBOL_GPL(__synchronize_sched); 1190EXPORT_SYMBOL_GPL(__synchronize_sched);
1171 1191
1172/* 1192/*
@@ -1420,7 +1440,7 @@ void __init __rcu_init(void)
1420 * We don't need protection against CPU-Hotplug here 1440 * We don't need protection against CPU-Hotplug here
1421 * since 1441 * since
1422 * a) If a CPU comes online while we are iterating over the 1442 * a) If a CPU comes online while we are iterating over the
1423 * cpu_online_map below, we would only end up making a 1443 * cpu_online_mask below, we would only end up making a
1424 * duplicate call to rcu_online_cpu() which sets the corresponding 1444 * duplicate call to rcu_online_cpu() which sets the corresponding
1425 * CPU's mask in the rcu_cpu_online_map. 1445 * CPU's mask in the rcu_cpu_online_map.
1426 * 1446 *
diff --git a/kernel/rcupreempt_trace.c b/kernel/rcupreempt_trace.c
index 35c2d3360ecf..7c2665cac172 100644
--- a/kernel/rcupreempt_trace.c
+++ b/kernel/rcupreempt_trace.c
@@ -149,12 +149,12 @@ static void rcupreempt_trace_sum(struct rcupreempt_trace *sp)
149 sp->done_length += cp->done_length; 149 sp->done_length += cp->done_length;
150 sp->done_add += cp->done_add; 150 sp->done_add += cp->done_add;
151 sp->done_remove += cp->done_remove; 151 sp->done_remove += cp->done_remove;
152 atomic_set(&sp->done_invoked, atomic_read(&cp->done_invoked)); 152 atomic_add(atomic_read(&cp->done_invoked), &sp->done_invoked);
153 sp->rcu_check_callbacks += cp->rcu_check_callbacks; 153 sp->rcu_check_callbacks += cp->rcu_check_callbacks;
154 atomic_set(&sp->rcu_try_flip_1, 154 atomic_add(atomic_read(&cp->rcu_try_flip_1),
155 atomic_read(&cp->rcu_try_flip_1)); 155 &sp->rcu_try_flip_1);
156 atomic_set(&sp->rcu_try_flip_e1, 156 atomic_add(atomic_read(&cp->rcu_try_flip_e1),
157 atomic_read(&cp->rcu_try_flip_e1)); 157 &sp->rcu_try_flip_e1);
158 sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1; 158 sp->rcu_try_flip_i1 += cp->rcu_try_flip_i1;
159 sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1; 159 sp->rcu_try_flip_ie1 += cp->rcu_try_flip_ie1;
160 sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1; 160 sp->rcu_try_flip_g1 += cp->rcu_try_flip_g1;
diff --git a/kernel/rcutorture.c b/kernel/rcutorture.c
index 85cb90588a55..1cff28db56b6 100644
--- a/kernel/rcutorture.c
+++ b/kernel/rcutorture.c
@@ -39,6 +39,7 @@
39#include <linux/moduleparam.h> 39#include <linux/moduleparam.h>
40#include <linux/percpu.h> 40#include <linux/percpu.h>
41#include <linux/notifier.h> 41#include <linux/notifier.h>
42#include <linux/reboot.h>
42#include <linux/freezer.h> 43#include <linux/freezer.h>
43#include <linux/cpu.h> 44#include <linux/cpu.h>
44#include <linux/delay.h> 45#include <linux/delay.h>
@@ -108,7 +109,6 @@ struct rcu_torture {
108 int rtort_mbtest; 109 int rtort_mbtest;
109}; 110};
110 111
111static int fullstop = 0; /* stop generating callbacks at test end. */
112static LIST_HEAD(rcu_torture_freelist); 112static LIST_HEAD(rcu_torture_freelist);
113static struct rcu_torture *rcu_torture_current = NULL; 113static struct rcu_torture *rcu_torture_current = NULL;
114static long rcu_torture_current_version = 0; 114static long rcu_torture_current_version = 0;
@@ -136,6 +136,28 @@ static int stutter_pause_test = 0;
136#endif 136#endif
137int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT; 137int rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;
138 138
139#define FULLSTOP_SHUTDOWN 1 /* Bail due to system shutdown/panic. */
140#define FULLSTOP_CLEANUP 2 /* Orderly shutdown. */
141static int fullstop; /* stop generating callbacks at test end. */
142DEFINE_MUTEX(fullstop_mutex); /* protect fullstop transitions and */
143 /* spawning of kthreads. */
144
145/*
146 * Detect and respond to a signal-based shutdown.
147 */
148static int
149rcutorture_shutdown_notify(struct notifier_block *unused1,
150 unsigned long unused2, void *unused3)
151{
152 if (fullstop)
153 return NOTIFY_DONE;
154 mutex_lock(&fullstop_mutex);
155 if (!fullstop)
156 fullstop = FULLSTOP_SHUTDOWN;
157 mutex_unlock(&fullstop_mutex);
158 return NOTIFY_DONE;
159}
160
139/* 161/*
140 * Allocate an element from the rcu_tortures pool. 162 * Allocate an element from the rcu_tortures pool.
141 */ 163 */
@@ -199,11 +221,12 @@ rcu_random(struct rcu_random_state *rrsp)
199static void 221static void
200rcu_stutter_wait(void) 222rcu_stutter_wait(void)
201{ 223{
202 while (stutter_pause_test || !rcutorture_runnable) 224 while ((stutter_pause_test || !rcutorture_runnable) && !fullstop) {
203 if (rcutorture_runnable) 225 if (rcutorture_runnable)
204 schedule_timeout_interruptible(1); 226 schedule_timeout_interruptible(1);
205 else 227 else
206 schedule_timeout_interruptible(round_jiffies_relative(HZ)); 228 schedule_timeout_interruptible(round_jiffies_relative(HZ));
229 }
207} 230}
208 231
209/* 232/*
@@ -599,7 +622,7 @@ rcu_torture_writer(void *arg)
599 rcu_stutter_wait(); 622 rcu_stutter_wait();
600 } while (!kthread_should_stop() && !fullstop); 623 } while (!kthread_should_stop() && !fullstop);
601 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping"); 624 VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
602 while (!kthread_should_stop()) 625 while (!kthread_should_stop() && fullstop != FULLSTOP_SHUTDOWN)
603 schedule_timeout_uninterruptible(1); 626 schedule_timeout_uninterruptible(1);
604 return 0; 627 return 0;
605} 628}
@@ -624,7 +647,7 @@ rcu_torture_fakewriter(void *arg)
624 } while (!kthread_should_stop() && !fullstop); 647 } while (!kthread_should_stop() && !fullstop);
625 648
626 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping"); 649 VERBOSE_PRINTK_STRING("rcu_torture_fakewriter task stopping");
627 while (!kthread_should_stop()) 650 while (!kthread_should_stop() && fullstop != FULLSTOP_SHUTDOWN)
628 schedule_timeout_uninterruptible(1); 651 schedule_timeout_uninterruptible(1);
629 return 0; 652 return 0;
630} 653}
@@ -734,7 +757,7 @@ rcu_torture_reader(void *arg)
734 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping"); 757 VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
735 if (irqreader && cur_ops->irqcapable) 758 if (irqreader && cur_ops->irqcapable)
736 del_timer_sync(&t); 759 del_timer_sync(&t);
737 while (!kthread_should_stop()) 760 while (!kthread_should_stop() && fullstop != FULLSTOP_SHUTDOWN)
738 schedule_timeout_uninterruptible(1); 761 schedule_timeout_uninterruptible(1);
739 return 0; 762 return 0;
740} 763}
@@ -831,7 +854,7 @@ rcu_torture_stats(void *arg)
831 do { 854 do {
832 schedule_timeout_interruptible(stat_interval * HZ); 855 schedule_timeout_interruptible(stat_interval * HZ);
833 rcu_torture_stats_print(); 856 rcu_torture_stats_print();
834 } while (!kthread_should_stop()); 857 } while (!kthread_should_stop() && !fullstop);
835 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping"); 858 VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
836 return 0; 859 return 0;
837} 860}
@@ -843,49 +866,52 @@ static int rcu_idle_cpu; /* Force all torture tasks off this CPU */
843 */ 866 */
844static void rcu_torture_shuffle_tasks(void) 867static void rcu_torture_shuffle_tasks(void)
845{ 868{
846 cpumask_t tmp_mask; 869 cpumask_var_t tmp_mask;
847 int i; 870 int i;
848 871
849 cpus_setall(tmp_mask); 872 if (!alloc_cpumask_var(&tmp_mask, GFP_KERNEL))
873 BUG();
874
875 cpumask_setall(tmp_mask);
850 get_online_cpus(); 876 get_online_cpus();
851 877
852 /* No point in shuffling if there is only one online CPU (ex: UP) */ 878 /* No point in shuffling if there is only one online CPU (ex: UP) */
853 if (num_online_cpus() == 1) { 879 if (num_online_cpus() == 1)
854 put_online_cpus(); 880 goto out;
855 return;
856 }
857 881
858 if (rcu_idle_cpu != -1) 882 if (rcu_idle_cpu != -1)
859 cpu_clear(rcu_idle_cpu, tmp_mask); 883 cpumask_clear_cpu(rcu_idle_cpu, tmp_mask);
860 884
861 set_cpus_allowed_ptr(current, &tmp_mask); 885 set_cpus_allowed_ptr(current, tmp_mask);
862 886
863 if (reader_tasks) { 887 if (reader_tasks) {
864 for (i = 0; i < nrealreaders; i++) 888 for (i = 0; i < nrealreaders; i++)
865 if (reader_tasks[i]) 889 if (reader_tasks[i])
866 set_cpus_allowed_ptr(reader_tasks[i], 890 set_cpus_allowed_ptr(reader_tasks[i],
867 &tmp_mask); 891 tmp_mask);
868 } 892 }
869 893
870 if (fakewriter_tasks) { 894 if (fakewriter_tasks) {
871 for (i = 0; i < nfakewriters; i++) 895 for (i = 0; i < nfakewriters; i++)
872 if (fakewriter_tasks[i]) 896 if (fakewriter_tasks[i])
873 set_cpus_allowed_ptr(fakewriter_tasks[i], 897 set_cpus_allowed_ptr(fakewriter_tasks[i],
874 &tmp_mask); 898 tmp_mask);
875 } 899 }
876 900
877 if (writer_task) 901 if (writer_task)
878 set_cpus_allowed_ptr(writer_task, &tmp_mask); 902 set_cpus_allowed_ptr(writer_task, tmp_mask);
879 903
880 if (stats_task) 904 if (stats_task)
881 set_cpus_allowed_ptr(stats_task, &tmp_mask); 905 set_cpus_allowed_ptr(stats_task, tmp_mask);
882 906
883 if (rcu_idle_cpu == -1) 907 if (rcu_idle_cpu == -1)
884 rcu_idle_cpu = num_online_cpus() - 1; 908 rcu_idle_cpu = num_online_cpus() - 1;
885 else 909 else
886 rcu_idle_cpu--; 910 rcu_idle_cpu--;
887 911
912out:
888 put_online_cpus(); 913 put_online_cpus();
914 free_cpumask_var(tmp_mask);
889} 915}
890 916
891/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the 917/* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
@@ -899,7 +925,7 @@ rcu_torture_shuffle(void *arg)
899 do { 925 do {
900 schedule_timeout_interruptible(shuffle_interval * HZ); 926 schedule_timeout_interruptible(shuffle_interval * HZ);
901 rcu_torture_shuffle_tasks(); 927 rcu_torture_shuffle_tasks();
902 } while (!kthread_should_stop()); 928 } while (!kthread_should_stop() && !fullstop);
903 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping"); 929 VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
904 return 0; 930 return 0;
905} 931}
@@ -914,10 +940,10 @@ rcu_torture_stutter(void *arg)
914 do { 940 do {
915 schedule_timeout_interruptible(stutter * HZ); 941 schedule_timeout_interruptible(stutter * HZ);
916 stutter_pause_test = 1; 942 stutter_pause_test = 1;
917 if (!kthread_should_stop()) 943 if (!kthread_should_stop() && !fullstop)
918 schedule_timeout_interruptible(stutter * HZ); 944 schedule_timeout_interruptible(stutter * HZ);
919 stutter_pause_test = 0; 945 stutter_pause_test = 0;
920 } while (!kthread_should_stop()); 946 } while (!kthread_should_stop() && !fullstop);
921 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping"); 947 VERBOSE_PRINTK_STRING("rcu_torture_stutter task stopping");
922 return 0; 948 return 0;
923} 949}
@@ -934,12 +960,27 @@ rcu_torture_print_module_parms(char *tag)
934 stutter, irqreader); 960 stutter, irqreader);
935} 961}
936 962
963static struct notifier_block rcutorture_nb = {
964 .notifier_call = rcutorture_shutdown_notify,
965};
966
937static void 967static void
938rcu_torture_cleanup(void) 968rcu_torture_cleanup(void)
939{ 969{
940 int i; 970 int i;
941 971
942 fullstop = 1; 972 mutex_lock(&fullstop_mutex);
973 if (!fullstop) {
974 /* If being signaled, let it happen, then exit. */
975 mutex_unlock(&fullstop_mutex);
976 schedule_timeout_interruptible(10 * HZ);
977 if (cur_ops->cb_barrier != NULL)
978 cur_ops->cb_barrier();
979 return;
980 }
981 fullstop = FULLSTOP_CLEANUP;
982 mutex_unlock(&fullstop_mutex);
983 unregister_reboot_notifier(&rcutorture_nb);
943 if (stutter_task) { 984 if (stutter_task) {
944 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task"); 985 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
945 kthread_stop(stutter_task); 986 kthread_stop(stutter_task);
@@ -1015,6 +1056,8 @@ rcu_torture_init(void)
1015 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops, 1056 { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops,
1016 &srcu_ops, &sched_ops, &sched_ops_sync, }; 1057 &srcu_ops, &sched_ops, &sched_ops_sync, };
1017 1058
1059 mutex_lock(&fullstop_mutex);
1060
1018 /* Process args and tell the world that the torturer is on the job. */ 1061 /* Process args and tell the world that the torturer is on the job. */
1019 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { 1062 for (i = 0; i < ARRAY_SIZE(torture_ops); i++) {
1020 cur_ops = torture_ops[i]; 1063 cur_ops = torture_ops[i];
@@ -1024,6 +1067,7 @@ rcu_torture_init(void)
1024 if (i == ARRAY_SIZE(torture_ops)) { 1067 if (i == ARRAY_SIZE(torture_ops)) {
1025 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n", 1068 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
1026 torture_type); 1069 torture_type);
1070 mutex_unlock(&fullstop_mutex);
1027 return (-EINVAL); 1071 return (-EINVAL);
1028 } 1072 }
1029 if (cur_ops->init) 1073 if (cur_ops->init)
@@ -1146,9 +1190,12 @@ rcu_torture_init(void)
1146 goto unwind; 1190 goto unwind;
1147 } 1191 }
1148 } 1192 }
1193 register_reboot_notifier(&rcutorture_nb);
1194 mutex_unlock(&fullstop_mutex);
1149 return 0; 1195 return 0;
1150 1196
1151unwind: 1197unwind:
1198 mutex_unlock(&fullstop_mutex);
1152 rcu_torture_cleanup(); 1199 rcu_torture_cleanup();
1153 return firsterr; 1200 return firsterr;
1154} 1201}
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
new file mode 100644
index 000000000000..f2d8638e6c60
--- /dev/null
+++ b/kernel/rcutree.c
@@ -0,0 +1,1532 @@
1/*
2 * Read-Copy Update mechanism for mutual exclusion
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Authors: Dipankar Sarma <dipankar@in.ibm.com>
21 * Manfred Spraul <manfred@colorfullife.com>
22 * Paul E. McKenney <paulmck@linux.vnet.ibm.com> Hierarchical version
23 *
24 * Based on the original work by Paul McKenney <paulmck@us.ibm.com>
25 * and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
26 *
27 * For detailed explanation of Read-Copy Update mechanism see -
28 * Documentation/RCU
29 */
30#include <linux/types.h>
31#include <linux/kernel.h>
32#include <linux/init.h>
33#include <linux/spinlock.h>
34#include <linux/smp.h>
35#include <linux/rcupdate.h>
36#include <linux/interrupt.h>
37#include <linux/sched.h>
38#include <asm/atomic.h>
39#include <linux/bitops.h>
40#include <linux/module.h>
41#include <linux/completion.h>
42#include <linux/moduleparam.h>
43#include <linux/percpu.h>
44#include <linux/notifier.h>
45#include <linux/cpu.h>
46#include <linux/mutex.h>
47#include <linux/time.h>
48
49#ifdef CONFIG_DEBUG_LOCK_ALLOC
50static struct lock_class_key rcu_lock_key;
51struct lockdep_map rcu_lock_map =
52 STATIC_LOCKDEP_MAP_INIT("rcu_read_lock", &rcu_lock_key);
53EXPORT_SYMBOL_GPL(rcu_lock_map);
54#endif
55
56/* Data structures. */
57
58#define RCU_STATE_INITIALIZER(name) { \
59 .level = { &name.node[0] }, \
60 .levelcnt = { \
61 NUM_RCU_LVL_0, /* root of hierarchy. */ \
62 NUM_RCU_LVL_1, \
63 NUM_RCU_LVL_2, \
64 NUM_RCU_LVL_3, /* == MAX_RCU_LVLS */ \
65 }, \
66 .signaled = RCU_SIGNAL_INIT, \
67 .gpnum = -300, \
68 .completed = -300, \
69 .onofflock = __SPIN_LOCK_UNLOCKED(&name.onofflock), \
70 .fqslock = __SPIN_LOCK_UNLOCKED(&name.fqslock), \
71 .n_force_qs = 0, \
72 .n_force_qs_ngp = 0, \
73}
74
75struct rcu_state rcu_state = RCU_STATE_INITIALIZER(rcu_state);
76DEFINE_PER_CPU(struct rcu_data, rcu_data);
77
78struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
79DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
80
81#ifdef CONFIG_NO_HZ
82DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
83 .dynticks_nesting = 1,
84 .dynticks = 1,
85};
86#endif /* #ifdef CONFIG_NO_HZ */
87
88static int blimit = 10; /* Maximum callbacks per softirq. */
89static int qhimark = 10000; /* If this many pending, ignore blimit. */
90static int qlowmark = 100; /* Once only this many pending, use blimit. */
91
92static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
93
94/*
95 * Return the number of RCU batches processed thus far for debug & stats.
96 */
97long rcu_batches_completed(void)
98{
99 return rcu_state.completed;
100}
101EXPORT_SYMBOL_GPL(rcu_batches_completed);
102
103/*
104 * Return the number of RCU BH batches processed thus far for debug & stats.
105 */
106long rcu_batches_completed_bh(void)
107{
108 return rcu_bh_state.completed;
109}
110EXPORT_SYMBOL_GPL(rcu_batches_completed_bh);
111
112/*
113 * Does the CPU have callbacks ready to be invoked?
114 */
115static int
116cpu_has_callbacks_ready_to_invoke(struct rcu_data *rdp)
117{
118 return &rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL];
119}
120
121/*
122 * Does the current CPU require a yet-as-unscheduled grace period?
123 */
124static int
125cpu_needs_another_gp(struct rcu_state *rsp, struct rcu_data *rdp)
126{
127 /* ACCESS_ONCE() because we are accessing outside of lock. */
128 return *rdp->nxttail[RCU_DONE_TAIL] &&
129 ACCESS_ONCE(rsp->completed) == ACCESS_ONCE(rsp->gpnum);
130}
131
132/*
133 * Return the root node of the specified rcu_state structure.
134 */
135static struct rcu_node *rcu_get_root(struct rcu_state *rsp)
136{
137 return &rsp->node[0];
138}
139
140#ifdef CONFIG_SMP
141
142/*
143 * If the specified CPU is offline, tell the caller that it is in
144 * a quiescent state. Otherwise, whack it with a reschedule IPI.
145 * Grace periods can end up waiting on an offline CPU when that
146 * CPU is in the process of coming online -- it will be added to the
147 * rcu_node bitmasks before it actually makes it online. The same thing
148 * can happen while a CPU is in the process of coming online. Because this
149 * race is quite rare, we check for it after detecting that the grace
150 * period has been delayed rather than checking each and every CPU
151 * each and every time we start a new grace period.
152 */
153static int rcu_implicit_offline_qs(struct rcu_data *rdp)
154{
155 /*
156 * If the CPU is offline, it is in a quiescent state. We can
157 * trust its state not to change because interrupts are disabled.
158 */
159 if (cpu_is_offline(rdp->cpu)) {
160 rdp->offline_fqs++;
161 return 1;
162 }
163
164 /* The CPU is online, so send it a reschedule IPI. */
165 if (rdp->cpu != smp_processor_id())
166 smp_send_reschedule(rdp->cpu);
167 else
168 set_need_resched();
169 rdp->resched_ipi++;
170 return 0;
171}
172
173#endif /* #ifdef CONFIG_SMP */
174
175#ifdef CONFIG_NO_HZ
176static DEFINE_RATELIMIT_STATE(rcu_rs, 10 * HZ, 5);
177
178/**
179 * rcu_enter_nohz - inform RCU that current CPU is entering nohz
180 *
181 * Enter nohz mode, in other words, -leave- the mode in which RCU
182 * read-side critical sections can occur. (Though RCU read-side
183 * critical sections can occur in irq handlers in nohz mode, a possibility
184 * handled by rcu_irq_enter() and rcu_irq_exit()).
185 */
186void rcu_enter_nohz(void)
187{
188 unsigned long flags;
189 struct rcu_dynticks *rdtp;
190
191 smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
192 local_irq_save(flags);
193 rdtp = &__get_cpu_var(rcu_dynticks);
194 rdtp->dynticks++;
195 rdtp->dynticks_nesting--;
196 WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
197 local_irq_restore(flags);
198}
199
200/*
201 * rcu_exit_nohz - inform RCU that current CPU is leaving nohz
202 *
203 * Exit nohz mode, in other words, -enter- the mode in which RCU
204 * read-side critical sections normally occur.
205 */
206void rcu_exit_nohz(void)
207{
208 unsigned long flags;
209 struct rcu_dynticks *rdtp;
210
211 local_irq_save(flags);
212 rdtp = &__get_cpu_var(rcu_dynticks);
213 rdtp->dynticks++;
214 rdtp->dynticks_nesting++;
215 WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
216 local_irq_restore(flags);
217 smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
218}
219
220/**
221 * rcu_nmi_enter - inform RCU of entry to NMI context
222 *
223 * If the CPU was idle with dynamic ticks active, and there is no
224 * irq handler running, this updates rdtp->dynticks_nmi to let the
225 * RCU grace-period handling know that the CPU is active.
226 */
227void rcu_nmi_enter(void)
228{
229 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
230
231 if (rdtp->dynticks & 0x1)
232 return;
233 rdtp->dynticks_nmi++;
234 WARN_ON_RATELIMIT(!(rdtp->dynticks_nmi & 0x1), &rcu_rs);
235 smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
236}
237
238/**
239 * rcu_nmi_exit - inform RCU of exit from NMI context
240 *
241 * If the CPU was idle with dynamic ticks active, and there is no
242 * irq handler running, this updates rdtp->dynticks_nmi to let the
243 * RCU grace-period handling know that the CPU is no longer active.
244 */
245void rcu_nmi_exit(void)
246{
247 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
248
249 if (rdtp->dynticks & 0x1)
250 return;
251 smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
252 rdtp->dynticks_nmi++;
253 WARN_ON_RATELIMIT(rdtp->dynticks_nmi & 0x1, &rcu_rs);
254}
255
256/**
257 * rcu_irq_enter - inform RCU of entry to hard irq context
258 *
259 * If the CPU was idle with dynamic ticks active, this updates the
260 * rdtp->dynticks to let the RCU handling know that the CPU is active.
261 */
262void rcu_irq_enter(void)
263{
264 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
265
266 if (rdtp->dynticks_nesting++)
267 return;
268 rdtp->dynticks++;
269 WARN_ON_RATELIMIT(!(rdtp->dynticks & 0x1), &rcu_rs);
270 smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
271}
272
273/**
274 * rcu_irq_exit - inform RCU of exit from hard irq context
275 *
276 * If the CPU was idle with dynamic ticks active, update the rdp->dynticks
277 * to put let the RCU handling be aware that the CPU is going back to idle
278 * with no ticks.
279 */
280void rcu_irq_exit(void)
281{
282 struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
283
284 if (--rdtp->dynticks_nesting)
285 return;
286 smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
287 rdtp->dynticks++;
288 WARN_ON_RATELIMIT(rdtp->dynticks & 0x1, &rcu_rs);
289
290 /* If the interrupt queued a callback, get out of dyntick mode. */
291 if (__get_cpu_var(rcu_data).nxtlist ||
292 __get_cpu_var(rcu_bh_data).nxtlist)
293 set_need_resched();
294}
295
296/*
297 * Record the specified "completed" value, which is later used to validate
298 * dynticks counter manipulations. Specify "rsp->completed - 1" to
299 * unconditionally invalidate any future dynticks manipulations (which is
300 * useful at the beginning of a grace period).
301 */
302static void dyntick_record_completed(struct rcu_state *rsp, long comp)
303{
304 rsp->dynticks_completed = comp;
305}
306
307#ifdef CONFIG_SMP
308
309/*
310 * Recall the previously recorded value of the completion for dynticks.
311 */
312static long dyntick_recall_completed(struct rcu_state *rsp)
313{
314 return rsp->dynticks_completed;
315}
316
317/*
318 * Snapshot the specified CPU's dynticks counter so that we can later
319 * credit them with an implicit quiescent state. Return 1 if this CPU
320 * is already in a quiescent state courtesy of dynticks idle mode.
321 */
322static int dyntick_save_progress_counter(struct rcu_data *rdp)
323{
324 int ret;
325 int snap;
326 int snap_nmi;
327
328 snap = rdp->dynticks->dynticks;
329 snap_nmi = rdp->dynticks->dynticks_nmi;
330 smp_mb(); /* Order sampling of snap with end of grace period. */
331 rdp->dynticks_snap = snap;
332 rdp->dynticks_nmi_snap = snap_nmi;
333 ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0);
334 if (ret)
335 rdp->dynticks_fqs++;
336 return ret;
337}
338
339/*
340 * Return true if the specified CPU has passed through a quiescent
341 * state by virtue of being in or having passed through an dynticks
342 * idle state since the last call to dyntick_save_progress_counter()
343 * for this same CPU.
344 */
345static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
346{
347 long curr;
348 long curr_nmi;
349 long snap;
350 long snap_nmi;
351
352 curr = rdp->dynticks->dynticks;
353 snap = rdp->dynticks_snap;
354 curr_nmi = rdp->dynticks->dynticks_nmi;
355 snap_nmi = rdp->dynticks_nmi_snap;
356 smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
357
358 /*
359 * If the CPU passed through or entered a dynticks idle phase with
360 * no active irq/NMI handlers, then we can safely pretend that the CPU
361 * already acknowledged the request to pass through a quiescent
362 * state. Either way, that CPU cannot possibly be in an RCU
363 * read-side critical section that started before the beginning
364 * of the current RCU grace period.
365 */
366 if ((curr != snap || (curr & 0x1) == 0) &&
367 (curr_nmi != snap_nmi || (curr_nmi & 0x1) == 0)) {
368 rdp->dynticks_fqs++;
369 return 1;
370 }
371
372 /* Go check for the CPU being offline. */
373 return rcu_implicit_offline_qs(rdp);
374}
375
376#endif /* #ifdef CONFIG_SMP */
377
378#else /* #ifdef CONFIG_NO_HZ */
379
380static void dyntick_record_completed(struct rcu_state *rsp, long comp)
381{
382}
383
384#ifdef CONFIG_SMP
385
386/*
387 * If there are no dynticks, then the only way that a CPU can passively
388 * be in a quiescent state is to be offline. Unlike dynticks idle, which
389 * is a point in time during the prior (already finished) grace period,
390 * an offline CPU is always in a quiescent state, and thus can be
391 * unconditionally applied. So just return the current value of completed.
392 */
393static long dyntick_recall_completed(struct rcu_state *rsp)
394{
395 return rsp->completed;
396}
397
398static int dyntick_save_progress_counter(struct rcu_data *rdp)
399{
400 return 0;
401}
402
403static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
404{
405 return rcu_implicit_offline_qs(rdp);
406}
407
408#endif /* #ifdef CONFIG_SMP */
409
410#endif /* #else #ifdef CONFIG_NO_HZ */
411
412#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
413
414static void record_gp_stall_check_time(struct rcu_state *rsp)
415{
416 rsp->gp_start = jiffies;
417 rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK;
418}
419
420static void print_other_cpu_stall(struct rcu_state *rsp)
421{
422 int cpu;
423 long delta;
424 unsigned long flags;
425 struct rcu_node *rnp = rcu_get_root(rsp);
426 struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS - 1];
427 struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES];
428
429 /* Only let one CPU complain about others per time interval. */
430
431 spin_lock_irqsave(&rnp->lock, flags);
432 delta = jiffies - rsp->jiffies_stall;
433 if (delta < RCU_STALL_RAT_DELAY || rsp->gpnum == rsp->completed) {
434 spin_unlock_irqrestore(&rnp->lock, flags);
435 return;
436 }
437 rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
438 spin_unlock_irqrestore(&rnp->lock, flags);
439
440 /* OK, time to rat on our buddy... */
441
442 printk(KERN_ERR "INFO: RCU detected CPU stalls:");
443 for (; rnp_cur < rnp_end; rnp_cur++) {
444 if (rnp_cur->qsmask == 0)
445 continue;
446 for (cpu = 0; cpu <= rnp_cur->grphi - rnp_cur->grplo; cpu++)
447 if (rnp_cur->qsmask & (1UL << cpu))
448 printk(" %d", rnp_cur->grplo + cpu);
449 }
450 printk(" (detected by %d, t=%ld jiffies)\n",
451 smp_processor_id(), (long)(jiffies - rsp->gp_start));
452 force_quiescent_state(rsp, 0); /* Kick them all. */
453}
454
455static void print_cpu_stall(struct rcu_state *rsp)
456{
457 unsigned long flags;
458 struct rcu_node *rnp = rcu_get_root(rsp);
459
460 printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n",
461 smp_processor_id(), jiffies - rsp->gp_start);
462 dump_stack();
463 spin_lock_irqsave(&rnp->lock, flags);
464 if ((long)(jiffies - rsp->jiffies_stall) >= 0)
465 rsp->jiffies_stall =
466 jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
467 spin_unlock_irqrestore(&rnp->lock, flags);
468 set_need_resched(); /* kick ourselves to get things going. */
469}
470
471static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
472{
473 long delta;
474 struct rcu_node *rnp;
475
476 delta = jiffies - rsp->jiffies_stall;
477 rnp = rdp->mynode;
478 if ((rnp->qsmask & rdp->grpmask) && delta >= 0) {
479
480 /* We haven't checked in, so go dump stack. */
481 print_cpu_stall(rsp);
482
483 } else if (rsp->gpnum != rsp->completed &&
484 delta >= RCU_STALL_RAT_DELAY) {
485
486 /* They had two time units to dump stack, so complain. */
487 print_other_cpu_stall(rsp);
488 }
489}
490
491#else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
492
493static void record_gp_stall_check_time(struct rcu_state *rsp)
494{
495}
496
497static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
498{
499}
500
501#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
502
503/*
504 * Update CPU-local rcu_data state to record the newly noticed grace period.
505 * This is used both when we started the grace period and when we notice
506 * that someone else started the grace period.
507 */
508static void note_new_gpnum(struct rcu_state *rsp, struct rcu_data *rdp)
509{
510 rdp->qs_pending = 1;
511 rdp->passed_quiesc = 0;
512 rdp->gpnum = rsp->gpnum;
513 rdp->n_rcu_pending_force_qs = rdp->n_rcu_pending +
514 RCU_JIFFIES_TILL_FORCE_QS;
515}
516
517/*
518 * Did someone else start a new RCU grace period start since we last
519 * checked? Update local state appropriately if so. Must be called
520 * on the CPU corresponding to rdp.
521 */
522static int
523check_for_new_grace_period(struct rcu_state *rsp, struct rcu_data *rdp)
524{
525 unsigned long flags;
526 int ret = 0;
527
528 local_irq_save(flags);
529 if (rdp->gpnum != rsp->gpnum) {
530 note_new_gpnum(rsp, rdp);
531 ret = 1;
532 }
533 local_irq_restore(flags);
534 return ret;
535}
536
537/*
538 * Start a new RCU grace period if warranted, re-initializing the hierarchy
539 * in preparation for detecting the next grace period. The caller must hold
540 * the root node's ->lock, which is released before return. Hard irqs must
541 * be disabled.
542 */
543static void
544rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
545 __releases(rcu_get_root(rsp)->lock)
546{
547 struct rcu_data *rdp = rsp->rda[smp_processor_id()];
548 struct rcu_node *rnp = rcu_get_root(rsp);
549 struct rcu_node *rnp_cur;
550 struct rcu_node *rnp_end;
551
552 if (!cpu_needs_another_gp(rsp, rdp)) {
553 spin_unlock_irqrestore(&rnp->lock, flags);
554 return;
555 }
556
557 /* Advance to a new grace period and initialize state. */
558 rsp->gpnum++;
559 rsp->signaled = RCU_GP_INIT; /* Hold off force_quiescent_state. */
560 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
561 rdp->n_rcu_pending_force_qs = rdp->n_rcu_pending +
562 RCU_JIFFIES_TILL_FORCE_QS;
563 record_gp_stall_check_time(rsp);
564 dyntick_record_completed(rsp, rsp->completed - 1);
565 note_new_gpnum(rsp, rdp);
566
567 /*
568 * Because we are first, we know that all our callbacks will
569 * be covered by this upcoming grace period, even the ones
570 * that were registered arbitrarily recently.
571 */
572 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
573 rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
574
575 /* Special-case the common single-level case. */
576 if (NUM_RCU_NODES == 1) {
577 rnp->qsmask = rnp->qsmaskinit;
578 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state OK. */
579 spin_unlock_irqrestore(&rnp->lock, flags);
580 return;
581 }
582
583 spin_unlock(&rnp->lock); /* leave irqs disabled. */
584
585
586 /* Exclude any concurrent CPU-hotplug operations. */
587 spin_lock(&rsp->onofflock); /* irqs already disabled. */
588
589 /*
590 * Set the quiescent-state-needed bits in all the non-leaf RCU
591 * nodes for all currently online CPUs. This operation relies
592 * on the layout of the hierarchy within the rsp->node[] array.
593 * Note that other CPUs will access only the leaves of the
594 * hierarchy, which still indicate that no grace period is in
595 * progress. In addition, we have excluded CPU-hotplug operations.
596 *
597 * We therefore do not need to hold any locks. Any required
598 * memory barriers will be supplied by the locks guarding the
599 * leaf rcu_nodes in the hierarchy.
600 */
601
602 rnp_end = rsp->level[NUM_RCU_LVLS - 1];
603 for (rnp_cur = &rsp->node[0]; rnp_cur < rnp_end; rnp_cur++)
604 rnp_cur->qsmask = rnp_cur->qsmaskinit;
605
606 /*
607 * Now set up the leaf nodes. Here we must be careful. First,
608 * we need to hold the lock in order to exclude other CPUs, which
609 * might be contending for the leaf nodes' locks. Second, as
610 * soon as we initialize a given leaf node, its CPUs might run
611 * up the rest of the hierarchy. We must therefore acquire locks
612 * for each node that we touch during this stage. (But we still
613 * are excluding CPU-hotplug operations.)
614 *
615 * Note that the grace period cannot complete until we finish
616 * the initialization process, as there will be at least one
617 * qsmask bit set in the root node until that time, namely the
618 * one corresponding to this CPU.
619 */
620 rnp_end = &rsp->node[NUM_RCU_NODES];
621 rnp_cur = rsp->level[NUM_RCU_LVLS - 1];
622 for (; rnp_cur < rnp_end; rnp_cur++) {
623 spin_lock(&rnp_cur->lock); /* irqs already disabled. */
624 rnp_cur->qsmask = rnp_cur->qsmaskinit;
625 spin_unlock(&rnp_cur->lock); /* irqs already disabled. */
626 }
627
628 rsp->signaled = RCU_SIGNAL_INIT; /* force_quiescent_state now OK. */
629 spin_unlock_irqrestore(&rsp->onofflock, flags);
630}
631
632/*
633 * Advance this CPU's callbacks, but only if the current grace period
634 * has ended. This may be called only from the CPU to whom the rdp
635 * belongs.
636 */
637static void
638rcu_process_gp_end(struct rcu_state *rsp, struct rcu_data *rdp)
639{
640 long completed_snap;
641 unsigned long flags;
642
643 local_irq_save(flags);
644 completed_snap = ACCESS_ONCE(rsp->completed); /* outside of lock. */
645
646 /* Did another grace period end? */
647 if (rdp->completed != completed_snap) {
648
649 /* Advance callbacks. No harm if list empty. */
650 rdp->nxttail[RCU_DONE_TAIL] = rdp->nxttail[RCU_WAIT_TAIL];
651 rdp->nxttail[RCU_WAIT_TAIL] = rdp->nxttail[RCU_NEXT_READY_TAIL];
652 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
653
654 /* Remember that we saw this grace-period completion. */
655 rdp->completed = completed_snap;
656 }
657 local_irq_restore(flags);
658}
659
660/*
661 * Similar to cpu_quiet(), for which it is a helper function. Allows
662 * a group of CPUs to be quieted at one go, though all the CPUs in the
663 * group must be represented by the same leaf rcu_node structure.
664 * That structure's lock must be held upon entry, and it is released
665 * before return.
666 */
667static void
668cpu_quiet_msk(unsigned long mask, struct rcu_state *rsp, struct rcu_node *rnp,
669 unsigned long flags)
670 __releases(rnp->lock)
671{
672 /* Walk up the rcu_node hierarchy. */
673 for (;;) {
674 if (!(rnp->qsmask & mask)) {
675
676 /* Our bit has already been cleared, so done. */
677 spin_unlock_irqrestore(&rnp->lock, flags);
678 return;
679 }
680 rnp->qsmask &= ~mask;
681 if (rnp->qsmask != 0) {
682
683 /* Other bits still set at this level, so done. */
684 spin_unlock_irqrestore(&rnp->lock, flags);
685 return;
686 }
687 mask = rnp->grpmask;
688 if (rnp->parent == NULL) {
689
690 /* No more levels. Exit loop holding root lock. */
691
692 break;
693 }
694 spin_unlock_irqrestore(&rnp->lock, flags);
695 rnp = rnp->parent;
696 spin_lock_irqsave(&rnp->lock, flags);
697 }
698
699 /*
700 * Get here if we are the last CPU to pass through a quiescent
701 * state for this grace period. Clean up and let rcu_start_gp()
702 * start up the next grace period if one is needed. Note that
703 * we still hold rnp->lock, as required by rcu_start_gp(), which
704 * will release it.
705 */
706 rsp->completed = rsp->gpnum;
707 rcu_process_gp_end(rsp, rsp->rda[smp_processor_id()]);
708 rcu_start_gp(rsp, flags); /* releases rnp->lock. */
709}
710
711/*
712 * Record a quiescent state for the specified CPU, which must either be
713 * the current CPU or an offline CPU. The lastcomp argument is used to
714 * make sure we are still in the grace period of interest. We don't want
715 * to end the current grace period based on quiescent states detected in
716 * an earlier grace period!
717 */
718static void
719cpu_quiet(int cpu, struct rcu_state *rsp, struct rcu_data *rdp, long lastcomp)
720{
721 unsigned long flags;
722 unsigned long mask;
723 struct rcu_node *rnp;
724
725 rnp = rdp->mynode;
726 spin_lock_irqsave(&rnp->lock, flags);
727 if (lastcomp != ACCESS_ONCE(rsp->completed)) {
728
729 /*
730 * Someone beat us to it for this grace period, so leave.
731 * The race with GP start is resolved by the fact that we
732 * hold the leaf rcu_node lock, so that the per-CPU bits
733 * cannot yet be initialized -- so we would simply find our
734 * CPU's bit already cleared in cpu_quiet_msk() if this race
735 * occurred.
736 */
737 rdp->passed_quiesc = 0; /* try again later! */
738 spin_unlock_irqrestore(&rnp->lock, flags);
739 return;
740 }
741 mask = rdp->grpmask;
742 if ((rnp->qsmask & mask) == 0) {
743 spin_unlock_irqrestore(&rnp->lock, flags);
744 } else {
745 rdp->qs_pending = 0;
746
747 /*
748 * This GP can't end until cpu checks in, so all of our
749 * callbacks can be processed during the next GP.
750 */
751 rdp = rsp->rda[smp_processor_id()];
752 rdp->nxttail[RCU_NEXT_READY_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
753
754 cpu_quiet_msk(mask, rsp, rnp, flags); /* releases rnp->lock */
755 }
756}
757
758/*
759 * Check to see if there is a new grace period of which this CPU
760 * is not yet aware, and if so, set up local rcu_data state for it.
761 * Otherwise, see if this CPU has just passed through its first
762 * quiescent state for this grace period, and record that fact if so.
763 */
764static void
765rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp)
766{
767 /* If there is now a new grace period, record and return. */
768 if (check_for_new_grace_period(rsp, rdp))
769 return;
770
771 /*
772 * Does this CPU still need to do its part for current grace period?
773 * If no, return and let the other CPUs do their part as well.
774 */
775 if (!rdp->qs_pending)
776 return;
777
778 /*
779 * Was there a quiescent state since the beginning of the grace
780 * period? If no, then exit and wait for the next call.
781 */
782 if (!rdp->passed_quiesc)
783 return;
784
785 /* Tell RCU we are done (but cpu_quiet() will be the judge of that). */
786 cpu_quiet(rdp->cpu, rsp, rdp, rdp->passed_quiesc_completed);
787}
788
789#ifdef CONFIG_HOTPLUG_CPU
790
791/*
792 * Remove the outgoing CPU from the bitmasks in the rcu_node hierarchy
793 * and move all callbacks from the outgoing CPU to the current one.
794 */
795static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
796{
797 int i;
798 unsigned long flags;
799 long lastcomp;
800 unsigned long mask;
801 struct rcu_data *rdp = rsp->rda[cpu];
802 struct rcu_data *rdp_me;
803 struct rcu_node *rnp;
804
805 /* Exclude any attempts to start a new grace period. */
806 spin_lock_irqsave(&rsp->onofflock, flags);
807
808 /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */
809 rnp = rdp->mynode;
810 mask = rdp->grpmask; /* rnp->grplo is constant. */
811 do {
812 spin_lock(&rnp->lock); /* irqs already disabled. */
813 rnp->qsmaskinit &= ~mask;
814 if (rnp->qsmaskinit != 0) {
815 spin_unlock(&rnp->lock); /* irqs already disabled. */
816 break;
817 }
818 mask = rnp->grpmask;
819 spin_unlock(&rnp->lock); /* irqs already disabled. */
820 rnp = rnp->parent;
821 } while (rnp != NULL);
822 lastcomp = rsp->completed;
823
824 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
825
826 /* Being offline is a quiescent state, so go record it. */
827 cpu_quiet(cpu, rsp, rdp, lastcomp);
828
829 /*
830 * Move callbacks from the outgoing CPU to the running CPU.
831 * Note that the outgoing CPU is now quiscent, so it is now
832 * (uncharacteristically) safe to access it rcu_data structure.
833 * Note also that we must carefully retain the order of the
834 * outgoing CPU's callbacks in order for rcu_barrier() to work
835 * correctly. Finally, note that we start all the callbacks
836 * afresh, even those that have passed through a grace period
837 * and are therefore ready to invoke. The theory is that hotplug
838 * events are rare, and that if they are frequent enough to
839 * indefinitely delay callbacks, you have far worse things to
840 * be worrying about.
841 */
842 rdp_me = rsp->rda[smp_processor_id()];
843 if (rdp->nxtlist != NULL) {
844 *rdp_me->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist;
845 rdp_me->nxttail[RCU_NEXT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
846 rdp->nxtlist = NULL;
847 for (i = 0; i < RCU_NEXT_SIZE; i++)
848 rdp->nxttail[i] = &rdp->nxtlist;
849 rdp_me->qlen += rdp->qlen;
850 rdp->qlen = 0;
851 }
852 local_irq_restore(flags);
853}
854
855/*
856 * Remove the specified CPU from the RCU hierarchy and move any pending
857 * callbacks that it might have to the current CPU. This code assumes
858 * that at least one CPU in the system will remain running at all times.
859 * Any attempt to offline -all- CPUs is likely to strand RCU callbacks.
860 */
861static void rcu_offline_cpu(int cpu)
862{
863 __rcu_offline_cpu(cpu, &rcu_state);
864 __rcu_offline_cpu(cpu, &rcu_bh_state);
865}
866
867#else /* #ifdef CONFIG_HOTPLUG_CPU */
868
869static void rcu_offline_cpu(int cpu)
870{
871}
872
873#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
874
875/*
876 * Invoke any RCU callbacks that have made it to the end of their grace
877 * period. Thottle as specified by rdp->blimit.
878 */
879static void rcu_do_batch(struct rcu_data *rdp)
880{
881 unsigned long flags;
882 struct rcu_head *next, *list, **tail;
883 int count;
884
885 /* If no callbacks are ready, just return.*/
886 if (!cpu_has_callbacks_ready_to_invoke(rdp))
887 return;
888
889 /*
890 * Extract the list of ready callbacks, disabling to prevent
891 * races with call_rcu() from interrupt handlers.
892 */
893 local_irq_save(flags);
894 list = rdp->nxtlist;
895 rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
896 *rdp->nxttail[RCU_DONE_TAIL] = NULL;
897 tail = rdp->nxttail[RCU_DONE_TAIL];
898 for (count = RCU_NEXT_SIZE - 1; count >= 0; count--)
899 if (rdp->nxttail[count] == rdp->nxttail[RCU_DONE_TAIL])
900 rdp->nxttail[count] = &rdp->nxtlist;
901 local_irq_restore(flags);
902
903 /* Invoke callbacks. */
904 count = 0;
905 while (list) {
906 next = list->next;
907 prefetch(next);
908 list->func(list);
909 list = next;
910 if (++count >= rdp->blimit)
911 break;
912 }
913
914 local_irq_save(flags);
915
916 /* Update count, and requeue any remaining callbacks. */
917 rdp->qlen -= count;
918 if (list != NULL) {
919 *tail = rdp->nxtlist;
920 rdp->nxtlist = list;
921 for (count = 0; count < RCU_NEXT_SIZE; count++)
922 if (&rdp->nxtlist == rdp->nxttail[count])
923 rdp->nxttail[count] = tail;
924 else
925 break;
926 }
927
928 /* Reinstate batch limit if we have worked down the excess. */
929 if (rdp->blimit == LONG_MAX && rdp->qlen <= qlowmark)
930 rdp->blimit = blimit;
931
932 local_irq_restore(flags);
933
934 /* Re-raise the RCU softirq if there are callbacks remaining. */
935 if (cpu_has_callbacks_ready_to_invoke(rdp))
936 raise_softirq(RCU_SOFTIRQ);
937}
938
939/*
940 * Check to see if this CPU is in a non-context-switch quiescent state
941 * (user mode or idle loop for rcu, non-softirq execution for rcu_bh).
942 * Also schedule the RCU softirq handler.
943 *
944 * This function must be called with hardirqs disabled. It is normally
945 * invoked from the scheduling-clock interrupt. If rcu_pending returns
946 * false, there is no point in invoking rcu_check_callbacks().
947 */
948void rcu_check_callbacks(int cpu, int user)
949{
950 if (user ||
951 (idle_cpu(cpu) && !in_softirq() &&
952 hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
953
954 /*
955 * Get here if this CPU took its interrupt from user
956 * mode or from the idle loop, and if this is not a
957 * nested interrupt. In this case, the CPU is in
958 * a quiescent state, so count it.
959 *
960 * No memory barrier is required here because both
961 * rcu_qsctr_inc() and rcu_bh_qsctr_inc() reference
962 * only CPU-local variables that other CPUs neither
963 * access nor modify, at least not while the corresponding
964 * CPU is online.
965 */
966
967 rcu_qsctr_inc(cpu);
968 rcu_bh_qsctr_inc(cpu);
969
970 } else if (!in_softirq()) {
971
972 /*
973 * Get here if this CPU did not take its interrupt from
974 * softirq, in other words, if it is not interrupting
975 * a rcu_bh read-side critical section. This is an _bh
976 * critical section, so count it.
977 */
978
979 rcu_bh_qsctr_inc(cpu);
980 }
981 raise_softirq(RCU_SOFTIRQ);
982}
983
984#ifdef CONFIG_SMP
985
986/*
987 * Scan the leaf rcu_node structures, processing dyntick state for any that
988 * have not yet encountered a quiescent state, using the function specified.
989 * Returns 1 if the current grace period ends while scanning (possibly
990 * because we made it end).
991 */
992static int rcu_process_dyntick(struct rcu_state *rsp, long lastcomp,
993 int (*f)(struct rcu_data *))
994{
995 unsigned long bit;
996 int cpu;
997 unsigned long flags;
998 unsigned long mask;
999 struct rcu_node *rnp_cur = rsp->level[NUM_RCU_LVLS - 1];
1000 struct rcu_node *rnp_end = &rsp->node[NUM_RCU_NODES];
1001
1002 for (; rnp_cur < rnp_end; rnp_cur++) {
1003 mask = 0;
1004 spin_lock_irqsave(&rnp_cur->lock, flags);
1005 if (rsp->completed != lastcomp) {
1006 spin_unlock_irqrestore(&rnp_cur->lock, flags);
1007 return 1;
1008 }
1009 if (rnp_cur->qsmask == 0) {
1010 spin_unlock_irqrestore(&rnp_cur->lock, flags);
1011 continue;
1012 }
1013 cpu = rnp_cur->grplo;
1014 bit = 1;
1015 for (; cpu <= rnp_cur->grphi; cpu++, bit <<= 1) {
1016 if ((rnp_cur->qsmask & bit) != 0 && f(rsp->rda[cpu]))
1017 mask |= bit;
1018 }
1019 if (mask != 0 && rsp->completed == lastcomp) {
1020
1021 /* cpu_quiet_msk() releases rnp_cur->lock. */
1022 cpu_quiet_msk(mask, rsp, rnp_cur, flags);
1023 continue;
1024 }
1025 spin_unlock_irqrestore(&rnp_cur->lock, flags);
1026 }
1027 return 0;
1028}
1029
1030/*
1031 * Force quiescent states on reluctant CPUs, and also detect which
1032 * CPUs are in dyntick-idle mode.
1033 */
1034static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1035{
1036 unsigned long flags;
1037 long lastcomp;
1038 struct rcu_data *rdp = rsp->rda[smp_processor_id()];
1039 struct rcu_node *rnp = rcu_get_root(rsp);
1040 u8 signaled;
1041
1042 if (ACCESS_ONCE(rsp->completed) == ACCESS_ONCE(rsp->gpnum))
1043 return; /* No grace period in progress, nothing to force. */
1044 if (!spin_trylock_irqsave(&rsp->fqslock, flags)) {
1045 rsp->n_force_qs_lh++; /* Inexact, can lose counts. Tough! */
1046 return; /* Someone else is already on the job. */
1047 }
1048 if (relaxed &&
1049 (long)(rsp->jiffies_force_qs - jiffies) >= 0 &&
1050 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) >= 0)
1051 goto unlock_ret; /* no emergency and done recently. */
1052 rsp->n_force_qs++;
1053 spin_lock(&rnp->lock);
1054 lastcomp = rsp->completed;
1055 signaled = rsp->signaled;
1056 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
1057 rdp->n_rcu_pending_force_qs = rdp->n_rcu_pending +
1058 RCU_JIFFIES_TILL_FORCE_QS;
1059 if (lastcomp == rsp->gpnum) {
1060 rsp->n_force_qs_ngp++;
1061 spin_unlock(&rnp->lock);
1062 goto unlock_ret; /* no GP in progress, time updated. */
1063 }
1064 spin_unlock(&rnp->lock);
1065 switch (signaled) {
1066 case RCU_GP_INIT:
1067
1068 break; /* grace period still initializing, ignore. */
1069
1070 case RCU_SAVE_DYNTICK:
1071
1072 if (RCU_SIGNAL_INIT != RCU_SAVE_DYNTICK)
1073 break; /* So gcc recognizes the dead code. */
1074
1075 /* Record dyntick-idle state. */
1076 if (rcu_process_dyntick(rsp, lastcomp,
1077 dyntick_save_progress_counter))
1078 goto unlock_ret;
1079
1080 /* Update state, record completion counter. */
1081 spin_lock(&rnp->lock);
1082 if (lastcomp == rsp->completed) {
1083 rsp->signaled = RCU_FORCE_QS;
1084 dyntick_record_completed(rsp, lastcomp);
1085 }
1086 spin_unlock(&rnp->lock);
1087 break;
1088
1089 case RCU_FORCE_QS:
1090
1091 /* Check dyntick-idle state, send IPI to laggarts. */
1092 if (rcu_process_dyntick(rsp, dyntick_recall_completed(rsp),
1093 rcu_implicit_dynticks_qs))
1094 goto unlock_ret;
1095
1096 /* Leave state in case more forcing is required. */
1097
1098 break;
1099 }
1100unlock_ret:
1101 spin_unlock_irqrestore(&rsp->fqslock, flags);
1102}
1103
1104#else /* #ifdef CONFIG_SMP */
1105
1106static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1107{
1108 set_need_resched();
1109}
1110
1111#endif /* #else #ifdef CONFIG_SMP */
1112
1113/*
1114 * This does the RCU processing work from softirq context for the
1115 * specified rcu_state and rcu_data structures. This may be called
1116 * only from the CPU to whom the rdp belongs.
1117 */
1118static void
1119__rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
1120{
1121 unsigned long flags;
1122
1123 /*
1124 * If an RCU GP has gone long enough, go check for dyntick
1125 * idle CPUs and, if needed, send resched IPIs.
1126 */
1127 if ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0 ||
1128 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) < 0)
1129 force_quiescent_state(rsp, 1);
1130
1131 /*
1132 * Advance callbacks in response to end of earlier grace
1133 * period that some other CPU ended.
1134 */
1135 rcu_process_gp_end(rsp, rdp);
1136
1137 /* Update RCU state based on any recent quiescent states. */
1138 rcu_check_quiescent_state(rsp, rdp);
1139
1140 /* Does this CPU require a not-yet-started grace period? */
1141 if (cpu_needs_another_gp(rsp, rdp)) {
1142 spin_lock_irqsave(&rcu_get_root(rsp)->lock, flags);
1143 rcu_start_gp(rsp, flags); /* releases above lock */
1144 }
1145
1146 /* If there are callbacks ready, invoke them. */
1147 rcu_do_batch(rdp);
1148}
1149
1150/*
1151 * Do softirq processing for the current CPU.
1152 */
1153static void rcu_process_callbacks(struct softirq_action *unused)
1154{
1155 /*
1156 * Memory references from any prior RCU read-side critical sections
1157 * executed by the interrupted code must be seen before any RCU
1158 * grace-period manipulations below.
1159 */
1160 smp_mb(); /* See above block comment. */
1161
1162 __rcu_process_callbacks(&rcu_state, &__get_cpu_var(rcu_data));
1163 __rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
1164
1165 /*
1166 * Memory references from any later RCU read-side critical sections
1167 * executed by the interrupted code must be seen after any RCU
1168 * grace-period manipulations above.
1169 */
1170 smp_mb(); /* See above block comment. */
1171}
1172
1173static void
1174__call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
1175 struct rcu_state *rsp)
1176{
1177 unsigned long flags;
1178 struct rcu_data *rdp;
1179
1180 head->func = func;
1181 head->next = NULL;
1182
1183 smp_mb(); /* Ensure RCU update seen before callback registry. */
1184
1185 /*
1186 * Opportunistically note grace-period endings and beginnings.
1187 * Note that we might see a beginning right after we see an
1188 * end, but never vice versa, since this CPU has to pass through
1189 * a quiescent state betweentimes.
1190 */
1191 local_irq_save(flags);
1192 rdp = rsp->rda[smp_processor_id()];
1193 rcu_process_gp_end(rsp, rdp);
1194 check_for_new_grace_period(rsp, rdp);
1195
1196 /* Add the callback to our list. */
1197 *rdp->nxttail[RCU_NEXT_TAIL] = head;
1198 rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
1199
1200 /* Start a new grace period if one not already started. */
1201 if (ACCESS_ONCE(rsp->completed) == ACCESS_ONCE(rsp->gpnum)) {
1202 unsigned long nestflag;
1203 struct rcu_node *rnp_root = rcu_get_root(rsp);
1204
1205 spin_lock_irqsave(&rnp_root->lock, nestflag);
1206 rcu_start_gp(rsp, nestflag); /* releases rnp_root->lock. */
1207 }
1208
1209 /* Force the grace period if too many callbacks or too long waiting. */
1210 if (unlikely(++rdp->qlen > qhimark)) {
1211 rdp->blimit = LONG_MAX;
1212 force_quiescent_state(rsp, 0);
1213 } else if ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0 ||
1214 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) < 0)
1215 force_quiescent_state(rsp, 1);
1216 local_irq_restore(flags);
1217}
1218
1219/*
1220 * Queue an RCU callback for invocation after a grace period.
1221 */
1222void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
1223{
1224 __call_rcu(head, func, &rcu_state);
1225}
1226EXPORT_SYMBOL_GPL(call_rcu);
1227
1228/*
1229 * Queue an RCU for invocation after a quicker grace period.
1230 */
1231void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
1232{
1233 __call_rcu(head, func, &rcu_bh_state);
1234}
1235EXPORT_SYMBOL_GPL(call_rcu_bh);
1236
1237/*
1238 * Check to see if there is any immediate RCU-related work to be done
1239 * by the current CPU, for the specified type of RCU, returning 1 if so.
1240 * The checks are in order of increasing expense: checks that can be
1241 * carried out against CPU-local state are performed first. However,
1242 * we must check for CPU stalls first, else we might not get a chance.
1243 */
1244static int __rcu_pending(struct rcu_state *rsp, struct rcu_data *rdp)
1245{
1246 rdp->n_rcu_pending++;
1247
1248 /* Check for CPU stalls, if enabled. */
1249 check_cpu_stall(rsp, rdp);
1250
1251 /* Is the RCU core waiting for a quiescent state from this CPU? */
1252 if (rdp->qs_pending)
1253 return 1;
1254
1255 /* Does this CPU have callbacks ready to invoke? */
1256 if (cpu_has_callbacks_ready_to_invoke(rdp))
1257 return 1;
1258
1259 /* Has RCU gone idle with this CPU needing another grace period? */
1260 if (cpu_needs_another_gp(rsp, rdp))
1261 return 1;
1262
1263 /* Has another RCU grace period completed? */
1264 if (ACCESS_ONCE(rsp->completed) != rdp->completed) /* outside of lock */
1265 return 1;
1266
1267 /* Has a new RCU grace period started? */
1268 if (ACCESS_ONCE(rsp->gpnum) != rdp->gpnum) /* outside of lock */
1269 return 1;
1270
1271 /* Has an RCU GP gone long enough to send resched IPIs &c? */
1272 if (ACCESS_ONCE(rsp->completed) != ACCESS_ONCE(rsp->gpnum) &&
1273 ((long)(ACCESS_ONCE(rsp->jiffies_force_qs) - jiffies) < 0 ||
1274 (rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending) < 0))
1275 return 1;
1276
1277 /* nothing to do */
1278 return 0;
1279}
1280
1281/*
1282 * Check to see if there is any immediate RCU-related work to be done
1283 * by the current CPU, returning 1 if so. This function is part of the
1284 * RCU implementation; it is -not- an exported member of the RCU API.
1285 */
1286int rcu_pending(int cpu)
1287{
1288 return __rcu_pending(&rcu_state, &per_cpu(rcu_data, cpu)) ||
1289 __rcu_pending(&rcu_bh_state, &per_cpu(rcu_bh_data, cpu));
1290}
1291
1292/*
1293 * Check to see if any future RCU-related work will need to be done
1294 * by the current CPU, even if none need be done immediately, returning
1295 * 1 if so. This function is part of the RCU implementation; it is -not-
1296 * an exported member of the RCU API.
1297 */
1298int rcu_needs_cpu(int cpu)
1299{
1300 /* RCU callbacks either ready or pending? */
1301 return per_cpu(rcu_data, cpu).nxtlist ||
1302 per_cpu(rcu_bh_data, cpu).nxtlist;
1303}
1304
1305/*
1306 * Initialize a CPU's per-CPU RCU data. We take this "scorched earth"
1307 * approach so that we don't have to worry about how long the CPU has
1308 * been gone, or whether it ever was online previously. We do trust the
1309 * ->mynode field, as it is constant for a given struct rcu_data and
1310 * initialized during early boot.
1311 *
1312 * Note that only one online or offline event can be happening at a given
1313 * time. Note also that we can accept some slop in the rsp->completed
1314 * access due to the fact that this CPU cannot possibly have any RCU
1315 * callbacks in flight yet.
1316 */
1317static void
1318rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
1319{
1320 unsigned long flags;
1321 int i;
1322 long lastcomp;
1323 unsigned long mask;
1324 struct rcu_data *rdp = rsp->rda[cpu];
1325 struct rcu_node *rnp = rcu_get_root(rsp);
1326
1327 /* Set up local state, ensuring consistent view of global state. */
1328 spin_lock_irqsave(&rnp->lock, flags);
1329 lastcomp = rsp->completed;
1330 rdp->completed = lastcomp;
1331 rdp->gpnum = lastcomp;
1332 rdp->passed_quiesc = 0; /* We could be racing with new GP, */
1333 rdp->qs_pending = 1; /* so set up to respond to current GP. */
1334 rdp->beenonline = 1; /* We have now been online. */
1335 rdp->passed_quiesc_completed = lastcomp - 1;
1336 rdp->grpmask = 1UL << (cpu - rdp->mynode->grplo);
1337 rdp->nxtlist = NULL;
1338 for (i = 0; i < RCU_NEXT_SIZE; i++)
1339 rdp->nxttail[i] = &rdp->nxtlist;
1340 rdp->qlen = 0;
1341 rdp->blimit = blimit;
1342#ifdef CONFIG_NO_HZ
1343 rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
1344#endif /* #ifdef CONFIG_NO_HZ */
1345 rdp->cpu = cpu;
1346 spin_unlock(&rnp->lock); /* irqs remain disabled. */
1347
1348 /*
1349 * A new grace period might start here. If so, we won't be part
1350 * of it, but that is OK, as we are currently in a quiescent state.
1351 */
1352
1353 /* Exclude any attempts to start a new GP on large systems. */
1354 spin_lock(&rsp->onofflock); /* irqs already disabled. */
1355
1356 /* Add CPU to rcu_node bitmasks. */
1357 rnp = rdp->mynode;
1358 mask = rdp->grpmask;
1359 do {
1360 /* Exclude any attempts to start a new GP on small systems. */
1361 spin_lock(&rnp->lock); /* irqs already disabled. */
1362 rnp->qsmaskinit |= mask;
1363 mask = rnp->grpmask;
1364 spin_unlock(&rnp->lock); /* irqs already disabled. */
1365 rnp = rnp->parent;
1366 } while (rnp != NULL && !(rnp->qsmaskinit & mask));
1367
1368 spin_unlock(&rsp->onofflock); /* irqs remain disabled. */
1369
1370 /*
1371 * A new grace period might start here. If so, we will be part of
1372 * it, and its gpnum will be greater than ours, so we will
1373 * participate. It is also possible for the gpnum to have been
1374 * incremented before this function was called, and the bitmasks
1375 * to not be filled out until now, in which case we will also
1376 * participate due to our gpnum being behind.
1377 */
1378
1379 /* Since it is coming online, the CPU is in a quiescent state. */
1380 cpu_quiet(cpu, rsp, rdp, lastcomp);
1381 local_irq_restore(flags);
1382}
1383
1384static void __cpuinit rcu_online_cpu(int cpu)
1385{
1386 rcu_init_percpu_data(cpu, &rcu_state);
1387 rcu_init_percpu_data(cpu, &rcu_bh_state);
1388 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
1389}
1390
1391/*
1392 * Handle CPU online/offline notifcation events.
1393 */
1394static int __cpuinit rcu_cpu_notify(struct notifier_block *self,
1395 unsigned long action, void *hcpu)
1396{
1397 long cpu = (long)hcpu;
1398
1399 switch (action) {
1400 case CPU_UP_PREPARE:
1401 case CPU_UP_PREPARE_FROZEN:
1402 rcu_online_cpu(cpu);
1403 break;
1404 case CPU_DEAD:
1405 case CPU_DEAD_FROZEN:
1406 case CPU_UP_CANCELED:
1407 case CPU_UP_CANCELED_FROZEN:
1408 rcu_offline_cpu(cpu);
1409 break;
1410 default:
1411 break;
1412 }
1413 return NOTIFY_OK;
1414}
1415
1416/*
1417 * Compute the per-level fanout, either using the exact fanout specified
1418 * or balancing the tree, depending on CONFIG_RCU_FANOUT_EXACT.
1419 */
1420#ifdef CONFIG_RCU_FANOUT_EXACT
1421static void __init rcu_init_levelspread(struct rcu_state *rsp)
1422{
1423 int i;
1424
1425 for (i = NUM_RCU_LVLS - 1; i >= 0; i--)
1426 rsp->levelspread[i] = CONFIG_RCU_FANOUT;
1427}
1428#else /* #ifdef CONFIG_RCU_FANOUT_EXACT */
1429static void __init rcu_init_levelspread(struct rcu_state *rsp)
1430{
1431 int ccur;
1432 int cprv;
1433 int i;
1434
1435 cprv = NR_CPUS;
1436 for (i = NUM_RCU_LVLS - 1; i >= 0; i--) {
1437 ccur = rsp->levelcnt[i];
1438 rsp->levelspread[i] = (cprv + ccur - 1) / ccur;
1439 cprv = ccur;
1440 }
1441}
1442#endif /* #else #ifdef CONFIG_RCU_FANOUT_EXACT */
1443
1444/*
1445 * Helper function for rcu_init() that initializes one rcu_state structure.
1446 */
1447static void __init rcu_init_one(struct rcu_state *rsp)
1448{
1449 int cpustride = 1;
1450 int i;
1451 int j;
1452 struct rcu_node *rnp;
1453
1454 /* Initialize the level-tracking arrays. */
1455
1456 for (i = 1; i < NUM_RCU_LVLS; i++)
1457 rsp->level[i] = rsp->level[i - 1] + rsp->levelcnt[i - 1];
1458 rcu_init_levelspread(rsp);
1459
1460 /* Initialize the elements themselves, starting from the leaves. */
1461
1462 for (i = NUM_RCU_LVLS - 1; i >= 0; i--) {
1463 cpustride *= rsp->levelspread[i];
1464 rnp = rsp->level[i];
1465 for (j = 0; j < rsp->levelcnt[i]; j++, rnp++) {
1466 spin_lock_init(&rnp->lock);
1467 rnp->qsmask = 0;
1468 rnp->qsmaskinit = 0;
1469 rnp->grplo = j * cpustride;
1470 rnp->grphi = (j + 1) * cpustride - 1;
1471 if (rnp->grphi >= NR_CPUS)
1472 rnp->grphi = NR_CPUS - 1;
1473 if (i == 0) {
1474 rnp->grpnum = 0;
1475 rnp->grpmask = 0;
1476 rnp->parent = NULL;
1477 } else {
1478 rnp->grpnum = j % rsp->levelspread[i - 1];
1479 rnp->grpmask = 1UL << rnp->grpnum;
1480 rnp->parent = rsp->level[i - 1] +
1481 j / rsp->levelspread[i - 1];
1482 }
1483 rnp->level = i;
1484 }
1485 }
1486}
1487
1488/*
1489 * Helper macro for __rcu_init(). To be used nowhere else!
1490 * Assigns leaf node pointers into each CPU's rcu_data structure.
1491 */
1492#define RCU_DATA_PTR_INIT(rsp, rcu_data) \
1493do { \
1494 rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \
1495 j = 0; \
1496 for_each_possible_cpu(i) { \
1497 if (i > rnp[j].grphi) \
1498 j++; \
1499 per_cpu(rcu_data, i).mynode = &rnp[j]; \
1500 (rsp)->rda[i] = &per_cpu(rcu_data, i); \
1501 } \
1502} while (0)
1503
1504static struct notifier_block __cpuinitdata rcu_nb = {
1505 .notifier_call = rcu_cpu_notify,
1506};
1507
1508void __init __rcu_init(void)
1509{
1510 int i; /* All used by RCU_DATA_PTR_INIT(). */
1511 int j;
1512 struct rcu_node *rnp;
1513
1514 printk(KERN_WARNING "Experimental hierarchical RCU implementation.\n");
1515#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
1516 printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
1517#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
1518 rcu_init_one(&rcu_state);
1519 RCU_DATA_PTR_INIT(&rcu_state, rcu_data);
1520 rcu_init_one(&rcu_bh_state);
1521 RCU_DATA_PTR_INIT(&rcu_bh_state, rcu_bh_data);
1522
1523 for_each_online_cpu(i)
1524 rcu_cpu_notify(&rcu_nb, CPU_UP_PREPARE, (void *)(long)i);
1525 /* Register notifier for non-boot CPUs */
1526 register_cpu_notifier(&rcu_nb);
1527 printk(KERN_WARNING "Experimental hierarchical RCU init done.\n");
1528}
1529
1530module_param(blimit, int, 0);
1531module_param(qhimark, int, 0);
1532module_param(qlowmark, int, 0);
diff --git a/kernel/rcutree_trace.c b/kernel/rcutree_trace.c
new file mode 100644
index 000000000000..d6db3e837826
--- /dev/null
+++ b/kernel/rcutree_trace.c
@@ -0,0 +1,271 @@
1/*
2 * Read-Copy Update tracing for classic implementation
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
8 *
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
13 *
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * Copyright IBM Corporation, 2008
19 *
20 * Papers: http://www.rdrop.com/users/paulmck/RCU
21 *
22 * For detailed explanation of Read-Copy Update mechanism see -
23 * Documentation/RCU
24 *
25 */
26#include <linux/types.h>
27#include <linux/kernel.h>
28#include <linux/init.h>
29#include <linux/spinlock.h>
30#include <linux/smp.h>
31#include <linux/rcupdate.h>
32#include <linux/interrupt.h>
33#include <linux/sched.h>
34#include <asm/atomic.h>
35#include <linux/bitops.h>
36#include <linux/module.h>
37#include <linux/completion.h>
38#include <linux/moduleparam.h>
39#include <linux/percpu.h>
40#include <linux/notifier.h>
41#include <linux/cpu.h>
42#include <linux/mutex.h>
43#include <linux/debugfs.h>
44#include <linux/seq_file.h>
45
46static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
47{
48 if (!rdp->beenonline)
49 return;
50 seq_printf(m, "%3d%cc=%ld g=%ld pq=%d pqc=%ld qp=%d rpfq=%ld rp=%x",
51 rdp->cpu,
52 cpu_is_offline(rdp->cpu) ? '!' : ' ',
53 rdp->completed, rdp->gpnum,
54 rdp->passed_quiesc, rdp->passed_quiesc_completed,
55 rdp->qs_pending,
56 rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending,
57 (int)(rdp->n_rcu_pending & 0xffff));
58#ifdef CONFIG_NO_HZ
59 seq_printf(m, " dt=%d/%d dn=%d df=%lu",
60 rdp->dynticks->dynticks,
61 rdp->dynticks->dynticks_nesting,
62 rdp->dynticks->dynticks_nmi,
63 rdp->dynticks_fqs);
64#endif /* #ifdef CONFIG_NO_HZ */
65 seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
66 seq_printf(m, " ql=%ld b=%ld\n", rdp->qlen, rdp->blimit);
67}
68
69#define PRINT_RCU_DATA(name, func, m) \
70 do { \
71 int _p_r_d_i; \
72 \
73 for_each_possible_cpu(_p_r_d_i) \
74 func(m, &per_cpu(name, _p_r_d_i)); \
75 } while (0)
76
77static int show_rcudata(struct seq_file *m, void *unused)
78{
79 seq_puts(m, "rcu:\n");
80 PRINT_RCU_DATA(rcu_data, print_one_rcu_data, m);
81 seq_puts(m, "rcu_bh:\n");
82 PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data, m);
83 return 0;
84}
85
86static int rcudata_open(struct inode *inode, struct file *file)
87{
88 return single_open(file, show_rcudata, NULL);
89}
90
91static struct file_operations rcudata_fops = {
92 .owner = THIS_MODULE,
93 .open = rcudata_open,
94 .read = seq_read,
95 .llseek = seq_lseek,
96 .release = single_release,
97};
98
99static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp)
100{
101 if (!rdp->beenonline)
102 return;
103 seq_printf(m, "%d,%s,%ld,%ld,%d,%ld,%d,%ld,%ld",
104 rdp->cpu,
105 cpu_is_offline(rdp->cpu) ? "\"Y\"" : "\"N\"",
106 rdp->completed, rdp->gpnum,
107 rdp->passed_quiesc, rdp->passed_quiesc_completed,
108 rdp->qs_pending,
109 rdp->n_rcu_pending_force_qs - rdp->n_rcu_pending,
110 rdp->n_rcu_pending);
111#ifdef CONFIG_NO_HZ
112 seq_printf(m, ",%d,%d,%d,%lu",
113 rdp->dynticks->dynticks,
114 rdp->dynticks->dynticks_nesting,
115 rdp->dynticks->dynticks_nmi,
116 rdp->dynticks_fqs);
117#endif /* #ifdef CONFIG_NO_HZ */
118 seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
119 seq_printf(m, ",%ld,%ld\n", rdp->qlen, rdp->blimit);
120}
121
122static int show_rcudata_csv(struct seq_file *m, void *unused)
123{
124 seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pqc\",\"pq\",\"rpfq\",\"rp\",");
125#ifdef CONFIG_NO_HZ
126 seq_puts(m, "\"dt\",\"dt nesting\",\"dn\",\"df\",");
127#endif /* #ifdef CONFIG_NO_HZ */
128 seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\"\n");
129 seq_puts(m, "\"rcu:\"\n");
130 PRINT_RCU_DATA(rcu_data, print_one_rcu_data_csv, m);
131 seq_puts(m, "\"rcu_bh:\"\n");
132 PRINT_RCU_DATA(rcu_bh_data, print_one_rcu_data_csv, m);
133 return 0;
134}
135
136static int rcudata_csv_open(struct inode *inode, struct file *file)
137{
138 return single_open(file, show_rcudata_csv, NULL);
139}
140
141static struct file_operations rcudata_csv_fops = {
142 .owner = THIS_MODULE,
143 .open = rcudata_csv_open,
144 .read = seq_read,
145 .llseek = seq_lseek,
146 .release = single_release,
147};
148
149static void print_one_rcu_state(struct seq_file *m, struct rcu_state *rsp)
150{
151 int level = 0;
152 struct rcu_node *rnp;
153
154 seq_printf(m, "c=%ld g=%ld s=%d jfq=%ld j=%x "
155 "nfqs=%lu/nfqsng=%lu(%lu) fqlh=%lu\n",
156 rsp->completed, rsp->gpnum, rsp->signaled,
157 (long)(rsp->jiffies_force_qs - jiffies),
158 (int)(jiffies & 0xffff),
159 rsp->n_force_qs, rsp->n_force_qs_ngp,
160 rsp->n_force_qs - rsp->n_force_qs_ngp,
161 rsp->n_force_qs_lh);
162 for (rnp = &rsp->node[0]; rnp - &rsp->node[0] < NUM_RCU_NODES; rnp++) {
163 if (rnp->level != level) {
164 seq_puts(m, "\n");
165 level = rnp->level;
166 }
167 seq_printf(m, "%lx/%lx %d:%d ^%d ",
168 rnp->qsmask, rnp->qsmaskinit,
169 rnp->grplo, rnp->grphi, rnp->grpnum);
170 }
171 seq_puts(m, "\n");
172}
173
174static int show_rcuhier(struct seq_file *m, void *unused)
175{
176 seq_puts(m, "rcu:\n");
177 print_one_rcu_state(m, &rcu_state);
178 seq_puts(m, "rcu_bh:\n");
179 print_one_rcu_state(m, &rcu_bh_state);
180 return 0;
181}
182
183static int rcuhier_open(struct inode *inode, struct file *file)
184{
185 return single_open(file, show_rcuhier, NULL);
186}
187
188static struct file_operations rcuhier_fops = {
189 .owner = THIS_MODULE,
190 .open = rcuhier_open,
191 .read = seq_read,
192 .llseek = seq_lseek,
193 .release = single_release,
194};
195
196static int show_rcugp(struct seq_file *m, void *unused)
197{
198 seq_printf(m, "rcu: completed=%ld gpnum=%ld\n",
199 rcu_state.completed, rcu_state.gpnum);
200 seq_printf(m, "rcu_bh: completed=%ld gpnum=%ld\n",
201 rcu_bh_state.completed, rcu_bh_state.gpnum);
202 return 0;
203}
204
205static int rcugp_open(struct inode *inode, struct file *file)
206{
207 return single_open(file, show_rcugp, NULL);
208}
209
210static struct file_operations rcugp_fops = {
211 .owner = THIS_MODULE,
212 .open = rcugp_open,
213 .read = seq_read,
214 .llseek = seq_lseek,
215 .release = single_release,
216};
217
218static struct dentry *rcudir, *datadir, *datadir_csv, *hierdir, *gpdir;
219static int __init rcuclassic_trace_init(void)
220{
221 rcudir = debugfs_create_dir("rcu", NULL);
222 if (!rcudir)
223 goto out;
224
225 datadir = debugfs_create_file("rcudata", 0444, rcudir,
226 NULL, &rcudata_fops);
227 if (!datadir)
228 goto free_out;
229
230 datadir_csv = debugfs_create_file("rcudata.csv", 0444, rcudir,
231 NULL, &rcudata_csv_fops);
232 if (!datadir_csv)
233 goto free_out;
234
235 gpdir = debugfs_create_file("rcugp", 0444, rcudir, NULL, &rcugp_fops);
236 if (!gpdir)
237 goto free_out;
238
239 hierdir = debugfs_create_file("rcuhier", 0444, rcudir,
240 NULL, &rcuhier_fops);
241 if (!hierdir)
242 goto free_out;
243 return 0;
244free_out:
245 if (datadir)
246 debugfs_remove(datadir);
247 if (datadir_csv)
248 debugfs_remove(datadir_csv);
249 if (gpdir)
250 debugfs_remove(gpdir);
251 debugfs_remove(rcudir);
252out:
253 return 1;
254}
255
256static void __exit rcuclassic_trace_cleanup(void)
257{
258 debugfs_remove(datadir);
259 debugfs_remove(datadir_csv);
260 debugfs_remove(gpdir);
261 debugfs_remove(hierdir);
262 debugfs_remove(rcudir);
263}
264
265
266module_init(rcuclassic_trace_init);
267module_exit(rcuclassic_trace_cleanup);
268
269MODULE_AUTHOR("Paul E. McKenney");
270MODULE_DESCRIPTION("Read-Copy Update tracing for hierarchical implementation");
271MODULE_LICENSE("GPL");
diff --git a/kernel/relay.c b/kernel/relay.c
index 32b0befdcb6a..09ac2008f77b 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -1317,12 +1317,9 @@ static ssize_t relay_file_splice_read(struct file *in,
1317 if (ret < 0) 1317 if (ret < 0)
1318 break; 1318 break;
1319 else if (!ret) { 1319 else if (!ret) {
1320 if (spliced) 1320 if (flags & SPLICE_F_NONBLOCK)
1321 break;
1322 if (flags & SPLICE_F_NONBLOCK) {
1323 ret = -EAGAIN; 1321 ret = -EAGAIN;
1324 break; 1322 break;
1325 }
1326 } 1323 }
1327 1324
1328 *ppos += ret; 1325 *ppos += ret;
diff --git a/kernel/resource.c b/kernel/resource.c
index 4337063663ef..e633106b12f6 100644
--- a/kernel/resource.c
+++ b/kernel/resource.c
@@ -853,6 +853,15 @@ int iomem_map_sanity_check(resource_size_t addr, unsigned long size)
853 if (PFN_DOWN(p->start) <= PFN_DOWN(addr) && 853 if (PFN_DOWN(p->start) <= PFN_DOWN(addr) &&
854 PFN_DOWN(p->end) >= PFN_DOWN(addr + size - 1)) 854 PFN_DOWN(p->end) >= PFN_DOWN(addr + size - 1))
855 continue; 855 continue;
856 /*
857 * if a resource is "BUSY", it's not a hardware resource
858 * but a driver mapping of such a resource; we don't want
859 * to warn for those; some drivers legitimately map only
860 * partial hardware resources. (example: vesafb)
861 */
862 if (p->flags & IORESOURCE_BUSY)
863 continue;
864
856 printk(KERN_WARNING "resource map sanity check conflict: " 865 printk(KERN_WARNING "resource map sanity check conflict: "
857 "0x%llx 0x%llx 0x%llx 0x%llx %s\n", 866 "0x%llx 0x%llx 0x%llx 0x%llx %s\n",
858 (unsigned long long)addr, 867 (unsigned long long)addr,
diff --git a/kernel/sched.c b/kernel/sched.c
index 9b1e79371c20..545c6fccd1dc 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -118,6 +118,12 @@
118 */ 118 */
119#define RUNTIME_INF ((u64)~0ULL) 119#define RUNTIME_INF ((u64)~0ULL)
120 120
121DEFINE_TRACE(sched_wait_task);
122DEFINE_TRACE(sched_wakeup);
123DEFINE_TRACE(sched_wakeup_new);
124DEFINE_TRACE(sched_switch);
125DEFINE_TRACE(sched_migrate_task);
126
121#ifdef CONFIG_SMP 127#ifdef CONFIG_SMP
122/* 128/*
123 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power) 129 * Divide a load by a sched group cpu_power : (load / sg->__cpu_power)
@@ -203,7 +209,6 @@ void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime)
203 hrtimer_init(&rt_b->rt_period_timer, 209 hrtimer_init(&rt_b->rt_period_timer,
204 CLOCK_MONOTONIC, HRTIMER_MODE_REL); 210 CLOCK_MONOTONIC, HRTIMER_MODE_REL);
205 rt_b->rt_period_timer.function = sched_rt_period_timer; 211 rt_b->rt_period_timer.function = sched_rt_period_timer;
206 rt_b->rt_period_timer.cb_mode = HRTIMER_CB_IRQSAFE_UNLOCKED;
207} 212}
208 213
209static inline int rt_bandwidth_enabled(void) 214static inline int rt_bandwidth_enabled(void)
@@ -261,6 +266,10 @@ struct task_group {
261 struct cgroup_subsys_state css; 266 struct cgroup_subsys_state css;
262#endif 267#endif
263 268
269#ifdef CONFIG_USER_SCHED
270 uid_t uid;
271#endif
272
264#ifdef CONFIG_FAIR_GROUP_SCHED 273#ifdef CONFIG_FAIR_GROUP_SCHED
265 /* schedulable entities of this group on each cpu */ 274 /* schedulable entities of this group on each cpu */
266 struct sched_entity **se; 275 struct sched_entity **se;
@@ -286,6 +295,12 @@ struct task_group {
286 295
287#ifdef CONFIG_USER_SCHED 296#ifdef CONFIG_USER_SCHED
288 297
298/* Helper function to pass uid information to create_sched_user() */
299void set_tg_uid(struct user_struct *user)
300{
301 user->tg->uid = user->uid;
302}
303
289/* 304/*
290 * Root task group. 305 * Root task group.
291 * Every UID task group (including init_task_group aka UID-0) will 306 * Every UID task group (including init_task_group aka UID-0) will
@@ -345,7 +360,9 @@ static inline struct task_group *task_group(struct task_struct *p)
345 struct task_group *tg; 360 struct task_group *tg;
346 361
347#ifdef CONFIG_USER_SCHED 362#ifdef CONFIG_USER_SCHED
348 tg = p->user->tg; 363 rcu_read_lock();
364 tg = __task_cred(p)->user->tg;
365 rcu_read_unlock();
349#elif defined(CONFIG_CGROUP_SCHED) 366#elif defined(CONFIG_CGROUP_SCHED)
350 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id), 367 tg = container_of(task_subsys_state(p, cpu_cgroup_subsys_id),
351 struct task_group, css); 368 struct task_group, css);
@@ -481,18 +498,26 @@ struct rt_rq {
481 */ 498 */
482struct root_domain { 499struct root_domain {
483 atomic_t refcount; 500 atomic_t refcount;
484 cpumask_t span; 501 cpumask_var_t span;
485 cpumask_t online; 502 cpumask_var_t online;
486 503
487 /* 504 /*
488 * The "RT overload" flag: it gets set if a CPU has more than 505 * The "RT overload" flag: it gets set if a CPU has more than
489 * one runnable RT task. 506 * one runnable RT task.
490 */ 507 */
491 cpumask_t rto_mask; 508 cpumask_var_t rto_mask;
492 atomic_t rto_count; 509 atomic_t rto_count;
493#ifdef CONFIG_SMP 510#ifdef CONFIG_SMP
494 struct cpupri cpupri; 511 struct cpupri cpupri;
495#endif 512#endif
513#if defined(CONFIG_SCHED_MC) || defined(CONFIG_SCHED_SMT)
514 /*
515 * Preferred wake up cpu nominated by sched_mc balance that will be
516 * used when most cpus are idle in the system indicating overall very
517 * low system utilisation. Triggered at POWERSAVINGS_BALANCE_WAKEUP(2)
518 */
519 unsigned int sched_mc_preferred_wakeup_cpu;
520#endif
496}; 521};
497 522
498/* 523/*
@@ -586,6 +611,8 @@ struct rq {
586#ifdef CONFIG_SCHEDSTATS 611#ifdef CONFIG_SCHEDSTATS
587 /* latency stats */ 612 /* latency stats */
588 struct sched_info rq_sched_info; 613 struct sched_info rq_sched_info;
614 unsigned long long rq_cpu_time;
615 /* could above be rq->cfs_rq.exec_clock + rq->rt_rq.rt_runtime ? */
589 616
590 /* sys_sched_yield() stats */ 617 /* sys_sched_yield() stats */
591 unsigned int yld_exp_empty; 618 unsigned int yld_exp_empty;
@@ -703,45 +730,18 @@ static __read_mostly char *sched_feat_names[] = {
703 730
704#undef SCHED_FEAT 731#undef SCHED_FEAT
705 732
706static int sched_feat_open(struct inode *inode, struct file *filp) 733static int sched_feat_show(struct seq_file *m, void *v)
707{ 734{
708 filp->private_data = inode->i_private;
709 return 0;
710}
711
712static ssize_t
713sched_feat_read(struct file *filp, char __user *ubuf,
714 size_t cnt, loff_t *ppos)
715{
716 char *buf;
717 int r = 0;
718 int len = 0;
719 int i; 735 int i;
720 736
721 for (i = 0; sched_feat_names[i]; i++) { 737 for (i = 0; sched_feat_names[i]; i++) {
722 len += strlen(sched_feat_names[i]); 738 if (!(sysctl_sched_features & (1UL << i)))
723 len += 4; 739 seq_puts(m, "NO_");
740 seq_printf(m, "%s ", sched_feat_names[i]);
724 } 741 }
742 seq_puts(m, "\n");
725 743
726 buf = kmalloc(len + 2, GFP_KERNEL); 744 return 0;
727 if (!buf)
728 return -ENOMEM;
729
730 for (i = 0; sched_feat_names[i]; i++) {
731 if (sysctl_sched_features & (1UL << i))
732 r += sprintf(buf + r, "%s ", sched_feat_names[i]);
733 else
734 r += sprintf(buf + r, "NO_%s ", sched_feat_names[i]);
735 }
736
737 r += sprintf(buf + r, "\n");
738 WARN_ON(r >= len + 2);
739
740 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
741
742 kfree(buf);
743
744 return r;
745} 745}
746 746
747static ssize_t 747static ssize_t
@@ -786,10 +786,17 @@ sched_feat_write(struct file *filp, const char __user *ubuf,
786 return cnt; 786 return cnt;
787} 787}
788 788
789static int sched_feat_open(struct inode *inode, struct file *filp)
790{
791 return single_open(filp, sched_feat_show, NULL);
792}
793
789static struct file_operations sched_feat_fops = { 794static struct file_operations sched_feat_fops = {
790 .open = sched_feat_open, 795 .open = sched_feat_open,
791 .read = sched_feat_read, 796 .write = sched_feat_write,
792 .write = sched_feat_write, 797 .read = seq_read,
798 .llseek = seq_lseek,
799 .release = single_release,
793}; 800};
794 801
795static __init int sched_init_debug(void) 802static __init int sched_init_debug(void)
@@ -1139,7 +1146,6 @@ static void init_rq_hrtick(struct rq *rq)
1139 1146
1140 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 1147 hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1141 rq->hrtick_timer.function = hrtick; 1148 rq->hrtick_timer.function = hrtick;
1142 rq->hrtick_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
1143} 1149}
1144#else /* CONFIG_SCHED_HRTICK */ 1150#else /* CONFIG_SCHED_HRTICK */
1145static inline void hrtick_clear(struct rq *rq) 1151static inline void hrtick_clear(struct rq *rq)
@@ -1453,9 +1459,10 @@ static int task_hot(struct task_struct *p, u64 now, struct sched_domain *sd);
1453static unsigned long cpu_avg_load_per_task(int cpu) 1459static unsigned long cpu_avg_load_per_task(int cpu)
1454{ 1460{
1455 struct rq *rq = cpu_rq(cpu); 1461 struct rq *rq = cpu_rq(cpu);
1462 unsigned long nr_running = ACCESS_ONCE(rq->nr_running);
1456 1463
1457 if (rq->nr_running) 1464 if (nr_running)
1458 rq->avg_load_per_task = rq->load.weight / rq->nr_running; 1465 rq->avg_load_per_task = rq->load.weight / nr_running;
1459 else 1466 else
1460 rq->avg_load_per_task = 0; 1467 rq->avg_load_per_task = 0;
1461 1468
@@ -1473,27 +1480,13 @@ static void
1473update_group_shares_cpu(struct task_group *tg, int cpu, 1480update_group_shares_cpu(struct task_group *tg, int cpu,
1474 unsigned long sd_shares, unsigned long sd_rq_weight) 1481 unsigned long sd_shares, unsigned long sd_rq_weight)
1475{ 1482{
1476 int boost = 0;
1477 unsigned long shares; 1483 unsigned long shares;
1478 unsigned long rq_weight; 1484 unsigned long rq_weight;
1479 1485
1480 if (!tg->se[cpu]) 1486 if (!tg->se[cpu])
1481 return; 1487 return;
1482 1488
1483 rq_weight = tg->cfs_rq[cpu]->load.weight; 1489 rq_weight = tg->cfs_rq[cpu]->rq_weight;
1484
1485 /*
1486 * If there are currently no tasks on the cpu pretend there is one of
1487 * average load so that when a new task gets to run here it will not
1488 * get delayed by group starvation.
1489 */
1490 if (!rq_weight) {
1491 boost = 1;
1492 rq_weight = NICE_0_LOAD;
1493 }
1494
1495 if (unlikely(rq_weight > sd_rq_weight))
1496 rq_weight = sd_rq_weight;
1497 1490
1498 /* 1491 /*
1499 * \Sum shares * rq_weight 1492 * \Sum shares * rq_weight
@@ -1501,7 +1494,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1501 * \Sum rq_weight 1494 * \Sum rq_weight
1502 * 1495 *
1503 */ 1496 */
1504 shares = (sd_shares * rq_weight) / (sd_rq_weight + 1); 1497 shares = (sd_shares * rq_weight) / sd_rq_weight;
1505 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES); 1498 shares = clamp_t(unsigned long, shares, MIN_SHARES, MAX_SHARES);
1506 1499
1507 if (abs(shares - tg->se[cpu]->load.weight) > 1500 if (abs(shares - tg->se[cpu]->load.weight) >
@@ -1510,11 +1503,7 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1510 unsigned long flags; 1503 unsigned long flags;
1511 1504
1512 spin_lock_irqsave(&rq->lock, flags); 1505 spin_lock_irqsave(&rq->lock, flags);
1513 /* 1506 tg->cfs_rq[cpu]->shares = shares;
1514 * record the actual number of shares, not the boosted amount.
1515 */
1516 tg->cfs_rq[cpu]->shares = boost ? 0 : shares;
1517 tg->cfs_rq[cpu]->rq_weight = rq_weight;
1518 1507
1519 __set_se_shares(tg->se[cpu], shares); 1508 __set_se_shares(tg->se[cpu], shares);
1520 spin_unlock_irqrestore(&rq->lock, flags); 1509 spin_unlock_irqrestore(&rq->lock, flags);
@@ -1528,13 +1517,23 @@ update_group_shares_cpu(struct task_group *tg, int cpu,
1528 */ 1517 */
1529static int tg_shares_up(struct task_group *tg, void *data) 1518static int tg_shares_up(struct task_group *tg, void *data)
1530{ 1519{
1531 unsigned long rq_weight = 0; 1520 unsigned long weight, rq_weight = 0;
1532 unsigned long shares = 0; 1521 unsigned long shares = 0;
1533 struct sched_domain *sd = data; 1522 struct sched_domain *sd = data;
1534 int i; 1523 int i;
1535 1524
1536 for_each_cpu_mask(i, sd->span) { 1525 for_each_cpu(i, sched_domain_span(sd)) {
1537 rq_weight += tg->cfs_rq[i]->load.weight; 1526 /*
1527 * If there are currently no tasks on the cpu pretend there
1528 * is one of average load so that when a new task gets to
1529 * run here it will not get delayed by group starvation.
1530 */
1531 weight = tg->cfs_rq[i]->load.weight;
1532 if (!weight)
1533 weight = NICE_0_LOAD;
1534
1535 tg->cfs_rq[i]->rq_weight = weight;
1536 rq_weight += weight;
1538 shares += tg->cfs_rq[i]->shares; 1537 shares += tg->cfs_rq[i]->shares;
1539 } 1538 }
1540 1539
@@ -1544,10 +1543,7 @@ static int tg_shares_up(struct task_group *tg, void *data)
1544 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE)) 1543 if (!sd->parent || !(sd->parent->flags & SD_LOAD_BALANCE))
1545 shares = tg->shares; 1544 shares = tg->shares;
1546 1545
1547 if (!rq_weight) 1546 for_each_cpu(i, sched_domain_span(sd))
1548 rq_weight = cpus_weight(sd->span) * NICE_0_LOAD;
1549
1550 for_each_cpu_mask(i, sd->span)
1551 update_group_shares_cpu(tg, i, shares, rq_weight); 1547 update_group_shares_cpu(tg, i, shares, rq_weight);
1552 1548
1553 return 0; 1549 return 0;
@@ -1611,6 +1607,39 @@ static inline void update_shares_locked(struct rq *rq, struct sched_domain *sd)
1611 1607
1612#endif 1608#endif
1613 1609
1610/*
1611 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
1612 */
1613static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
1614 __releases(this_rq->lock)
1615 __acquires(busiest->lock)
1616 __acquires(this_rq->lock)
1617{
1618 int ret = 0;
1619
1620 if (unlikely(!irqs_disabled())) {
1621 /* printk() doesn't work good under rq->lock */
1622 spin_unlock(&this_rq->lock);
1623 BUG_ON(1);
1624 }
1625 if (unlikely(!spin_trylock(&busiest->lock))) {
1626 if (busiest < this_rq) {
1627 spin_unlock(&this_rq->lock);
1628 spin_lock(&busiest->lock);
1629 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
1630 ret = 1;
1631 } else
1632 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
1633 }
1634 return ret;
1635}
1636
1637static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
1638 __releases(busiest->lock)
1639{
1640 spin_unlock(&busiest->lock);
1641 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
1642}
1614#endif 1643#endif
1615 1644
1616#ifdef CONFIG_FAIR_GROUP_SCHED 1645#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -1844,6 +1873,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
1844 1873
1845 clock_offset = old_rq->clock - new_rq->clock; 1874 clock_offset = old_rq->clock - new_rq->clock;
1846 1875
1876 trace_sched_migrate_task(p, task_cpu(p), new_cpu);
1877
1847#ifdef CONFIG_SCHEDSTATS 1878#ifdef CONFIG_SCHEDSTATS
1848 if (p->se.wait_start) 1879 if (p->se.wait_start)
1849 p->se.wait_start -= clock_offset; 1880 p->se.wait_start -= clock_offset;
@@ -2078,15 +2109,17 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2078 int i; 2109 int i;
2079 2110
2080 /* Skip over this group if it has no CPUs allowed */ 2111 /* Skip over this group if it has no CPUs allowed */
2081 if (!cpus_intersects(group->cpumask, p->cpus_allowed)) 2112 if (!cpumask_intersects(sched_group_cpus(group),
2113 &p->cpus_allowed))
2082 continue; 2114 continue;
2083 2115
2084 local_group = cpu_isset(this_cpu, group->cpumask); 2116 local_group = cpumask_test_cpu(this_cpu,
2117 sched_group_cpus(group));
2085 2118
2086 /* Tally up the load of all CPUs in the group */ 2119 /* Tally up the load of all CPUs in the group */
2087 avg_load = 0; 2120 avg_load = 0;
2088 2121
2089 for_each_cpu_mask_nr(i, group->cpumask) { 2122 for_each_cpu(i, sched_group_cpus(group)) {
2090 /* Bias balancing toward cpus of our domain */ 2123 /* Bias balancing toward cpus of our domain */
2091 if (local_group) 2124 if (local_group)
2092 load = source_load(i, load_idx); 2125 load = source_load(i, load_idx);
@@ -2118,17 +2151,14 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
2118 * find_idlest_cpu - find the idlest cpu among the cpus in group. 2151 * find_idlest_cpu - find the idlest cpu among the cpus in group.
2119 */ 2152 */
2120static int 2153static int
2121find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, 2154find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
2122 cpumask_t *tmp)
2123{ 2155{
2124 unsigned long load, min_load = ULONG_MAX; 2156 unsigned long load, min_load = ULONG_MAX;
2125 int idlest = -1; 2157 int idlest = -1;
2126 int i; 2158 int i;
2127 2159
2128 /* Traverse only the allowed CPUs */ 2160 /* Traverse only the allowed CPUs */
2129 cpus_and(*tmp, group->cpumask, p->cpus_allowed); 2161 for_each_cpu_and(i, sched_group_cpus(group), &p->cpus_allowed) {
2130
2131 for_each_cpu_mask_nr(i, *tmp) {
2132 load = weighted_cpuload(i); 2162 load = weighted_cpuload(i);
2133 2163
2134 if (load < min_load || (load == min_load && i == this_cpu)) { 2164 if (load < min_load || (load == min_load && i == this_cpu)) {
@@ -2170,7 +2200,6 @@ static int sched_balance_self(int cpu, int flag)
2170 update_shares(sd); 2200 update_shares(sd);
2171 2201
2172 while (sd) { 2202 while (sd) {
2173 cpumask_t span, tmpmask;
2174 struct sched_group *group; 2203 struct sched_group *group;
2175 int new_cpu, weight; 2204 int new_cpu, weight;
2176 2205
@@ -2179,14 +2208,13 @@ static int sched_balance_self(int cpu, int flag)
2179 continue; 2208 continue;
2180 } 2209 }
2181 2210
2182 span = sd->span;
2183 group = find_idlest_group(sd, t, cpu); 2211 group = find_idlest_group(sd, t, cpu);
2184 if (!group) { 2212 if (!group) {
2185 sd = sd->child; 2213 sd = sd->child;
2186 continue; 2214 continue;
2187 } 2215 }
2188 2216
2189 new_cpu = find_idlest_cpu(group, t, cpu, &tmpmask); 2217 new_cpu = find_idlest_cpu(group, t, cpu);
2190 if (new_cpu == -1 || new_cpu == cpu) { 2218 if (new_cpu == -1 || new_cpu == cpu) {
2191 /* Now try balancing at a lower domain level of cpu */ 2219 /* Now try balancing at a lower domain level of cpu */
2192 sd = sd->child; 2220 sd = sd->child;
@@ -2195,10 +2223,10 @@ static int sched_balance_self(int cpu, int flag)
2195 2223
2196 /* Now try balancing at a lower domain level of new_cpu */ 2224 /* Now try balancing at a lower domain level of new_cpu */
2197 cpu = new_cpu; 2225 cpu = new_cpu;
2226 weight = cpumask_weight(sched_domain_span(sd));
2198 sd = NULL; 2227 sd = NULL;
2199 weight = cpus_weight(span);
2200 for_each_domain(cpu, tmp) { 2228 for_each_domain(cpu, tmp) {
2201 if (weight <= cpus_weight(tmp->span)) 2229 if (weight <= cpumask_weight(sched_domain_span(tmp)))
2202 break; 2230 break;
2203 if (tmp->flags & flag) 2231 if (tmp->flags & flag)
2204 sd = tmp; 2232 sd = tmp;
@@ -2243,7 +2271,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2243 cpu = task_cpu(p); 2271 cpu = task_cpu(p);
2244 2272
2245 for_each_domain(this_cpu, sd) { 2273 for_each_domain(this_cpu, sd) {
2246 if (cpu_isset(cpu, sd->span)) { 2274 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2247 update_shares(sd); 2275 update_shares(sd);
2248 break; 2276 break;
2249 } 2277 }
@@ -2253,6 +2281,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2253 2281
2254 smp_wmb(); 2282 smp_wmb();
2255 rq = task_rq_lock(p, &flags); 2283 rq = task_rq_lock(p, &flags);
2284 update_rq_clock(rq);
2256 old_state = p->state; 2285 old_state = p->state;
2257 if (!(old_state & state)) 2286 if (!(old_state & state))
2258 goto out; 2287 goto out;
@@ -2291,7 +2320,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
2291 else { 2320 else {
2292 struct sched_domain *sd; 2321 struct sched_domain *sd;
2293 for_each_domain(this_cpu, sd) { 2322 for_each_domain(this_cpu, sd) {
2294 if (cpu_isset(cpu, sd->span)) { 2323 if (cpumask_test_cpu(cpu, sched_domain_span(sd))) {
2295 schedstat_inc(sd, ttwu_wake_remote); 2324 schedstat_inc(sd, ttwu_wake_remote);
2296 break; 2325 break;
2297 } 2326 }
@@ -2310,12 +2339,11 @@ out_activate:
2310 schedstat_inc(p, se.nr_wakeups_local); 2339 schedstat_inc(p, se.nr_wakeups_local);
2311 else 2340 else
2312 schedstat_inc(p, se.nr_wakeups_remote); 2341 schedstat_inc(p, se.nr_wakeups_remote);
2313 update_rq_clock(rq);
2314 activate_task(rq, p, 1); 2342 activate_task(rq, p, 1);
2315 success = 1; 2343 success = 1;
2316 2344
2317out_running: 2345out_running:
2318 trace_sched_wakeup(rq, p); 2346 trace_sched_wakeup(rq, p, success);
2319 check_preempt_curr(rq, p, sync); 2347 check_preempt_curr(rq, p, sync);
2320 2348
2321 p->state = TASK_RUNNING; 2349 p->state = TASK_RUNNING;
@@ -2448,7 +2476,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags)
2448 p->sched_class->task_new(rq, p); 2476 p->sched_class->task_new(rq, p);
2449 inc_nr_running(rq); 2477 inc_nr_running(rq);
2450 } 2478 }
2451 trace_sched_wakeup_new(rq, p); 2479 trace_sched_wakeup_new(rq, p, 1);
2452 check_preempt_curr(rq, p, 0); 2480 check_preempt_curr(rq, p, 0);
2453#ifdef CONFIG_SMP 2481#ifdef CONFIG_SMP
2454 if (p->sched_class->task_wake_up) 2482 if (p->sched_class->task_wake_up)
@@ -2811,40 +2839,6 @@ static void double_rq_unlock(struct rq *rq1, struct rq *rq2)
2811} 2839}
2812 2840
2813/* 2841/*
2814 * double_lock_balance - lock the busiest runqueue, this_rq is locked already.
2815 */
2816static int double_lock_balance(struct rq *this_rq, struct rq *busiest)
2817 __releases(this_rq->lock)
2818 __acquires(busiest->lock)
2819 __acquires(this_rq->lock)
2820{
2821 int ret = 0;
2822
2823 if (unlikely(!irqs_disabled())) {
2824 /* printk() doesn't work good under rq->lock */
2825 spin_unlock(&this_rq->lock);
2826 BUG_ON(1);
2827 }
2828 if (unlikely(!spin_trylock(&busiest->lock))) {
2829 if (busiest < this_rq) {
2830 spin_unlock(&this_rq->lock);
2831 spin_lock(&busiest->lock);
2832 spin_lock_nested(&this_rq->lock, SINGLE_DEPTH_NESTING);
2833 ret = 1;
2834 } else
2835 spin_lock_nested(&busiest->lock, SINGLE_DEPTH_NESTING);
2836 }
2837 return ret;
2838}
2839
2840static void double_unlock_balance(struct rq *this_rq, struct rq *busiest)
2841 __releases(busiest->lock)
2842{
2843 spin_unlock(&busiest->lock);
2844 lock_set_subclass(&this_rq->lock.dep_map, 0, _RET_IP_);
2845}
2846
2847/*
2848 * If dest_cpu is allowed for this process, migrate the task to it. 2842 * If dest_cpu is allowed for this process, migrate the task to it.
2849 * This is accomplished by forcing the cpu_allowed mask to only 2843 * This is accomplished by forcing the cpu_allowed mask to only
2850 * allow dest_cpu, which will force the cpu onto dest_cpu. Then 2844 * allow dest_cpu, which will force the cpu onto dest_cpu. Then
@@ -2857,11 +2851,10 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu)
2857 struct rq *rq; 2851 struct rq *rq;
2858 2852
2859 rq = task_rq_lock(p, &flags); 2853 rq = task_rq_lock(p, &flags);
2860 if (!cpu_isset(dest_cpu, p->cpus_allowed) 2854 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed)
2861 || unlikely(!cpu_active(dest_cpu))) 2855 || unlikely(!cpu_active(dest_cpu)))
2862 goto out; 2856 goto out;
2863 2857
2864 trace_sched_migrate_task(rq, p, dest_cpu);
2865 /* force the process onto the specified CPU */ 2858 /* force the process onto the specified CPU */
2866 if (migrate_task(p, dest_cpu, &req)) { 2859 if (migrate_task(p, dest_cpu, &req)) {
2867 /* Need to wait for migration thread (might exit: take ref). */ 2860 /* Need to wait for migration thread (might exit: take ref). */
@@ -2923,7 +2916,7 @@ int can_migrate_task(struct task_struct *p, struct rq *rq, int this_cpu,
2923 * 2) cannot be migrated to this CPU due to cpus_allowed, or 2916 * 2) cannot be migrated to this CPU due to cpus_allowed, or
2924 * 3) are cache-hot on their current CPU. 2917 * 3) are cache-hot on their current CPU.
2925 */ 2918 */
2926 if (!cpu_isset(this_cpu, p->cpus_allowed)) { 2919 if (!cpumask_test_cpu(this_cpu, &p->cpus_allowed)) {
2927 schedstat_inc(p, se.nr_failed_migrations_affine); 2920 schedstat_inc(p, se.nr_failed_migrations_affine);
2928 return 0; 2921 return 0;
2929 } 2922 }
@@ -3098,7 +3091,7 @@ static int move_one_task(struct rq *this_rq, int this_cpu, struct rq *busiest,
3098static struct sched_group * 3091static struct sched_group *
3099find_busiest_group(struct sched_domain *sd, int this_cpu, 3092find_busiest_group(struct sched_domain *sd, int this_cpu,
3100 unsigned long *imbalance, enum cpu_idle_type idle, 3093 unsigned long *imbalance, enum cpu_idle_type idle,
3101 int *sd_idle, const cpumask_t *cpus, int *balance) 3094 int *sd_idle, const struct cpumask *cpus, int *balance)
3102{ 3095{
3103 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups; 3096 struct sched_group *busiest = NULL, *this = NULL, *group = sd->groups;
3104 unsigned long max_load, avg_load, total_load, this_load, total_pwr; 3097 unsigned long max_load, avg_load, total_load, this_load, total_pwr;
@@ -3134,10 +3127,11 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3134 unsigned long sum_avg_load_per_task; 3127 unsigned long sum_avg_load_per_task;
3135 unsigned long avg_load_per_task; 3128 unsigned long avg_load_per_task;
3136 3129
3137 local_group = cpu_isset(this_cpu, group->cpumask); 3130 local_group = cpumask_test_cpu(this_cpu,
3131 sched_group_cpus(group));
3138 3132
3139 if (local_group) 3133 if (local_group)
3140 balance_cpu = first_cpu(group->cpumask); 3134 balance_cpu = cpumask_first(sched_group_cpus(group));
3141 3135
3142 /* Tally up the load of all CPUs in the group */ 3136 /* Tally up the load of all CPUs in the group */
3143 sum_weighted_load = sum_nr_running = avg_load = 0; 3137 sum_weighted_load = sum_nr_running = avg_load = 0;
@@ -3146,13 +3140,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3146 max_cpu_load = 0; 3140 max_cpu_load = 0;
3147 min_cpu_load = ~0UL; 3141 min_cpu_load = ~0UL;
3148 3142
3149 for_each_cpu_mask_nr(i, group->cpumask) { 3143 for_each_cpu_and(i, sched_group_cpus(group), cpus) {
3150 struct rq *rq; 3144 struct rq *rq = cpu_rq(i);
3151
3152 if (!cpu_isset(i, *cpus))
3153 continue;
3154
3155 rq = cpu_rq(i);
3156 3145
3157 if (*sd_idle && rq->nr_running) 3146 if (*sd_idle && rq->nr_running)
3158 *sd_idle = 0; 3147 *sd_idle = 0;
@@ -3263,8 +3252,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3263 */ 3252 */
3264 if ((sum_nr_running < min_nr_running) || 3253 if ((sum_nr_running < min_nr_running) ||
3265 (sum_nr_running == min_nr_running && 3254 (sum_nr_running == min_nr_running &&
3266 first_cpu(group->cpumask) < 3255 cpumask_first(sched_group_cpus(group)) >
3267 first_cpu(group_min->cpumask))) { 3256 cpumask_first(sched_group_cpus(group_min)))) {
3268 group_min = group; 3257 group_min = group;
3269 min_nr_running = sum_nr_running; 3258 min_nr_running = sum_nr_running;
3270 min_load_per_task = sum_weighted_load / 3259 min_load_per_task = sum_weighted_load /
@@ -3279,8 +3268,8 @@ find_busiest_group(struct sched_domain *sd, int this_cpu,
3279 if (sum_nr_running <= group_capacity - 1) { 3268 if (sum_nr_running <= group_capacity - 1) {
3280 if (sum_nr_running > leader_nr_running || 3269 if (sum_nr_running > leader_nr_running ||
3281 (sum_nr_running == leader_nr_running && 3270 (sum_nr_running == leader_nr_running &&
3282 first_cpu(group->cpumask) > 3271 cpumask_first(sched_group_cpus(group)) <
3283 first_cpu(group_leader->cpumask))) { 3272 cpumask_first(sched_group_cpus(group_leader)))) {
3284 group_leader = group; 3273 group_leader = group;
3285 leader_nr_running = sum_nr_running; 3274 leader_nr_running = sum_nr_running;
3286 } 3275 }
@@ -3406,6 +3395,10 @@ out_balanced:
3406 3395
3407 if (this == group_leader && group_leader != group_min) { 3396 if (this == group_leader && group_leader != group_min) {
3408 *imbalance = min_load_per_task; 3397 *imbalance = min_load_per_task;
3398 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP) {
3399 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu =
3400 cpumask_first(sched_group_cpus(group_leader));
3401 }
3409 return group_min; 3402 return group_min;
3410 } 3403 }
3411#endif 3404#endif
@@ -3419,16 +3412,16 @@ ret:
3419 */ 3412 */
3420static struct rq * 3413static struct rq *
3421find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, 3414find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3422 unsigned long imbalance, const cpumask_t *cpus) 3415 unsigned long imbalance, const struct cpumask *cpus)
3423{ 3416{
3424 struct rq *busiest = NULL, *rq; 3417 struct rq *busiest = NULL, *rq;
3425 unsigned long max_load = 0; 3418 unsigned long max_load = 0;
3426 int i; 3419 int i;
3427 3420
3428 for_each_cpu_mask_nr(i, group->cpumask) { 3421 for_each_cpu(i, sched_group_cpus(group)) {
3429 unsigned long wl; 3422 unsigned long wl;
3430 3423
3431 if (!cpu_isset(i, *cpus)) 3424 if (!cpumask_test_cpu(i, cpus))
3432 continue; 3425 continue;
3433 3426
3434 rq = cpu_rq(i); 3427 rq = cpu_rq(i);
@@ -3458,7 +3451,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle,
3458 */ 3451 */
3459static int load_balance(int this_cpu, struct rq *this_rq, 3452static int load_balance(int this_cpu, struct rq *this_rq,
3460 struct sched_domain *sd, enum cpu_idle_type idle, 3453 struct sched_domain *sd, enum cpu_idle_type idle,
3461 int *balance, cpumask_t *cpus) 3454 int *balance, struct cpumask *cpus)
3462{ 3455{
3463 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0; 3456 int ld_moved, all_pinned = 0, active_balance = 0, sd_idle = 0;
3464 struct sched_group *group; 3457 struct sched_group *group;
@@ -3466,7 +3459,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
3466 struct rq *busiest; 3459 struct rq *busiest;
3467 unsigned long flags; 3460 unsigned long flags;
3468 3461
3469 cpus_setall(*cpus); 3462 cpumask_setall(cpus);
3470 3463
3471 /* 3464 /*
3472 * When power savings policy is enabled for the parent domain, idle 3465 * When power savings policy is enabled for the parent domain, idle
@@ -3526,8 +3519,8 @@ redo:
3526 3519
3527 /* All tasks on this runqueue were pinned by CPU affinity */ 3520 /* All tasks on this runqueue were pinned by CPU affinity */
3528 if (unlikely(all_pinned)) { 3521 if (unlikely(all_pinned)) {
3529 cpu_clear(cpu_of(busiest), *cpus); 3522 cpumask_clear_cpu(cpu_of(busiest), cpus);
3530 if (!cpus_empty(*cpus)) 3523 if (!cpumask_empty(cpus))
3531 goto redo; 3524 goto redo;
3532 goto out_balanced; 3525 goto out_balanced;
3533 } 3526 }
@@ -3544,7 +3537,8 @@ redo:
3544 /* don't kick the migration_thread, if the curr 3537 /* don't kick the migration_thread, if the curr
3545 * task on busiest cpu can't be moved to this_cpu 3538 * task on busiest cpu can't be moved to this_cpu
3546 */ 3539 */
3547 if (!cpu_isset(this_cpu, busiest->curr->cpus_allowed)) { 3540 if (!cpumask_test_cpu(this_cpu,
3541 &busiest->curr->cpus_allowed)) {
3548 spin_unlock_irqrestore(&busiest->lock, flags); 3542 spin_unlock_irqrestore(&busiest->lock, flags);
3549 all_pinned = 1; 3543 all_pinned = 1;
3550 goto out_one_pinned; 3544 goto out_one_pinned;
@@ -3619,7 +3613,7 @@ out:
3619 */ 3613 */
3620static int 3614static int
3621load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd, 3615load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3622 cpumask_t *cpus) 3616 struct cpumask *cpus)
3623{ 3617{
3624 struct sched_group *group; 3618 struct sched_group *group;
3625 struct rq *busiest = NULL; 3619 struct rq *busiest = NULL;
@@ -3628,7 +3622,7 @@ load_balance_newidle(int this_cpu, struct rq *this_rq, struct sched_domain *sd,
3628 int sd_idle = 0; 3622 int sd_idle = 0;
3629 int all_pinned = 0; 3623 int all_pinned = 0;
3630 3624
3631 cpus_setall(*cpus); 3625 cpumask_setall(cpus);
3632 3626
3633 /* 3627 /*
3634 * When power savings policy is enabled for the parent domain, idle 3628 * When power savings policy is enabled for the parent domain, idle
@@ -3672,17 +3666,71 @@ redo:
3672 double_unlock_balance(this_rq, busiest); 3666 double_unlock_balance(this_rq, busiest);
3673 3667
3674 if (unlikely(all_pinned)) { 3668 if (unlikely(all_pinned)) {
3675 cpu_clear(cpu_of(busiest), *cpus); 3669 cpumask_clear_cpu(cpu_of(busiest), cpus);
3676 if (!cpus_empty(*cpus)) 3670 if (!cpumask_empty(cpus))
3677 goto redo; 3671 goto redo;
3678 } 3672 }
3679 } 3673 }
3680 3674
3681 if (!ld_moved) { 3675 if (!ld_moved) {
3676 int active_balance = 0;
3677
3682 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]); 3678 schedstat_inc(sd, lb_failed[CPU_NEWLY_IDLE]);
3683 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER && 3679 if (!sd_idle && sd->flags & SD_SHARE_CPUPOWER &&
3684 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE)) 3680 !test_sd_parent(sd, SD_POWERSAVINGS_BALANCE))
3685 return -1; 3681 return -1;
3682
3683 if (sched_mc_power_savings < POWERSAVINGS_BALANCE_WAKEUP)
3684 return -1;
3685
3686 if (sd->nr_balance_failed++ < 2)
3687 return -1;
3688
3689 /*
3690 * The only task running in a non-idle cpu can be moved to this
3691 * cpu in an attempt to completely freeup the other CPU
3692 * package. The same method used to move task in load_balance()
3693 * have been extended for load_balance_newidle() to speedup
3694 * consolidation at sched_mc=POWERSAVINGS_BALANCE_WAKEUP (2)
3695 *
3696 * The package power saving logic comes from
3697 * find_busiest_group(). If there are no imbalance, then
3698 * f_b_g() will return NULL. However when sched_mc={1,2} then
3699 * f_b_g() will select a group from which a running task may be
3700 * pulled to this cpu in order to make the other package idle.
3701 * If there is no opportunity to make a package idle and if
3702 * there are no imbalance, then f_b_g() will return NULL and no
3703 * action will be taken in load_balance_newidle().
3704 *
3705 * Under normal task pull operation due to imbalance, there
3706 * will be more than one task in the source run queue and
3707 * move_tasks() will succeed. ld_moved will be true and this
3708 * active balance code will not be triggered.
3709 */
3710
3711 /* Lock busiest in correct order while this_rq is held */
3712 double_lock_balance(this_rq, busiest);
3713
3714 /*
3715 * don't kick the migration_thread, if the curr
3716 * task on busiest cpu can't be moved to this_cpu
3717 */
3718 if (!cpumask_test_cpu(this_cpu, &busiest->curr->cpus_allowed)) {
3719 double_unlock_balance(this_rq, busiest);
3720 all_pinned = 1;
3721 return ld_moved;
3722 }
3723
3724 if (!busiest->active_balance) {
3725 busiest->active_balance = 1;
3726 busiest->push_cpu = this_cpu;
3727 active_balance = 1;
3728 }
3729
3730 double_unlock_balance(this_rq, busiest);
3731 if (active_balance)
3732 wake_up_process(busiest->migration_thread);
3733
3686 } else 3734 } else
3687 sd->nr_balance_failed = 0; 3735 sd->nr_balance_failed = 0;
3688 3736
@@ -3706,9 +3754,12 @@ out_balanced:
3706static void idle_balance(int this_cpu, struct rq *this_rq) 3754static void idle_balance(int this_cpu, struct rq *this_rq)
3707{ 3755{
3708 struct sched_domain *sd; 3756 struct sched_domain *sd;
3709 int pulled_task = -1; 3757 int pulled_task = 0;
3710 unsigned long next_balance = jiffies + HZ; 3758 unsigned long next_balance = jiffies + HZ;
3711 cpumask_t tmpmask; 3759 cpumask_var_t tmpmask;
3760
3761 if (!alloc_cpumask_var(&tmpmask, GFP_ATOMIC))
3762 return;
3712 3763
3713 for_each_domain(this_cpu, sd) { 3764 for_each_domain(this_cpu, sd) {
3714 unsigned long interval; 3765 unsigned long interval;
@@ -3719,7 +3770,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3719 if (sd->flags & SD_BALANCE_NEWIDLE) 3770 if (sd->flags & SD_BALANCE_NEWIDLE)
3720 /* If we've pulled tasks over stop searching: */ 3771 /* If we've pulled tasks over stop searching: */
3721 pulled_task = load_balance_newidle(this_cpu, this_rq, 3772 pulled_task = load_balance_newidle(this_cpu, this_rq,
3722 sd, &tmpmask); 3773 sd, tmpmask);
3723 3774
3724 interval = msecs_to_jiffies(sd->balance_interval); 3775 interval = msecs_to_jiffies(sd->balance_interval);
3725 if (time_after(next_balance, sd->last_balance + interval)) 3776 if (time_after(next_balance, sd->last_balance + interval))
@@ -3734,6 +3785,7 @@ static void idle_balance(int this_cpu, struct rq *this_rq)
3734 */ 3785 */
3735 this_rq->next_balance = next_balance; 3786 this_rq->next_balance = next_balance;
3736 } 3787 }
3788 free_cpumask_var(tmpmask);
3737} 3789}
3738 3790
3739/* 3791/*
@@ -3771,7 +3823,7 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3771 /* Search for an sd spanning us and the target CPU. */ 3823 /* Search for an sd spanning us and the target CPU. */
3772 for_each_domain(target_cpu, sd) { 3824 for_each_domain(target_cpu, sd) {
3773 if ((sd->flags & SD_LOAD_BALANCE) && 3825 if ((sd->flags & SD_LOAD_BALANCE) &&
3774 cpu_isset(busiest_cpu, sd->span)) 3826 cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3775 break; 3827 break;
3776 } 3828 }
3777 3829
@@ -3790,10 +3842,9 @@ static void active_load_balance(struct rq *busiest_rq, int busiest_cpu)
3790#ifdef CONFIG_NO_HZ 3842#ifdef CONFIG_NO_HZ
3791static struct { 3843static struct {
3792 atomic_t load_balancer; 3844 atomic_t load_balancer;
3793 cpumask_t cpu_mask; 3845 cpumask_var_t cpu_mask;
3794} nohz ____cacheline_aligned = { 3846} nohz ____cacheline_aligned = {
3795 .load_balancer = ATOMIC_INIT(-1), 3847 .load_balancer = ATOMIC_INIT(-1),
3796 .cpu_mask = CPU_MASK_NONE,
3797}; 3848};
3798 3849
3799/* 3850/*
@@ -3821,7 +3872,7 @@ int select_nohz_load_balancer(int stop_tick)
3821 int cpu = smp_processor_id(); 3872 int cpu = smp_processor_id();
3822 3873
3823 if (stop_tick) { 3874 if (stop_tick) {
3824 cpu_set(cpu, nohz.cpu_mask); 3875 cpumask_set_cpu(cpu, nohz.cpu_mask);
3825 cpu_rq(cpu)->in_nohz_recently = 1; 3876 cpu_rq(cpu)->in_nohz_recently = 1;
3826 3877
3827 /* 3878 /*
@@ -3835,7 +3886,7 @@ int select_nohz_load_balancer(int stop_tick)
3835 } 3886 }
3836 3887
3837 /* time for ilb owner also to sleep */ 3888 /* time for ilb owner also to sleep */
3838 if (cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 3889 if (cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
3839 if (atomic_read(&nohz.load_balancer) == cpu) 3890 if (atomic_read(&nohz.load_balancer) == cpu)
3840 atomic_set(&nohz.load_balancer, -1); 3891 atomic_set(&nohz.load_balancer, -1);
3841 return 0; 3892 return 0;
@@ -3848,10 +3899,10 @@ int select_nohz_load_balancer(int stop_tick)
3848 } else if (atomic_read(&nohz.load_balancer) == cpu) 3899 } else if (atomic_read(&nohz.load_balancer) == cpu)
3849 return 1; 3900 return 1;
3850 } else { 3901 } else {
3851 if (!cpu_isset(cpu, nohz.cpu_mask)) 3902 if (!cpumask_test_cpu(cpu, nohz.cpu_mask))
3852 return 0; 3903 return 0;
3853 3904
3854 cpu_clear(cpu, nohz.cpu_mask); 3905 cpumask_clear_cpu(cpu, nohz.cpu_mask);
3855 3906
3856 if (atomic_read(&nohz.load_balancer) == cpu) 3907 if (atomic_read(&nohz.load_balancer) == cpu)
3857 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu) 3908 if (atomic_cmpxchg(&nohz.load_balancer, cpu, -1) != cpu)
@@ -3879,7 +3930,11 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3879 unsigned long next_balance = jiffies + 60*HZ; 3930 unsigned long next_balance = jiffies + 60*HZ;
3880 int update_next_balance = 0; 3931 int update_next_balance = 0;
3881 int need_serialize; 3932 int need_serialize;
3882 cpumask_t tmp; 3933 cpumask_var_t tmp;
3934
3935 /* Fails alloc? Rebalancing probably not a priority right now. */
3936 if (!alloc_cpumask_var(&tmp, GFP_ATOMIC))
3937 return;
3883 3938
3884 for_each_domain(cpu, sd) { 3939 for_each_domain(cpu, sd) {
3885 if (!(sd->flags & SD_LOAD_BALANCE)) 3940 if (!(sd->flags & SD_LOAD_BALANCE))
@@ -3904,7 +3959,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle)
3904 } 3959 }
3905 3960
3906 if (time_after_eq(jiffies, sd->last_balance + interval)) { 3961 if (time_after_eq(jiffies, sd->last_balance + interval)) {
3907 if (load_balance(cpu, rq, sd, idle, &balance, &tmp)) { 3962 if (load_balance(cpu, rq, sd, idle, &balance, tmp)) {
3908 /* 3963 /*
3909 * We've pulled tasks over so either we're no 3964 * We've pulled tasks over so either we're no
3910 * longer idle, or one of our SMT siblings is 3965 * longer idle, or one of our SMT siblings is
@@ -3938,6 +3993,8 @@ out:
3938 */ 3993 */
3939 if (likely(update_next_balance)) 3994 if (likely(update_next_balance))
3940 rq->next_balance = next_balance; 3995 rq->next_balance = next_balance;
3996
3997 free_cpumask_var(tmp);
3941} 3998}
3942 3999
3943/* 4000/*
@@ -3962,12 +4019,13 @@ static void run_rebalance_domains(struct softirq_action *h)
3962 */ 4019 */
3963 if (this_rq->idle_at_tick && 4020 if (this_rq->idle_at_tick &&
3964 atomic_read(&nohz.load_balancer) == this_cpu) { 4021 atomic_read(&nohz.load_balancer) == this_cpu) {
3965 cpumask_t cpus = nohz.cpu_mask;
3966 struct rq *rq; 4022 struct rq *rq;
3967 int balance_cpu; 4023 int balance_cpu;
3968 4024
3969 cpu_clear(this_cpu, cpus); 4025 for_each_cpu(balance_cpu, nohz.cpu_mask) {
3970 for_each_cpu_mask_nr(balance_cpu, cpus) { 4026 if (balance_cpu == this_cpu)
4027 continue;
4028
3971 /* 4029 /*
3972 * If this cpu gets work to do, stop the load balancing 4030 * If this cpu gets work to do, stop the load balancing
3973 * work being done for other cpus. Next load 4031 * work being done for other cpus. Next load
@@ -4005,7 +4063,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4005 rq->in_nohz_recently = 0; 4063 rq->in_nohz_recently = 0;
4006 4064
4007 if (atomic_read(&nohz.load_balancer) == cpu) { 4065 if (atomic_read(&nohz.load_balancer) == cpu) {
4008 cpu_clear(cpu, nohz.cpu_mask); 4066 cpumask_clear_cpu(cpu, nohz.cpu_mask);
4009 atomic_set(&nohz.load_balancer, -1); 4067 atomic_set(&nohz.load_balancer, -1);
4010 } 4068 }
4011 4069
@@ -4018,7 +4076,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4018 * TBD: Traverse the sched domains and nominate 4076 * TBD: Traverse the sched domains and nominate
4019 * the nearest cpu in the nohz.cpu_mask. 4077 * the nearest cpu in the nohz.cpu_mask.
4020 */ 4078 */
4021 int ilb = first_cpu(nohz.cpu_mask); 4079 int ilb = cpumask_first(nohz.cpu_mask);
4022 4080
4023 if (ilb < nr_cpu_ids) 4081 if (ilb < nr_cpu_ids)
4024 resched_cpu(ilb); 4082 resched_cpu(ilb);
@@ -4030,7 +4088,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4030 * cpus with ticks stopped, is it time for that to stop? 4088 * cpus with ticks stopped, is it time for that to stop?
4031 */ 4089 */
4032 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu && 4090 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) == cpu &&
4033 cpus_weight(nohz.cpu_mask) == num_online_cpus()) { 4091 cpumask_weight(nohz.cpu_mask) == num_online_cpus()) {
4034 resched_cpu(cpu); 4092 resched_cpu(cpu);
4035 return; 4093 return;
4036 } 4094 }
@@ -4040,7 +4098,7 @@ static inline void trigger_load_balance(struct rq *rq, int cpu)
4040 * someone else, then no need raise the SCHED_SOFTIRQ 4098 * someone else, then no need raise the SCHED_SOFTIRQ
4041 */ 4099 */
4042 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu && 4100 if (rq->idle_at_tick && atomic_read(&nohz.load_balancer) != cpu &&
4043 cpu_isset(cpu, nohz.cpu_mask)) 4101 cpumask_test_cpu(cpu, nohz.cpu_mask))
4044 return; 4102 return;
4045#endif 4103#endif
4046 if (time_after_eq(jiffies, rq->next_balance)) 4104 if (time_after_eq(jiffies, rq->next_balance))
@@ -4092,13 +4150,17 @@ unsigned long long task_delta_exec(struct task_struct *p)
4092 * Account user cpu time to a process. 4150 * Account user cpu time to a process.
4093 * @p: the process that the cpu time gets accounted to 4151 * @p: the process that the cpu time gets accounted to
4094 * @cputime: the cpu time spent in user space since the last update 4152 * @cputime: the cpu time spent in user space since the last update
4153 * @cputime_scaled: cputime scaled by cpu frequency
4095 */ 4154 */
4096void account_user_time(struct task_struct *p, cputime_t cputime) 4155void account_user_time(struct task_struct *p, cputime_t cputime,
4156 cputime_t cputime_scaled)
4097{ 4157{
4098 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4158 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4099 cputime64_t tmp; 4159 cputime64_t tmp;
4100 4160
4161 /* Add user time to process. */
4101 p->utime = cputime_add(p->utime, cputime); 4162 p->utime = cputime_add(p->utime, cputime);
4163 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4102 account_group_user_time(p, cputime); 4164 account_group_user_time(p, cputime);
4103 4165
4104 /* Add user time to cpustat. */ 4166 /* Add user time to cpustat. */
@@ -4115,51 +4177,48 @@ void account_user_time(struct task_struct *p, cputime_t cputime)
4115 * Account guest cpu time to a process. 4177 * Account guest cpu time to a process.
4116 * @p: the process that the cpu time gets accounted to 4178 * @p: the process that the cpu time gets accounted to
4117 * @cputime: the cpu time spent in virtual machine since the last update 4179 * @cputime: the cpu time spent in virtual machine since the last update
4180 * @cputime_scaled: cputime scaled by cpu frequency
4118 */ 4181 */
4119static void account_guest_time(struct task_struct *p, cputime_t cputime) 4182static void account_guest_time(struct task_struct *p, cputime_t cputime,
4183 cputime_t cputime_scaled)
4120{ 4184{
4121 cputime64_t tmp; 4185 cputime64_t tmp;
4122 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4186 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4123 4187
4124 tmp = cputime_to_cputime64(cputime); 4188 tmp = cputime_to_cputime64(cputime);
4125 4189
4190 /* Add guest time to process. */
4126 p->utime = cputime_add(p->utime, cputime); 4191 p->utime = cputime_add(p->utime, cputime);
4192 p->utimescaled = cputime_add(p->utimescaled, cputime_scaled);
4127 account_group_user_time(p, cputime); 4193 account_group_user_time(p, cputime);
4128 p->gtime = cputime_add(p->gtime, cputime); 4194 p->gtime = cputime_add(p->gtime, cputime);
4129 4195
4196 /* Add guest time to cpustat. */
4130 cpustat->user = cputime64_add(cpustat->user, tmp); 4197 cpustat->user = cputime64_add(cpustat->user, tmp);
4131 cpustat->guest = cputime64_add(cpustat->guest, tmp); 4198 cpustat->guest = cputime64_add(cpustat->guest, tmp);
4132} 4199}
4133 4200
4134/* 4201/*
4135 * Account scaled user cpu time to a process.
4136 * @p: the process that the cpu time gets accounted to
4137 * @cputime: the cpu time spent in user space since the last update
4138 */
4139void account_user_time_scaled(struct task_struct *p, cputime_t cputime)
4140{
4141 p->utimescaled = cputime_add(p->utimescaled, cputime);
4142}
4143
4144/*
4145 * Account system cpu time to a process. 4202 * Account system cpu time to a process.
4146 * @p: the process that the cpu time gets accounted to 4203 * @p: the process that the cpu time gets accounted to
4147 * @hardirq_offset: the offset to subtract from hardirq_count() 4204 * @hardirq_offset: the offset to subtract from hardirq_count()
4148 * @cputime: the cpu time spent in kernel space since the last update 4205 * @cputime: the cpu time spent in kernel space since the last update
4206 * @cputime_scaled: cputime scaled by cpu frequency
4149 */ 4207 */
4150void account_system_time(struct task_struct *p, int hardirq_offset, 4208void account_system_time(struct task_struct *p, int hardirq_offset,
4151 cputime_t cputime) 4209 cputime_t cputime, cputime_t cputime_scaled)
4152{ 4210{
4153 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4211 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4154 struct rq *rq = this_rq();
4155 cputime64_t tmp; 4212 cputime64_t tmp;
4156 4213
4157 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) { 4214 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
4158 account_guest_time(p, cputime); 4215 account_guest_time(p, cputime, cputime_scaled);
4159 return; 4216 return;
4160 } 4217 }
4161 4218
4219 /* Add system time to process. */
4162 p->stime = cputime_add(p->stime, cputime); 4220 p->stime = cputime_add(p->stime, cputime);
4221 p->stimescaled = cputime_add(p->stimescaled, cputime_scaled);
4163 account_group_system_time(p, cputime); 4222 account_group_system_time(p, cputime);
4164 4223
4165 /* Add system time to cpustat. */ 4224 /* Add system time to cpustat. */
@@ -4168,50 +4227,85 @@ void account_system_time(struct task_struct *p, int hardirq_offset,
4168 cpustat->irq = cputime64_add(cpustat->irq, tmp); 4227 cpustat->irq = cputime64_add(cpustat->irq, tmp);
4169 else if (softirq_count()) 4228 else if (softirq_count())
4170 cpustat->softirq = cputime64_add(cpustat->softirq, tmp); 4229 cpustat->softirq = cputime64_add(cpustat->softirq, tmp);
4171 else if (p != rq->idle)
4172 cpustat->system = cputime64_add(cpustat->system, tmp);
4173 else if (atomic_read(&rq->nr_iowait) > 0)
4174 cpustat->iowait = cputime64_add(cpustat->iowait, tmp);
4175 else 4230 else
4176 cpustat->idle = cputime64_add(cpustat->idle, tmp); 4231 cpustat->system = cputime64_add(cpustat->system, tmp);
4232
4177 /* Account for system time used */ 4233 /* Account for system time used */
4178 acct_update_integrals(p); 4234 acct_update_integrals(p);
4179} 4235}
4180 4236
4181/* 4237/*
4182 * Account scaled system cpu time to a process. 4238 * Account for involuntary wait time.
4183 * @p: the process that the cpu time gets accounted to 4239 * @steal: the cpu time spent in involuntary wait
4184 * @hardirq_offset: the offset to subtract from hardirq_count()
4185 * @cputime: the cpu time spent in kernel space since the last update
4186 */ 4240 */
4187void account_system_time_scaled(struct task_struct *p, cputime_t cputime) 4241void account_steal_time(cputime_t cputime)
4188{ 4242{
4189 p->stimescaled = cputime_add(p->stimescaled, cputime); 4243 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4244 cputime64_t cputime64 = cputime_to_cputime64(cputime);
4245
4246 cpustat->steal = cputime64_add(cpustat->steal, cputime64);
4190} 4247}
4191 4248
4192/* 4249/*
4193 * Account for involuntary wait time. 4250 * Account for idle time.
4194 * @p: the process from which the cpu time has been stolen 4251 * @cputime: the cpu time spent in idle wait
4195 * @steal: the cpu time spent in involuntary wait
4196 */ 4252 */
4197void account_steal_time(struct task_struct *p, cputime_t steal) 4253void account_idle_time(cputime_t cputime)
4198{ 4254{
4199 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat; 4255 struct cpu_usage_stat *cpustat = &kstat_this_cpu.cpustat;
4200 cputime64_t tmp = cputime_to_cputime64(steal); 4256 cputime64_t cputime64 = cputime_to_cputime64(cputime);
4201 struct rq *rq = this_rq(); 4257 struct rq *rq = this_rq();
4202 4258
4203 if (p == rq->idle) { 4259 if (atomic_read(&rq->nr_iowait) > 0)
4204 p->stime = cputime_add(p->stime, steal); 4260 cpustat->iowait = cputime64_add(cpustat->iowait, cputime64);
4205 account_group_system_time(p, steal); 4261 else
4206 if (atomic_read(&rq->nr_iowait) > 0) 4262 cpustat->idle = cputime64_add(cpustat->idle, cputime64);
4207 cpustat->iowait = cputime64_add(cpustat->iowait, tmp); 4263}
4208 else 4264
4209 cpustat->idle = cputime64_add(cpustat->idle, tmp); 4265#ifndef CONFIG_VIRT_CPU_ACCOUNTING
4210 } else 4266
4211 cpustat->steal = cputime64_add(cpustat->steal, tmp); 4267/*
4268 * Account a single tick of cpu time.
4269 * @p: the process that the cpu time gets accounted to
4270 * @user_tick: indicates if the tick is a user or a system tick
4271 */
4272void account_process_tick(struct task_struct *p, int user_tick)
4273{
4274 cputime_t one_jiffy = jiffies_to_cputime(1);
4275 cputime_t one_jiffy_scaled = cputime_to_scaled(one_jiffy);
4276 struct rq *rq = this_rq();
4277
4278 if (user_tick)
4279 account_user_time(p, one_jiffy, one_jiffy_scaled);
4280 else if (p != rq->idle)
4281 account_system_time(p, HARDIRQ_OFFSET, one_jiffy,
4282 one_jiffy_scaled);
4283 else
4284 account_idle_time(one_jiffy);
4285}
4286
4287/*
4288 * Account multiple ticks of steal time.
4289 * @p: the process from which the cpu time has been stolen
4290 * @ticks: number of stolen ticks
4291 */
4292void account_steal_ticks(unsigned long ticks)
4293{
4294 account_steal_time(jiffies_to_cputime(ticks));
4212} 4295}
4213 4296
4214/* 4297/*
4298 * Account multiple ticks of idle time.
4299 * @ticks: number of stolen ticks
4300 */
4301void account_idle_ticks(unsigned long ticks)
4302{
4303 account_idle_time(jiffies_to_cputime(ticks));
4304}
4305
4306#endif
4307
4308/*
4215 * Use precise platform statistics if available: 4309 * Use precise platform statistics if available:
4216 */ 4310 */
4217#ifdef CONFIG_VIRT_CPU_ACCOUNTING 4311#ifdef CONFIG_VIRT_CPU_ACCOUNTING
@@ -4338,7 +4432,7 @@ void __kprobes sub_preempt_count(int val)
4338 /* 4432 /*
4339 * Underflow? 4433 * Underflow?
4340 */ 4434 */
4341 if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) 4435 if (DEBUG_LOCKS_WARN_ON(val > preempt_count() - (!!kernel_locked())))
4342 return; 4436 return;
4343 /* 4437 /*
4344 * Is the spinlock portion underflowing? 4438 * Is the spinlock portion underflowing?
@@ -5133,6 +5227,22 @@ __setscheduler(struct rq *rq, struct task_struct *p, int policy, int prio)
5133 set_load_weight(p); 5227 set_load_weight(p);
5134} 5228}
5135 5229
5230/*
5231 * check the target process has a UID that matches the current process's
5232 */
5233static bool check_same_owner(struct task_struct *p)
5234{
5235 const struct cred *cred = current_cred(), *pcred;
5236 bool match;
5237
5238 rcu_read_lock();
5239 pcred = __task_cred(p);
5240 match = (cred->euid == pcred->euid ||
5241 cred->euid == pcred->uid);
5242 rcu_read_unlock();
5243 return match;
5244}
5245
5136static int __sched_setscheduler(struct task_struct *p, int policy, 5246static int __sched_setscheduler(struct task_struct *p, int policy,
5137 struct sched_param *param, bool user) 5247 struct sched_param *param, bool user)
5138{ 5248{
@@ -5192,8 +5302,7 @@ recheck:
5192 return -EPERM; 5302 return -EPERM;
5193 5303
5194 /* can't change other user's priorities */ 5304 /* can't change other user's priorities */
5195 if ((current->euid != p->euid) && 5305 if (!check_same_owner(p))
5196 (current->euid != p->uid))
5197 return -EPERM; 5306 return -EPERM;
5198 } 5307 }
5199 5308
@@ -5399,10 +5508,9 @@ out_unlock:
5399 return retval; 5508 return retval;
5400} 5509}
5401 5510
5402long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) 5511long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
5403{ 5512{
5404 cpumask_t cpus_allowed; 5513 cpumask_var_t cpus_allowed, new_mask;
5405 cpumask_t new_mask = *in_mask;
5406 struct task_struct *p; 5514 struct task_struct *p;
5407 int retval; 5515 int retval;
5408 5516
@@ -5424,46 +5532,57 @@ long sched_setaffinity(pid_t pid, const cpumask_t *in_mask)
5424 get_task_struct(p); 5532 get_task_struct(p);
5425 read_unlock(&tasklist_lock); 5533 read_unlock(&tasklist_lock);
5426 5534
5535 if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) {
5536 retval = -ENOMEM;
5537 goto out_put_task;
5538 }
5539 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) {
5540 retval = -ENOMEM;
5541 goto out_free_cpus_allowed;
5542 }
5427 retval = -EPERM; 5543 retval = -EPERM;
5428 if ((current->euid != p->euid) && (current->euid != p->uid) && 5544 if (!check_same_owner(p) && !capable(CAP_SYS_NICE))
5429 !capable(CAP_SYS_NICE))
5430 goto out_unlock; 5545 goto out_unlock;
5431 5546
5432 retval = security_task_setscheduler(p, 0, NULL); 5547 retval = security_task_setscheduler(p, 0, NULL);
5433 if (retval) 5548 if (retval)
5434 goto out_unlock; 5549 goto out_unlock;
5435 5550
5436 cpuset_cpus_allowed(p, &cpus_allowed); 5551 cpuset_cpus_allowed(p, cpus_allowed);
5437 cpus_and(new_mask, new_mask, cpus_allowed); 5552 cpumask_and(new_mask, in_mask, cpus_allowed);
5438 again: 5553 again:
5439 retval = set_cpus_allowed_ptr(p, &new_mask); 5554 retval = set_cpus_allowed_ptr(p, new_mask);
5440 5555
5441 if (!retval) { 5556 if (!retval) {
5442 cpuset_cpus_allowed(p, &cpus_allowed); 5557 cpuset_cpus_allowed(p, cpus_allowed);
5443 if (!cpus_subset(new_mask, cpus_allowed)) { 5558 if (!cpumask_subset(new_mask, cpus_allowed)) {
5444 /* 5559 /*
5445 * We must have raced with a concurrent cpuset 5560 * We must have raced with a concurrent cpuset
5446 * update. Just reset the cpus_allowed to the 5561 * update. Just reset the cpus_allowed to the
5447 * cpuset's cpus_allowed 5562 * cpuset's cpus_allowed
5448 */ 5563 */
5449 new_mask = cpus_allowed; 5564 cpumask_copy(new_mask, cpus_allowed);
5450 goto again; 5565 goto again;
5451 } 5566 }
5452 } 5567 }
5453out_unlock: 5568out_unlock:
5569 free_cpumask_var(new_mask);
5570out_free_cpus_allowed:
5571 free_cpumask_var(cpus_allowed);
5572out_put_task:
5454 put_task_struct(p); 5573 put_task_struct(p);
5455 put_online_cpus(); 5574 put_online_cpus();
5456 return retval; 5575 return retval;
5457} 5576}
5458 5577
5459static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, 5578static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5460 cpumask_t *new_mask) 5579 struct cpumask *new_mask)
5461{ 5580{
5462 if (len < sizeof(cpumask_t)) { 5581 if (len < cpumask_size())
5463 memset(new_mask, 0, sizeof(cpumask_t)); 5582 cpumask_clear(new_mask);
5464 } else if (len > sizeof(cpumask_t)) { 5583 else if (len > cpumask_size())
5465 len = sizeof(cpumask_t); 5584 len = cpumask_size();
5466 } 5585
5467 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; 5586 return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0;
5468} 5587}
5469 5588
@@ -5476,17 +5595,20 @@ static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len,
5476asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, 5595asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len,
5477 unsigned long __user *user_mask_ptr) 5596 unsigned long __user *user_mask_ptr)
5478{ 5597{
5479 cpumask_t new_mask; 5598 cpumask_var_t new_mask;
5480 int retval; 5599 int retval;
5481 5600
5482 retval = get_user_cpu_mask(user_mask_ptr, len, &new_mask); 5601 if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
5483 if (retval) 5602 return -ENOMEM;
5484 return retval;
5485 5603
5486 return sched_setaffinity(pid, &new_mask); 5604 retval = get_user_cpu_mask(user_mask_ptr, len, new_mask);
5605 if (retval == 0)
5606 retval = sched_setaffinity(pid, new_mask);
5607 free_cpumask_var(new_mask);
5608 return retval;
5487} 5609}
5488 5610
5489long sched_getaffinity(pid_t pid, cpumask_t *mask) 5611long sched_getaffinity(pid_t pid, struct cpumask *mask)
5490{ 5612{
5491 struct task_struct *p; 5613 struct task_struct *p;
5492 int retval; 5614 int retval;
@@ -5503,7 +5625,7 @@ long sched_getaffinity(pid_t pid, cpumask_t *mask)
5503 if (retval) 5625 if (retval)
5504 goto out_unlock; 5626 goto out_unlock;
5505 5627
5506 cpus_and(*mask, p->cpus_allowed, cpu_online_map); 5628 cpumask_and(mask, &p->cpus_allowed, cpu_online_mask);
5507 5629
5508out_unlock: 5630out_unlock:
5509 read_unlock(&tasklist_lock); 5631 read_unlock(&tasklist_lock);
@@ -5522,19 +5644,24 @@ asmlinkage long sys_sched_getaffinity(pid_t pid, unsigned int len,
5522 unsigned long __user *user_mask_ptr) 5644 unsigned long __user *user_mask_ptr)
5523{ 5645{
5524 int ret; 5646 int ret;
5525 cpumask_t mask; 5647 cpumask_var_t mask;
5526 5648
5527 if (len < sizeof(cpumask_t)) 5649 if (len < cpumask_size())
5528 return -EINVAL; 5650 return -EINVAL;
5529 5651
5530 ret = sched_getaffinity(pid, &mask); 5652 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
5531 if (ret < 0) 5653 return -ENOMEM;
5532 return ret;
5533 5654
5534 if (copy_to_user(user_mask_ptr, &mask, sizeof(cpumask_t))) 5655 ret = sched_getaffinity(pid, mask);
5535 return -EFAULT; 5656 if (ret == 0) {
5657 if (copy_to_user(user_mask_ptr, mask, cpumask_size()))
5658 ret = -EFAULT;
5659 else
5660 ret = cpumask_size();
5661 }
5662 free_cpumask_var(mask);
5536 5663
5537 return sizeof(cpumask_t); 5664 return ret;
5538} 5665}
5539 5666
5540/** 5667/**
@@ -5876,7 +6003,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5876 idle->se.exec_start = sched_clock(); 6003 idle->se.exec_start = sched_clock();
5877 6004
5878 idle->prio = idle->normal_prio = MAX_PRIO; 6005 idle->prio = idle->normal_prio = MAX_PRIO;
5879 idle->cpus_allowed = cpumask_of_cpu(cpu); 6006 cpumask_copy(&idle->cpus_allowed, cpumask_of(cpu));
5880 __set_task_cpu(idle, cpu); 6007 __set_task_cpu(idle, cpu);
5881 6008
5882 rq->curr = rq->idle = idle; 6009 rq->curr = rq->idle = idle;
@@ -5895,6 +6022,7 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5895 * The idle tasks have their own, simple scheduling class: 6022 * The idle tasks have their own, simple scheduling class:
5896 */ 6023 */
5897 idle->sched_class = &idle_sched_class; 6024 idle->sched_class = &idle_sched_class;
6025 ftrace_graph_init_task(idle);
5898} 6026}
5899 6027
5900/* 6028/*
@@ -5902,9 +6030,9 @@ void __cpuinit init_idle(struct task_struct *idle, int cpu)
5902 * indicates which cpus entered this state. This is used 6030 * indicates which cpus entered this state. This is used
5903 * in the rcu update to wait only for active cpus. For system 6031 * in the rcu update to wait only for active cpus. For system
5904 * which do not switch off the HZ timer nohz_cpu_mask should 6032 * which do not switch off the HZ timer nohz_cpu_mask should
5905 * always be CPU_MASK_NONE. 6033 * always be CPU_BITS_NONE.
5906 */ 6034 */
5907cpumask_t nohz_cpu_mask = CPU_MASK_NONE; 6035cpumask_var_t nohz_cpu_mask;
5908 6036
5909/* 6037/*
5910 * Increase the granularity value when there are more CPUs, 6038 * Increase the granularity value when there are more CPUs,
@@ -5959,7 +6087,7 @@ static inline void sched_init_granularity(void)
5959 * task must not exit() & deallocate itself prematurely. The 6087 * task must not exit() & deallocate itself prematurely. The
5960 * call is not atomic; no spinlocks may be held. 6088 * call is not atomic; no spinlocks may be held.
5961 */ 6089 */
5962int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) 6090int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
5963{ 6091{
5964 struct migration_req req; 6092 struct migration_req req;
5965 unsigned long flags; 6093 unsigned long flags;
@@ -5967,13 +6095,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5967 int ret = 0; 6095 int ret = 0;
5968 6096
5969 rq = task_rq_lock(p, &flags); 6097 rq = task_rq_lock(p, &flags);
5970 if (!cpus_intersects(*new_mask, cpu_online_map)) { 6098 if (!cpumask_intersects(new_mask, cpu_online_mask)) {
5971 ret = -EINVAL; 6099 ret = -EINVAL;
5972 goto out; 6100 goto out;
5973 } 6101 }
5974 6102
5975 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && 6103 if (unlikely((p->flags & PF_THREAD_BOUND) && p != current &&
5976 !cpus_equal(p->cpus_allowed, *new_mask))) { 6104 !cpumask_equal(&p->cpus_allowed, new_mask))) {
5977 ret = -EINVAL; 6105 ret = -EINVAL;
5978 goto out; 6106 goto out;
5979 } 6107 }
@@ -5981,15 +6109,15 @@ int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask)
5981 if (p->sched_class->set_cpus_allowed) 6109 if (p->sched_class->set_cpus_allowed)
5982 p->sched_class->set_cpus_allowed(p, new_mask); 6110 p->sched_class->set_cpus_allowed(p, new_mask);
5983 else { 6111 else {
5984 p->cpus_allowed = *new_mask; 6112 cpumask_copy(&p->cpus_allowed, new_mask);
5985 p->rt.nr_cpus_allowed = cpus_weight(*new_mask); 6113 p->rt.nr_cpus_allowed = cpumask_weight(new_mask);
5986 } 6114 }
5987 6115
5988 /* Can the task run on the task's current CPU? If so, we're done */ 6116 /* Can the task run on the task's current CPU? If so, we're done */
5989 if (cpu_isset(task_cpu(p), *new_mask)) 6117 if (cpumask_test_cpu(task_cpu(p), new_mask))
5990 goto out; 6118 goto out;
5991 6119
5992 if (migrate_task(p, any_online_cpu(*new_mask), &req)) { 6120 if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) {
5993 /* Need help from migration thread: drop lock and wait. */ 6121 /* Need help from migration thread: drop lock and wait. */
5994 task_rq_unlock(rq, &flags); 6122 task_rq_unlock(rq, &flags);
5995 wake_up_process(rq->migration_thread); 6123 wake_up_process(rq->migration_thread);
@@ -6031,7 +6159,7 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu)
6031 if (task_cpu(p) != src_cpu) 6159 if (task_cpu(p) != src_cpu)
6032 goto done; 6160 goto done;
6033 /* Affinity changed (again). */ 6161 /* Affinity changed (again). */
6034 if (!cpu_isset(dest_cpu, p->cpus_allowed)) 6162 if (!cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6035 goto fail; 6163 goto fail;
6036 6164
6037 on_rq = p->se.on_rq; 6165 on_rq = p->se.on_rq;
@@ -6125,54 +6253,44 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu)
6125 6253
6126/* 6254/*
6127 * Figure out where task on dead CPU should go, use force if necessary. 6255 * Figure out where task on dead CPU should go, use force if necessary.
6128 * NOTE: interrupts should be disabled by the caller
6129 */ 6256 */
6130static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p) 6257static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6131{ 6258{
6132 unsigned long flags;
6133 cpumask_t mask;
6134 struct rq *rq;
6135 int dest_cpu; 6259 int dest_cpu;
6260 const struct cpumask *nodemask = cpumask_of_node(cpu_to_node(dead_cpu));
6136 6261
6137 do { 6262again:
6138 /* On same node? */ 6263 /* Look for allowed, online CPU in same node. */
6139 mask = node_to_cpumask(cpu_to_node(dead_cpu)); 6264 for_each_cpu_and(dest_cpu, nodemask, cpu_online_mask)
6140 cpus_and(mask, mask, p->cpus_allowed); 6265 if (cpumask_test_cpu(dest_cpu, &p->cpus_allowed))
6141 dest_cpu = any_online_cpu(mask); 6266 goto move;
6142 6267
6143 /* On any allowed CPU? */ 6268 /* Any allowed, online CPU? */
6144 if (dest_cpu >= nr_cpu_ids) 6269 dest_cpu = cpumask_any_and(&p->cpus_allowed, cpu_online_mask);
6145 dest_cpu = any_online_cpu(p->cpus_allowed); 6270 if (dest_cpu < nr_cpu_ids)
6271 goto move;
6146 6272
6147 /* No more Mr. Nice Guy. */ 6273 /* No more Mr. Nice Guy. */
6148 if (dest_cpu >= nr_cpu_ids) { 6274 if (dest_cpu >= nr_cpu_ids) {
6149 cpumask_t cpus_allowed; 6275 cpuset_cpus_allowed_locked(p, &p->cpus_allowed);
6276 dest_cpu = cpumask_any_and(cpu_online_mask, &p->cpus_allowed);
6150 6277
6151 cpuset_cpus_allowed_locked(p, &cpus_allowed); 6278 /*
6152 /* 6279 * Don't tell them about moving exiting tasks or
6153 * Try to stay on the same cpuset, where the 6280 * kernel threads (both mm NULL), since they never
6154 * current cpuset may be a subset of all cpus. 6281 * leave kernel.
6155 * The cpuset_cpus_allowed_locked() variant of 6282 */
6156 * cpuset_cpus_allowed() will not block. It must be 6283 if (p->mm && printk_ratelimit()) {
6157 * called within calls to cpuset_lock/cpuset_unlock. 6284 printk(KERN_INFO "process %d (%s) no "
6158 */ 6285 "longer affine to cpu%d\n",
6159 rq = task_rq_lock(p, &flags); 6286 task_pid_nr(p), p->comm, dead_cpu);
6160 p->cpus_allowed = cpus_allowed;
6161 dest_cpu = any_online_cpu(p->cpus_allowed);
6162 task_rq_unlock(rq, &flags);
6163
6164 /*
6165 * Don't tell them about moving exiting tasks or
6166 * kernel threads (both mm NULL), since they never
6167 * leave kernel.
6168 */
6169 if (p->mm && printk_ratelimit()) {
6170 printk(KERN_INFO "process %d (%s) no "
6171 "longer affine to cpu%d\n",
6172 task_pid_nr(p), p->comm, dead_cpu);
6173 }
6174 } 6287 }
6175 } while (!__migrate_task_irq(p, dead_cpu, dest_cpu)); 6288 }
6289
6290move:
6291 /* It can have affinity changed while we were choosing. */
6292 if (unlikely(!__migrate_task_irq(p, dead_cpu, dest_cpu)))
6293 goto again;
6176} 6294}
6177 6295
6178/* 6296/*
@@ -6184,7 +6302,7 @@ static void move_task_off_dead_cpu(int dead_cpu, struct task_struct *p)
6184 */ 6302 */
6185static void migrate_nr_uninterruptible(struct rq *rq_src) 6303static void migrate_nr_uninterruptible(struct rq *rq_src)
6186{ 6304{
6187 struct rq *rq_dest = cpu_rq(any_online_cpu(*CPU_MASK_ALL_PTR)); 6305 struct rq *rq_dest = cpu_rq(cpumask_any(cpu_online_mask));
6188 unsigned long flags; 6306 unsigned long flags;
6189 6307
6190 local_irq_save(flags); 6308 local_irq_save(flags);
@@ -6474,7 +6592,7 @@ static void set_rq_online(struct rq *rq)
6474 if (!rq->online) { 6592 if (!rq->online) {
6475 const struct sched_class *class; 6593 const struct sched_class *class;
6476 6594
6477 cpu_set(rq->cpu, rq->rd->online); 6595 cpumask_set_cpu(rq->cpu, rq->rd->online);
6478 rq->online = 1; 6596 rq->online = 1;
6479 6597
6480 for_each_class(class) { 6598 for_each_class(class) {
@@ -6494,7 +6612,7 @@ static void set_rq_offline(struct rq *rq)
6494 class->rq_offline(rq); 6612 class->rq_offline(rq);
6495 } 6613 }
6496 6614
6497 cpu_clear(rq->cpu, rq->rd->online); 6615 cpumask_clear_cpu(rq->cpu, rq->rd->online);
6498 rq->online = 0; 6616 rq->online = 0;
6499 } 6617 }
6500} 6618}
@@ -6535,7 +6653,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6535 rq = cpu_rq(cpu); 6653 rq = cpu_rq(cpu);
6536 spin_lock_irqsave(&rq->lock, flags); 6654 spin_lock_irqsave(&rq->lock, flags);
6537 if (rq->rd) { 6655 if (rq->rd) {
6538 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6656 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6539 6657
6540 set_rq_online(rq); 6658 set_rq_online(rq);
6541 } 6659 }
@@ -6549,7 +6667,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6549 break; 6667 break;
6550 /* Unbind it from offline cpu so it can run. Fall thru. */ 6668 /* Unbind it from offline cpu so it can run. Fall thru. */
6551 kthread_bind(cpu_rq(cpu)->migration_thread, 6669 kthread_bind(cpu_rq(cpu)->migration_thread,
6552 any_online_cpu(cpu_online_map)); 6670 cpumask_any(cpu_online_mask));
6553 kthread_stop(cpu_rq(cpu)->migration_thread); 6671 kthread_stop(cpu_rq(cpu)->migration_thread);
6554 cpu_rq(cpu)->migration_thread = NULL; 6672 cpu_rq(cpu)->migration_thread = NULL;
6555 break; 6673 break;
@@ -6586,7 +6704,9 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6586 req = list_entry(rq->migration_queue.next, 6704 req = list_entry(rq->migration_queue.next,
6587 struct migration_req, list); 6705 struct migration_req, list);
6588 list_del_init(&req->list); 6706 list_del_init(&req->list);
6707 spin_unlock_irq(&rq->lock);
6589 complete(&req->done); 6708 complete(&req->done);
6709 spin_lock_irq(&rq->lock);
6590 } 6710 }
6591 spin_unlock_irq(&rq->lock); 6711 spin_unlock_irq(&rq->lock);
6592 break; 6712 break;
@@ -6597,7 +6717,7 @@ migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu)
6597 rq = cpu_rq(cpu); 6717 rq = cpu_rq(cpu);
6598 spin_lock_irqsave(&rq->lock, flags); 6718 spin_lock_irqsave(&rq->lock, flags);
6599 if (rq->rd) { 6719 if (rq->rd) {
6600 BUG_ON(!cpu_isset(cpu, rq->rd->span)); 6720 BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
6601 set_rq_offline(rq); 6721 set_rq_offline(rq);
6602 } 6722 }
6603 spin_unlock_irqrestore(&rq->lock, flags); 6723 spin_unlock_irqrestore(&rq->lock, flags);
@@ -6635,36 +6755,14 @@ early_initcall(migration_init);
6635 6755
6636#ifdef CONFIG_SCHED_DEBUG 6756#ifdef CONFIG_SCHED_DEBUG
6637 6757
6638static inline const char *sd_level_to_string(enum sched_domain_level lvl)
6639{
6640 switch (lvl) {
6641 case SD_LV_NONE:
6642 return "NONE";
6643 case SD_LV_SIBLING:
6644 return "SIBLING";
6645 case SD_LV_MC:
6646 return "MC";
6647 case SD_LV_CPU:
6648 return "CPU";
6649 case SD_LV_NODE:
6650 return "NODE";
6651 case SD_LV_ALLNODES:
6652 return "ALLNODES";
6653 case SD_LV_MAX:
6654 return "MAX";
6655
6656 }
6657 return "MAX";
6658}
6659
6660static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, 6758static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6661 cpumask_t *groupmask) 6759 struct cpumask *groupmask)
6662{ 6760{
6663 struct sched_group *group = sd->groups; 6761 struct sched_group *group = sd->groups;
6664 char str[256]; 6762 char str[256];
6665 6763
6666 cpulist_scnprintf(str, sizeof(str), sd->span); 6764 cpulist_scnprintf(str, sizeof(str), sched_domain_span(sd));
6667 cpus_clear(*groupmask); 6765 cpumask_clear(groupmask);
6668 6766
6669 printk(KERN_DEBUG "%*s domain %d: ", level, "", level); 6767 printk(KERN_DEBUG "%*s domain %d: ", level, "", level);
6670 6768
@@ -6676,14 +6774,13 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6676 return -1; 6774 return -1;
6677 } 6775 }
6678 6776
6679 printk(KERN_CONT "span %s level %s\n", 6777 printk(KERN_CONT "span %s level %s\n", str, sd->name);
6680 str, sd_level_to_string(sd->level));
6681 6778
6682 if (!cpu_isset(cpu, sd->span)) { 6779 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
6683 printk(KERN_ERR "ERROR: domain->span does not contain " 6780 printk(KERN_ERR "ERROR: domain->span does not contain "
6684 "CPU%d\n", cpu); 6781 "CPU%d\n", cpu);
6685 } 6782 }
6686 if (!cpu_isset(cpu, group->cpumask)) { 6783 if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) {
6687 printk(KERN_ERR "ERROR: domain->groups does not contain" 6784 printk(KERN_ERR "ERROR: domain->groups does not contain"
6688 " CPU%d\n", cpu); 6785 " CPU%d\n", cpu);
6689 } 6786 }
@@ -6703,31 +6800,32 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6703 break; 6800 break;
6704 } 6801 }
6705 6802
6706 if (!cpus_weight(group->cpumask)) { 6803 if (!cpumask_weight(sched_group_cpus(group))) {
6707 printk(KERN_CONT "\n"); 6804 printk(KERN_CONT "\n");
6708 printk(KERN_ERR "ERROR: empty group\n"); 6805 printk(KERN_ERR "ERROR: empty group\n");
6709 break; 6806 break;
6710 } 6807 }
6711 6808
6712 if (cpus_intersects(*groupmask, group->cpumask)) { 6809 if (cpumask_intersects(groupmask, sched_group_cpus(group))) {
6713 printk(KERN_CONT "\n"); 6810 printk(KERN_CONT "\n");
6714 printk(KERN_ERR "ERROR: repeated CPUs\n"); 6811 printk(KERN_ERR "ERROR: repeated CPUs\n");
6715 break; 6812 break;
6716 } 6813 }
6717 6814
6718 cpus_or(*groupmask, *groupmask, group->cpumask); 6815 cpumask_or(groupmask, groupmask, sched_group_cpus(group));
6719 6816
6720 cpulist_scnprintf(str, sizeof(str), group->cpumask); 6817 cpulist_scnprintf(str, sizeof(str), sched_group_cpus(group));
6721 printk(KERN_CONT " %s", str); 6818 printk(KERN_CONT " %s", str);
6722 6819
6723 group = group->next; 6820 group = group->next;
6724 } while (group != sd->groups); 6821 } while (group != sd->groups);
6725 printk(KERN_CONT "\n"); 6822 printk(KERN_CONT "\n");
6726 6823
6727 if (!cpus_equal(sd->span, *groupmask)) 6824 if (!cpumask_equal(sched_domain_span(sd), groupmask))
6728 printk(KERN_ERR "ERROR: groups don't span domain->span\n"); 6825 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
6729 6826
6730 if (sd->parent && !cpus_subset(*groupmask, sd->parent->span)) 6827 if (sd->parent &&
6828 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
6731 printk(KERN_ERR "ERROR: parent span is not a superset " 6829 printk(KERN_ERR "ERROR: parent span is not a superset "
6732 "of domain->span\n"); 6830 "of domain->span\n");
6733 return 0; 6831 return 0;
@@ -6735,7 +6833,7 @@ static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
6735 6833
6736static void sched_domain_debug(struct sched_domain *sd, int cpu) 6834static void sched_domain_debug(struct sched_domain *sd, int cpu)
6737{ 6835{
6738 cpumask_t *groupmask; 6836 cpumask_var_t groupmask;
6739 int level = 0; 6837 int level = 0;
6740 6838
6741 if (!sd) { 6839 if (!sd) {
@@ -6745,8 +6843,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6745 6843
6746 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); 6844 printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu);
6747 6845
6748 groupmask = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 6846 if (!alloc_cpumask_var(&groupmask, GFP_KERNEL)) {
6749 if (!groupmask) {
6750 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n"); 6847 printk(KERN_DEBUG "Cannot load-balance (out of memory)\n");
6751 return; 6848 return;
6752 } 6849 }
@@ -6759,7 +6856,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6759 if (!sd) 6856 if (!sd)
6760 break; 6857 break;
6761 } 6858 }
6762 kfree(groupmask); 6859 free_cpumask_var(groupmask);
6763} 6860}
6764#else /* !CONFIG_SCHED_DEBUG */ 6861#else /* !CONFIG_SCHED_DEBUG */
6765# define sched_domain_debug(sd, cpu) do { } while (0) 6862# define sched_domain_debug(sd, cpu) do { } while (0)
@@ -6767,7 +6864,7 @@ static void sched_domain_debug(struct sched_domain *sd, int cpu)
6767 6864
6768static int sd_degenerate(struct sched_domain *sd) 6865static int sd_degenerate(struct sched_domain *sd)
6769{ 6866{
6770 if (cpus_weight(sd->span) == 1) 6867 if (cpumask_weight(sched_domain_span(sd)) == 1)
6771 return 1; 6868 return 1;
6772 6869
6773 /* Following flags need at least 2 groups */ 6870 /* Following flags need at least 2 groups */
@@ -6798,7 +6895,7 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6798 if (sd_degenerate(parent)) 6895 if (sd_degenerate(parent))
6799 return 1; 6896 return 1;
6800 6897
6801 if (!cpus_equal(sd->span, parent->span)) 6898 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent)))
6802 return 0; 6899 return 0;
6803 6900
6804 /* Does parent contain flags not in child? */ 6901 /* Does parent contain flags not in child? */
@@ -6813,6 +6910,8 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6813 SD_BALANCE_EXEC | 6910 SD_BALANCE_EXEC |
6814 SD_SHARE_CPUPOWER | 6911 SD_SHARE_CPUPOWER |
6815 SD_SHARE_PKG_RESOURCES); 6912 SD_SHARE_PKG_RESOURCES);
6913 if (nr_node_ids == 1)
6914 pflags &= ~SD_SERIALIZE;
6816 } 6915 }
6817 if (~cflags & pflags) 6916 if (~cflags & pflags)
6818 return 0; 6917 return 0;
@@ -6820,6 +6919,16 @@ sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent)
6820 return 1; 6919 return 1;
6821} 6920}
6822 6921
6922static void free_rootdomain(struct root_domain *rd)
6923{
6924 cpupri_cleanup(&rd->cpupri);
6925
6926 free_cpumask_var(rd->rto_mask);
6927 free_cpumask_var(rd->online);
6928 free_cpumask_var(rd->span);
6929 kfree(rd);
6930}
6931
6823static void rq_attach_root(struct rq *rq, struct root_domain *rd) 6932static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6824{ 6933{
6825 unsigned long flags; 6934 unsigned long flags;
@@ -6829,38 +6938,63 @@ static void rq_attach_root(struct rq *rq, struct root_domain *rd)
6829 if (rq->rd) { 6938 if (rq->rd) {
6830 struct root_domain *old_rd = rq->rd; 6939 struct root_domain *old_rd = rq->rd;
6831 6940
6832 if (cpu_isset(rq->cpu, old_rd->online)) 6941 if (cpumask_test_cpu(rq->cpu, old_rd->online))
6833 set_rq_offline(rq); 6942 set_rq_offline(rq);
6834 6943
6835 cpu_clear(rq->cpu, old_rd->span); 6944 cpumask_clear_cpu(rq->cpu, old_rd->span);
6836 6945
6837 if (atomic_dec_and_test(&old_rd->refcount)) 6946 if (atomic_dec_and_test(&old_rd->refcount))
6838 kfree(old_rd); 6947 free_rootdomain(old_rd);
6839 } 6948 }
6840 6949
6841 atomic_inc(&rd->refcount); 6950 atomic_inc(&rd->refcount);
6842 rq->rd = rd; 6951 rq->rd = rd;
6843 6952
6844 cpu_set(rq->cpu, rd->span); 6953 cpumask_set_cpu(rq->cpu, rd->span);
6845 if (cpu_isset(rq->cpu, cpu_online_map)) 6954 if (cpumask_test_cpu(rq->cpu, cpu_online_mask))
6846 set_rq_online(rq); 6955 set_rq_online(rq);
6847 6956
6848 spin_unlock_irqrestore(&rq->lock, flags); 6957 spin_unlock_irqrestore(&rq->lock, flags);
6849} 6958}
6850 6959
6851static void init_rootdomain(struct root_domain *rd) 6960static int init_rootdomain(struct root_domain *rd, bool bootmem)
6852{ 6961{
6853 memset(rd, 0, sizeof(*rd)); 6962 memset(rd, 0, sizeof(*rd));
6854 6963
6855 cpus_clear(rd->span); 6964 if (bootmem) {
6856 cpus_clear(rd->online); 6965 alloc_bootmem_cpumask_var(&def_root_domain.span);
6966 alloc_bootmem_cpumask_var(&def_root_domain.online);
6967 alloc_bootmem_cpumask_var(&def_root_domain.rto_mask);
6968 cpupri_init(&rd->cpupri, true);
6969 return 0;
6970 }
6857 6971
6858 cpupri_init(&rd->cpupri); 6972 if (!alloc_cpumask_var(&rd->span, GFP_KERNEL))
6973 goto free_rd;
6974 if (!alloc_cpumask_var(&rd->online, GFP_KERNEL))
6975 goto free_span;
6976 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
6977 goto free_online;
6978
6979 if (cpupri_init(&rd->cpupri, false) != 0)
6980 goto free_rto_mask;
6981 return 0;
6982
6983free_rto_mask:
6984 free_cpumask_var(rd->rto_mask);
6985free_online:
6986 free_cpumask_var(rd->online);
6987free_span:
6988 free_cpumask_var(rd->span);
6989free_rd:
6990 kfree(rd);
6991 return -ENOMEM;
6859} 6992}
6860 6993
6861static void init_defrootdomain(void) 6994static void init_defrootdomain(void)
6862{ 6995{
6863 init_rootdomain(&def_root_domain); 6996 init_rootdomain(&def_root_domain, true);
6997
6864 atomic_set(&def_root_domain.refcount, 1); 6998 atomic_set(&def_root_domain.refcount, 1);
6865} 6999}
6866 7000
@@ -6872,7 +7006,10 @@ static struct root_domain *alloc_rootdomain(void)
6872 if (!rd) 7006 if (!rd)
6873 return NULL; 7007 return NULL;
6874 7008
6875 init_rootdomain(rd); 7009 if (init_rootdomain(rd, false) != 0) {
7010 kfree(rd);
7011 return NULL;
7012 }
6876 7013
6877 return rd; 7014 return rd;
6878} 7015}
@@ -6914,19 +7051,12 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
6914} 7051}
6915 7052
6916/* cpus with isolated domains */ 7053/* cpus with isolated domains */
6917static cpumask_t cpu_isolated_map = CPU_MASK_NONE; 7054static cpumask_var_t cpu_isolated_map;
6918 7055
6919/* Setup the mask of cpus configured for isolated domains */ 7056/* Setup the mask of cpus configured for isolated domains */
6920static int __init isolated_cpu_setup(char *str) 7057static int __init isolated_cpu_setup(char *str)
6921{ 7058{
6922 static int __initdata ints[NR_CPUS]; 7059 cpulist_parse(str, cpu_isolated_map);
6923 int i;
6924
6925 str = get_options(str, ARRAY_SIZE(ints), ints);
6926 cpus_clear(cpu_isolated_map);
6927 for (i = 1; i <= ints[0]; i++)
6928 if (ints[i] < NR_CPUS)
6929 cpu_set(ints[i], cpu_isolated_map);
6930 return 1; 7060 return 1;
6931} 7061}
6932 7062
@@ -6935,42 +7065,43 @@ __setup("isolcpus=", isolated_cpu_setup);
6935/* 7065/*
6936 * init_sched_build_groups takes the cpumask we wish to span, and a pointer 7066 * init_sched_build_groups takes the cpumask we wish to span, and a pointer
6937 * to a function which identifies what group(along with sched group) a CPU 7067 * to a function which identifies what group(along with sched group) a CPU
6938 * belongs to. The return value of group_fn must be a >= 0 and < NR_CPUS 7068 * belongs to. The return value of group_fn must be a >= 0 and < nr_cpu_ids
6939 * (due to the fact that we keep track of groups covered with a cpumask_t). 7069 * (due to the fact that we keep track of groups covered with a struct cpumask).
6940 * 7070 *
6941 * init_sched_build_groups will build a circular linked list of the groups 7071 * init_sched_build_groups will build a circular linked list of the groups
6942 * covered by the given span, and will set each group's ->cpumask correctly, 7072 * covered by the given span, and will set each group's ->cpumask correctly,
6943 * and ->cpu_power to 0. 7073 * and ->cpu_power to 0.
6944 */ 7074 */
6945static void 7075static void
6946init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, 7076init_sched_build_groups(const struct cpumask *span,
6947 int (*group_fn)(int cpu, const cpumask_t *cpu_map, 7077 const struct cpumask *cpu_map,
7078 int (*group_fn)(int cpu, const struct cpumask *cpu_map,
6948 struct sched_group **sg, 7079 struct sched_group **sg,
6949 cpumask_t *tmpmask), 7080 struct cpumask *tmpmask),
6950 cpumask_t *covered, cpumask_t *tmpmask) 7081 struct cpumask *covered, struct cpumask *tmpmask)
6951{ 7082{
6952 struct sched_group *first = NULL, *last = NULL; 7083 struct sched_group *first = NULL, *last = NULL;
6953 int i; 7084 int i;
6954 7085
6955 cpus_clear(*covered); 7086 cpumask_clear(covered);
6956 7087
6957 for_each_cpu_mask_nr(i, *span) { 7088 for_each_cpu(i, span) {
6958 struct sched_group *sg; 7089 struct sched_group *sg;
6959 int group = group_fn(i, cpu_map, &sg, tmpmask); 7090 int group = group_fn(i, cpu_map, &sg, tmpmask);
6960 int j; 7091 int j;
6961 7092
6962 if (cpu_isset(i, *covered)) 7093 if (cpumask_test_cpu(i, covered))
6963 continue; 7094 continue;
6964 7095
6965 cpus_clear(sg->cpumask); 7096 cpumask_clear(sched_group_cpus(sg));
6966 sg->__cpu_power = 0; 7097 sg->__cpu_power = 0;
6967 7098
6968 for_each_cpu_mask_nr(j, *span) { 7099 for_each_cpu(j, span) {
6969 if (group_fn(j, cpu_map, NULL, tmpmask) != group) 7100 if (group_fn(j, cpu_map, NULL, tmpmask) != group)
6970 continue; 7101 continue;
6971 7102
6972 cpu_set(j, *covered); 7103 cpumask_set_cpu(j, covered);
6973 cpu_set(j, sg->cpumask); 7104 cpumask_set_cpu(j, sched_group_cpus(sg));
6974 } 7105 }
6975 if (!first) 7106 if (!first)
6976 first = sg; 7107 first = sg;
@@ -7034,23 +7165,21 @@ static int find_next_best_node(int node, nodemask_t *used_nodes)
7034 * should be one that prevents unnecessary balancing, but also spreads tasks 7165 * should be one that prevents unnecessary balancing, but also spreads tasks
7035 * out optimally. 7166 * out optimally.
7036 */ 7167 */
7037static void sched_domain_node_span(int node, cpumask_t *span) 7168static void sched_domain_node_span(int node, struct cpumask *span)
7038{ 7169{
7039 nodemask_t used_nodes; 7170 nodemask_t used_nodes;
7040 node_to_cpumask_ptr(nodemask, node);
7041 int i; 7171 int i;
7042 7172
7043 cpus_clear(*span); 7173 cpumask_clear(span);
7044 nodes_clear(used_nodes); 7174 nodes_clear(used_nodes);
7045 7175
7046 cpus_or(*span, *span, *nodemask); 7176 cpumask_or(span, span, cpumask_of_node(node));
7047 node_set(node, used_nodes); 7177 node_set(node, used_nodes);
7048 7178
7049 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) { 7179 for (i = 1; i < SD_NODES_PER_DOMAIN; i++) {
7050 int next_node = find_next_best_node(node, &used_nodes); 7180 int next_node = find_next_best_node(node, &used_nodes);
7051 7181
7052 node_to_cpumask_ptr_next(nodemask, next_node); 7182 cpumask_or(span, span, cpumask_of_node(next_node));
7053 cpus_or(*span, *span, *nodemask);
7054 } 7183 }
7055} 7184}
7056#endif /* CONFIG_NUMA */ 7185#endif /* CONFIG_NUMA */
@@ -7058,18 +7187,33 @@ static void sched_domain_node_span(int node, cpumask_t *span)
7058int sched_smt_power_savings = 0, sched_mc_power_savings = 0; 7187int sched_smt_power_savings = 0, sched_mc_power_savings = 0;
7059 7188
7060/* 7189/*
7190 * The cpus mask in sched_group and sched_domain hangs off the end.
7191 * FIXME: use cpumask_var_t or dynamic percpu alloc to avoid wasting space
7192 * for nr_cpu_ids < CONFIG_NR_CPUS.
7193 */
7194struct static_sched_group {
7195 struct sched_group sg;
7196 DECLARE_BITMAP(cpus, CONFIG_NR_CPUS);
7197};
7198
7199struct static_sched_domain {
7200 struct sched_domain sd;
7201 DECLARE_BITMAP(span, CONFIG_NR_CPUS);
7202};
7203
7204/*
7061 * SMT sched-domains: 7205 * SMT sched-domains:
7062 */ 7206 */
7063#ifdef CONFIG_SCHED_SMT 7207#ifdef CONFIG_SCHED_SMT
7064static DEFINE_PER_CPU(struct sched_domain, cpu_domains); 7208static DEFINE_PER_CPU(struct static_sched_domain, cpu_domains);
7065static DEFINE_PER_CPU(struct sched_group, sched_group_cpus); 7209static DEFINE_PER_CPU(struct static_sched_group, sched_group_cpus);
7066 7210
7067static int 7211static int
7068cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7212cpu_to_cpu_group(int cpu, const struct cpumask *cpu_map,
7069 cpumask_t *unused) 7213 struct sched_group **sg, struct cpumask *unused)
7070{ 7214{
7071 if (sg) 7215 if (sg)
7072 *sg = &per_cpu(sched_group_cpus, cpu); 7216 *sg = &per_cpu(sched_group_cpus, cpu).sg;
7073 return cpu; 7217 return cpu;
7074} 7218}
7075#endif /* CONFIG_SCHED_SMT */ 7219#endif /* CONFIG_SCHED_SMT */
@@ -7078,56 +7222,53 @@ cpu_to_cpu_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg,
7078 * multi-core sched-domains: 7222 * multi-core sched-domains:
7079 */ 7223 */
7080#ifdef CONFIG_SCHED_MC 7224#ifdef CONFIG_SCHED_MC
7081static DEFINE_PER_CPU(struct sched_domain, core_domains); 7225static DEFINE_PER_CPU(struct static_sched_domain, core_domains);
7082static DEFINE_PER_CPU(struct sched_group, sched_group_core); 7226static DEFINE_PER_CPU(struct static_sched_group, sched_group_core);
7083#endif /* CONFIG_SCHED_MC */ 7227#endif /* CONFIG_SCHED_MC */
7084 7228
7085#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT) 7229#if defined(CONFIG_SCHED_MC) && defined(CONFIG_SCHED_SMT)
7086static int 7230static int
7087cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7231cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7088 cpumask_t *mask) 7232 struct sched_group **sg, struct cpumask *mask)
7089{ 7233{
7090 int group; 7234 int group;
7091 7235
7092 *mask = per_cpu(cpu_sibling_map, cpu); 7236 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7093 cpus_and(*mask, *mask, *cpu_map); 7237 group = cpumask_first(mask);
7094 group = first_cpu(*mask);
7095 if (sg) 7238 if (sg)
7096 *sg = &per_cpu(sched_group_core, group); 7239 *sg = &per_cpu(sched_group_core, group).sg;
7097 return group; 7240 return group;
7098} 7241}
7099#elif defined(CONFIG_SCHED_MC) 7242#elif defined(CONFIG_SCHED_MC)
7100static int 7243static int
7101cpu_to_core_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7244cpu_to_core_group(int cpu, const struct cpumask *cpu_map,
7102 cpumask_t *unused) 7245 struct sched_group **sg, struct cpumask *unused)
7103{ 7246{
7104 if (sg) 7247 if (sg)
7105 *sg = &per_cpu(sched_group_core, cpu); 7248 *sg = &per_cpu(sched_group_core, cpu).sg;
7106 return cpu; 7249 return cpu;
7107} 7250}
7108#endif 7251#endif
7109 7252
7110static DEFINE_PER_CPU(struct sched_domain, phys_domains); 7253static DEFINE_PER_CPU(struct static_sched_domain, phys_domains);
7111static DEFINE_PER_CPU(struct sched_group, sched_group_phys); 7254static DEFINE_PER_CPU(struct static_sched_group, sched_group_phys);
7112 7255
7113static int 7256static int
7114cpu_to_phys_group(int cpu, const cpumask_t *cpu_map, struct sched_group **sg, 7257cpu_to_phys_group(int cpu, const struct cpumask *cpu_map,
7115 cpumask_t *mask) 7258 struct sched_group **sg, struct cpumask *mask)
7116{ 7259{
7117 int group; 7260 int group;
7118#ifdef CONFIG_SCHED_MC 7261#ifdef CONFIG_SCHED_MC
7119 *mask = cpu_coregroup_map(cpu); 7262 cpumask_and(mask, cpu_coregroup_mask(cpu), cpu_map);
7120 cpus_and(*mask, *mask, *cpu_map); 7263 group = cpumask_first(mask);
7121 group = first_cpu(*mask);
7122#elif defined(CONFIG_SCHED_SMT) 7264#elif defined(CONFIG_SCHED_SMT)
7123 *mask = per_cpu(cpu_sibling_map, cpu); 7265 cpumask_and(mask, &per_cpu(cpu_sibling_map, cpu), cpu_map);
7124 cpus_and(*mask, *mask, *cpu_map); 7266 group = cpumask_first(mask);
7125 group = first_cpu(*mask);
7126#else 7267#else
7127 group = cpu; 7268 group = cpu;
7128#endif 7269#endif
7129 if (sg) 7270 if (sg)
7130 *sg = &per_cpu(sched_group_phys, group); 7271 *sg = &per_cpu(sched_group_phys, group).sg;
7131 return group; 7272 return group;
7132} 7273}
7133 7274
@@ -7141,19 +7282,19 @@ static DEFINE_PER_CPU(struct sched_domain, node_domains);
7141static struct sched_group ***sched_group_nodes_bycpu; 7282static struct sched_group ***sched_group_nodes_bycpu;
7142 7283
7143static DEFINE_PER_CPU(struct sched_domain, allnodes_domains); 7284static DEFINE_PER_CPU(struct sched_domain, allnodes_domains);
7144static DEFINE_PER_CPU(struct sched_group, sched_group_allnodes); 7285static DEFINE_PER_CPU(struct static_sched_group, sched_group_allnodes);
7145 7286
7146static int cpu_to_allnodes_group(int cpu, const cpumask_t *cpu_map, 7287static int cpu_to_allnodes_group(int cpu, const struct cpumask *cpu_map,
7147 struct sched_group **sg, cpumask_t *nodemask) 7288 struct sched_group **sg,
7289 struct cpumask *nodemask)
7148{ 7290{
7149 int group; 7291 int group;
7150 7292
7151 *nodemask = node_to_cpumask(cpu_to_node(cpu)); 7293 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(cpu)), cpu_map);
7152 cpus_and(*nodemask, *nodemask, *cpu_map); 7294 group = cpumask_first(nodemask);
7153 group = first_cpu(*nodemask);
7154 7295
7155 if (sg) 7296 if (sg)
7156 *sg = &per_cpu(sched_group_allnodes, group); 7297 *sg = &per_cpu(sched_group_allnodes, group).sg;
7157 return group; 7298 return group;
7158} 7299}
7159 7300
@@ -7165,11 +7306,11 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7165 if (!sg) 7306 if (!sg)
7166 return; 7307 return;
7167 do { 7308 do {
7168 for_each_cpu_mask_nr(j, sg->cpumask) { 7309 for_each_cpu(j, sched_group_cpus(sg)) {
7169 struct sched_domain *sd; 7310 struct sched_domain *sd;
7170 7311
7171 sd = &per_cpu(phys_domains, j); 7312 sd = &per_cpu(phys_domains, j).sd;
7172 if (j != first_cpu(sd->groups->cpumask)) { 7313 if (j != cpumask_first(sched_group_cpus(sd->groups))) {
7173 /* 7314 /*
7174 * Only add "power" once for each 7315 * Only add "power" once for each
7175 * physical package. 7316 * physical package.
@@ -7186,11 +7327,12 @@ static void init_numa_sched_groups_power(struct sched_group *group_head)
7186 7327
7187#ifdef CONFIG_NUMA 7328#ifdef CONFIG_NUMA
7188/* Free memory allocated for various sched_group structures */ 7329/* Free memory allocated for various sched_group structures */
7189static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7330static void free_sched_groups(const struct cpumask *cpu_map,
7331 struct cpumask *nodemask)
7190{ 7332{
7191 int cpu, i; 7333 int cpu, i;
7192 7334
7193 for_each_cpu_mask_nr(cpu, *cpu_map) { 7335 for_each_cpu(cpu, cpu_map) {
7194 struct sched_group **sched_group_nodes 7336 struct sched_group **sched_group_nodes
7195 = sched_group_nodes_bycpu[cpu]; 7337 = sched_group_nodes_bycpu[cpu];
7196 7338
@@ -7200,9 +7342,8 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask)
7200 for (i = 0; i < nr_node_ids; i++) { 7342 for (i = 0; i < nr_node_ids; i++) {
7201 struct sched_group *oldsg, *sg = sched_group_nodes[i]; 7343 struct sched_group *oldsg, *sg = sched_group_nodes[i];
7202 7344
7203 *nodemask = node_to_cpumask(i); 7345 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7204 cpus_and(*nodemask, *nodemask, *cpu_map); 7346 if (cpumask_empty(nodemask))
7205 if (cpus_empty(*nodemask))
7206 continue; 7347 continue;
7207 7348
7208 if (sg == NULL) 7349 if (sg == NULL)
@@ -7220,7 +7361,8 @@ next_sg:
7220 } 7361 }
7221} 7362}
7222#else /* !CONFIG_NUMA */ 7363#else /* !CONFIG_NUMA */
7223static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) 7364static void free_sched_groups(const struct cpumask *cpu_map,
7365 struct cpumask *nodemask)
7224{ 7366{
7225} 7367}
7226#endif /* CONFIG_NUMA */ 7368#endif /* CONFIG_NUMA */
@@ -7246,7 +7388,7 @@ static void init_sched_groups_power(int cpu, struct sched_domain *sd)
7246 7388
7247 WARN_ON(!sd || !sd->groups); 7389 WARN_ON(!sd || !sd->groups);
7248 7390
7249 if (cpu != first_cpu(sd->groups->cpumask)) 7391 if (cpu != cpumask_first(sched_group_cpus(sd->groups)))
7250 return; 7392 return;
7251 7393
7252 child = sd->child; 7394 child = sd->child;
@@ -7311,40 +7453,6 @@ SD_INIT_FUNC(CPU)
7311 SD_INIT_FUNC(MC) 7453 SD_INIT_FUNC(MC)
7312#endif 7454#endif
7313 7455
7314/*
7315 * To minimize stack usage kmalloc room for cpumasks and share the
7316 * space as the usage in build_sched_domains() dictates. Used only
7317 * if the amount of space is significant.
7318 */
7319struct allmasks {
7320 cpumask_t tmpmask; /* make this one first */
7321 union {
7322 cpumask_t nodemask;
7323 cpumask_t this_sibling_map;
7324 cpumask_t this_core_map;
7325 };
7326 cpumask_t send_covered;
7327
7328#ifdef CONFIG_NUMA
7329 cpumask_t domainspan;
7330 cpumask_t covered;
7331 cpumask_t notcovered;
7332#endif
7333};
7334
7335#if NR_CPUS > 128
7336#define SCHED_CPUMASK_ALLOC 1
7337#define SCHED_CPUMASK_FREE(v) kfree(v)
7338#define SCHED_CPUMASK_DECLARE(v) struct allmasks *v
7339#else
7340#define SCHED_CPUMASK_ALLOC 0
7341#define SCHED_CPUMASK_FREE(v)
7342#define SCHED_CPUMASK_DECLARE(v) struct allmasks _v, *v = &_v
7343#endif
7344
7345#define SCHED_CPUMASK_VAR(v, a) cpumask_t *v = (cpumask_t *) \
7346 ((unsigned long)(a) + offsetof(struct allmasks, v))
7347
7348static int default_relax_domain_level = -1; 7456static int default_relax_domain_level = -1;
7349 7457
7350static int __init setup_relax_domain_level(char *str) 7458static int __init setup_relax_domain_level(char *str)
@@ -7384,17 +7492,38 @@ static void set_domain_attribute(struct sched_domain *sd,
7384 * Build sched domains for a given set of cpus and attach the sched domains 7492 * Build sched domains for a given set of cpus and attach the sched domains
7385 * to the individual cpus 7493 * to the individual cpus
7386 */ 7494 */
7387static int __build_sched_domains(const cpumask_t *cpu_map, 7495static int __build_sched_domains(const struct cpumask *cpu_map,
7388 struct sched_domain_attr *attr) 7496 struct sched_domain_attr *attr)
7389{ 7497{
7390 int i; 7498 int i, err = -ENOMEM;
7391 struct root_domain *rd; 7499 struct root_domain *rd;
7392 SCHED_CPUMASK_DECLARE(allmasks); 7500 cpumask_var_t nodemask, this_sibling_map, this_core_map, send_covered,
7393 cpumask_t *tmpmask; 7501 tmpmask;
7394#ifdef CONFIG_NUMA 7502#ifdef CONFIG_NUMA
7503 cpumask_var_t domainspan, covered, notcovered;
7395 struct sched_group **sched_group_nodes = NULL; 7504 struct sched_group **sched_group_nodes = NULL;
7396 int sd_allnodes = 0; 7505 int sd_allnodes = 0;
7397 7506
7507 if (!alloc_cpumask_var(&domainspan, GFP_KERNEL))
7508 goto out;
7509 if (!alloc_cpumask_var(&covered, GFP_KERNEL))
7510 goto free_domainspan;
7511 if (!alloc_cpumask_var(&notcovered, GFP_KERNEL))
7512 goto free_covered;
7513#endif
7514
7515 if (!alloc_cpumask_var(&nodemask, GFP_KERNEL))
7516 goto free_notcovered;
7517 if (!alloc_cpumask_var(&this_sibling_map, GFP_KERNEL))
7518 goto free_nodemask;
7519 if (!alloc_cpumask_var(&this_core_map, GFP_KERNEL))
7520 goto free_this_sibling_map;
7521 if (!alloc_cpumask_var(&send_covered, GFP_KERNEL))
7522 goto free_this_core_map;
7523 if (!alloc_cpumask_var(&tmpmask, GFP_KERNEL))
7524 goto free_send_covered;
7525
7526#ifdef CONFIG_NUMA
7398 /* 7527 /*
7399 * Allocate the per-node list of sched groups 7528 * Allocate the per-node list of sched groups
7400 */ 7529 */
@@ -7402,55 +7531,35 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7402 GFP_KERNEL); 7531 GFP_KERNEL);
7403 if (!sched_group_nodes) { 7532 if (!sched_group_nodes) {
7404 printk(KERN_WARNING "Can not alloc sched group node list\n"); 7533 printk(KERN_WARNING "Can not alloc sched group node list\n");
7405 return -ENOMEM; 7534 goto free_tmpmask;
7406 } 7535 }
7407#endif 7536#endif
7408 7537
7409 rd = alloc_rootdomain(); 7538 rd = alloc_rootdomain();
7410 if (!rd) { 7539 if (!rd) {
7411 printk(KERN_WARNING "Cannot alloc root domain\n"); 7540 printk(KERN_WARNING "Cannot alloc root domain\n");
7412#ifdef CONFIG_NUMA 7541 goto free_sched_groups;
7413 kfree(sched_group_nodes);
7414#endif
7415 return -ENOMEM;
7416 } 7542 }
7417 7543
7418#if SCHED_CPUMASK_ALLOC
7419 /* get space for all scratch cpumask variables */
7420 allmasks = kmalloc(sizeof(*allmasks), GFP_KERNEL);
7421 if (!allmasks) {
7422 printk(KERN_WARNING "Cannot alloc cpumask array\n");
7423 kfree(rd);
7424#ifdef CONFIG_NUMA 7544#ifdef CONFIG_NUMA
7425 kfree(sched_group_nodes); 7545 sched_group_nodes_bycpu[cpumask_first(cpu_map)] = sched_group_nodes;
7426#endif
7427 return -ENOMEM;
7428 }
7429#endif
7430 tmpmask = (cpumask_t *)allmasks;
7431
7432
7433#ifdef CONFIG_NUMA
7434 sched_group_nodes_bycpu[first_cpu(*cpu_map)] = sched_group_nodes;
7435#endif 7546#endif
7436 7547
7437 /* 7548 /*
7438 * Set up domains for cpus specified by the cpu_map. 7549 * Set up domains for cpus specified by the cpu_map.
7439 */ 7550 */
7440 for_each_cpu_mask_nr(i, *cpu_map) { 7551 for_each_cpu(i, cpu_map) {
7441 struct sched_domain *sd = NULL, *p; 7552 struct sched_domain *sd = NULL, *p;
7442 SCHED_CPUMASK_VAR(nodemask, allmasks);
7443 7553
7444 *nodemask = node_to_cpumask(cpu_to_node(i)); 7554 cpumask_and(nodemask, cpumask_of_node(cpu_to_node(i)), cpu_map);
7445 cpus_and(*nodemask, *nodemask, *cpu_map);
7446 7555
7447#ifdef CONFIG_NUMA 7556#ifdef CONFIG_NUMA
7448 if (cpus_weight(*cpu_map) > 7557 if (cpumask_weight(cpu_map) >
7449 SD_NODES_PER_DOMAIN*cpus_weight(*nodemask)) { 7558 SD_NODES_PER_DOMAIN*cpumask_weight(nodemask)) {
7450 sd = &per_cpu(allnodes_domains, i); 7559 sd = &per_cpu(allnodes_domains, i);
7451 SD_INIT(sd, ALLNODES); 7560 SD_INIT(sd, ALLNODES);
7452 set_domain_attribute(sd, attr); 7561 set_domain_attribute(sd, attr);
7453 sd->span = *cpu_map; 7562 cpumask_copy(sched_domain_span(sd), cpu_map);
7454 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask); 7563 cpu_to_allnodes_group(i, cpu_map, &sd->groups, tmpmask);
7455 p = sd; 7564 p = sd;
7456 sd_allnodes = 1; 7565 sd_allnodes = 1;
@@ -7460,18 +7569,19 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7460 sd = &per_cpu(node_domains, i); 7569 sd = &per_cpu(node_domains, i);
7461 SD_INIT(sd, NODE); 7570 SD_INIT(sd, NODE);
7462 set_domain_attribute(sd, attr); 7571 set_domain_attribute(sd, attr);
7463 sched_domain_node_span(cpu_to_node(i), &sd->span); 7572 sched_domain_node_span(cpu_to_node(i), sched_domain_span(sd));
7464 sd->parent = p; 7573 sd->parent = p;
7465 if (p) 7574 if (p)
7466 p->child = sd; 7575 p->child = sd;
7467 cpus_and(sd->span, sd->span, *cpu_map); 7576 cpumask_and(sched_domain_span(sd),
7577 sched_domain_span(sd), cpu_map);
7468#endif 7578#endif
7469 7579
7470 p = sd; 7580 p = sd;
7471 sd = &per_cpu(phys_domains, i); 7581 sd = &per_cpu(phys_domains, i).sd;
7472 SD_INIT(sd, CPU); 7582 SD_INIT(sd, CPU);
7473 set_domain_attribute(sd, attr); 7583 set_domain_attribute(sd, attr);
7474 sd->span = *nodemask; 7584 cpumask_copy(sched_domain_span(sd), nodemask);
7475 sd->parent = p; 7585 sd->parent = p;
7476 if (p) 7586 if (p)
7477 p->child = sd; 7587 p->child = sd;
@@ -7479,11 +7589,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7479 7589
7480#ifdef CONFIG_SCHED_MC 7590#ifdef CONFIG_SCHED_MC
7481 p = sd; 7591 p = sd;
7482 sd = &per_cpu(core_domains, i); 7592 sd = &per_cpu(core_domains, i).sd;
7483 SD_INIT(sd, MC); 7593 SD_INIT(sd, MC);
7484 set_domain_attribute(sd, attr); 7594 set_domain_attribute(sd, attr);
7485 sd->span = cpu_coregroup_map(i); 7595 cpumask_and(sched_domain_span(sd), cpu_map,
7486 cpus_and(sd->span, sd->span, *cpu_map); 7596 cpu_coregroup_mask(i));
7487 sd->parent = p; 7597 sd->parent = p;
7488 p->child = sd; 7598 p->child = sd;
7489 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask); 7599 cpu_to_core_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7491,11 +7601,11 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7491 7601
7492#ifdef CONFIG_SCHED_SMT 7602#ifdef CONFIG_SCHED_SMT
7493 p = sd; 7603 p = sd;
7494 sd = &per_cpu(cpu_domains, i); 7604 sd = &per_cpu(cpu_domains, i).sd;
7495 SD_INIT(sd, SIBLING); 7605 SD_INIT(sd, SIBLING);
7496 set_domain_attribute(sd, attr); 7606 set_domain_attribute(sd, attr);
7497 sd->span = per_cpu(cpu_sibling_map, i); 7607 cpumask_and(sched_domain_span(sd),
7498 cpus_and(sd->span, sd->span, *cpu_map); 7608 &per_cpu(cpu_sibling_map, i), cpu_map);
7499 sd->parent = p; 7609 sd->parent = p;
7500 p->child = sd; 7610 p->child = sd;
7501 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask); 7611 cpu_to_cpu_group(i, cpu_map, &sd->groups, tmpmask);
@@ -7504,13 +7614,10 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7504 7614
7505#ifdef CONFIG_SCHED_SMT 7615#ifdef CONFIG_SCHED_SMT
7506 /* Set up CPU (sibling) groups */ 7616 /* Set up CPU (sibling) groups */
7507 for_each_cpu_mask_nr(i, *cpu_map) { 7617 for_each_cpu(i, cpu_map) {
7508 SCHED_CPUMASK_VAR(this_sibling_map, allmasks); 7618 cpumask_and(this_sibling_map,
7509 SCHED_CPUMASK_VAR(send_covered, allmasks); 7619 &per_cpu(cpu_sibling_map, i), cpu_map);
7510 7620 if (i != cpumask_first(this_sibling_map))
7511 *this_sibling_map = per_cpu(cpu_sibling_map, i);
7512 cpus_and(*this_sibling_map, *this_sibling_map, *cpu_map);
7513 if (i != first_cpu(*this_sibling_map))
7514 continue; 7621 continue;
7515 7622
7516 init_sched_build_groups(this_sibling_map, cpu_map, 7623 init_sched_build_groups(this_sibling_map, cpu_map,
@@ -7521,13 +7628,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7521 7628
7522#ifdef CONFIG_SCHED_MC 7629#ifdef CONFIG_SCHED_MC
7523 /* Set up multi-core groups */ 7630 /* Set up multi-core groups */
7524 for_each_cpu_mask_nr(i, *cpu_map) { 7631 for_each_cpu(i, cpu_map) {
7525 SCHED_CPUMASK_VAR(this_core_map, allmasks); 7632 cpumask_and(this_core_map, cpu_coregroup_mask(i), cpu_map);
7526 SCHED_CPUMASK_VAR(send_covered, allmasks); 7633 if (i != cpumask_first(this_core_map))
7527
7528 *this_core_map = cpu_coregroup_map(i);
7529 cpus_and(*this_core_map, *this_core_map, *cpu_map);
7530 if (i != first_cpu(*this_core_map))
7531 continue; 7634 continue;
7532 7635
7533 init_sched_build_groups(this_core_map, cpu_map, 7636 init_sched_build_groups(this_core_map, cpu_map,
@@ -7538,12 +7641,8 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7538 7641
7539 /* Set up physical groups */ 7642 /* Set up physical groups */
7540 for (i = 0; i < nr_node_ids; i++) { 7643 for (i = 0; i < nr_node_ids; i++) {
7541 SCHED_CPUMASK_VAR(nodemask, allmasks); 7644 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7542 SCHED_CPUMASK_VAR(send_covered, allmasks); 7645 if (cpumask_empty(nodemask))
7543
7544 *nodemask = node_to_cpumask(i);
7545 cpus_and(*nodemask, *nodemask, *cpu_map);
7546 if (cpus_empty(*nodemask))
7547 continue; 7646 continue;
7548 7647
7549 init_sched_build_groups(nodemask, cpu_map, 7648 init_sched_build_groups(nodemask, cpu_map,
@@ -7554,8 +7653,6 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7554#ifdef CONFIG_NUMA 7653#ifdef CONFIG_NUMA
7555 /* Set up node groups */ 7654 /* Set up node groups */
7556 if (sd_allnodes) { 7655 if (sd_allnodes) {
7557 SCHED_CPUMASK_VAR(send_covered, allmasks);
7558
7559 init_sched_build_groups(cpu_map, cpu_map, 7656 init_sched_build_groups(cpu_map, cpu_map,
7560 &cpu_to_allnodes_group, 7657 &cpu_to_allnodes_group,
7561 send_covered, tmpmask); 7658 send_covered, tmpmask);
@@ -7564,58 +7661,53 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7564 for (i = 0; i < nr_node_ids; i++) { 7661 for (i = 0; i < nr_node_ids; i++) {
7565 /* Set up node groups */ 7662 /* Set up node groups */
7566 struct sched_group *sg, *prev; 7663 struct sched_group *sg, *prev;
7567 SCHED_CPUMASK_VAR(nodemask, allmasks);
7568 SCHED_CPUMASK_VAR(domainspan, allmasks);
7569 SCHED_CPUMASK_VAR(covered, allmasks);
7570 int j; 7664 int j;
7571 7665
7572 *nodemask = node_to_cpumask(i); 7666 cpumask_clear(covered);
7573 cpus_clear(*covered); 7667 cpumask_and(nodemask, cpumask_of_node(i), cpu_map);
7574 7668 if (cpumask_empty(nodemask)) {
7575 cpus_and(*nodemask, *nodemask, *cpu_map);
7576 if (cpus_empty(*nodemask)) {
7577 sched_group_nodes[i] = NULL; 7669 sched_group_nodes[i] = NULL;
7578 continue; 7670 continue;
7579 } 7671 }
7580 7672
7581 sched_domain_node_span(i, domainspan); 7673 sched_domain_node_span(i, domainspan);
7582 cpus_and(*domainspan, *domainspan, *cpu_map); 7674 cpumask_and(domainspan, domainspan, cpu_map);
7583 7675
7584 sg = kmalloc_node(sizeof(struct sched_group), GFP_KERNEL, i); 7676 sg = kmalloc_node(sizeof(struct sched_group) + cpumask_size(),
7677 GFP_KERNEL, i);
7585 if (!sg) { 7678 if (!sg) {
7586 printk(KERN_WARNING "Can not alloc domain group for " 7679 printk(KERN_WARNING "Can not alloc domain group for "
7587 "node %d\n", i); 7680 "node %d\n", i);
7588 goto error; 7681 goto error;
7589 } 7682 }
7590 sched_group_nodes[i] = sg; 7683 sched_group_nodes[i] = sg;
7591 for_each_cpu_mask_nr(j, *nodemask) { 7684 for_each_cpu(j, nodemask) {
7592 struct sched_domain *sd; 7685 struct sched_domain *sd;
7593 7686
7594 sd = &per_cpu(node_domains, j); 7687 sd = &per_cpu(node_domains, j);
7595 sd->groups = sg; 7688 sd->groups = sg;
7596 } 7689 }
7597 sg->__cpu_power = 0; 7690 sg->__cpu_power = 0;
7598 sg->cpumask = *nodemask; 7691 cpumask_copy(sched_group_cpus(sg), nodemask);
7599 sg->next = sg; 7692 sg->next = sg;
7600 cpus_or(*covered, *covered, *nodemask); 7693 cpumask_or(covered, covered, nodemask);
7601 prev = sg; 7694 prev = sg;
7602 7695
7603 for (j = 0; j < nr_node_ids; j++) { 7696 for (j = 0; j < nr_node_ids; j++) {
7604 SCHED_CPUMASK_VAR(notcovered, allmasks);
7605 int n = (i + j) % nr_node_ids; 7697 int n = (i + j) % nr_node_ids;
7606 node_to_cpumask_ptr(pnodemask, n);
7607 7698
7608 cpus_complement(*notcovered, *covered); 7699 cpumask_complement(notcovered, covered);
7609 cpus_and(*tmpmask, *notcovered, *cpu_map); 7700 cpumask_and(tmpmask, notcovered, cpu_map);
7610 cpus_and(*tmpmask, *tmpmask, *domainspan); 7701 cpumask_and(tmpmask, tmpmask, domainspan);
7611 if (cpus_empty(*tmpmask)) 7702 if (cpumask_empty(tmpmask))
7612 break; 7703 break;
7613 7704
7614 cpus_and(*tmpmask, *tmpmask, *pnodemask); 7705 cpumask_and(tmpmask, tmpmask, cpumask_of_node(n));
7615 if (cpus_empty(*tmpmask)) 7706 if (cpumask_empty(tmpmask))
7616 continue; 7707 continue;
7617 7708
7618 sg = kmalloc_node(sizeof(struct sched_group), 7709 sg = kmalloc_node(sizeof(struct sched_group) +
7710 cpumask_size(),
7619 GFP_KERNEL, i); 7711 GFP_KERNEL, i);
7620 if (!sg) { 7712 if (!sg) {
7621 printk(KERN_WARNING 7713 printk(KERN_WARNING
@@ -7623,9 +7715,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7623 goto error; 7715 goto error;
7624 } 7716 }
7625 sg->__cpu_power = 0; 7717 sg->__cpu_power = 0;
7626 sg->cpumask = *tmpmask; 7718 cpumask_copy(sched_group_cpus(sg), tmpmask);
7627 sg->next = prev->next; 7719 sg->next = prev->next;
7628 cpus_or(*covered, *covered, *tmpmask); 7720 cpumask_or(covered, covered, tmpmask);
7629 prev->next = sg; 7721 prev->next = sg;
7630 prev = sg; 7722 prev = sg;
7631 } 7723 }
@@ -7634,22 +7726,22 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7634 7726
7635 /* Calculate CPU power for physical packages and nodes */ 7727 /* Calculate CPU power for physical packages and nodes */
7636#ifdef CONFIG_SCHED_SMT 7728#ifdef CONFIG_SCHED_SMT
7637 for_each_cpu_mask_nr(i, *cpu_map) { 7729 for_each_cpu(i, cpu_map) {
7638 struct sched_domain *sd = &per_cpu(cpu_domains, i); 7730 struct sched_domain *sd = &per_cpu(cpu_domains, i).sd;
7639 7731
7640 init_sched_groups_power(i, sd); 7732 init_sched_groups_power(i, sd);
7641 } 7733 }
7642#endif 7734#endif
7643#ifdef CONFIG_SCHED_MC 7735#ifdef CONFIG_SCHED_MC
7644 for_each_cpu_mask_nr(i, *cpu_map) { 7736 for_each_cpu(i, cpu_map) {
7645 struct sched_domain *sd = &per_cpu(core_domains, i); 7737 struct sched_domain *sd = &per_cpu(core_domains, i).sd;
7646 7738
7647 init_sched_groups_power(i, sd); 7739 init_sched_groups_power(i, sd);
7648 } 7740 }
7649#endif 7741#endif
7650 7742
7651 for_each_cpu_mask_nr(i, *cpu_map) { 7743 for_each_cpu(i, cpu_map) {
7652 struct sched_domain *sd = &per_cpu(phys_domains, i); 7744 struct sched_domain *sd = &per_cpu(phys_domains, i).sd;
7653 7745
7654 init_sched_groups_power(i, sd); 7746 init_sched_groups_power(i, sd);
7655 } 7747 }
@@ -7661,56 +7753,87 @@ static int __build_sched_domains(const cpumask_t *cpu_map,
7661 if (sd_allnodes) { 7753 if (sd_allnodes) {
7662 struct sched_group *sg; 7754 struct sched_group *sg;
7663 7755
7664 cpu_to_allnodes_group(first_cpu(*cpu_map), cpu_map, &sg, 7756 cpu_to_allnodes_group(cpumask_first(cpu_map), cpu_map, &sg,
7665 tmpmask); 7757 tmpmask);
7666 init_numa_sched_groups_power(sg); 7758 init_numa_sched_groups_power(sg);
7667 } 7759 }
7668#endif 7760#endif
7669 7761
7670 /* Attach the domains */ 7762 /* Attach the domains */
7671 for_each_cpu_mask_nr(i, *cpu_map) { 7763 for_each_cpu(i, cpu_map) {
7672 struct sched_domain *sd; 7764 struct sched_domain *sd;
7673#ifdef CONFIG_SCHED_SMT 7765#ifdef CONFIG_SCHED_SMT
7674 sd = &per_cpu(cpu_domains, i); 7766 sd = &per_cpu(cpu_domains, i).sd;
7675#elif defined(CONFIG_SCHED_MC) 7767#elif defined(CONFIG_SCHED_MC)
7676 sd = &per_cpu(core_domains, i); 7768 sd = &per_cpu(core_domains, i).sd;
7677#else 7769#else
7678 sd = &per_cpu(phys_domains, i); 7770 sd = &per_cpu(phys_domains, i).sd;
7679#endif 7771#endif
7680 cpu_attach_domain(sd, rd, i); 7772 cpu_attach_domain(sd, rd, i);
7681 } 7773 }
7682 7774
7683 SCHED_CPUMASK_FREE((void *)allmasks); 7775 err = 0;
7684 return 0; 7776
7777free_tmpmask:
7778 free_cpumask_var(tmpmask);
7779free_send_covered:
7780 free_cpumask_var(send_covered);
7781free_this_core_map:
7782 free_cpumask_var(this_core_map);
7783free_this_sibling_map:
7784 free_cpumask_var(this_sibling_map);
7785free_nodemask:
7786 free_cpumask_var(nodemask);
7787free_notcovered:
7788#ifdef CONFIG_NUMA
7789 free_cpumask_var(notcovered);
7790free_covered:
7791 free_cpumask_var(covered);
7792free_domainspan:
7793 free_cpumask_var(domainspan);
7794out:
7795#endif
7796 return err;
7797
7798free_sched_groups:
7799#ifdef CONFIG_NUMA
7800 kfree(sched_group_nodes);
7801#endif
7802 goto free_tmpmask;
7685 7803
7686#ifdef CONFIG_NUMA 7804#ifdef CONFIG_NUMA
7687error: 7805error:
7688 free_sched_groups(cpu_map, tmpmask); 7806 free_sched_groups(cpu_map, tmpmask);
7689 SCHED_CPUMASK_FREE((void *)allmasks); 7807 free_rootdomain(rd);
7690 kfree(rd); 7808 goto free_tmpmask;
7691 return -ENOMEM;
7692#endif 7809#endif
7693} 7810}
7694 7811
7695static int build_sched_domains(const cpumask_t *cpu_map) 7812static int build_sched_domains(const struct cpumask *cpu_map)
7696{ 7813{
7697 return __build_sched_domains(cpu_map, NULL); 7814 return __build_sched_domains(cpu_map, NULL);
7698} 7815}
7699 7816
7700static cpumask_t *doms_cur; /* current sched domains */ 7817static struct cpumask *doms_cur; /* current sched domains */
7701static int ndoms_cur; /* number of sched domains in 'doms_cur' */ 7818static int ndoms_cur; /* number of sched domains in 'doms_cur' */
7702static struct sched_domain_attr *dattr_cur; 7819static struct sched_domain_attr *dattr_cur;
7703 /* attribues of custom domains in 'doms_cur' */ 7820 /* attribues of custom domains in 'doms_cur' */
7704 7821
7705/* 7822/*
7706 * Special case: If a kmalloc of a doms_cur partition (array of 7823 * Special case: If a kmalloc of a doms_cur partition (array of
7707 * cpumask_t) fails, then fallback to a single sched domain, 7824 * cpumask) fails, then fallback to a single sched domain,
7708 * as determined by the single cpumask_t fallback_doms. 7825 * as determined by the single cpumask fallback_doms.
7709 */ 7826 */
7710static cpumask_t fallback_doms; 7827static cpumask_var_t fallback_doms;
7711 7828
7712void __attribute__((weak)) arch_update_cpu_topology(void) 7829/*
7830 * arch_update_cpu_topology lets virtualized architectures update the
7831 * cpu core maps. It is supposed to return 1 if the topology changed
7832 * or 0 if it stayed the same.
7833 */
7834int __attribute__((weak)) arch_update_cpu_topology(void)
7713{ 7835{
7836 return 0;
7714} 7837}
7715 7838
7716/* 7839/*
@@ -7718,16 +7841,16 @@ void __attribute__((weak)) arch_update_cpu_topology(void)
7718 * For now this just excludes isolated cpus, but could be used to 7841 * For now this just excludes isolated cpus, but could be used to
7719 * exclude other special cases in the future. 7842 * exclude other special cases in the future.
7720 */ 7843 */
7721static int arch_init_sched_domains(const cpumask_t *cpu_map) 7844static int arch_init_sched_domains(const struct cpumask *cpu_map)
7722{ 7845{
7723 int err; 7846 int err;
7724 7847
7725 arch_update_cpu_topology(); 7848 arch_update_cpu_topology();
7726 ndoms_cur = 1; 7849 ndoms_cur = 1;
7727 doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); 7850 doms_cur = kmalloc(cpumask_size(), GFP_KERNEL);
7728 if (!doms_cur) 7851 if (!doms_cur)
7729 doms_cur = &fallback_doms; 7852 doms_cur = fallback_doms;
7730 cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); 7853 cpumask_andnot(doms_cur, cpu_map, cpu_isolated_map);
7731 dattr_cur = NULL; 7854 dattr_cur = NULL;
7732 err = build_sched_domains(doms_cur); 7855 err = build_sched_domains(doms_cur);
7733 register_sched_domain_sysctl(); 7856 register_sched_domain_sysctl();
@@ -7735,8 +7858,8 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map)
7735 return err; 7858 return err;
7736} 7859}
7737 7860
7738static void arch_destroy_sched_domains(const cpumask_t *cpu_map, 7861static void arch_destroy_sched_domains(const struct cpumask *cpu_map,
7739 cpumask_t *tmpmask) 7862 struct cpumask *tmpmask)
7740{ 7863{
7741 free_sched_groups(cpu_map, tmpmask); 7864 free_sched_groups(cpu_map, tmpmask);
7742} 7865}
@@ -7745,17 +7868,16 @@ static void arch_destroy_sched_domains(const cpumask_t *cpu_map,
7745 * Detach sched domains from a group of cpus specified in cpu_map 7868 * Detach sched domains from a group of cpus specified in cpu_map
7746 * These cpus will now be attached to the NULL domain 7869 * These cpus will now be attached to the NULL domain
7747 */ 7870 */
7748static void detach_destroy_domains(const cpumask_t *cpu_map) 7871static void detach_destroy_domains(const struct cpumask *cpu_map)
7749{ 7872{
7750 cpumask_t tmpmask; 7873 /* Save because hotplug lock held. */
7874 static DECLARE_BITMAP(tmpmask, CONFIG_NR_CPUS);
7751 int i; 7875 int i;
7752 7876
7753 unregister_sched_domain_sysctl(); 7877 for_each_cpu(i, cpu_map)
7754
7755 for_each_cpu_mask_nr(i, *cpu_map)
7756 cpu_attach_domain(NULL, &def_root_domain, i); 7878 cpu_attach_domain(NULL, &def_root_domain, i);
7757 synchronize_sched(); 7879 synchronize_sched();
7758 arch_destroy_sched_domains(cpu_map, &tmpmask); 7880 arch_destroy_sched_domains(cpu_map, to_cpumask(tmpmask));
7759} 7881}
7760 7882
7761/* handle null as "default" */ 7883/* handle null as "default" */
@@ -7780,7 +7902,7 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7780 * doms_new[] to the current sched domain partitioning, doms_cur[]. 7902 * doms_new[] to the current sched domain partitioning, doms_cur[].
7781 * It destroys each deleted domain and builds each new domain. 7903 * It destroys each deleted domain and builds each new domain.
7782 * 7904 *
7783 * 'doms_new' is an array of cpumask_t's of length 'ndoms_new'. 7905 * 'doms_new' is an array of cpumask's of length 'ndoms_new'.
7784 * The masks don't intersect (don't overlap.) We should setup one 7906 * The masks don't intersect (don't overlap.) We should setup one
7785 * sched domain for each mask. CPUs not in any of the cpumasks will 7907 * sched domain for each mask. CPUs not in any of the cpumasks will
7786 * not be load balanced. If the same cpumask appears both in the 7908 * not be load balanced. If the same cpumask appears both in the
@@ -7794,28 +7916,33 @@ static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur,
7794 * the single partition 'fallback_doms', it also forces the domains 7916 * the single partition 'fallback_doms', it also forces the domains
7795 * to be rebuilt. 7917 * to be rebuilt.
7796 * 7918 *
7797 * If doms_new == NULL it will be replaced with cpu_online_map. 7919 * If doms_new == NULL it will be replaced with cpu_online_mask.
7798 * ndoms_new == 0 is a special case for destroying existing domains, 7920 * ndoms_new == 0 is a special case for destroying existing domains,
7799 * and it will not create the default domain. 7921 * and it will not create the default domain.
7800 * 7922 *
7801 * Call with hotplug lock held 7923 * Call with hotplug lock held
7802 */ 7924 */
7803void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, 7925/* FIXME: Change to struct cpumask *doms_new[] */
7926void partition_sched_domains(int ndoms_new, struct cpumask *doms_new,
7804 struct sched_domain_attr *dattr_new) 7927 struct sched_domain_attr *dattr_new)
7805{ 7928{
7806 int i, j, n; 7929 int i, j, n;
7930 int new_topology;
7807 7931
7808 mutex_lock(&sched_domains_mutex); 7932 mutex_lock(&sched_domains_mutex);
7809 7933
7810 /* always unregister in case we don't destroy any domains */ 7934 /* always unregister in case we don't destroy any domains */
7811 unregister_sched_domain_sysctl(); 7935 unregister_sched_domain_sysctl();
7812 7936
7937 /* Let architecture update cpu core mappings. */
7938 new_topology = arch_update_cpu_topology();
7939
7813 n = doms_new ? ndoms_new : 0; 7940 n = doms_new ? ndoms_new : 0;
7814 7941
7815 /* Destroy deleted domains */ 7942 /* Destroy deleted domains */
7816 for (i = 0; i < ndoms_cur; i++) { 7943 for (i = 0; i < ndoms_cur; i++) {
7817 for (j = 0; j < n; j++) { 7944 for (j = 0; j < n && !new_topology; j++) {
7818 if (cpus_equal(doms_cur[i], doms_new[j]) 7945 if (cpumask_equal(&doms_cur[i], &doms_new[j])
7819 && dattrs_equal(dattr_cur, i, dattr_new, j)) 7946 && dattrs_equal(dattr_cur, i, dattr_new, j))
7820 goto match1; 7947 goto match1;
7821 } 7948 }
@@ -7827,15 +7954,15 @@ match1:
7827 7954
7828 if (doms_new == NULL) { 7955 if (doms_new == NULL) {
7829 ndoms_cur = 0; 7956 ndoms_cur = 0;
7830 doms_new = &fallback_doms; 7957 doms_new = fallback_doms;
7831 cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); 7958 cpumask_andnot(&doms_new[0], cpu_online_mask, cpu_isolated_map);
7832 dattr_new = NULL; 7959 WARN_ON_ONCE(dattr_new);
7833 } 7960 }
7834 7961
7835 /* Build new domains */ 7962 /* Build new domains */
7836 for (i = 0; i < ndoms_new; i++) { 7963 for (i = 0; i < ndoms_new; i++) {
7837 for (j = 0; j < ndoms_cur; j++) { 7964 for (j = 0; j < ndoms_cur && !new_topology; j++) {
7838 if (cpus_equal(doms_new[i], doms_cur[j]) 7965 if (cpumask_equal(&doms_new[i], &doms_cur[j])
7839 && dattrs_equal(dattr_new, i, dattr_cur, j)) 7966 && dattrs_equal(dattr_new, i, dattr_cur, j))
7840 goto match2; 7967 goto match2;
7841 } 7968 }
@@ -7847,7 +7974,7 @@ match2:
7847 } 7974 }
7848 7975
7849 /* Remember the new sched domains */ 7976 /* Remember the new sched domains */
7850 if (doms_cur != &fallback_doms) 7977 if (doms_cur != fallback_doms)
7851 kfree(doms_cur); 7978 kfree(doms_cur);
7852 kfree(dattr_cur); /* kfree(NULL) is safe */ 7979 kfree(dattr_cur); /* kfree(NULL) is safe */
7853 doms_cur = doms_new; 7980 doms_cur = doms_new;
@@ -7876,14 +8003,25 @@ int arch_reinit_sched_domains(void)
7876static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt) 8003static ssize_t sched_power_savings_store(const char *buf, size_t count, int smt)
7877{ 8004{
7878 int ret; 8005 int ret;
8006 unsigned int level = 0;
8007
8008 if (sscanf(buf, "%u", &level) != 1)
8009 return -EINVAL;
8010
8011 /*
8012 * level is always be positive so don't check for
8013 * level < POWERSAVINGS_BALANCE_NONE which is 0
8014 * What happens on 0 or 1 byte write,
8015 * need to check for count as well?
8016 */
7879 8017
7880 if (buf[0] != '0' && buf[0] != '1') 8018 if (level >= MAX_POWERSAVINGS_BALANCE_LEVELS)
7881 return -EINVAL; 8019 return -EINVAL;
7882 8020
7883 if (smt) 8021 if (smt)
7884 sched_smt_power_savings = (buf[0] == '1'); 8022 sched_smt_power_savings = level;
7885 else 8023 else
7886 sched_mc_power_savings = (buf[0] == '1'); 8024 sched_mc_power_savings = level;
7887 8025
7888 ret = arch_reinit_sched_domains(); 8026 ret = arch_reinit_sched_domains();
7889 8027
@@ -7987,7 +8125,9 @@ static int update_runtime(struct notifier_block *nfb,
7987 8125
7988void __init sched_init_smp(void) 8126void __init sched_init_smp(void)
7989{ 8127{
7990 cpumask_t non_isolated_cpus; 8128 cpumask_var_t non_isolated_cpus;
8129
8130 alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL);
7991 8131
7992#if defined(CONFIG_NUMA) 8132#if defined(CONFIG_NUMA)
7993 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **), 8133 sched_group_nodes_bycpu = kzalloc(nr_cpu_ids * sizeof(void **),
@@ -7996,10 +8136,10 @@ void __init sched_init_smp(void)
7996#endif 8136#endif
7997 get_online_cpus(); 8137 get_online_cpus();
7998 mutex_lock(&sched_domains_mutex); 8138 mutex_lock(&sched_domains_mutex);
7999 arch_init_sched_domains(&cpu_online_map); 8139 arch_init_sched_domains(cpu_online_mask);
8000 cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); 8140 cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map);
8001 if (cpus_empty(non_isolated_cpus)) 8141 if (cpumask_empty(non_isolated_cpus))
8002 cpu_set(smp_processor_id(), non_isolated_cpus); 8142 cpumask_set_cpu(smp_processor_id(), non_isolated_cpus);
8003 mutex_unlock(&sched_domains_mutex); 8143 mutex_unlock(&sched_domains_mutex);
8004 put_online_cpus(); 8144 put_online_cpus();
8005 8145
@@ -8014,9 +8154,13 @@ void __init sched_init_smp(void)
8014 init_hrtick(); 8154 init_hrtick();
8015 8155
8016 /* Move init over to a non-isolated CPU */ 8156 /* Move init over to a non-isolated CPU */
8017 if (set_cpus_allowed_ptr(current, &non_isolated_cpus) < 0) 8157 if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0)
8018 BUG(); 8158 BUG();
8019 sched_init_granularity(); 8159 sched_init_granularity();
8160 free_cpumask_var(non_isolated_cpus);
8161
8162 alloc_cpumask_var(&fallback_doms, GFP_KERNEL);
8163 init_sched_rt_class();
8020} 8164}
8021#else 8165#else
8022void __init sched_init_smp(void) 8166void __init sched_init_smp(void)
@@ -8331,6 +8475,15 @@ void __init sched_init(void)
8331 */ 8475 */
8332 current->sched_class = &fair_sched_class; 8476 current->sched_class = &fair_sched_class;
8333 8477
8478 /* Allocate the nohz_cpu_mask if CONFIG_CPUMASK_OFFSTACK */
8479 alloc_bootmem_cpumask_var(&nohz_cpu_mask);
8480#ifdef CONFIG_SMP
8481#ifdef CONFIG_NO_HZ
8482 alloc_bootmem_cpumask_var(&nohz.cpu_mask);
8483#endif
8484 alloc_bootmem_cpumask_var(&cpu_isolated_map);
8485#endif /* SMP */
8486
8334 scheduler_running = 1; 8487 scheduler_running = 1;
8335} 8488}
8336 8489
@@ -8489,7 +8642,7 @@ static
8489int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent) 8642int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8490{ 8643{
8491 struct cfs_rq *cfs_rq; 8644 struct cfs_rq *cfs_rq;
8492 struct sched_entity *se, *parent_se; 8645 struct sched_entity *se;
8493 struct rq *rq; 8646 struct rq *rq;
8494 int i; 8647 int i;
8495 8648
@@ -8505,18 +8658,17 @@ int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
8505 for_each_possible_cpu(i) { 8658 for_each_possible_cpu(i) {
8506 rq = cpu_rq(i); 8659 rq = cpu_rq(i);
8507 8660
8508 cfs_rq = kmalloc_node(sizeof(struct cfs_rq), 8661 cfs_rq = kzalloc_node(sizeof(struct cfs_rq),
8509 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8662 GFP_KERNEL, cpu_to_node(i));
8510 if (!cfs_rq) 8663 if (!cfs_rq)
8511 goto err; 8664 goto err;
8512 8665
8513 se = kmalloc_node(sizeof(struct sched_entity), 8666 se = kzalloc_node(sizeof(struct sched_entity),
8514 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8667 GFP_KERNEL, cpu_to_node(i));
8515 if (!se) 8668 if (!se)
8516 goto err; 8669 goto err;
8517 8670
8518 parent_se = parent ? parent->se[i] : NULL; 8671 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent->se[i]);
8519 init_tg_cfs_entry(tg, cfs_rq, se, i, 0, parent_se);
8520 } 8672 }
8521 8673
8522 return 1; 8674 return 1;
@@ -8577,7 +8729,7 @@ static
8577int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent) 8729int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8578{ 8730{
8579 struct rt_rq *rt_rq; 8731 struct rt_rq *rt_rq;
8580 struct sched_rt_entity *rt_se, *parent_se; 8732 struct sched_rt_entity *rt_se;
8581 struct rq *rq; 8733 struct rq *rq;
8582 int i; 8734 int i;
8583 8735
@@ -8594,18 +8746,17 @@ int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
8594 for_each_possible_cpu(i) { 8746 for_each_possible_cpu(i) {
8595 rq = cpu_rq(i); 8747 rq = cpu_rq(i);
8596 8748
8597 rt_rq = kmalloc_node(sizeof(struct rt_rq), 8749 rt_rq = kzalloc_node(sizeof(struct rt_rq),
8598 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8750 GFP_KERNEL, cpu_to_node(i));
8599 if (!rt_rq) 8751 if (!rt_rq)
8600 goto err; 8752 goto err;
8601 8753
8602 rt_se = kmalloc_node(sizeof(struct sched_rt_entity), 8754 rt_se = kzalloc_node(sizeof(struct sched_rt_entity),
8603 GFP_KERNEL|__GFP_ZERO, cpu_to_node(i)); 8755 GFP_KERNEL, cpu_to_node(i));
8604 if (!rt_se) 8756 if (!rt_se)
8605 goto err; 8757 goto err;
8606 8758
8607 parent_se = parent ? parent->rt_se[i] : NULL; 8759 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent->rt_se[i]);
8608 init_tg_rt_entry(tg, rt_rq, rt_se, i, 0, parent_se);
8609 } 8760 }
8610 8761
8611 return 1; 8762 return 1;
@@ -9248,11 +9399,12 @@ struct cgroup_subsys cpu_cgroup_subsys = {
9248 * (balbir@in.ibm.com). 9399 * (balbir@in.ibm.com).
9249 */ 9400 */
9250 9401
9251/* track cpu usage of a group of tasks */ 9402/* track cpu usage of a group of tasks and its child groups */
9252struct cpuacct { 9403struct cpuacct {
9253 struct cgroup_subsys_state css; 9404 struct cgroup_subsys_state css;
9254 /* cpuusage holds pointer to a u64-type object on every cpu */ 9405 /* cpuusage holds pointer to a u64-type object on every cpu */
9255 u64 *cpuusage; 9406 u64 *cpuusage;
9407 struct cpuacct *parent;
9256}; 9408};
9257 9409
9258struct cgroup_subsys cpuacct_subsys; 9410struct cgroup_subsys cpuacct_subsys;
@@ -9286,6 +9438,9 @@ static struct cgroup_subsys_state *cpuacct_create(
9286 return ERR_PTR(-ENOMEM); 9438 return ERR_PTR(-ENOMEM);
9287 } 9439 }
9288 9440
9441 if (cgrp->parent)
9442 ca->parent = cgroup_ca(cgrp->parent);
9443
9289 return &ca->css; 9444 return &ca->css;
9290} 9445}
9291 9446
@@ -9299,6 +9454,41 @@ cpuacct_destroy(struct cgroup_subsys *ss, struct cgroup *cgrp)
9299 kfree(ca); 9454 kfree(ca);
9300} 9455}
9301 9456
9457static u64 cpuacct_cpuusage_read(struct cpuacct *ca, int cpu)
9458{
9459 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9460 u64 data;
9461
9462#ifndef CONFIG_64BIT
9463 /*
9464 * Take rq->lock to make 64-bit read safe on 32-bit platforms.
9465 */
9466 spin_lock_irq(&cpu_rq(cpu)->lock);
9467 data = *cpuusage;
9468 spin_unlock_irq(&cpu_rq(cpu)->lock);
9469#else
9470 data = *cpuusage;
9471#endif
9472
9473 return data;
9474}
9475
9476static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
9477{
9478 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9479
9480#ifndef CONFIG_64BIT
9481 /*
9482 * Take rq->lock to make 64-bit write safe on 32-bit platforms.
9483 */
9484 spin_lock_irq(&cpu_rq(cpu)->lock);
9485 *cpuusage = val;
9486 spin_unlock_irq(&cpu_rq(cpu)->lock);
9487#else
9488 *cpuusage = val;
9489#endif
9490}
9491
9302/* return total cpu usage (in nanoseconds) of a group */ 9492/* return total cpu usage (in nanoseconds) of a group */
9303static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft) 9493static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9304{ 9494{
@@ -9306,17 +9496,8 @@ static u64 cpuusage_read(struct cgroup *cgrp, struct cftype *cft)
9306 u64 totalcpuusage = 0; 9496 u64 totalcpuusage = 0;
9307 int i; 9497 int i;
9308 9498
9309 for_each_possible_cpu(i) { 9499 for_each_present_cpu(i)
9310 u64 *cpuusage = percpu_ptr(ca->cpuusage, i); 9500 totalcpuusage += cpuacct_cpuusage_read(ca, i);
9311
9312 /*
9313 * Take rq->lock to make 64-bit addition safe on 32-bit
9314 * platforms.
9315 */
9316 spin_lock_irq(&cpu_rq(i)->lock);
9317 totalcpuusage += *cpuusage;
9318 spin_unlock_irq(&cpu_rq(i)->lock);
9319 }
9320 9501
9321 return totalcpuusage; 9502 return totalcpuusage;
9322} 9503}
@@ -9333,23 +9514,39 @@ static int cpuusage_write(struct cgroup *cgrp, struct cftype *cftype,
9333 goto out; 9514 goto out;
9334 } 9515 }
9335 9516
9336 for_each_possible_cpu(i) { 9517 for_each_present_cpu(i)
9337 u64 *cpuusage = percpu_ptr(ca->cpuusage, i); 9518 cpuacct_cpuusage_write(ca, i, 0);
9338 9519
9339 spin_lock_irq(&cpu_rq(i)->lock);
9340 *cpuusage = 0;
9341 spin_unlock_irq(&cpu_rq(i)->lock);
9342 }
9343out: 9520out:
9344 return err; 9521 return err;
9345} 9522}
9346 9523
9524static int cpuacct_percpu_seq_read(struct cgroup *cgroup, struct cftype *cft,
9525 struct seq_file *m)
9526{
9527 struct cpuacct *ca = cgroup_ca(cgroup);
9528 u64 percpu;
9529 int i;
9530
9531 for_each_present_cpu(i) {
9532 percpu = cpuacct_cpuusage_read(ca, i);
9533 seq_printf(m, "%llu ", (unsigned long long) percpu);
9534 }
9535 seq_printf(m, "\n");
9536 return 0;
9537}
9538
9347static struct cftype files[] = { 9539static struct cftype files[] = {
9348 { 9540 {
9349 .name = "usage", 9541 .name = "usage",
9350 .read_u64 = cpuusage_read, 9542 .read_u64 = cpuusage_read,
9351 .write_u64 = cpuusage_write, 9543 .write_u64 = cpuusage_write,
9352 }, 9544 },
9545 {
9546 .name = "usage_percpu",
9547 .read_seq_string = cpuacct_percpu_seq_read,
9548 },
9549
9353}; 9550};
9354 9551
9355static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp) 9552static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
@@ -9365,14 +9562,16 @@ static int cpuacct_populate(struct cgroup_subsys *ss, struct cgroup *cgrp)
9365static void cpuacct_charge(struct task_struct *tsk, u64 cputime) 9562static void cpuacct_charge(struct task_struct *tsk, u64 cputime)
9366{ 9563{
9367 struct cpuacct *ca; 9564 struct cpuacct *ca;
9565 int cpu;
9368 9566
9369 if (!cpuacct_subsys.active) 9567 if (!cpuacct_subsys.active)
9370 return; 9568 return;
9371 9569
9570 cpu = task_cpu(tsk);
9372 ca = task_ca(tsk); 9571 ca = task_ca(tsk);
9373 if (ca) {
9374 u64 *cpuusage = percpu_ptr(ca->cpuusage, task_cpu(tsk));
9375 9572
9573 for (; ca; ca = ca->parent) {
9574 u64 *cpuusage = percpu_ptr(ca->cpuusage, cpu);
9376 *cpuusage += cputime; 9575 *cpuusage += cputime;
9377 } 9576 }
9378} 9577}
diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c
index 81787248b60f..e8ab096ddfe3 100644
--- a/kernel/sched_clock.c
+++ b/kernel/sched_clock.c
@@ -118,13 +118,13 @@ static u64 __update_sched_clock(struct sched_clock_data *scd, u64 now)
118 118
119 /* 119 /*
120 * scd->clock = clamp(scd->tick_gtod + delta, 120 * scd->clock = clamp(scd->tick_gtod + delta,
121 * max(scd->tick_gtod, scd->clock), 121 * max(scd->tick_gtod, scd->clock),
122 * max(scd->clock, scd->tick_gtod + TICK_NSEC)); 122 * scd->tick_gtod + TICK_NSEC);
123 */ 123 */
124 124
125 clock = scd->tick_gtod + delta; 125 clock = scd->tick_gtod + delta;
126 min_clock = wrap_max(scd->tick_gtod, scd->clock); 126 min_clock = wrap_max(scd->tick_gtod, scd->clock);
127 max_clock = wrap_max(scd->clock, scd->tick_gtod + TICK_NSEC); 127 max_clock = scd->tick_gtod + TICK_NSEC;
128 128
129 clock = wrap_max(clock, min_clock); 129 clock = wrap_max(clock, min_clock);
130 clock = wrap_min(clock, max_clock); 130 clock = wrap_min(clock, max_clock);
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
index 52154fefab7e..018b7be1db2e 100644
--- a/kernel/sched_cpupri.c
+++ b/kernel/sched_cpupri.c
@@ -67,24 +67,21 @@ static int convert_prio(int prio)
67 * Returns: (int)bool - CPUs were found 67 * Returns: (int)bool - CPUs were found
68 */ 68 */
69int cpupri_find(struct cpupri *cp, struct task_struct *p, 69int cpupri_find(struct cpupri *cp, struct task_struct *p,
70 cpumask_t *lowest_mask) 70 struct cpumask *lowest_mask)
71{ 71{
72 int idx = 0; 72 int idx = 0;
73 int task_pri = convert_prio(p->prio); 73 int task_pri = convert_prio(p->prio);
74 74
75 for_each_cpupri_active(cp->pri_active, idx) { 75 for_each_cpupri_active(cp->pri_active, idx) {
76 struct cpupri_vec *vec = &cp->pri_to_cpu[idx]; 76 struct cpupri_vec *vec = &cp->pri_to_cpu[idx];
77 cpumask_t mask;
78 77
79 if (idx >= task_pri) 78 if (idx >= task_pri)
80 break; 79 break;
81 80
82 cpus_and(mask, p->cpus_allowed, vec->mask); 81 if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
83
84 if (cpus_empty(mask))
85 continue; 82 continue;
86 83
87 *lowest_mask = mask; 84 cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
88 return 1; 85 return 1;
89 } 86 }
90 87
@@ -126,7 +123,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
126 vec->count--; 123 vec->count--;
127 if (!vec->count) 124 if (!vec->count)
128 clear_bit(oldpri, cp->pri_active); 125 clear_bit(oldpri, cp->pri_active);
129 cpu_clear(cpu, vec->mask); 126 cpumask_clear_cpu(cpu, vec->mask);
130 127
131 spin_unlock_irqrestore(&vec->lock, flags); 128 spin_unlock_irqrestore(&vec->lock, flags);
132 } 129 }
@@ -136,7 +133,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
136 133
137 spin_lock_irqsave(&vec->lock, flags); 134 spin_lock_irqsave(&vec->lock, flags);
138 135
139 cpu_set(cpu, vec->mask); 136 cpumask_set_cpu(cpu, vec->mask);
140 vec->count++; 137 vec->count++;
141 if (vec->count == 1) 138 if (vec->count == 1)
142 set_bit(newpri, cp->pri_active); 139 set_bit(newpri, cp->pri_active);
@@ -150,10 +147,11 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
150/** 147/**
151 * cpupri_init - initialize the cpupri structure 148 * cpupri_init - initialize the cpupri structure
152 * @cp: The cpupri context 149 * @cp: The cpupri context
150 * @bootmem: true if allocations need to use bootmem
153 * 151 *
154 * Returns: (void) 152 * Returns: -ENOMEM if memory fails.
155 */ 153 */
156void cpupri_init(struct cpupri *cp) 154int cpupri_init(struct cpupri *cp, bool bootmem)
157{ 155{
158 int i; 156 int i;
159 157
@@ -164,11 +162,30 @@ void cpupri_init(struct cpupri *cp)
164 162
165 spin_lock_init(&vec->lock); 163 spin_lock_init(&vec->lock);
166 vec->count = 0; 164 vec->count = 0;
167 cpus_clear(vec->mask); 165 if (bootmem)
166 alloc_bootmem_cpumask_var(&vec->mask);
167 else if (!alloc_cpumask_var(&vec->mask, GFP_KERNEL))
168 goto cleanup;
168 } 169 }
169 170
170 for_each_possible_cpu(i) 171 for_each_possible_cpu(i)
171 cp->cpu_to_pri[i] = CPUPRI_INVALID; 172 cp->cpu_to_pri[i] = CPUPRI_INVALID;
173 return 0;
174
175cleanup:
176 for (i--; i >= 0; i--)
177 free_cpumask_var(cp->pri_to_cpu[i].mask);
178 return -ENOMEM;
172} 179}
173 180
181/**
182 * cpupri_cleanup - clean up the cpupri structure
183 * @cp: The cpupri context
184 */
185void cpupri_cleanup(struct cpupri *cp)
186{
187 int i;
174 188
189 for (i = 0; i < CPUPRI_NR_PRIORITIES; i++)
190 free_cpumask_var(cp->pri_to_cpu[i].mask);
191}
diff --git a/kernel/sched_cpupri.h b/kernel/sched_cpupri.h
index f25811b0f931..642a94ef8a0a 100644
--- a/kernel/sched_cpupri.h
+++ b/kernel/sched_cpupri.h
@@ -14,7 +14,7 @@
14struct cpupri_vec { 14struct cpupri_vec {
15 spinlock_t lock; 15 spinlock_t lock;
16 int count; 16 int count;
17 cpumask_t mask; 17 cpumask_var_t mask;
18}; 18};
19 19
20struct cpupri { 20struct cpupri {
@@ -27,7 +27,8 @@ struct cpupri {
27int cpupri_find(struct cpupri *cp, 27int cpupri_find(struct cpupri *cp,
28 struct task_struct *p, cpumask_t *lowest_mask); 28 struct task_struct *p, cpumask_t *lowest_mask);
29void cpupri_set(struct cpupri *cp, int cpu, int pri); 29void cpupri_set(struct cpupri *cp, int cpu, int pri);
30void cpupri_init(struct cpupri *cp); 30int cpupri_init(struct cpupri *cp, bool bootmem);
31void cpupri_cleanup(struct cpupri *cp);
31#else 32#else
32#define cpupri_set(cp, cpu, pri) do { } while (0) 33#define cpupri_set(cp, cpu, pri) do { } while (0)
33#define cpupri_init() do { } while (0) 34#define cpupri_init() do { } while (0)
diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c
index 26ed8e3d1c15..4293cfa9681d 100644
--- a/kernel/sched_debug.c
+++ b/kernel/sched_debug.c
@@ -53,6 +53,40 @@ static unsigned long nsec_low(unsigned long long nsec)
53 53
54#define SPLIT_NS(x) nsec_high(x), nsec_low(x) 54#define SPLIT_NS(x) nsec_high(x), nsec_low(x)
55 55
56#ifdef CONFIG_FAIR_GROUP_SCHED
57static void print_cfs_group_stats(struct seq_file *m, int cpu,
58 struct task_group *tg)
59{
60 struct sched_entity *se = tg->se[cpu];
61 if (!se)
62 return;
63
64#define P(F) \
65 SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
66#define PN(F) \
67 SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
68
69 PN(se->exec_start);
70 PN(se->vruntime);
71 PN(se->sum_exec_runtime);
72#ifdef CONFIG_SCHEDSTATS
73 PN(se->wait_start);
74 PN(se->sleep_start);
75 PN(se->block_start);
76 PN(se->sleep_max);
77 PN(se->block_max);
78 PN(se->exec_max);
79 PN(se->slice_max);
80 PN(se->wait_max);
81 PN(se->wait_sum);
82 P(se->wait_count);
83#endif
84 P(se->load.weight);
85#undef PN
86#undef P
87}
88#endif
89
56static void 90static void
57print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) 91print_task(struct seq_file *m, struct rq *rq, struct task_struct *p)
58{ 92{
@@ -121,20 +155,19 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
121 155
122#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED) 156#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED)
123 char path[128] = ""; 157 char path[128] = "";
124 struct cgroup *cgroup = NULL;
125 struct task_group *tg = cfs_rq->tg; 158 struct task_group *tg = cfs_rq->tg;
126 159
127 if (tg) 160 cgroup_path(tg->css.cgroup, path, sizeof(path));
128 cgroup = tg->css.cgroup;
129
130 if (cgroup)
131 cgroup_path(cgroup, path, sizeof(path));
132 161
133 SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path); 162 SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path);
163#elif defined(CONFIG_USER_SCHED) && defined(CONFIG_FAIR_GROUP_SCHED)
164 {
165 uid_t uid = cfs_rq->tg->uid;
166 SEQ_printf(m, "\ncfs_rq[%d] for UID: %u\n", cpu, uid);
167 }
134#else 168#else
135 SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu); 169 SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu);
136#endif 170#endif
137
138 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", 171 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock",
139 SPLIT_NS(cfs_rq->exec_clock)); 172 SPLIT_NS(cfs_rq->exec_clock));
140 173
@@ -168,6 +201,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq)
168#ifdef CONFIG_SMP 201#ifdef CONFIG_SMP
169 SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares); 202 SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares);
170#endif 203#endif
204 print_cfs_group_stats(m, cpu, cfs_rq->tg);
171#endif 205#endif
172} 206}
173 207
@@ -175,14 +209,9 @@ void print_rt_rq(struct seq_file *m, int cpu, struct rt_rq *rt_rq)
175{ 209{
176#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED) 210#if defined(CONFIG_CGROUP_SCHED) && defined(CONFIG_RT_GROUP_SCHED)
177 char path[128] = ""; 211 char path[128] = "";
178 struct cgroup *cgroup = NULL;
179 struct task_group *tg = rt_rq->tg; 212 struct task_group *tg = rt_rq->tg;
180 213
181 if (tg) 214 cgroup_path(tg->css.cgroup, path, sizeof(path));
182 cgroup = tg->css.cgroup;
183
184 if (cgroup)
185 cgroup_path(cgroup, path, sizeof(path));
186 215
187 SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path); 216 SEQ_printf(m, "\nrt_rq[%d]:%s\n", cpu, path);
188#else 217#else
@@ -272,7 +301,7 @@ static int sched_debug_show(struct seq_file *m, void *v)
272 u64 now = ktime_to_ns(ktime_get()); 301 u64 now = ktime_to_ns(ktime_get());
273 int cpu; 302 int cpu;
274 303
275 SEQ_printf(m, "Sched Debug Version: v0.07, %s %.*s\n", 304 SEQ_printf(m, "Sched Debug Version: v0.08, %s %.*s\n",
276 init_utsname()->release, 305 init_utsname()->release,
277 (int)strcspn(init_utsname()->version, " "), 306 (int)strcspn(init_utsname()->version, " "),
278 init_utsname()->version); 307 init_utsname()->version);
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 98345e45b059..56c0efe902a7 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -492,6 +492,8 @@ static void update_curr(struct cfs_rq *cfs_rq)
492 * overflow on 32 bits): 492 * overflow on 32 bits):
493 */ 493 */
494 delta_exec = (unsigned long)(now - curr->exec_start); 494 delta_exec = (unsigned long)(now - curr->exec_start);
495 if (!delta_exec)
496 return;
495 497
496 __update_curr(cfs_rq, curr, delta_exec); 498 __update_curr(cfs_rq, curr, delta_exec);
497 curr->exec_start = now; 499 curr->exec_start = now;
@@ -1017,16 +1019,33 @@ static void yield_task_fair(struct rq *rq)
1017 * search starts with cpus closest then further out as needed, 1019 * search starts with cpus closest then further out as needed,
1018 * so we always favor a closer, idle cpu. 1020 * so we always favor a closer, idle cpu.
1019 * Domains may include CPUs that are not usable for migration, 1021 * Domains may include CPUs that are not usable for migration,
1020 * hence we need to mask them out (cpu_active_map) 1022 * hence we need to mask them out (cpu_active_mask)
1021 * 1023 *
1022 * Returns the CPU we should wake onto. 1024 * Returns the CPU we should wake onto.
1023 */ 1025 */
1024#if defined(ARCH_HAS_SCHED_WAKE_IDLE) 1026#if defined(ARCH_HAS_SCHED_WAKE_IDLE)
1025static int wake_idle(int cpu, struct task_struct *p) 1027static int wake_idle(int cpu, struct task_struct *p)
1026{ 1028{
1027 cpumask_t tmp;
1028 struct sched_domain *sd; 1029 struct sched_domain *sd;
1029 int i; 1030 int i;
1031 unsigned int chosen_wakeup_cpu;
1032 int this_cpu;
1033
1034 /*
1035 * At POWERSAVINGS_BALANCE_WAKEUP level, if both this_cpu and prev_cpu
1036 * are idle and this is not a kernel thread and this task's affinity
1037 * allows it to be moved to preferred cpu, then just move!
1038 */
1039
1040 this_cpu = smp_processor_id();
1041 chosen_wakeup_cpu =
1042 cpu_rq(this_cpu)->rd->sched_mc_preferred_wakeup_cpu;
1043
1044 if (sched_mc_power_savings >= POWERSAVINGS_BALANCE_WAKEUP &&
1045 idle_cpu(cpu) && idle_cpu(this_cpu) &&
1046 p->mm && !(p->flags & PF_KTHREAD) &&
1047 cpu_isset(chosen_wakeup_cpu, p->cpus_allowed))
1048 return chosen_wakeup_cpu;
1030 1049
1031 /* 1050 /*
1032 * If it is idle, then it is the best cpu to run this task. 1051 * If it is idle, then it is the best cpu to run this task.
@@ -1044,10 +1063,9 @@ static int wake_idle(int cpu, struct task_struct *p)
1044 if ((sd->flags & SD_WAKE_IDLE) 1063 if ((sd->flags & SD_WAKE_IDLE)
1045 || ((sd->flags & SD_WAKE_IDLE_FAR) 1064 || ((sd->flags & SD_WAKE_IDLE_FAR)
1046 && !task_hot(p, task_rq(p)->clock, sd))) { 1065 && !task_hot(p, task_rq(p)->clock, sd))) {
1047 cpus_and(tmp, sd->span, p->cpus_allowed); 1066 for_each_cpu_and(i, sched_domain_span(sd),
1048 cpus_and(tmp, tmp, cpu_active_map); 1067 &p->cpus_allowed) {
1049 for_each_cpu_mask_nr(i, tmp) { 1068 if (cpu_active(i) && idle_cpu(i)) {
1050 if (idle_cpu(i)) {
1051 if (i != task_cpu(p)) { 1069 if (i != task_cpu(p)) {
1052 schedstat_inc(p, 1070 schedstat_inc(p,
1053 se.nr_wakeups_idle); 1071 se.nr_wakeups_idle);
@@ -1240,13 +1258,13 @@ static int select_task_rq_fair(struct task_struct *p, int sync)
1240 * this_cpu and prev_cpu are present in: 1258 * this_cpu and prev_cpu are present in:
1241 */ 1259 */
1242 for_each_domain(this_cpu, sd) { 1260 for_each_domain(this_cpu, sd) {
1243 if (cpu_isset(prev_cpu, sd->span)) { 1261 if (cpumask_test_cpu(prev_cpu, sched_domain_span(sd))) {
1244 this_sd = sd; 1262 this_sd = sd;
1245 break; 1263 break;
1246 } 1264 }
1247 } 1265 }
1248 1266
1249 if (unlikely(!cpu_isset(this_cpu, p->cpus_allowed))) 1267 if (unlikely(!cpumask_test_cpu(this_cpu, &p->cpus_allowed)))
1250 goto out; 1268 goto out;
1251 1269
1252 /* 1270 /*
@@ -1345,12 +1363,11 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p, int sync)
1345{ 1363{
1346 struct task_struct *curr = rq->curr; 1364 struct task_struct *curr = rq->curr;
1347 struct sched_entity *se = &curr->se, *pse = &p->se; 1365 struct sched_entity *se = &curr->se, *pse = &p->se;
1366 struct cfs_rq *cfs_rq = task_cfs_rq(curr);
1348 1367
1349 if (unlikely(rt_prio(p->prio))) { 1368 update_curr(cfs_rq);
1350 struct cfs_rq *cfs_rq = task_cfs_rq(curr);
1351 1369
1352 update_rq_clock(rq); 1370 if (unlikely(rt_prio(p->prio))) {
1353 update_curr(cfs_rq);
1354 resched_task(curr); 1371 resched_task(curr);
1355 return; 1372 return;
1356 } 1373 }
diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c
index d9ba9d5f99d6..954e1a81b796 100644
--- a/kernel/sched_rt.c
+++ b/kernel/sched_rt.c
@@ -15,7 +15,7 @@ static inline void rt_set_overload(struct rq *rq)
15 if (!rq->online) 15 if (!rq->online)
16 return; 16 return;
17 17
18 cpu_set(rq->cpu, rq->rd->rto_mask); 18 cpumask_set_cpu(rq->cpu, rq->rd->rto_mask);
19 /* 19 /*
20 * Make sure the mask is visible before we set 20 * Make sure the mask is visible before we set
21 * the overload count. That is checked to determine 21 * the overload count. That is checked to determine
@@ -34,7 +34,7 @@ static inline void rt_clear_overload(struct rq *rq)
34 34
35 /* the order here really doesn't matter */ 35 /* the order here really doesn't matter */
36 atomic_dec(&rq->rd->rto_count); 36 atomic_dec(&rq->rd->rto_count);
37 cpu_clear(rq->cpu, rq->rd->rto_mask); 37 cpumask_clear_cpu(rq->cpu, rq->rd->rto_mask);
38} 38}
39 39
40static void update_rt_migration(struct rq *rq) 40static void update_rt_migration(struct rq *rq)
@@ -77,7 +77,7 @@ static inline u64 sched_rt_period(struct rt_rq *rt_rq)
77} 77}
78 78
79#define for_each_leaf_rt_rq(rt_rq, rq) \ 79#define for_each_leaf_rt_rq(rt_rq, rq) \
80 list_for_each_entry(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list) 80 list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list)
81 81
82static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq) 82static inline struct rq *rq_of_rt_rq(struct rt_rq *rt_rq)
83{ 83{
@@ -139,14 +139,14 @@ static int rt_se_boosted(struct sched_rt_entity *rt_se)
139} 139}
140 140
141#ifdef CONFIG_SMP 141#ifdef CONFIG_SMP
142static inline cpumask_t sched_rt_period_mask(void) 142static inline const struct cpumask *sched_rt_period_mask(void)
143{ 143{
144 return cpu_rq(smp_processor_id())->rd->span; 144 return cpu_rq(smp_processor_id())->rd->span;
145} 145}
146#else 146#else
147static inline cpumask_t sched_rt_period_mask(void) 147static inline const struct cpumask *sched_rt_period_mask(void)
148{ 148{
149 return cpu_online_map; 149 return cpu_online_mask;
150} 150}
151#endif 151#endif
152 152
@@ -212,9 +212,9 @@ static inline int rt_rq_throttled(struct rt_rq *rt_rq)
212 return rt_rq->rt_throttled; 212 return rt_rq->rt_throttled;
213} 213}
214 214
215static inline cpumask_t sched_rt_period_mask(void) 215static inline const struct cpumask *sched_rt_period_mask(void)
216{ 216{
217 return cpu_online_map; 217 return cpu_online_mask;
218} 218}
219 219
220static inline 220static inline
@@ -241,11 +241,11 @@ static int do_balance_runtime(struct rt_rq *rt_rq)
241 int i, weight, more = 0; 241 int i, weight, more = 0;
242 u64 rt_period; 242 u64 rt_period;
243 243
244 weight = cpus_weight(rd->span); 244 weight = cpumask_weight(rd->span);
245 245
246 spin_lock(&rt_b->rt_runtime_lock); 246 spin_lock(&rt_b->rt_runtime_lock);
247 rt_period = ktime_to_ns(rt_b->rt_period); 247 rt_period = ktime_to_ns(rt_b->rt_period);
248 for_each_cpu_mask_nr(i, rd->span) { 248 for_each_cpu(i, rd->span) {
249 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 249 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
250 s64 diff; 250 s64 diff;
251 251
@@ -324,7 +324,7 @@ static void __disable_runtime(struct rq *rq)
324 /* 324 /*
325 * Greedy reclaim, take back as much as we can. 325 * Greedy reclaim, take back as much as we can.
326 */ 326 */
327 for_each_cpu_mask(i, rd->span) { 327 for_each_cpu(i, rd->span) {
328 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); 328 struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
329 s64 diff; 329 s64 diff;
330 330
@@ -429,13 +429,13 @@ static inline int balance_runtime(struct rt_rq *rt_rq)
429static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) 429static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun)
430{ 430{
431 int i, idle = 1; 431 int i, idle = 1;
432 cpumask_t span; 432 const struct cpumask *span;
433 433
434 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF) 434 if (!rt_bandwidth_enabled() || rt_b->rt_runtime == RUNTIME_INF)
435 return 1; 435 return 1;
436 436
437 span = sched_rt_period_mask(); 437 span = sched_rt_period_mask();
438 for_each_cpu_mask(i, span) { 438 for_each_cpu(i, span) {
439 int enqueue = 0; 439 int enqueue = 0;
440 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); 440 struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
441 struct rq *rq = rq_of_rt_rq(rt_rq); 441 struct rq *rq = rq_of_rt_rq(rt_rq);
@@ -537,13 +537,13 @@ static void update_curr_rt(struct rq *rq)
537 for_each_sched_rt_entity(rt_se) { 537 for_each_sched_rt_entity(rt_se) {
538 rt_rq = rt_rq_of_se(rt_se); 538 rt_rq = rt_rq_of_se(rt_se);
539 539
540 spin_lock(&rt_rq->rt_runtime_lock);
541 if (sched_rt_runtime(rt_rq) != RUNTIME_INF) { 540 if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
541 spin_lock(&rt_rq->rt_runtime_lock);
542 rt_rq->rt_time += delta_exec; 542 rt_rq->rt_time += delta_exec;
543 if (sched_rt_runtime_exceeded(rt_rq)) 543 if (sched_rt_runtime_exceeded(rt_rq))
544 resched_task(curr); 544 resched_task(curr);
545 spin_unlock(&rt_rq->rt_runtime_lock);
545 } 546 }
546 spin_unlock(&rt_rq->rt_runtime_lock);
547 } 547 }
548} 548}
549 549
@@ -805,17 +805,20 @@ static int select_task_rq_rt(struct task_struct *p, int sync)
805 805
806static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p) 806static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
807{ 807{
808 cpumask_t mask; 808 cpumask_var_t mask;
809 809
810 if (rq->curr->rt.nr_cpus_allowed == 1) 810 if (rq->curr->rt.nr_cpus_allowed == 1)
811 return; 811 return;
812 812
813 if (p->rt.nr_cpus_allowed != 1 813 if (!alloc_cpumask_var(&mask, GFP_ATOMIC))
814 && cpupri_find(&rq->rd->cpupri, p, &mask))
815 return; 814 return;
816 815
817 if (!cpupri_find(&rq->rd->cpupri, rq->curr, &mask)) 816 if (p->rt.nr_cpus_allowed != 1
818 return; 817 && cpupri_find(&rq->rd->cpupri, p, mask))
818 goto free;
819
820 if (!cpupri_find(&rq->rd->cpupri, rq->curr, mask))
821 goto free;
819 822
820 /* 823 /*
821 * There appears to be other cpus that can accept 824 * There appears to be other cpus that can accept
@@ -824,6 +827,8 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
824 */ 827 */
825 requeue_task_rt(rq, p, 1); 828 requeue_task_rt(rq, p, 1);
826 resched_task(rq->curr); 829 resched_task(rq->curr);
830free:
831 free_cpumask_var(mask);
827} 832}
828 833
829#endif /* CONFIG_SMP */ 834#endif /* CONFIG_SMP */
@@ -909,15 +914,12 @@ static void put_prev_task_rt(struct rq *rq, struct task_struct *p)
909/* Only try algorithms three times */ 914/* Only try algorithms three times */
910#define RT_MAX_TRIES 3 915#define RT_MAX_TRIES 3
911 916
912static int double_lock_balance(struct rq *this_rq, struct rq *busiest);
913static void double_unlock_balance(struct rq *this_rq, struct rq *busiest);
914
915static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep); 917static void deactivate_task(struct rq *rq, struct task_struct *p, int sleep);
916 918
917static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu) 919static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
918{ 920{
919 if (!task_running(rq, p) && 921 if (!task_running(rq, p) &&
920 (cpu < 0 || cpu_isset(cpu, p->cpus_allowed)) && 922 (cpu < 0 || cpumask_test_cpu(cpu, &p->cpus_allowed)) &&
921 (p->rt.nr_cpus_allowed > 1)) 923 (p->rt.nr_cpus_allowed > 1))
922 return 1; 924 return 1;
923 return 0; 925 return 0;
@@ -956,7 +958,7 @@ static struct task_struct *pick_next_highest_task_rt(struct rq *rq, int cpu)
956 return next; 958 return next;
957} 959}
958 960
959static DEFINE_PER_CPU(cpumask_t, local_cpu_mask); 961static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
960 962
961static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask) 963static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
962{ 964{
@@ -976,7 +978,7 @@ static inline int pick_optimal_cpu(int this_cpu, cpumask_t *mask)
976static int find_lowest_rq(struct task_struct *task) 978static int find_lowest_rq(struct task_struct *task)
977{ 979{
978 struct sched_domain *sd; 980 struct sched_domain *sd;
979 cpumask_t *lowest_mask = &__get_cpu_var(local_cpu_mask); 981 struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);
980 int this_cpu = smp_processor_id(); 982 int this_cpu = smp_processor_id();
981 int cpu = task_cpu(task); 983 int cpu = task_cpu(task);
982 984
@@ -991,7 +993,7 @@ static int find_lowest_rq(struct task_struct *task)
991 * I guess we might want to change cpupri_find() to ignore those 993 * I guess we might want to change cpupri_find() to ignore those
992 * in the first place. 994 * in the first place.
993 */ 995 */
994 cpus_and(*lowest_mask, *lowest_mask, cpu_active_map); 996 cpumask_and(lowest_mask, lowest_mask, cpu_active_mask);
995 997
996 /* 998 /*
997 * At this point we have built a mask of cpus representing the 999 * At this point we have built a mask of cpus representing the
@@ -1001,7 +1003,7 @@ static int find_lowest_rq(struct task_struct *task)
1001 * We prioritize the last cpu that the task executed on since 1003 * We prioritize the last cpu that the task executed on since
1002 * it is most likely cache-hot in that location. 1004 * it is most likely cache-hot in that location.
1003 */ 1005 */
1004 if (cpu_isset(cpu, *lowest_mask)) 1006 if (cpumask_test_cpu(cpu, lowest_mask))
1005 return cpu; 1007 return cpu;
1006 1008
1007 /* 1009 /*
@@ -1016,7 +1018,8 @@ static int find_lowest_rq(struct task_struct *task)
1016 cpumask_t domain_mask; 1018 cpumask_t domain_mask;
1017 int best_cpu; 1019 int best_cpu;
1018 1020
1019 cpus_and(domain_mask, sd->span, *lowest_mask); 1021 cpumask_and(&domain_mask, sched_domain_span(sd),
1022 lowest_mask);
1020 1023
1021 best_cpu = pick_optimal_cpu(this_cpu, 1024 best_cpu = pick_optimal_cpu(this_cpu,
1022 &domain_mask); 1025 &domain_mask);
@@ -1057,8 +1060,8 @@ static struct rq *find_lock_lowest_rq(struct task_struct *task, struct rq *rq)
1057 * Also make sure that it wasn't scheduled on its rq. 1060 * Also make sure that it wasn't scheduled on its rq.
1058 */ 1061 */
1059 if (unlikely(task_rq(task) != rq || 1062 if (unlikely(task_rq(task) != rq ||
1060 !cpu_isset(lowest_rq->cpu, 1063 !cpumask_test_cpu(lowest_rq->cpu,
1061 task->cpus_allowed) || 1064 &task->cpus_allowed) ||
1062 task_running(rq, task) || 1065 task_running(rq, task) ||
1063 !task->se.on_rq)) { 1066 !task->se.on_rq)) {
1064 1067
@@ -1179,7 +1182,7 @@ static int pull_rt_task(struct rq *this_rq)
1179 1182
1180 next = pick_next_task_rt(this_rq); 1183 next = pick_next_task_rt(this_rq);
1181 1184
1182 for_each_cpu_mask_nr(cpu, this_rq->rd->rto_mask) { 1185 for_each_cpu(cpu, this_rq->rd->rto_mask) {
1183 if (this_cpu == cpu) 1186 if (this_cpu == cpu)
1184 continue; 1187 continue;
1185 1188
@@ -1308,9 +1311,9 @@ move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
1308} 1311}
1309 1312
1310static void set_cpus_allowed_rt(struct task_struct *p, 1313static void set_cpus_allowed_rt(struct task_struct *p,
1311 const cpumask_t *new_mask) 1314 const struct cpumask *new_mask)
1312{ 1315{
1313 int weight = cpus_weight(*new_mask); 1316 int weight = cpumask_weight(new_mask);
1314 1317
1315 BUG_ON(!rt_task(p)); 1318 BUG_ON(!rt_task(p));
1316 1319
@@ -1331,7 +1334,7 @@ static void set_cpus_allowed_rt(struct task_struct *p,
1331 update_rt_migration(rq); 1334 update_rt_migration(rq);
1332 } 1335 }
1333 1336
1334 p->cpus_allowed = *new_mask; 1337 cpumask_copy(&p->cpus_allowed, new_mask);
1335 p->rt.nr_cpus_allowed = weight; 1338 p->rt.nr_cpus_allowed = weight;
1336} 1339}
1337 1340
@@ -1374,6 +1377,15 @@ static void switched_from_rt(struct rq *rq, struct task_struct *p,
1374 if (!rq->rt.rt_nr_running) 1377 if (!rq->rt.rt_nr_running)
1375 pull_rt_task(rq); 1378 pull_rt_task(rq);
1376} 1379}
1380
1381static inline void init_sched_rt_class(void)
1382{
1383 unsigned int i;
1384
1385 for_each_possible_cpu(i)
1386 alloc_cpumask_var_node(&per_cpu(local_cpu_mask, i),
1387 GFP_KERNEL, cpu_to_node(i));
1388}
1377#endif /* CONFIG_SMP */ 1389#endif /* CONFIG_SMP */
1378 1390
1379/* 1391/*
@@ -1544,3 +1556,4 @@ static void print_rt_stats(struct seq_file *m, int cpu)
1544 rcu_read_unlock(); 1556 rcu_read_unlock();
1545} 1557}
1546#endif /* CONFIG_SCHED_DEBUG */ 1558#endif /* CONFIG_SCHED_DEBUG */
1559
diff --git a/kernel/sched_stats.h b/kernel/sched_stats.h
index 7dbf72a2b02c..f2773b5d1226 100644
--- a/kernel/sched_stats.h
+++ b/kernel/sched_stats.h
@@ -31,7 +31,7 @@ static int show_schedstat(struct seq_file *seq, void *v)
31 rq->yld_act_empty, rq->yld_exp_empty, rq->yld_count, 31 rq->yld_act_empty, rq->yld_exp_empty, rq->yld_count,
32 rq->sched_switch, rq->sched_count, rq->sched_goidle, 32 rq->sched_switch, rq->sched_count, rq->sched_goidle,
33 rq->ttwu_count, rq->ttwu_local, 33 rq->ttwu_count, rq->ttwu_local,
34 rq->rq_sched_info.cpu_time, 34 rq->rq_cpu_time,
35 rq->rq_sched_info.run_delay, rq->rq_sched_info.pcount); 35 rq->rq_sched_info.run_delay, rq->rq_sched_info.pcount);
36 36
37 seq_printf(seq, "\n"); 37 seq_printf(seq, "\n");
@@ -42,7 +42,8 @@ static int show_schedstat(struct seq_file *seq, void *v)
42 for_each_domain(cpu, sd) { 42 for_each_domain(cpu, sd) {
43 enum cpu_idle_type itype; 43 enum cpu_idle_type itype;
44 44
45 cpumask_scnprintf(mask_str, mask_len, sd->span); 45 cpumask_scnprintf(mask_str, mask_len,
46 sched_domain_span(sd));
46 seq_printf(seq, "domain%d %s", dcount++, mask_str); 47 seq_printf(seq, "domain%d %s", dcount++, mask_str);
47 for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES; 48 for (itype = CPU_IDLE; itype < CPU_MAX_IDLE_TYPES;
48 itype++) { 49 itype++) {
@@ -123,7 +124,7 @@ static inline void
123rq_sched_info_depart(struct rq *rq, unsigned long long delta) 124rq_sched_info_depart(struct rq *rq, unsigned long long delta)
124{ 125{
125 if (rq) 126 if (rq)
126 rq->rq_sched_info.cpu_time += delta; 127 rq->rq_cpu_time += delta;
127} 128}
128 129
129static inline void 130static inline void
@@ -236,7 +237,6 @@ static inline void sched_info_depart(struct task_struct *t)
236 unsigned long long delta = task_rq(t)->clock - 237 unsigned long long delta = task_rq(t)->clock -
237 t->sched_info.last_arrival; 238 t->sched_info.last_arrival;
238 239
239 t->sched_info.cpu_time += delta;
240 rq_sched_info_depart(task_rq(t), delta); 240 rq_sched_info_depart(task_rq(t), delta);
241 241
242 if (t->state == TASK_RUNNING) 242 if (t->state == TASK_RUNNING)
diff --git a/kernel/signal.c b/kernel/signal.c
index 4530fc654455..8e95855ff3cf 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -41,6 +41,8 @@
41 41
42static struct kmem_cache *sigqueue_cachep; 42static struct kmem_cache *sigqueue_cachep;
43 43
44DEFINE_TRACE(sched_signal_send);
45
44static void __user *sig_handler(struct task_struct *t, int sig) 46static void __user *sig_handler(struct task_struct *t, int sig)
45{ 47{
46 return t->sighand->action[sig - 1].sa.sa_handler; 48 return t->sighand->action[sig - 1].sa.sa_handler;
@@ -177,6 +179,11 @@ int next_signal(struct sigpending *pending, sigset_t *mask)
177 return sig; 179 return sig;
178} 180}
179 181
182/*
183 * allocate a new signal queue record
184 * - this may be called without locks if and only if t == current, otherwise an
185 * appopriate lock must be held to stop the target task from exiting
186 */
180static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags, 187static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags,
181 int override_rlimit) 188 int override_rlimit)
182{ 189{
@@ -184,11 +191,12 @@ static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags,
184 struct user_struct *user; 191 struct user_struct *user;
185 192
186 /* 193 /*
187 * In order to avoid problems with "switch_user()", we want to make 194 * We won't get problems with the target's UID changing under us
188 * sure that the compiler doesn't re-load "t->user" 195 * because changing it requires RCU be used, and if t != current, the
196 * caller must be holding the RCU readlock (by way of a spinlock) and
197 * we use RCU protection here
189 */ 198 */
190 user = t->user; 199 user = get_uid(__task_cred(t)->user);
191 barrier();
192 atomic_inc(&user->sigpending); 200 atomic_inc(&user->sigpending);
193 if (override_rlimit || 201 if (override_rlimit ||
194 atomic_read(&user->sigpending) <= 202 atomic_read(&user->sigpending) <=
@@ -196,12 +204,14 @@ static struct sigqueue *__sigqueue_alloc(struct task_struct *t, gfp_t flags,
196 q = kmem_cache_alloc(sigqueue_cachep, flags); 204 q = kmem_cache_alloc(sigqueue_cachep, flags);
197 if (unlikely(q == NULL)) { 205 if (unlikely(q == NULL)) {
198 atomic_dec(&user->sigpending); 206 atomic_dec(&user->sigpending);
207 free_uid(user);
199 } else { 208 } else {
200 INIT_LIST_HEAD(&q->list); 209 INIT_LIST_HEAD(&q->list);
201 q->flags = 0; 210 q->flags = 0;
202 q->user = get_uid(user); 211 q->user = user;
203 } 212 }
204 return(q); 213
214 return q;
205} 215}
206 216
207static void __sigqueue_free(struct sigqueue *q) 217static void __sigqueue_free(struct sigqueue *q)
@@ -562,10 +572,12 @@ static int rm_from_queue(unsigned long mask, struct sigpending *s)
562 572
563/* 573/*
564 * Bad permissions for sending the signal 574 * Bad permissions for sending the signal
575 * - the caller must hold at least the RCU read lock
565 */ 576 */
566static int check_kill_permission(int sig, struct siginfo *info, 577static int check_kill_permission(int sig, struct siginfo *info,
567 struct task_struct *t) 578 struct task_struct *t)
568{ 579{
580 const struct cred *cred = current_cred(), *tcred;
569 struct pid *sid; 581 struct pid *sid;
570 int error; 582 int error;
571 583
@@ -579,8 +591,11 @@ static int check_kill_permission(int sig, struct siginfo *info,
579 if (error) 591 if (error)
580 return error; 592 return error;
581 593
582 if ((current->euid ^ t->suid) && (current->euid ^ t->uid) && 594 tcred = __task_cred(t);
583 (current->uid ^ t->suid) && (current->uid ^ t->uid) && 595 if ((cred->euid ^ tcred->suid) &&
596 (cred->euid ^ tcred->uid) &&
597 (cred->uid ^ tcred->suid) &&
598 (cred->uid ^ tcred->uid) &&
584 !capable(CAP_KILL)) { 599 !capable(CAP_KILL)) {
585 switch (sig) { 600 switch (sig) {
586 case SIGCONT: 601 case SIGCONT:
@@ -844,7 +859,7 @@ static int send_signal(int sig, struct siginfo *info, struct task_struct *t,
844 q->info.si_errno = 0; 859 q->info.si_errno = 0;
845 q->info.si_code = SI_USER; 860 q->info.si_code = SI_USER;
846 q->info.si_pid = task_pid_vnr(current); 861 q->info.si_pid = task_pid_vnr(current);
847 q->info.si_uid = current->uid; 862 q->info.si_uid = current_uid();
848 break; 863 break;
849 case (unsigned long) SEND_SIG_PRIV: 864 case (unsigned long) SEND_SIG_PRIV:
850 q->info.si_signo = sig; 865 q->info.si_signo = sig;
@@ -1008,6 +1023,10 @@ struct sighand_struct *lock_task_sighand(struct task_struct *tsk, unsigned long
1008 return sighand; 1023 return sighand;
1009} 1024}
1010 1025
1026/*
1027 * send signal info to all the members of a group
1028 * - the caller must hold the RCU read lock at least
1029 */
1011int group_send_sig_info(int sig, struct siginfo *info, struct task_struct *p) 1030int group_send_sig_info(int sig, struct siginfo *info, struct task_struct *p)
1012{ 1031{
1013 unsigned long flags; 1032 unsigned long flags;
@@ -1029,8 +1048,8 @@ int group_send_sig_info(int sig, struct siginfo *info, struct task_struct *p)
1029/* 1048/*
1030 * __kill_pgrp_info() sends a signal to a process group: this is what the tty 1049 * __kill_pgrp_info() sends a signal to a process group: this is what the tty
1031 * control characters do (^C, ^Z etc) 1050 * control characters do (^C, ^Z etc)
1051 * - the caller must hold at least a readlock on tasklist_lock
1032 */ 1052 */
1033
1034int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp) 1053int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp)
1035{ 1054{
1036 struct task_struct *p = NULL; 1055 struct task_struct *p = NULL;
@@ -1086,6 +1105,7 @@ int kill_pid_info_as_uid(int sig, struct siginfo *info, struct pid *pid,
1086{ 1105{
1087 int ret = -EINVAL; 1106 int ret = -EINVAL;
1088 struct task_struct *p; 1107 struct task_struct *p;
1108 const struct cred *pcred;
1089 1109
1090 if (!valid_signal(sig)) 1110 if (!valid_signal(sig))
1091 return ret; 1111 return ret;
@@ -1096,9 +1116,11 @@ int kill_pid_info_as_uid(int sig, struct siginfo *info, struct pid *pid,
1096 ret = -ESRCH; 1116 ret = -ESRCH;
1097 goto out_unlock; 1117 goto out_unlock;
1098 } 1118 }
1099 if ((info == SEND_SIG_NOINFO || (!is_si_special(info) && SI_FROMUSER(info))) 1119 pcred = __task_cred(p);
1100 && (euid != p->suid) && (euid != p->uid) 1120 if ((info == SEND_SIG_NOINFO ||
1101 && (uid != p->suid) && (uid != p->uid)) { 1121 (!is_si_special(info) && SI_FROMUSER(info))) &&
1122 euid != pcred->suid && euid != pcred->uid &&
1123 uid != pcred->suid && uid != pcred->uid) {
1102 ret = -EPERM; 1124 ret = -EPERM;
1103 goto out_unlock; 1125 goto out_unlock;
1104 } 1126 }
@@ -1369,10 +1391,9 @@ int do_notify_parent(struct task_struct *tsk, int sig)
1369 */ 1391 */
1370 rcu_read_lock(); 1392 rcu_read_lock();
1371 info.si_pid = task_pid_nr_ns(tsk, tsk->parent->nsproxy->pid_ns); 1393 info.si_pid = task_pid_nr_ns(tsk, tsk->parent->nsproxy->pid_ns);
1394 info.si_uid = __task_cred(tsk)->uid;
1372 rcu_read_unlock(); 1395 rcu_read_unlock();
1373 1396
1374 info.si_uid = tsk->uid;
1375
1376 thread_group_cputime(tsk, &cputime); 1397 thread_group_cputime(tsk, &cputime);
1377 info.si_utime = cputime_to_jiffies(cputime.utime); 1398 info.si_utime = cputime_to_jiffies(cputime.utime);
1378 info.si_stime = cputime_to_jiffies(cputime.stime); 1399 info.si_stime = cputime_to_jiffies(cputime.stime);
@@ -1440,10 +1461,9 @@ static void do_notify_parent_cldstop(struct task_struct *tsk, int why)
1440 */ 1461 */
1441 rcu_read_lock(); 1462 rcu_read_lock();
1442 info.si_pid = task_pid_nr_ns(tsk, tsk->parent->nsproxy->pid_ns); 1463 info.si_pid = task_pid_nr_ns(tsk, tsk->parent->nsproxy->pid_ns);
1464 info.si_uid = __task_cred(tsk)->uid;
1443 rcu_read_unlock(); 1465 rcu_read_unlock();
1444 1466
1445 info.si_uid = tsk->uid;
1446
1447 info.si_utime = cputime_to_clock_t(tsk->utime); 1467 info.si_utime = cputime_to_clock_t(tsk->utime);
1448 info.si_stime = cputime_to_clock_t(tsk->stime); 1468 info.si_stime = cputime_to_clock_t(tsk->stime);
1449 1469
@@ -1598,7 +1618,7 @@ void ptrace_notify(int exit_code)
1598 info.si_signo = SIGTRAP; 1618 info.si_signo = SIGTRAP;
1599 info.si_code = exit_code; 1619 info.si_code = exit_code;
1600 info.si_pid = task_pid_vnr(current); 1620 info.si_pid = task_pid_vnr(current);
1601 info.si_uid = current->uid; 1621 info.si_uid = current_uid();
1602 1622
1603 /* Let the debugger run. */ 1623 /* Let the debugger run. */
1604 spin_lock_irq(&current->sighand->siglock); 1624 spin_lock_irq(&current->sighand->siglock);
@@ -1710,7 +1730,7 @@ static int ptrace_signal(int signr, siginfo_t *info,
1710 info->si_errno = 0; 1730 info->si_errno = 0;
1711 info->si_code = SI_USER; 1731 info->si_code = SI_USER;
1712 info->si_pid = task_pid_vnr(current->parent); 1732 info->si_pid = task_pid_vnr(current->parent);
1713 info->si_uid = current->parent->uid; 1733 info->si_uid = task_uid(current->parent);
1714 } 1734 }
1715 1735
1716 /* If the (new) signal is now blocked, requeue it. */ 1736 /* If the (new) signal is now blocked, requeue it. */
@@ -2211,7 +2231,7 @@ sys_kill(pid_t pid, int sig)
2211 info.si_errno = 0; 2231 info.si_errno = 0;
2212 info.si_code = SI_USER; 2232 info.si_code = SI_USER;
2213 info.si_pid = task_tgid_vnr(current); 2233 info.si_pid = task_tgid_vnr(current);
2214 info.si_uid = current->uid; 2234 info.si_uid = current_uid();
2215 2235
2216 return kill_something_info(sig, &info, pid); 2236 return kill_something_info(sig, &info, pid);
2217} 2237}
@@ -2228,7 +2248,7 @@ static int do_tkill(pid_t tgid, pid_t pid, int sig)
2228 info.si_errno = 0; 2248 info.si_errno = 0;
2229 info.si_code = SI_TKILL; 2249 info.si_code = SI_TKILL;
2230 info.si_pid = task_tgid_vnr(current); 2250 info.si_pid = task_tgid_vnr(current);
2231 info.si_uid = current->uid; 2251 info.si_uid = current_uid();
2232 2252
2233 rcu_read_lock(); 2253 rcu_read_lock();
2234 p = find_task_by_vpid(pid); 2254 p = find_task_by_vpid(pid);
diff --git a/kernel/smp.c b/kernel/smp.c
index 75c8dde58c55..5cfa0e5e3e88 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -24,8 +24,8 @@ struct call_function_data {
24 struct call_single_data csd; 24 struct call_single_data csd;
25 spinlock_t lock; 25 spinlock_t lock;
26 unsigned int refs; 26 unsigned int refs;
27 cpumask_t cpumask;
28 struct rcu_head rcu_head; 27 struct rcu_head rcu_head;
28 unsigned long cpumask_bits[];
29}; 29};
30 30
31struct call_single_queue { 31struct call_single_queue {
@@ -110,13 +110,13 @@ void generic_smp_call_function_interrupt(void)
110 list_for_each_entry_rcu(data, &call_function_queue, csd.list) { 110 list_for_each_entry_rcu(data, &call_function_queue, csd.list) {
111 int refs; 111 int refs;
112 112
113 if (!cpu_isset(cpu, data->cpumask)) 113 if (!cpumask_test_cpu(cpu, to_cpumask(data->cpumask_bits)))
114 continue; 114 continue;
115 115
116 data->csd.func(data->csd.info); 116 data->csd.func(data->csd.info);
117 117
118 spin_lock(&data->lock); 118 spin_lock(&data->lock);
119 cpu_clear(cpu, data->cpumask); 119 cpumask_clear_cpu(cpu, to_cpumask(data->cpumask_bits));
120 WARN_ON(data->refs == 0); 120 WARN_ON(data->refs == 0);
121 data->refs--; 121 data->refs--;
122 refs = data->refs; 122 refs = data->refs;
@@ -223,7 +223,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
223 local_irq_save(flags); 223 local_irq_save(flags);
224 func(info); 224 func(info);
225 local_irq_restore(flags); 225 local_irq_restore(flags);
226 } else if ((unsigned)cpu < NR_CPUS && cpu_online(cpu)) { 226 } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
227 struct call_single_data *data = NULL; 227 struct call_single_data *data = NULL;
228 228
229 if (!wait) { 229 if (!wait) {
@@ -266,51 +266,19 @@ void __smp_call_function_single(int cpu, struct call_single_data *data)
266 generic_exec_single(cpu, data); 266 generic_exec_single(cpu, data);
267} 267}
268 268
269/* Dummy function */ 269/* FIXME: Shim for archs using old arch_send_call_function_ipi API. */
270static void quiesce_dummy(void *unused) 270#ifndef arch_send_call_function_ipi_mask
271{ 271#define arch_send_call_function_ipi_mask(maskp) \
272} 272 arch_send_call_function_ipi(*(maskp))
273 273#endif
274/*
275 * Ensure stack based data used in call function mask is safe to free.
276 *
277 * This is needed by smp_call_function_mask when using on-stack data, because
278 * a single call function queue is shared by all CPUs, and any CPU may pick up
279 * the data item on the queue at any time before it is deleted. So we need to
280 * ensure that all CPUs have transitioned through a quiescent state after
281 * this call.
282 *
283 * This is a very slow function, implemented by sending synchronous IPIs to
284 * all possible CPUs. For this reason, we have to alloc data rather than use
285 * stack based data even in the case of synchronous calls. The stack based
286 * data is then just used for deadlock/oom fallback which will be very rare.
287 *
288 * If a faster scheme can be made, we could go back to preferring stack based
289 * data -- the data allocation/free is non-zero cost.
290 */
291static void smp_call_function_mask_quiesce_stack(cpumask_t mask)
292{
293 struct call_single_data data;
294 int cpu;
295
296 data.func = quiesce_dummy;
297 data.info = NULL;
298
299 for_each_cpu_mask(cpu, mask) {
300 data.flags = CSD_FLAG_WAIT;
301 generic_exec_single(cpu, &data);
302 }
303}
304 274
305/** 275/**
306 * smp_call_function_mask(): Run a function on a set of other CPUs. 276 * smp_call_function_many(): Run a function on a set of other CPUs.
307 * @mask: The set of cpus to run on. 277 * @mask: The set of cpus to run on (only runs on online subset).
308 * @func: The function to run. This must be fast and non-blocking. 278 * @func: The function to run. This must be fast and non-blocking.
309 * @info: An arbitrary pointer to pass to the function. 279 * @info: An arbitrary pointer to pass to the function.
310 * @wait: If true, wait (atomically) until function has completed on other CPUs. 280 * @wait: If true, wait (atomically) until function has completed on other CPUs.
311 * 281 *
312 * Returns 0 on success, else a negative status code.
313 *
314 * If @wait is true, then returns once @func has returned. Note that @wait 282 * If @wait is true, then returns once @func has returned. Note that @wait
315 * will be implicitly turned on in case of allocation failures, since 283 * will be implicitly turned on in case of allocation failures, since
316 * we fall back to on-stack allocation. 284 * we fall back to on-stack allocation.
@@ -319,53 +287,57 @@ static void smp_call_function_mask_quiesce_stack(cpumask_t mask)
319 * hardware interrupt handler or from a bottom half handler. Preemption 287 * hardware interrupt handler or from a bottom half handler. Preemption
320 * must be disabled when calling this function. 288 * must be disabled when calling this function.
321 */ 289 */
322int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info, 290void smp_call_function_many(const struct cpumask *mask,
323 int wait) 291 void (*func)(void *), void *info,
292 bool wait)
324{ 293{
325 struct call_function_data d; 294 struct call_function_data *data;
326 struct call_function_data *data = NULL;
327 cpumask_t allbutself;
328 unsigned long flags; 295 unsigned long flags;
329 int cpu, num_cpus; 296 int cpu, next_cpu;
330 int slowpath = 0;
331 297
332 /* Can deadlock when called with interrupts disabled */ 298 /* Can deadlock when called with interrupts disabled */
333 WARN_ON(irqs_disabled()); 299 WARN_ON(irqs_disabled());
334 300
335 cpu = smp_processor_id(); 301 /* So, what's a CPU they want? Ignoring this one. */
336 allbutself = cpu_online_map; 302 cpu = cpumask_first_and(mask, cpu_online_mask);
337 cpu_clear(cpu, allbutself); 303 if (cpu == smp_processor_id())
338 cpus_and(mask, mask, allbutself); 304 cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
339 num_cpus = cpus_weight(mask); 305 /* No online cpus? We're done. */
340 306 if (cpu >= nr_cpu_ids)
341 /* 307 return;
342 * If zero CPUs, return. If just a single CPU, turn this request 308
343 * into a targetted single call instead since it's faster. 309 /* Do we have another CPU which isn't us? */
344 */ 310 next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
345 if (!num_cpus) 311 if (next_cpu == smp_processor_id())
346 return 0; 312 next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask);
347 else if (num_cpus == 1) { 313
348 cpu = first_cpu(mask); 314 /* Fastpath: do that cpu by itself. */
349 return smp_call_function_single(cpu, func, info, wait); 315 if (next_cpu >= nr_cpu_ids) {
316 smp_call_function_single(cpu, func, info, wait);
317 return;
350 } 318 }
351 319
352 data = kmalloc(sizeof(*data), GFP_ATOMIC); 320 data = kmalloc(sizeof(*data) + cpumask_size(), GFP_ATOMIC);
353 if (data) { 321 if (unlikely(!data)) {
354 data->csd.flags = CSD_FLAG_ALLOC; 322 /* Slow path. */
355 if (wait) 323 for_each_online_cpu(cpu) {
356 data->csd.flags |= CSD_FLAG_WAIT; 324 if (cpu == smp_processor_id())
357 } else { 325 continue;
358 data = &d; 326 if (cpumask_test_cpu(cpu, mask))
359 data->csd.flags = CSD_FLAG_WAIT; 327 smp_call_function_single(cpu, func, info, wait);
360 wait = 1; 328 }
361 slowpath = 1; 329 return;
362 } 330 }
363 331
364 spin_lock_init(&data->lock); 332 spin_lock_init(&data->lock);
333 data->csd.flags = CSD_FLAG_ALLOC;
334 if (wait)
335 data->csd.flags |= CSD_FLAG_WAIT;
365 data->csd.func = func; 336 data->csd.func = func;
366 data->csd.info = info; 337 data->csd.info = info;
367 data->refs = num_cpus; 338 cpumask_and(to_cpumask(data->cpumask_bits), mask, cpu_online_mask);
368 data->cpumask = mask; 339 cpumask_clear_cpu(smp_processor_id(), to_cpumask(data->cpumask_bits));
340 data->refs = cpumask_weight(to_cpumask(data->cpumask_bits));
369 341
370 spin_lock_irqsave(&call_function_lock, flags); 342 spin_lock_irqsave(&call_function_lock, flags);
371 list_add_tail_rcu(&data->csd.list, &call_function_queue); 343 list_add_tail_rcu(&data->csd.list, &call_function_queue);
@@ -377,18 +349,13 @@ int smp_call_function_mask(cpumask_t mask, void (*func)(void *), void *info,
377 smp_mb(); 349 smp_mb();
378 350
379 /* Send a message to all CPUs in the map */ 351 /* Send a message to all CPUs in the map */
380 arch_send_call_function_ipi(mask); 352 arch_send_call_function_ipi_mask(to_cpumask(data->cpumask_bits));
381 353
382 /* optionally wait for the CPUs to complete */ 354 /* optionally wait for the CPUs to complete */
383 if (wait) { 355 if (wait)
384 csd_flag_wait(&data->csd); 356 csd_flag_wait(&data->csd);
385 if (unlikely(slowpath))
386 smp_call_function_mask_quiesce_stack(mask);
387 }
388
389 return 0;
390} 357}
391EXPORT_SYMBOL(smp_call_function_mask); 358EXPORT_SYMBOL(smp_call_function_many);
392 359
393/** 360/**
394 * smp_call_function(): Run a function on all other CPUs. 361 * smp_call_function(): Run a function on all other CPUs.
@@ -396,7 +363,7 @@ EXPORT_SYMBOL(smp_call_function_mask);
396 * @info: An arbitrary pointer to pass to the function. 363 * @info: An arbitrary pointer to pass to the function.
397 * @wait: If true, wait (atomically) until function has completed on other CPUs. 364 * @wait: If true, wait (atomically) until function has completed on other CPUs.
398 * 365 *
399 * Returns 0 on success, else a negative status code. 366 * Returns 0.
400 * 367 *
401 * If @wait is true, then returns once @func has returned; otherwise 368 * If @wait is true, then returns once @func has returned; otherwise
402 * it returns just before the target cpu calls @func. In case of allocation 369 * it returns just before the target cpu calls @func. In case of allocation
@@ -407,12 +374,10 @@ EXPORT_SYMBOL(smp_call_function_mask);
407 */ 374 */
408int smp_call_function(void (*func)(void *), void *info, int wait) 375int smp_call_function(void (*func)(void *), void *info, int wait)
409{ 376{
410 int ret;
411
412 preempt_disable(); 377 preempt_disable();
413 ret = smp_call_function_mask(cpu_online_map, func, info, wait); 378 smp_call_function_many(cpu_online_mask, func, info, wait);
414 preempt_enable(); 379 preempt_enable();
415 return ret; 380 return 0;
416} 381}
417EXPORT_SYMBOL(smp_call_function); 382EXPORT_SYMBOL(smp_call_function);
418 383
diff --git a/kernel/softirq.c b/kernel/softirq.c
index e7c69a720d69..bdbe9de9cd8d 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -102,20 +102,6 @@ void local_bh_disable(void)
102 102
103EXPORT_SYMBOL(local_bh_disable); 103EXPORT_SYMBOL(local_bh_disable);
104 104
105void __local_bh_enable(void)
106{
107 WARN_ON_ONCE(in_irq());
108
109 /*
110 * softirqs should never be enabled by __local_bh_enable(),
111 * it always nests inside local_bh_enable() sections:
112 */
113 WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
114
115 sub_preempt_count(SOFTIRQ_OFFSET);
116}
117EXPORT_SYMBOL_GPL(__local_bh_enable);
118
119/* 105/*
120 * Special-case - softirqs can safely be enabled in 106 * Special-case - softirqs can safely be enabled in
121 * cond_resched_softirq(), or by __do_softirq(), 107 * cond_resched_softirq(), or by __do_softirq(),
@@ -269,6 +255,7 @@ void irq_enter(void)
269{ 255{
270 int cpu = smp_processor_id(); 256 int cpu = smp_processor_id();
271 257
258 rcu_irq_enter();
272 if (idle_cpu(cpu) && !in_interrupt()) { 259 if (idle_cpu(cpu) && !in_interrupt()) {
273 __irq_enter(); 260 __irq_enter();
274 tick_check_idle(cpu); 261 tick_check_idle(cpu);
@@ -295,9 +282,9 @@ void irq_exit(void)
295 282
296#ifdef CONFIG_NO_HZ 283#ifdef CONFIG_NO_HZ
297 /* Make sure that timer wheel updates are propagated */ 284 /* Make sure that timer wheel updates are propagated */
298 if (!in_interrupt() && idle_cpu(smp_processor_id()) && !need_resched())
299 tick_nohz_stop_sched_tick(0);
300 rcu_irq_exit(); 285 rcu_irq_exit();
286 if (idle_cpu(smp_processor_id()) && !in_interrupt() && !need_resched())
287 tick_nohz_stop_sched_tick(0);
301#endif 288#endif
302 preempt_enable_no_resched(); 289 preempt_enable_no_resched();
303} 290}
@@ -746,7 +733,7 @@ static int __cpuinit cpu_callback(struct notifier_block *nfb,
746 break; 733 break;
747 /* Unbind so it can run. Fall thru. */ 734 /* Unbind so it can run. Fall thru. */
748 kthread_bind(per_cpu(ksoftirqd, hotcpu), 735 kthread_bind(per_cpu(ksoftirqd, hotcpu),
749 any_online_cpu(cpu_online_map)); 736 cpumask_any(cpu_online_mask));
750 case CPU_DEAD: 737 case CPU_DEAD:
751 case CPU_DEAD_FROZEN: { 738 case CPU_DEAD_FROZEN: {
752 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 739 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
@@ -797,3 +784,23 @@ int on_each_cpu(void (*func) (void *info), void *info, int wait)
797} 784}
798EXPORT_SYMBOL(on_each_cpu); 785EXPORT_SYMBOL(on_each_cpu);
799#endif 786#endif
787
788/*
789 * [ These __weak aliases are kept in a separate compilation unit, so that
790 * GCC does not inline them incorrectly. ]
791 */
792
793int __init __weak early_irq_init(void)
794{
795 return 0;
796}
797
798int __init __weak arch_early_irq_init(void)
799{
800 return 0;
801}
802
803int __weak arch_init_chip_data(struct irq_desc *desc, int cpu)
804{
805 return 0;
806}
diff --git a/kernel/softlockup.c b/kernel/softlockup.c
index 3953e4aed733..d9188c66278a 100644
--- a/kernel/softlockup.c
+++ b/kernel/softlockup.c
@@ -164,7 +164,7 @@ unsigned long __read_mostly sysctl_hung_task_check_count = 1024;
164/* 164/*
165 * Zero means infinite timeout - no checking done: 165 * Zero means infinite timeout - no checking done:
166 */ 166 */
167unsigned long __read_mostly sysctl_hung_task_timeout_secs = 120; 167unsigned long __read_mostly sysctl_hung_task_timeout_secs = 480;
168 168
169unsigned long __read_mostly sysctl_hung_task_warnings = 10; 169unsigned long __read_mostly sysctl_hung_task_warnings = 10;
170 170
@@ -188,7 +188,7 @@ static void check_hung_task(struct task_struct *t, unsigned long now)
188 if ((long)(now - t->last_switch_timestamp) < 188 if ((long)(now - t->last_switch_timestamp) <
189 sysctl_hung_task_timeout_secs) 189 sysctl_hung_task_timeout_secs)
190 return; 190 return;
191 if (sysctl_hung_task_warnings < 0) 191 if (!sysctl_hung_task_warnings)
192 return; 192 return;
193 sysctl_hung_task_warnings--; 193 sysctl_hung_task_warnings--;
194 194
@@ -303,17 +303,15 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
303 break; 303 break;
304 case CPU_ONLINE: 304 case CPU_ONLINE:
305 case CPU_ONLINE_FROZEN: 305 case CPU_ONLINE_FROZEN:
306 check_cpu = any_online_cpu(cpu_online_map); 306 check_cpu = cpumask_any(cpu_online_mask);
307 wake_up_process(per_cpu(watchdog_task, hotcpu)); 307 wake_up_process(per_cpu(watchdog_task, hotcpu));
308 break; 308 break;
309#ifdef CONFIG_HOTPLUG_CPU 309#ifdef CONFIG_HOTPLUG_CPU
310 case CPU_DOWN_PREPARE: 310 case CPU_DOWN_PREPARE:
311 case CPU_DOWN_PREPARE_FROZEN: 311 case CPU_DOWN_PREPARE_FROZEN:
312 if (hotcpu == check_cpu) { 312 if (hotcpu == check_cpu) {
313 cpumask_t temp_cpu_online_map = cpu_online_map; 313 /* Pick any other online cpu. */
314 314 check_cpu = cpumask_any_but(cpu_online_mask, hotcpu);
315 cpu_clear(hotcpu, temp_cpu_online_map);
316 check_cpu = any_online_cpu(temp_cpu_online_map);
317 } 315 }
318 break; 316 break;
319 317
@@ -323,7 +321,7 @@ cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
323 break; 321 break;
324 /* Unbind so it can run. Fall thru. */ 322 /* Unbind so it can run. Fall thru. */
325 kthread_bind(per_cpu(watchdog_task, hotcpu), 323 kthread_bind(per_cpu(watchdog_task, hotcpu),
326 any_online_cpu(cpu_online_map)); 324 cpumask_any(cpu_online_mask));
327 case CPU_DEAD: 325 case CPU_DEAD:
328 case CPU_DEAD_FROZEN: 326 case CPU_DEAD_FROZEN:
329 p = per_cpu(watchdog_task, hotcpu); 327 p = per_cpu(watchdog_task, hotcpu);
diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c
index 94b527ef1d1e..eb212f8f8bc8 100644
--- a/kernel/stacktrace.c
+++ b/kernel/stacktrace.c
@@ -6,6 +6,7 @@
6 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com> 6 * Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
7 */ 7 */
8#include <linux/sched.h> 8#include <linux/sched.h>
9#include <linux/kernel.h>
9#include <linux/module.h> 10#include <linux/module.h>
10#include <linux/kallsyms.h> 11#include <linux/kallsyms.h>
11#include <linux/stacktrace.h> 12#include <linux/stacktrace.h>
@@ -24,3 +25,13 @@ void print_stack_trace(struct stack_trace *trace, int spaces)
24} 25}
25EXPORT_SYMBOL_GPL(print_stack_trace); 26EXPORT_SYMBOL_GPL(print_stack_trace);
26 27
28/*
29 * Architectures that do not implement save_stack_trace_tsk get this
30 * weak alias and a once-per-bootup warning (whenever this facility
31 * is utilized - for example by procfs):
32 */
33__weak void
34save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
35{
36 WARN_ONCE(1, KERN_INFO "save_stack_trace_tsk() not implemented yet.\n");
37}
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 24e8ceacc388..0cd415ee62a2 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -38,7 +38,10 @@ struct stop_machine_data {
38static unsigned int num_threads; 38static unsigned int num_threads;
39static atomic_t thread_ack; 39static atomic_t thread_ack;
40static DEFINE_MUTEX(lock); 40static DEFINE_MUTEX(lock);
41 41/* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */
42static DEFINE_MUTEX(setup_lock);
43/* Users of stop_machine. */
44static int refcount;
42static struct workqueue_struct *stop_machine_wq; 45static struct workqueue_struct *stop_machine_wq;
43static struct stop_machine_data active, idle; 46static struct stop_machine_data active, idle;
44static const cpumask_t *active_cpus; 47static const cpumask_t *active_cpus;
@@ -69,10 +72,10 @@ static void stop_cpu(struct work_struct *unused)
69 int err; 72 int err;
70 73
71 if (!active_cpus) { 74 if (!active_cpus) {
72 if (cpu == first_cpu(cpu_online_map)) 75 if (cpu == cpumask_first(cpu_online_mask))
73 smdata = &active; 76 smdata = &active;
74 } else { 77 } else {
75 if (cpu_isset(cpu, *active_cpus)) 78 if (cpumask_test_cpu(cpu, active_cpus))
76 smdata = &active; 79 smdata = &active;
77 } 80 }
78 /* Simple state machine */ 81 /* Simple state machine */
@@ -109,7 +112,44 @@ static int chill(void *unused)
109 return 0; 112 return 0;
110} 113}
111 114
112int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) 115int stop_machine_create(void)
116{
117 mutex_lock(&setup_lock);
118 if (refcount)
119 goto done;
120 stop_machine_wq = create_rt_workqueue("kstop");
121 if (!stop_machine_wq)
122 goto err_out;
123 stop_machine_work = alloc_percpu(struct work_struct);
124 if (!stop_machine_work)
125 goto err_out;
126done:
127 refcount++;
128 mutex_unlock(&setup_lock);
129 return 0;
130
131err_out:
132 if (stop_machine_wq)
133 destroy_workqueue(stop_machine_wq);
134 mutex_unlock(&setup_lock);
135 return -ENOMEM;
136}
137EXPORT_SYMBOL_GPL(stop_machine_create);
138
139void stop_machine_destroy(void)
140{
141 mutex_lock(&setup_lock);
142 refcount--;
143 if (refcount)
144 goto done;
145 destroy_workqueue(stop_machine_wq);
146 free_percpu(stop_machine_work);
147done:
148 mutex_unlock(&setup_lock);
149}
150EXPORT_SYMBOL_GPL(stop_machine_destroy);
151
152int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
113{ 153{
114 struct work_struct *sm_work; 154 struct work_struct *sm_work;
115 int i, ret; 155 int i, ret;
@@ -142,23 +182,18 @@ int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus)
142 return ret; 182 return ret;
143} 183}
144 184
145int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) 185int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus)
146{ 186{
147 int ret; 187 int ret;
148 188
189 ret = stop_machine_create();
190 if (ret)
191 return ret;
149 /* No CPUs can come up or down during this. */ 192 /* No CPUs can come up or down during this. */
150 get_online_cpus(); 193 get_online_cpus();
151 ret = __stop_machine(fn, data, cpus); 194 ret = __stop_machine(fn, data, cpus);
152 put_online_cpus(); 195 put_online_cpus();
153 196 stop_machine_destroy();
154 return ret; 197 return ret;
155} 198}
156EXPORT_SYMBOL_GPL(stop_machine); 199EXPORT_SYMBOL_GPL(stop_machine);
157
158static int __init stop_machine_init(void)
159{
160 stop_machine_wq = create_rt_workqueue("kstop");
161 stop_machine_work = alloc_percpu(struct work_struct);
162 return 0;
163}
164core_initcall(stop_machine_init);
diff --git a/kernel/sys.c b/kernel/sys.c
index 31deba8f7d16..d356d79e84ac 100644
--- a/kernel/sys.c
+++ b/kernel/sys.c
@@ -112,12 +112,17 @@ EXPORT_SYMBOL(cad_pid);
112 112
113void (*pm_power_off_prepare)(void); 113void (*pm_power_off_prepare)(void);
114 114
115/*
116 * set the priority of a task
117 * - the caller must hold the RCU read lock
118 */
115static int set_one_prio(struct task_struct *p, int niceval, int error) 119static int set_one_prio(struct task_struct *p, int niceval, int error)
116{ 120{
121 const struct cred *cred = current_cred(), *pcred = __task_cred(p);
117 int no_nice; 122 int no_nice;
118 123
119 if (p->uid != current->euid && 124 if (pcred->uid != cred->euid &&
120 p->euid != current->euid && !capable(CAP_SYS_NICE)) { 125 pcred->euid != cred->euid && !capable(CAP_SYS_NICE)) {
121 error = -EPERM; 126 error = -EPERM;
122 goto out; 127 goto out;
123 } 128 }
@@ -141,6 +146,7 @@ asmlinkage long sys_setpriority(int which, int who, int niceval)
141{ 146{
142 struct task_struct *g, *p; 147 struct task_struct *g, *p;
143 struct user_struct *user; 148 struct user_struct *user;
149 const struct cred *cred = current_cred();
144 int error = -EINVAL; 150 int error = -EINVAL;
145 struct pid *pgrp; 151 struct pid *pgrp;
146 152
@@ -174,18 +180,18 @@ asmlinkage long sys_setpriority(int which, int who, int niceval)
174 } while_each_pid_thread(pgrp, PIDTYPE_PGID, p); 180 } while_each_pid_thread(pgrp, PIDTYPE_PGID, p);
175 break; 181 break;
176 case PRIO_USER: 182 case PRIO_USER:
177 user = current->user; 183 user = (struct user_struct *) cred->user;
178 if (!who) 184 if (!who)
179 who = current->uid; 185 who = cred->uid;
180 else 186 else if ((who != cred->uid) &&
181 if ((who != current->uid) && !(user = find_user(who))) 187 !(user = find_user(who)))
182 goto out_unlock; /* No processes for this user */ 188 goto out_unlock; /* No processes for this user */
183 189
184 do_each_thread(g, p) 190 do_each_thread(g, p)
185 if (p->uid == who) 191 if (__task_cred(p)->uid == who)
186 error = set_one_prio(p, niceval, error); 192 error = set_one_prio(p, niceval, error);
187 while_each_thread(g, p); 193 while_each_thread(g, p);
188 if (who != current->uid) 194 if (who != cred->uid)
189 free_uid(user); /* For find_user() */ 195 free_uid(user); /* For find_user() */
190 break; 196 break;
191 } 197 }
@@ -205,6 +211,7 @@ asmlinkage long sys_getpriority(int which, int who)
205{ 211{
206 struct task_struct *g, *p; 212 struct task_struct *g, *p;
207 struct user_struct *user; 213 struct user_struct *user;
214 const struct cred *cred = current_cred();
208 long niceval, retval = -ESRCH; 215 long niceval, retval = -ESRCH;
209 struct pid *pgrp; 216 struct pid *pgrp;
210 217
@@ -236,21 +243,21 @@ asmlinkage long sys_getpriority(int which, int who)
236 } while_each_pid_thread(pgrp, PIDTYPE_PGID, p); 243 } while_each_pid_thread(pgrp, PIDTYPE_PGID, p);
237 break; 244 break;
238 case PRIO_USER: 245 case PRIO_USER:
239 user = current->user; 246 user = (struct user_struct *) cred->user;
240 if (!who) 247 if (!who)
241 who = current->uid; 248 who = cred->uid;
242 else 249 else if ((who != cred->uid) &&
243 if ((who != current->uid) && !(user = find_user(who))) 250 !(user = find_user(who)))
244 goto out_unlock; /* No processes for this user */ 251 goto out_unlock; /* No processes for this user */
245 252
246 do_each_thread(g, p) 253 do_each_thread(g, p)
247 if (p->uid == who) { 254 if (__task_cred(p)->uid == who) {
248 niceval = 20 - task_nice(p); 255 niceval = 20 - task_nice(p);
249 if (niceval > retval) 256 if (niceval > retval)
250 retval = niceval; 257 retval = niceval;
251 } 258 }
252 while_each_thread(g, p); 259 while_each_thread(g, p);
253 if (who != current->uid) 260 if (who != cred->uid)
254 free_uid(user); /* for find_user() */ 261 free_uid(user); /* for find_user() */
255 break; 262 break;
256 } 263 }
@@ -472,46 +479,48 @@ void ctrl_alt_del(void)
472 */ 479 */
473asmlinkage long sys_setregid(gid_t rgid, gid_t egid) 480asmlinkage long sys_setregid(gid_t rgid, gid_t egid)
474{ 481{
475 int old_rgid = current->gid; 482 const struct cred *old;
476 int old_egid = current->egid; 483 struct cred *new;
477 int new_rgid = old_rgid;
478 int new_egid = old_egid;
479 int retval; 484 int retval;
480 485
486 new = prepare_creds();
487 if (!new)
488 return -ENOMEM;
489 old = current_cred();
490
481 retval = security_task_setgid(rgid, egid, (gid_t)-1, LSM_SETID_RE); 491 retval = security_task_setgid(rgid, egid, (gid_t)-1, LSM_SETID_RE);
482 if (retval) 492 if (retval)
483 return retval; 493 goto error;
484 494
495 retval = -EPERM;
485 if (rgid != (gid_t) -1) { 496 if (rgid != (gid_t) -1) {
486 if ((old_rgid == rgid) || 497 if (old->gid == rgid ||
487 (current->egid==rgid) || 498 old->egid == rgid ||
488 capable(CAP_SETGID)) 499 capable(CAP_SETGID))
489 new_rgid = rgid; 500 new->gid = rgid;
490 else 501 else
491 return -EPERM; 502 goto error;
492 } 503 }
493 if (egid != (gid_t) -1) { 504 if (egid != (gid_t) -1) {
494 if ((old_rgid == egid) || 505 if (old->gid == egid ||
495 (current->egid == egid) || 506 old->egid == egid ||
496 (current->sgid == egid) || 507 old->sgid == egid ||
497 capable(CAP_SETGID)) 508 capable(CAP_SETGID))
498 new_egid = egid; 509 new->egid = egid;
499 else 510 else
500 return -EPERM; 511 goto error;
501 }
502 if (new_egid != old_egid) {
503 set_dumpable(current->mm, suid_dumpable);
504 smp_wmb();
505 } 512 }
513
506 if (rgid != (gid_t) -1 || 514 if (rgid != (gid_t) -1 ||
507 (egid != (gid_t) -1 && egid != old_rgid)) 515 (egid != (gid_t) -1 && egid != old->gid))
508 current->sgid = new_egid; 516 new->sgid = new->egid;
509 current->fsgid = new_egid; 517 new->fsgid = new->egid;
510 current->egid = new_egid; 518
511 current->gid = new_rgid; 519 return commit_creds(new);
512 key_fsgid_changed(current); 520
513 proc_id_connector(current, PROC_EVENT_GID); 521error:
514 return 0; 522 abort_creds(new);
523 return retval;
515} 524}
516 525
517/* 526/*
@@ -521,56 +530,54 @@ asmlinkage long sys_setregid(gid_t rgid, gid_t egid)
521 */ 530 */
522asmlinkage long sys_setgid(gid_t gid) 531asmlinkage long sys_setgid(gid_t gid)
523{ 532{
524 int old_egid = current->egid; 533 const struct cred *old;
534 struct cred *new;
525 int retval; 535 int retval;
526 536
537 new = prepare_creds();
538 if (!new)
539 return -ENOMEM;
540 old = current_cred();
541
527 retval = security_task_setgid(gid, (gid_t)-1, (gid_t)-1, LSM_SETID_ID); 542 retval = security_task_setgid(gid, (gid_t)-1, (gid_t)-1, LSM_SETID_ID);
528 if (retval) 543 if (retval)
529 return retval; 544 goto error;
530 545
531 if (capable(CAP_SETGID)) { 546 retval = -EPERM;
532 if (old_egid != gid) { 547 if (capable(CAP_SETGID))
533 set_dumpable(current->mm, suid_dumpable); 548 new->gid = new->egid = new->sgid = new->fsgid = gid;
534 smp_wmb(); 549 else if (gid == old->gid || gid == old->sgid)
535 } 550 new->egid = new->fsgid = gid;
536 current->gid = current->egid = current->sgid = current->fsgid = gid;
537 } else if ((gid == current->gid) || (gid == current->sgid)) {
538 if (old_egid != gid) {
539 set_dumpable(current->mm, suid_dumpable);
540 smp_wmb();
541 }
542 current->egid = current->fsgid = gid;
543 }
544 else 551 else
545 return -EPERM; 552 goto error;
546 553
547 key_fsgid_changed(current); 554 return commit_creds(new);
548 proc_id_connector(current, PROC_EVENT_GID); 555
549 return 0; 556error:
557 abort_creds(new);
558 return retval;
550} 559}
551 560
552static int set_user(uid_t new_ruid, int dumpclear) 561/*
562 * change the user struct in a credentials set to match the new UID
563 */
564static int set_user(struct cred *new)
553{ 565{
554 struct user_struct *new_user; 566 struct user_struct *new_user;
555 567
556 new_user = alloc_uid(current->nsproxy->user_ns, new_ruid); 568 new_user = alloc_uid(current_user_ns(), new->uid);
557 if (!new_user) 569 if (!new_user)
558 return -EAGAIN; 570 return -EAGAIN;
559 571
560 if (atomic_read(&new_user->processes) >= 572 if (atomic_read(&new_user->processes) >=
561 current->signal->rlim[RLIMIT_NPROC].rlim_cur && 573 current->signal->rlim[RLIMIT_NPROC].rlim_cur &&
562 new_user != current->nsproxy->user_ns->root_user) { 574 new_user != INIT_USER) {
563 free_uid(new_user); 575 free_uid(new_user);
564 return -EAGAIN; 576 return -EAGAIN;
565 } 577 }
566 578
567 switch_uid(new_user); 579 free_uid(new->user);
568 580 new->user = new_user;
569 if (dumpclear) {
570 set_dumpable(current->mm, suid_dumpable);
571 smp_wmb();
572 }
573 current->uid = new_ruid;
574 return 0; 581 return 0;
575} 582}
576 583
@@ -591,54 +598,56 @@ static int set_user(uid_t new_ruid, int dumpclear)
591 */ 598 */
592asmlinkage long sys_setreuid(uid_t ruid, uid_t euid) 599asmlinkage long sys_setreuid(uid_t ruid, uid_t euid)
593{ 600{
594 int old_ruid, old_euid, old_suid, new_ruid, new_euid; 601 const struct cred *old;
602 struct cred *new;
595 int retval; 603 int retval;
596 604
605 new = prepare_creds();
606 if (!new)
607 return -ENOMEM;
608 old = current_cred();
609
597 retval = security_task_setuid(ruid, euid, (uid_t)-1, LSM_SETID_RE); 610 retval = security_task_setuid(ruid, euid, (uid_t)-1, LSM_SETID_RE);
598 if (retval) 611 if (retval)
599 return retval; 612 goto error;
600
601 new_ruid = old_ruid = current->uid;
602 new_euid = old_euid = current->euid;
603 old_suid = current->suid;
604 613
614 retval = -EPERM;
605 if (ruid != (uid_t) -1) { 615 if (ruid != (uid_t) -1) {
606 new_ruid = ruid; 616 new->uid = ruid;
607 if ((old_ruid != ruid) && 617 if (old->uid != ruid &&
608 (current->euid != ruid) && 618 old->euid != ruid &&
609 !capable(CAP_SETUID)) 619 !capable(CAP_SETUID))
610 return -EPERM; 620 goto error;
611 } 621 }
612 622
613 if (euid != (uid_t) -1) { 623 if (euid != (uid_t) -1) {
614 new_euid = euid; 624 new->euid = euid;
615 if ((old_ruid != euid) && 625 if (old->uid != euid &&
616 (current->euid != euid) && 626 old->euid != euid &&
617 (current->suid != euid) && 627 old->suid != euid &&
618 !capable(CAP_SETUID)) 628 !capable(CAP_SETUID))
619 return -EPERM; 629 goto error;
620 } 630 }
621 631
622 if (new_ruid != old_ruid && set_user(new_ruid, new_euid != old_euid) < 0) 632 retval = -EAGAIN;
623 return -EAGAIN; 633 if (new->uid != old->uid && set_user(new) < 0)
634 goto error;
624 635
625 if (new_euid != old_euid) {
626 set_dumpable(current->mm, suid_dumpable);
627 smp_wmb();
628 }
629 current->fsuid = current->euid = new_euid;
630 if (ruid != (uid_t) -1 || 636 if (ruid != (uid_t) -1 ||
631 (euid != (uid_t) -1 && euid != old_ruid)) 637 (euid != (uid_t) -1 && euid != old->uid))
632 current->suid = current->euid; 638 new->suid = new->euid;
633 current->fsuid = current->euid; 639 new->fsuid = new->euid;
634 640
635 key_fsuid_changed(current); 641 retval = security_task_fix_setuid(new, old, LSM_SETID_RE);
636 proc_id_connector(current, PROC_EVENT_UID); 642 if (retval < 0)
637 643 goto error;
638 return security_task_post_setuid(old_ruid, old_euid, old_suid, LSM_SETID_RE);
639}
640 644
645 return commit_creds(new);
641 646
647error:
648 abort_creds(new);
649 return retval;
650}
642 651
643/* 652/*
644 * setuid() is implemented like SysV with SAVED_IDS 653 * setuid() is implemented like SysV with SAVED_IDS
@@ -653,36 +662,41 @@ asmlinkage long sys_setreuid(uid_t ruid, uid_t euid)
653 */ 662 */
654asmlinkage long sys_setuid(uid_t uid) 663asmlinkage long sys_setuid(uid_t uid)
655{ 664{
656 int old_euid = current->euid; 665 const struct cred *old;
657 int old_ruid, old_suid, new_suid; 666 struct cred *new;
658 int retval; 667 int retval;
659 668
669 new = prepare_creds();
670 if (!new)
671 return -ENOMEM;
672 old = current_cred();
673
660 retval = security_task_setuid(uid, (uid_t)-1, (uid_t)-1, LSM_SETID_ID); 674 retval = security_task_setuid(uid, (uid_t)-1, (uid_t)-1, LSM_SETID_ID);
661 if (retval) 675 if (retval)
662 return retval; 676 goto error;
663 677
664 old_ruid = current->uid; 678 retval = -EPERM;
665 old_suid = current->suid;
666 new_suid = old_suid;
667
668 if (capable(CAP_SETUID)) { 679 if (capable(CAP_SETUID)) {
669 if (uid != old_ruid && set_user(uid, old_euid != uid) < 0) 680 new->suid = new->uid = uid;
670 return -EAGAIN; 681 if (uid != old->uid && set_user(new) < 0) {
671 new_suid = uid; 682 retval = -EAGAIN;
672 } else if ((uid != current->uid) && (uid != new_suid)) 683 goto error;
673 return -EPERM; 684 }
674 685 } else if (uid != old->uid && uid != new->suid) {
675 if (old_euid != uid) { 686 goto error;
676 set_dumpable(current->mm, suid_dumpable);
677 smp_wmb();
678 } 687 }
679 current->fsuid = current->euid = uid;
680 current->suid = new_suid;
681 688
682 key_fsuid_changed(current); 689 new->fsuid = new->euid = uid;
683 proc_id_connector(current, PROC_EVENT_UID); 690
691 retval = security_task_fix_setuid(new, old, LSM_SETID_ID);
692 if (retval < 0)
693 goto error;
684 694
685 return security_task_post_setuid(old_ruid, old_euid, old_suid, LSM_SETID_ID); 695 return commit_creds(new);
696
697error:
698 abort_creds(new);
699 return retval;
686} 700}
687 701
688 702
@@ -692,54 +706,63 @@ asmlinkage long sys_setuid(uid_t uid)
692 */ 706 */
693asmlinkage long sys_setresuid(uid_t ruid, uid_t euid, uid_t suid) 707asmlinkage long sys_setresuid(uid_t ruid, uid_t euid, uid_t suid)
694{ 708{
695 int old_ruid = current->uid; 709 const struct cred *old;
696 int old_euid = current->euid; 710 struct cred *new;
697 int old_suid = current->suid;
698 int retval; 711 int retval;
699 712
713 new = prepare_creds();
714 if (!new)
715 return -ENOMEM;
716
700 retval = security_task_setuid(ruid, euid, suid, LSM_SETID_RES); 717 retval = security_task_setuid(ruid, euid, suid, LSM_SETID_RES);
701 if (retval) 718 if (retval)
702 return retval; 719 goto error;
720 old = current_cred();
703 721
722 retval = -EPERM;
704 if (!capable(CAP_SETUID)) { 723 if (!capable(CAP_SETUID)) {
705 if ((ruid != (uid_t) -1) && (ruid != current->uid) && 724 if (ruid != (uid_t) -1 && ruid != old->uid &&
706 (ruid != current->euid) && (ruid != current->suid)) 725 ruid != old->euid && ruid != old->suid)
707 return -EPERM; 726 goto error;
708 if ((euid != (uid_t) -1) && (euid != current->uid) && 727 if (euid != (uid_t) -1 && euid != old->uid &&
709 (euid != current->euid) && (euid != current->suid)) 728 euid != old->euid && euid != old->suid)
710 return -EPERM; 729 goto error;
711 if ((suid != (uid_t) -1) && (suid != current->uid) && 730 if (suid != (uid_t) -1 && suid != old->uid &&
712 (suid != current->euid) && (suid != current->suid)) 731 suid != old->euid && suid != old->suid)
713 return -EPERM; 732 goto error;
714 } 733 }
734
735 retval = -EAGAIN;
715 if (ruid != (uid_t) -1) { 736 if (ruid != (uid_t) -1) {
716 if (ruid != current->uid && set_user(ruid, euid != current->euid) < 0) 737 new->uid = ruid;
717 return -EAGAIN; 738 if (ruid != old->uid && set_user(new) < 0)
739 goto error;
718 } 740 }
719 if (euid != (uid_t) -1) { 741 if (euid != (uid_t) -1)
720 if (euid != current->euid) { 742 new->euid = euid;
721 set_dumpable(current->mm, suid_dumpable);
722 smp_wmb();
723 }
724 current->euid = euid;
725 }
726 current->fsuid = current->euid;
727 if (suid != (uid_t) -1) 743 if (suid != (uid_t) -1)
728 current->suid = suid; 744 new->suid = suid;
745 new->fsuid = new->euid;
746
747 retval = security_task_fix_setuid(new, old, LSM_SETID_RES);
748 if (retval < 0)
749 goto error;
729 750
730 key_fsuid_changed(current); 751 return commit_creds(new);
731 proc_id_connector(current, PROC_EVENT_UID);
732 752
733 return security_task_post_setuid(old_ruid, old_euid, old_suid, LSM_SETID_RES); 753error:
754 abort_creds(new);
755 return retval;
734} 756}
735 757
736asmlinkage long sys_getresuid(uid_t __user *ruid, uid_t __user *euid, uid_t __user *suid) 758asmlinkage long sys_getresuid(uid_t __user *ruid, uid_t __user *euid, uid_t __user *suid)
737{ 759{
760 const struct cred *cred = current_cred();
738 int retval; 761 int retval;
739 762
740 if (!(retval = put_user(current->uid, ruid)) && 763 if (!(retval = put_user(cred->uid, ruid)) &&
741 !(retval = put_user(current->euid, euid))) 764 !(retval = put_user(cred->euid, euid)))
742 retval = put_user(current->suid, suid); 765 retval = put_user(cred->suid, suid);
743 766
744 return retval; 767 return retval;
745} 768}
@@ -749,48 +772,55 @@ asmlinkage long sys_getresuid(uid_t __user *ruid, uid_t __user *euid, uid_t __us
749 */ 772 */
750asmlinkage long sys_setresgid(gid_t rgid, gid_t egid, gid_t sgid) 773asmlinkage long sys_setresgid(gid_t rgid, gid_t egid, gid_t sgid)
751{ 774{
775 const struct cred *old;
776 struct cred *new;
752 int retval; 777 int retval;
753 778
779 new = prepare_creds();
780 if (!new)
781 return -ENOMEM;
782 old = current_cred();
783
754 retval = security_task_setgid(rgid, egid, sgid, LSM_SETID_RES); 784 retval = security_task_setgid(rgid, egid, sgid, LSM_SETID_RES);
755 if (retval) 785 if (retval)
756 return retval; 786 goto error;
757 787
788 retval = -EPERM;
758 if (!capable(CAP_SETGID)) { 789 if (!capable(CAP_SETGID)) {
759 if ((rgid != (gid_t) -1) && (rgid != current->gid) && 790 if (rgid != (gid_t) -1 && rgid != old->gid &&
760 (rgid != current->egid) && (rgid != current->sgid)) 791 rgid != old->egid && rgid != old->sgid)
761 return -EPERM; 792 goto error;
762 if ((egid != (gid_t) -1) && (egid != current->gid) && 793 if (egid != (gid_t) -1 && egid != old->gid &&
763 (egid != current->egid) && (egid != current->sgid)) 794 egid != old->egid && egid != old->sgid)
764 return -EPERM; 795 goto error;
765 if ((sgid != (gid_t) -1) && (sgid != current->gid) && 796 if (sgid != (gid_t) -1 && sgid != old->gid &&
766 (sgid != current->egid) && (sgid != current->sgid)) 797 sgid != old->egid && sgid != old->sgid)
767 return -EPERM; 798 goto error;
768 }
769 if (egid != (gid_t) -1) {
770 if (egid != current->egid) {
771 set_dumpable(current->mm, suid_dumpable);
772 smp_wmb();
773 }
774 current->egid = egid;
775 } 799 }
776 current->fsgid = current->egid; 800
777 if (rgid != (gid_t) -1) 801 if (rgid != (gid_t) -1)
778 current->gid = rgid; 802 new->gid = rgid;
803 if (egid != (gid_t) -1)
804 new->egid = egid;
779 if (sgid != (gid_t) -1) 805 if (sgid != (gid_t) -1)
780 current->sgid = sgid; 806 new->sgid = sgid;
807 new->fsgid = new->egid;
781 808
782 key_fsgid_changed(current); 809 return commit_creds(new);
783 proc_id_connector(current, PROC_EVENT_GID); 810
784 return 0; 811error:
812 abort_creds(new);
813 return retval;
785} 814}
786 815
787asmlinkage long sys_getresgid(gid_t __user *rgid, gid_t __user *egid, gid_t __user *sgid) 816asmlinkage long sys_getresgid(gid_t __user *rgid, gid_t __user *egid, gid_t __user *sgid)
788{ 817{
818 const struct cred *cred = current_cred();
789 int retval; 819 int retval;
790 820
791 if (!(retval = put_user(current->gid, rgid)) && 821 if (!(retval = put_user(cred->gid, rgid)) &&
792 !(retval = put_user(current->egid, egid))) 822 !(retval = put_user(cred->egid, egid)))
793 retval = put_user(current->sgid, sgid); 823 retval = put_user(cred->sgid, sgid);
794 824
795 return retval; 825 return retval;
796} 826}
@@ -804,27 +834,35 @@ asmlinkage long sys_getresgid(gid_t __user *rgid, gid_t __user *egid, gid_t __us
804 */ 834 */
805asmlinkage long sys_setfsuid(uid_t uid) 835asmlinkage long sys_setfsuid(uid_t uid)
806{ 836{
807 int old_fsuid; 837 const struct cred *old;
838 struct cred *new;
839 uid_t old_fsuid;
840
841 new = prepare_creds();
842 if (!new)
843 return current_fsuid();
844 old = current_cred();
845 old_fsuid = old->fsuid;
808 846
809 old_fsuid = current->fsuid; 847 if (security_task_setuid(uid, (uid_t)-1, (uid_t)-1, LSM_SETID_FS) < 0)
810 if (security_task_setuid(uid, (uid_t)-1, (uid_t)-1, LSM_SETID_FS)) 848 goto error;
811 return old_fsuid;
812 849
813 if (uid == current->uid || uid == current->euid || 850 if (uid == old->uid || uid == old->euid ||
814 uid == current->suid || uid == current->fsuid || 851 uid == old->suid || uid == old->fsuid ||
815 capable(CAP_SETUID)) { 852 capable(CAP_SETUID)) {
816 if (uid != old_fsuid) { 853 if (uid != old_fsuid) {
817 set_dumpable(current->mm, suid_dumpable); 854 new->fsuid = uid;
818 smp_wmb(); 855 if (security_task_fix_setuid(new, old, LSM_SETID_FS) == 0)
856 goto change_okay;
819 } 857 }
820 current->fsuid = uid;
821 } 858 }
822 859
823 key_fsuid_changed(current); 860error:
824 proc_id_connector(current, PROC_EVENT_UID); 861 abort_creds(new);
825 862 return old_fsuid;
826 security_task_post_setuid(old_fsuid, (uid_t)-1, (uid_t)-1, LSM_SETID_FS);
827 863
864change_okay:
865 commit_creds(new);
828 return old_fsuid; 866 return old_fsuid;
829} 867}
830 868
@@ -833,23 +871,34 @@ asmlinkage long sys_setfsuid(uid_t uid)
833 */ 871 */
834asmlinkage long sys_setfsgid(gid_t gid) 872asmlinkage long sys_setfsgid(gid_t gid)
835{ 873{
836 int old_fsgid; 874 const struct cred *old;
875 struct cred *new;
876 gid_t old_fsgid;
877
878 new = prepare_creds();
879 if (!new)
880 return current_fsgid();
881 old = current_cred();
882 old_fsgid = old->fsgid;
837 883
838 old_fsgid = current->fsgid;
839 if (security_task_setgid(gid, (gid_t)-1, (gid_t)-1, LSM_SETID_FS)) 884 if (security_task_setgid(gid, (gid_t)-1, (gid_t)-1, LSM_SETID_FS))
840 return old_fsgid; 885 goto error;
841 886
842 if (gid == current->gid || gid == current->egid || 887 if (gid == old->gid || gid == old->egid ||
843 gid == current->sgid || gid == current->fsgid || 888 gid == old->sgid || gid == old->fsgid ||
844 capable(CAP_SETGID)) { 889 capable(CAP_SETGID)) {
845 if (gid != old_fsgid) { 890 if (gid != old_fsgid) {
846 set_dumpable(current->mm, suid_dumpable); 891 new->fsgid = gid;
847 smp_wmb(); 892 goto change_okay;
848 } 893 }
849 current->fsgid = gid;
850 key_fsgid_changed(current);
851 proc_id_connector(current, PROC_EVENT_GID);
852 } 894 }
895
896error:
897 abort_creds(new);
898 return old_fsgid;
899
900change_okay:
901 commit_creds(new);
853 return old_fsgid; 902 return old_fsgid;
854} 903}
855 904
@@ -858,8 +907,8 @@ void do_sys_times(struct tms *tms)
858 struct task_cputime cputime; 907 struct task_cputime cputime;
859 cputime_t cutime, cstime; 908 cputime_t cutime, cstime;
860 909
861 spin_lock_irq(&current->sighand->siglock);
862 thread_group_cputime(current, &cputime); 910 thread_group_cputime(current, &cputime);
911 spin_lock_irq(&current->sighand->siglock);
863 cutime = current->signal->cutime; 912 cutime = current->signal->cutime;
864 cstime = current->signal->cstime; 913 cstime = current->signal->cstime;
865 spin_unlock_irq(&current->sighand->siglock); 914 spin_unlock_irq(&current->sighand->siglock);
@@ -1118,7 +1167,7 @@ EXPORT_SYMBOL(groups_free);
1118 1167
1119/* export the group_info to a user-space array */ 1168/* export the group_info to a user-space array */
1120static int groups_to_user(gid_t __user *grouplist, 1169static int groups_to_user(gid_t __user *grouplist,
1121 struct group_info *group_info) 1170 const struct group_info *group_info)
1122{ 1171{
1123 int i; 1172 int i;
1124 unsigned int count = group_info->ngroups; 1173 unsigned int count = group_info->ngroups;
@@ -1186,7 +1235,7 @@ static void groups_sort(struct group_info *group_info)
1186} 1235}
1187 1236
1188/* a simple bsearch */ 1237/* a simple bsearch */
1189int groups_search(struct group_info *group_info, gid_t grp) 1238int groups_search(const struct group_info *group_info, gid_t grp)
1190{ 1239{
1191 unsigned int left, right; 1240 unsigned int left, right;
1192 1241
@@ -1208,51 +1257,74 @@ int groups_search(struct group_info *group_info, gid_t grp)
1208 return 0; 1257 return 0;
1209} 1258}
1210 1259
1211/* validate and set current->group_info */ 1260/**
1212int set_current_groups(struct group_info *group_info) 1261 * set_groups - Change a group subscription in a set of credentials
1262 * @new: The newly prepared set of credentials to alter
1263 * @group_info: The group list to install
1264 *
1265 * Validate a group subscription and, if valid, insert it into a set
1266 * of credentials.
1267 */
1268int set_groups(struct cred *new, struct group_info *group_info)
1213{ 1269{
1214 int retval; 1270 int retval;
1215 struct group_info *old_info;
1216 1271
1217 retval = security_task_setgroups(group_info); 1272 retval = security_task_setgroups(group_info);
1218 if (retval) 1273 if (retval)
1219 return retval; 1274 return retval;
1220 1275
1276 put_group_info(new->group_info);
1221 groups_sort(group_info); 1277 groups_sort(group_info);
1222 get_group_info(group_info); 1278 get_group_info(group_info);
1279 new->group_info = group_info;
1280 return 0;
1281}
1223 1282
1224 task_lock(current); 1283EXPORT_SYMBOL(set_groups);
1225 old_info = current->group_info;
1226 current->group_info = group_info;
1227 task_unlock(current);
1228 1284
1229 put_group_info(old_info); 1285/**
1286 * set_current_groups - Change current's group subscription
1287 * @group_info: The group list to impose
1288 *
1289 * Validate a group subscription and, if valid, impose it upon current's task
1290 * security record.
1291 */
1292int set_current_groups(struct group_info *group_info)
1293{
1294 struct cred *new;
1295 int ret;
1230 1296
1231 return 0; 1297 new = prepare_creds();
1298 if (!new)
1299 return -ENOMEM;
1300
1301 ret = set_groups(new, group_info);
1302 if (ret < 0) {
1303 abort_creds(new);
1304 return ret;
1305 }
1306
1307 return commit_creds(new);
1232} 1308}
1233 1309
1234EXPORT_SYMBOL(set_current_groups); 1310EXPORT_SYMBOL(set_current_groups);
1235 1311
1236asmlinkage long sys_getgroups(int gidsetsize, gid_t __user *grouplist) 1312asmlinkage long sys_getgroups(int gidsetsize, gid_t __user *grouplist)
1237{ 1313{
1238 int i = 0; 1314 const struct cred *cred = current_cred();
1239 1315 int i;
1240 /*
1241 * SMP: Nobody else can change our grouplist. Thus we are
1242 * safe.
1243 */
1244 1316
1245 if (gidsetsize < 0) 1317 if (gidsetsize < 0)
1246 return -EINVAL; 1318 return -EINVAL;
1247 1319
1248 /* no need to grab task_lock here; it cannot change */ 1320 /* no need to grab task_lock here; it cannot change */
1249 i = current->group_info->ngroups; 1321 i = cred->group_info->ngroups;
1250 if (gidsetsize) { 1322 if (gidsetsize) {
1251 if (i > gidsetsize) { 1323 if (i > gidsetsize) {
1252 i = -EINVAL; 1324 i = -EINVAL;
1253 goto out; 1325 goto out;
1254 } 1326 }
1255 if (groups_to_user(grouplist, current->group_info)) { 1327 if (groups_to_user(grouplist, cred->group_info)) {
1256 i = -EFAULT; 1328 i = -EFAULT;
1257 goto out; 1329 goto out;
1258 } 1330 }
@@ -1296,9 +1368,11 @@ asmlinkage long sys_setgroups(int gidsetsize, gid_t __user *grouplist)
1296 */ 1368 */
1297int in_group_p(gid_t grp) 1369int in_group_p(gid_t grp)
1298{ 1370{
1371 const struct cred *cred = current_cred();
1299 int retval = 1; 1372 int retval = 1;
1300 if (grp != current->fsgid) 1373
1301 retval = groups_search(current->group_info, grp); 1374 if (grp != cred->fsgid)
1375 retval = groups_search(cred->group_info, grp);
1302 return retval; 1376 return retval;
1303} 1377}
1304 1378
@@ -1306,9 +1380,11 @@ EXPORT_SYMBOL(in_group_p);
1306 1380
1307int in_egroup_p(gid_t grp) 1381int in_egroup_p(gid_t grp)
1308{ 1382{
1383 const struct cred *cred = current_cred();
1309 int retval = 1; 1384 int retval = 1;
1310 if (grp != current->egid) 1385
1311 retval = groups_search(current->group_info, grp); 1386 if (grp != cred->egid)
1387 retval = groups_search(cred->group_info, grp);
1312 return retval; 1388 return retval;
1313} 1389}
1314 1390
@@ -1624,50 +1700,56 @@ asmlinkage long sys_umask(int mask)
1624asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3, 1700asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3,
1625 unsigned long arg4, unsigned long arg5) 1701 unsigned long arg4, unsigned long arg5)
1626{ 1702{
1627 long error = 0; 1703 struct task_struct *me = current;
1704 unsigned char comm[sizeof(me->comm)];
1705 long error;
1628 1706
1629 if (security_task_prctl(option, arg2, arg3, arg4, arg5, &error)) 1707 error = security_task_prctl(option, arg2, arg3, arg4, arg5);
1708 if (error != -ENOSYS)
1630 return error; 1709 return error;
1631 1710
1711 error = 0;
1632 switch (option) { 1712 switch (option) {
1633 case PR_SET_PDEATHSIG: 1713 case PR_SET_PDEATHSIG:
1634 if (!valid_signal(arg2)) { 1714 if (!valid_signal(arg2)) {
1635 error = -EINVAL; 1715 error = -EINVAL;
1636 break; 1716 break;
1637 } 1717 }
1638 current->pdeath_signal = arg2; 1718 me->pdeath_signal = arg2;
1719 error = 0;
1639 break; 1720 break;
1640 case PR_GET_PDEATHSIG: 1721 case PR_GET_PDEATHSIG:
1641 error = put_user(current->pdeath_signal, (int __user *)arg2); 1722 error = put_user(me->pdeath_signal, (int __user *)arg2);
1642 break; 1723 break;
1643 case PR_GET_DUMPABLE: 1724 case PR_GET_DUMPABLE:
1644 error = get_dumpable(current->mm); 1725 error = get_dumpable(me->mm);
1645 break; 1726 break;
1646 case PR_SET_DUMPABLE: 1727 case PR_SET_DUMPABLE:
1647 if (arg2 < 0 || arg2 > 1) { 1728 if (arg2 < 0 || arg2 > 1) {
1648 error = -EINVAL; 1729 error = -EINVAL;
1649 break; 1730 break;
1650 } 1731 }
1651 set_dumpable(current->mm, arg2); 1732 set_dumpable(me->mm, arg2);
1733 error = 0;
1652 break; 1734 break;
1653 1735
1654 case PR_SET_UNALIGN: 1736 case PR_SET_UNALIGN:
1655 error = SET_UNALIGN_CTL(current, arg2); 1737 error = SET_UNALIGN_CTL(me, arg2);
1656 break; 1738 break;
1657 case PR_GET_UNALIGN: 1739 case PR_GET_UNALIGN:
1658 error = GET_UNALIGN_CTL(current, arg2); 1740 error = GET_UNALIGN_CTL(me, arg2);
1659 break; 1741 break;
1660 case PR_SET_FPEMU: 1742 case PR_SET_FPEMU:
1661 error = SET_FPEMU_CTL(current, arg2); 1743 error = SET_FPEMU_CTL(me, arg2);
1662 break; 1744 break;
1663 case PR_GET_FPEMU: 1745 case PR_GET_FPEMU:
1664 error = GET_FPEMU_CTL(current, arg2); 1746 error = GET_FPEMU_CTL(me, arg2);
1665 break; 1747 break;
1666 case PR_SET_FPEXC: 1748 case PR_SET_FPEXC:
1667 error = SET_FPEXC_CTL(current, arg2); 1749 error = SET_FPEXC_CTL(me, arg2);
1668 break; 1750 break;
1669 case PR_GET_FPEXC: 1751 case PR_GET_FPEXC:
1670 error = GET_FPEXC_CTL(current, arg2); 1752 error = GET_FPEXC_CTL(me, arg2);
1671 break; 1753 break;
1672 case PR_GET_TIMING: 1754 case PR_GET_TIMING:
1673 error = PR_TIMING_STATISTICAL; 1755 error = PR_TIMING_STATISTICAL;
@@ -1675,33 +1757,28 @@ asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3,
1675 case PR_SET_TIMING: 1757 case PR_SET_TIMING:
1676 if (arg2 != PR_TIMING_STATISTICAL) 1758 if (arg2 != PR_TIMING_STATISTICAL)
1677 error = -EINVAL; 1759 error = -EINVAL;
1760 else
1761 error = 0;
1678 break; 1762 break;
1679 1763
1680 case PR_SET_NAME: { 1764 case PR_SET_NAME:
1681 struct task_struct *me = current; 1765 comm[sizeof(me->comm)-1] = 0;
1682 unsigned char ncomm[sizeof(me->comm)]; 1766 if (strncpy_from_user(comm, (char __user *)arg2,
1683 1767 sizeof(me->comm) - 1) < 0)
1684 ncomm[sizeof(me->comm)-1] = 0;
1685 if (strncpy_from_user(ncomm, (char __user *)arg2,
1686 sizeof(me->comm)-1) < 0)
1687 return -EFAULT; 1768 return -EFAULT;
1688 set_task_comm(me, ncomm); 1769 set_task_comm(me, comm);
1689 return 0; 1770 return 0;
1690 } 1771 case PR_GET_NAME:
1691 case PR_GET_NAME: { 1772 get_task_comm(comm, me);
1692 struct task_struct *me = current; 1773 if (copy_to_user((char __user *)arg2, comm,
1693 unsigned char tcomm[sizeof(me->comm)]; 1774 sizeof(comm)))
1694
1695 get_task_comm(tcomm, me);
1696 if (copy_to_user((char __user *)arg2, tcomm, sizeof(tcomm)))
1697 return -EFAULT; 1775 return -EFAULT;
1698 return 0; 1776 return 0;
1699 }
1700 case PR_GET_ENDIAN: 1777 case PR_GET_ENDIAN:
1701 error = GET_ENDIAN(current, arg2); 1778 error = GET_ENDIAN(me, arg2);
1702 break; 1779 break;
1703 case PR_SET_ENDIAN: 1780 case PR_SET_ENDIAN:
1704 error = SET_ENDIAN(current, arg2); 1781 error = SET_ENDIAN(me, arg2);
1705 break; 1782 break;
1706 1783
1707 case PR_GET_SECCOMP: 1784 case PR_GET_SECCOMP:
@@ -1725,6 +1802,7 @@ asmlinkage long sys_prctl(int option, unsigned long arg2, unsigned long arg3,
1725 current->default_timer_slack_ns; 1802 current->default_timer_slack_ns;
1726 else 1803 else
1727 current->timer_slack_ns = arg2; 1804 current->timer_slack_ns = arg2;
1805 error = 0;
1728 break; 1806 break;
1729 default: 1807 default:
1730 error = -EINVAL; 1808 error = -EINVAL;
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 9d048fa2d902..ff6d45c7626f 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -121,6 +121,10 @@ extern int sg_big_buff;
121#include <asm/system.h> 121#include <asm/system.h>
122#endif 122#endif
123 123
124#ifdef CONFIG_SPARC64
125extern int sysctl_tsb_ratio;
126#endif
127
124#ifdef __hppa__ 128#ifdef __hppa__
125extern int pwrsw_enabled; 129extern int pwrsw_enabled;
126extern int unaligned_enabled; 130extern int unaligned_enabled;
@@ -176,6 +180,9 @@ extern struct ctl_table random_table[];
176#ifdef CONFIG_INOTIFY_USER 180#ifdef CONFIG_INOTIFY_USER
177extern struct ctl_table inotify_table[]; 181extern struct ctl_table inotify_table[];
178#endif 182#endif
183#ifdef CONFIG_EPOLL
184extern struct ctl_table epoll_table[];
185#endif
179 186
180#ifdef HAVE_ARCH_PICK_MMAP_LAYOUT 187#ifdef HAVE_ARCH_PICK_MMAP_LAYOUT
181int sysctl_legacy_va_layout; 188int sysctl_legacy_va_layout;
@@ -448,6 +455,16 @@ static struct ctl_table kern_table[] = {
448 .proc_handler = &proc_dointvec, 455 .proc_handler = &proc_dointvec,
449 }, 456 },
450#endif 457#endif
458#ifdef CONFIG_SPARC64
459 {
460 .ctl_name = CTL_UNNUMBERED,
461 .procname = "tsb-ratio",
462 .data = &sysctl_tsb_ratio,
463 .maxlen = sizeof (int),
464 .mode = 0644,
465 .proc_handler = &proc_dointvec,
466 },
467#endif
451#ifdef __hppa__ 468#ifdef __hppa__
452 { 469 {
453 .ctl_name = KERN_HPPA_PWRSW, 470 .ctl_name = KERN_HPPA_PWRSW,
@@ -484,6 +501,26 @@ static struct ctl_table kern_table[] = {
484 .proc_handler = &ftrace_enable_sysctl, 501 .proc_handler = &ftrace_enable_sysctl,
485 }, 502 },
486#endif 503#endif
504#ifdef CONFIG_STACK_TRACER
505 {
506 .ctl_name = CTL_UNNUMBERED,
507 .procname = "stack_tracer_enabled",
508 .data = &stack_tracer_enabled,
509 .maxlen = sizeof(int),
510 .mode = 0644,
511 .proc_handler = &stack_trace_sysctl,
512 },
513#endif
514#ifdef CONFIG_TRACING
515 {
516 .ctl_name = CTL_UNNUMBERED,
517 .procname = "ftrace_dump_on_oops",
518 .data = &ftrace_dump_on_oops,
519 .maxlen = sizeof(int),
520 .mode = 0644,
521 .proc_handler = &proc_dointvec,
522 },
523#endif
487#ifdef CONFIG_MODULES 524#ifdef CONFIG_MODULES
488 { 525 {
489 .ctl_name = KERN_MODPROBE, 526 .ctl_name = KERN_MODPROBE,
@@ -1325,6 +1362,13 @@ static struct ctl_table fs_table[] = {
1325 .child = inotify_table, 1362 .child = inotify_table,
1326 }, 1363 },
1327#endif 1364#endif
1365#ifdef CONFIG_EPOLL
1366 {
1367 .procname = "epoll",
1368 .mode = 0555,
1369 .child = epoll_table,
1370 },
1371#endif
1328#endif 1372#endif
1329 { 1373 {
1330 .ctl_name = KERN_SETUID_DUMPABLE, 1374 .ctl_name = KERN_SETUID_DUMPABLE,
@@ -1641,7 +1685,7 @@ out:
1641 1685
1642static int test_perm(int mode, int op) 1686static int test_perm(int mode, int op)
1643{ 1687{
1644 if (!current->euid) 1688 if (!current_euid())
1645 mode >>= 6; 1689 mode >>= 6;
1646 else if (in_egroup_p(0)) 1690 else if (in_egroup_p(0))
1647 mode >>= 3; 1691 mode >>= 3;
diff --git a/kernel/sysctl_check.c b/kernel/sysctl_check.c
index c35da23ab8fb..fafeb48f27c0 100644
--- a/kernel/sysctl_check.c
+++ b/kernel/sysctl_check.c
@@ -730,7 +730,6 @@ static const struct trans_ctl_table trans_fs_quota_table[] = {
730}; 730};
731 731
732static const struct trans_ctl_table trans_fs_xfs_table[] = { 732static const struct trans_ctl_table trans_fs_xfs_table[] = {
733 { XFS_RESTRICT_CHOWN, "restrict_chown" },
734 { XFS_SGID_INHERIT, "irix_sgid_inherit" }, 733 { XFS_SGID_INHERIT, "irix_sgid_inherit" },
735 { XFS_SYMLINK_MODE, "irix_symlink_mode" }, 734 { XFS_SYMLINK_MODE, "irix_symlink_mode" },
736 { XFS_PANIC_MASK, "panic_mask" }, 735 { XFS_PANIC_MASK, "panic_mask" },
diff --git a/kernel/taskstats.c b/kernel/taskstats.c
index bd6be76303cf..888adbcca30c 100644
--- a/kernel/taskstats.c
+++ b/kernel/taskstats.c
@@ -290,18 +290,17 @@ ret:
290 return; 290 return;
291} 291}
292 292
293static int add_del_listener(pid_t pid, cpumask_t *maskp, int isadd) 293static int add_del_listener(pid_t pid, const struct cpumask *mask, int isadd)
294{ 294{
295 struct listener_list *listeners; 295 struct listener_list *listeners;
296 struct listener *s, *tmp; 296 struct listener *s, *tmp;
297 unsigned int cpu; 297 unsigned int cpu;
298 cpumask_t mask = *maskp;
299 298
300 if (!cpus_subset(mask, cpu_possible_map)) 299 if (!cpumask_subset(mask, cpu_possible_mask))
301 return -EINVAL; 300 return -EINVAL;
302 301
303 if (isadd == REGISTER) { 302 if (isadd == REGISTER) {
304 for_each_cpu_mask_nr(cpu, mask) { 303 for_each_cpu(cpu, mask) {
305 s = kmalloc_node(sizeof(struct listener), GFP_KERNEL, 304 s = kmalloc_node(sizeof(struct listener), GFP_KERNEL,
306 cpu_to_node(cpu)); 305 cpu_to_node(cpu));
307 if (!s) 306 if (!s)
@@ -320,7 +319,7 @@ static int add_del_listener(pid_t pid, cpumask_t *maskp, int isadd)
320 319
321 /* Deregister or cleanup */ 320 /* Deregister or cleanup */
322cleanup: 321cleanup:
323 for_each_cpu_mask_nr(cpu, mask) { 322 for_each_cpu(cpu, mask) {
324 listeners = &per_cpu(listener_array, cpu); 323 listeners = &per_cpu(listener_array, cpu);
325 down_write(&listeners->sem); 324 down_write(&listeners->sem);
326 list_for_each_entry_safe(s, tmp, &listeners->list, list) { 325 list_for_each_entry_safe(s, tmp, &listeners->list, list) {
@@ -335,7 +334,7 @@ cleanup:
335 return 0; 334 return 0;
336} 335}
337 336
338static int parse(struct nlattr *na, cpumask_t *mask) 337static int parse(struct nlattr *na, struct cpumask *mask)
339{ 338{
340 char *data; 339 char *data;
341 int len; 340 int len;
@@ -352,7 +351,7 @@ static int parse(struct nlattr *na, cpumask_t *mask)
352 if (!data) 351 if (!data)
353 return -ENOMEM; 352 return -ENOMEM;
354 nla_strlcpy(data, na, len); 353 nla_strlcpy(data, na, len);
355 ret = cpulist_parse(data, *mask); 354 ret = cpulist_parse(data, mask);
356 kfree(data); 355 kfree(data);
357 return ret; 356 return ret;
358} 357}
@@ -428,23 +427,33 @@ err:
428 427
429static int taskstats_user_cmd(struct sk_buff *skb, struct genl_info *info) 428static int taskstats_user_cmd(struct sk_buff *skb, struct genl_info *info)
430{ 429{
431 int rc = 0; 430 int rc;
432 struct sk_buff *rep_skb; 431 struct sk_buff *rep_skb;
433 struct taskstats *stats; 432 struct taskstats *stats;
434 size_t size; 433 size_t size;
435 cpumask_t mask; 434 cpumask_var_t mask;
435
436 if (!alloc_cpumask_var(&mask, GFP_KERNEL))
437 return -ENOMEM;
436 438
437 rc = parse(info->attrs[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK], &mask); 439 rc = parse(info->attrs[TASKSTATS_CMD_ATTR_REGISTER_CPUMASK], mask);
438 if (rc < 0) 440 if (rc < 0)
439 return rc; 441 goto free_return_rc;
440 if (rc == 0) 442 if (rc == 0) {
441 return add_del_listener(info->snd_pid, &mask, REGISTER); 443 rc = add_del_listener(info->snd_pid, mask, REGISTER);
444 goto free_return_rc;
445 }
442 446
443 rc = parse(info->attrs[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK], &mask); 447 rc = parse(info->attrs[TASKSTATS_CMD_ATTR_DEREGISTER_CPUMASK], mask);
444 if (rc < 0) 448 if (rc < 0)
449 goto free_return_rc;
450 if (rc == 0) {
451 rc = add_del_listener(info->snd_pid, mask, DEREGISTER);
452free_return_rc:
453 free_cpumask_var(mask);
445 return rc; 454 return rc;
446 if (rc == 0) 455 }
447 return add_del_listener(info->snd_pid, &mask, DEREGISTER); 456 free_cpumask_var(mask);
448 457
449 /* 458 /*
450 * Size includes space for nested attributes 459 * Size includes space for nested attributes
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index f8d968063cea..ea2f48af83cf 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -166,6 +166,8 @@ static void clockevents_notify_released(void)
166void clockevents_register_device(struct clock_event_device *dev) 166void clockevents_register_device(struct clock_event_device *dev)
167{ 167{
168 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); 168 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
169 BUG_ON(!dev->cpumask);
170
169 /* 171 /*
170 * A nsec2cyc multiplicator of 0 is invalid and we'd crash 172 * A nsec2cyc multiplicator of 0 is invalid and we'd crash
171 * on it, so fix it up and emit a warning: 173 * on it, so fix it up and emit a warning:
diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c
index 9ed2eec97526..ca89e1593f08 100644
--- a/kernel/time/clocksource.c
+++ b/kernel/time/clocksource.c
@@ -145,10 +145,11 @@ static void clocksource_watchdog(unsigned long data)
145 * Cycle through CPUs to check if the CPUs stay 145 * Cycle through CPUs to check if the CPUs stay
146 * synchronized to each other. 146 * synchronized to each other.
147 */ 147 */
148 int next_cpu = next_cpu_nr(raw_smp_processor_id(), cpu_online_map); 148 int next_cpu = cpumask_next(raw_smp_processor_id(),
149 cpu_online_mask);
149 150
150 if (next_cpu >= nr_cpu_ids) 151 if (next_cpu >= nr_cpu_ids)
151 next_cpu = first_cpu(cpu_online_map); 152 next_cpu = cpumask_first(cpu_online_mask);
152 watchdog_timer.expires += WATCHDOG_INTERVAL; 153 watchdog_timer.expires += WATCHDOG_INTERVAL;
153 add_timer_on(&watchdog_timer, next_cpu); 154 add_timer_on(&watchdog_timer, next_cpu);
154 } 155 }
@@ -173,7 +174,7 @@ static void clocksource_check_watchdog(struct clocksource *cs)
173 watchdog_last = watchdog->read(); 174 watchdog_last = watchdog->read();
174 watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL; 175 watchdog_timer.expires = jiffies + WATCHDOG_INTERVAL;
175 add_timer_on(&watchdog_timer, 176 add_timer_on(&watchdog_timer,
176 first_cpu(cpu_online_map)); 177 cpumask_first(cpu_online_mask));
177 } 178 }
178 } else { 179 } else {
179 if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) 180 if (cs->flags & CLOCK_SOURCE_IS_CONTINUOUS)
@@ -195,7 +196,7 @@ static void clocksource_check_watchdog(struct clocksource *cs)
195 watchdog_timer.expires = 196 watchdog_timer.expires =
196 jiffies + WATCHDOG_INTERVAL; 197 jiffies + WATCHDOG_INTERVAL;
197 add_timer_on(&watchdog_timer, 198 add_timer_on(&watchdog_timer,
198 first_cpu(cpu_online_map)); 199 cpumask_first(cpu_online_mask));
199 } 200 }
200 } 201 }
201 } 202 }
diff --git a/kernel/time/ntp.c b/kernel/time/ntp.c
index 8ff15e5d486b..f5f793d92415 100644
--- a/kernel/time/ntp.c
+++ b/kernel/time/ntp.c
@@ -131,7 +131,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer)
131{ 131{
132 enum hrtimer_restart res = HRTIMER_NORESTART; 132 enum hrtimer_restart res = HRTIMER_NORESTART;
133 133
134 write_seqlock_irq(&xtime_lock); 134 write_seqlock(&xtime_lock);
135 135
136 switch (time_state) { 136 switch (time_state) {
137 case TIME_OK: 137 case TIME_OK:
@@ -164,7 +164,7 @@ static enum hrtimer_restart ntp_leap_second(struct hrtimer *timer)
164 } 164 }
165 update_vsyscall(&xtime, clock); 165 update_vsyscall(&xtime, clock);
166 166
167 write_sequnlock_irq(&xtime_lock); 167 write_sequnlock(&xtime_lock);
168 168
169 return res; 169 return res;
170} 170}
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index f98a1b7b16e9..118a3b3b3f9a 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -28,7 +28,9 @@
28 */ 28 */
29 29
30struct tick_device tick_broadcast_device; 30struct tick_device tick_broadcast_device;
31static cpumask_t tick_broadcast_mask; 31/* FIXME: Use cpumask_var_t. */
32static DECLARE_BITMAP(tick_broadcast_mask, NR_CPUS);
33static DECLARE_BITMAP(tmpmask, NR_CPUS);
32static DEFINE_SPINLOCK(tick_broadcast_lock); 34static DEFINE_SPINLOCK(tick_broadcast_lock);
33static int tick_broadcast_force; 35static int tick_broadcast_force;
34 36
@@ -46,9 +48,9 @@ struct tick_device *tick_get_broadcast_device(void)
46 return &tick_broadcast_device; 48 return &tick_broadcast_device;
47} 49}
48 50
49cpumask_t *tick_get_broadcast_mask(void) 51struct cpumask *tick_get_broadcast_mask(void)
50{ 52{
51 return &tick_broadcast_mask; 53 return to_cpumask(tick_broadcast_mask);
52} 54}
53 55
54/* 56/*
@@ -72,7 +74,7 @@ int tick_check_broadcast_device(struct clock_event_device *dev)
72 74
73 clockevents_exchange_device(NULL, dev); 75 clockevents_exchange_device(NULL, dev);
74 tick_broadcast_device.evtdev = dev; 76 tick_broadcast_device.evtdev = dev;
75 if (!cpus_empty(tick_broadcast_mask)) 77 if (!cpumask_empty(tick_get_broadcast_mask()))
76 tick_broadcast_start_periodic(dev); 78 tick_broadcast_start_periodic(dev);
77 return 1; 79 return 1;
78} 80}
@@ -104,7 +106,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
104 */ 106 */
105 if (!tick_device_is_functional(dev)) { 107 if (!tick_device_is_functional(dev)) {
106 dev->event_handler = tick_handle_periodic; 108 dev->event_handler = tick_handle_periodic;
107 cpu_set(cpu, tick_broadcast_mask); 109 cpumask_set_cpu(cpu, tick_get_broadcast_mask());
108 tick_broadcast_start_periodic(tick_broadcast_device.evtdev); 110 tick_broadcast_start_periodic(tick_broadcast_device.evtdev);
109 ret = 1; 111 ret = 1;
110 } else { 112 } else {
@@ -116,7 +118,7 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
116 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) { 118 if (!(dev->features & CLOCK_EVT_FEAT_C3STOP)) {
117 int cpu = smp_processor_id(); 119 int cpu = smp_processor_id();
118 120
119 cpu_clear(cpu, tick_broadcast_mask); 121 cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
120 tick_broadcast_clear_oneshot(cpu); 122 tick_broadcast_clear_oneshot(cpu);
121 } 123 }
122 } 124 }
@@ -125,9 +127,9 @@ int tick_device_uses_broadcast(struct clock_event_device *dev, int cpu)
125} 127}
126 128
127/* 129/*
128 * Broadcast the event to the cpus, which are set in the mask 130 * Broadcast the event to the cpus, which are set in the mask (mangled).
129 */ 131 */
130static void tick_do_broadcast(cpumask_t mask) 132static void tick_do_broadcast(struct cpumask *mask)
131{ 133{
132 int cpu = smp_processor_id(); 134 int cpu = smp_processor_id();
133 struct tick_device *td; 135 struct tick_device *td;
@@ -135,21 +137,20 @@ static void tick_do_broadcast(cpumask_t mask)
135 /* 137 /*
136 * Check, if the current cpu is in the mask 138 * Check, if the current cpu is in the mask
137 */ 139 */
138 if (cpu_isset(cpu, mask)) { 140 if (cpumask_test_cpu(cpu, mask)) {
139 cpu_clear(cpu, mask); 141 cpumask_clear_cpu(cpu, mask);
140 td = &per_cpu(tick_cpu_device, cpu); 142 td = &per_cpu(tick_cpu_device, cpu);
141 td->evtdev->event_handler(td->evtdev); 143 td->evtdev->event_handler(td->evtdev);
142 } 144 }
143 145
144 if (!cpus_empty(mask)) { 146 if (!cpumask_empty(mask)) {
145 /* 147 /*
146 * It might be necessary to actually check whether the devices 148 * It might be necessary to actually check whether the devices
147 * have different broadcast functions. For now, just use the 149 * have different broadcast functions. For now, just use the
148 * one of the first device. This works as long as we have this 150 * one of the first device. This works as long as we have this
149 * misfeature only on x86 (lapic) 151 * misfeature only on x86 (lapic)
150 */ 152 */
151 cpu = first_cpu(mask); 153 td = &per_cpu(tick_cpu_device, cpumask_first(mask));
152 td = &per_cpu(tick_cpu_device, cpu);
153 td->evtdev->broadcast(mask); 154 td->evtdev->broadcast(mask);
154 } 155 }
155} 156}
@@ -160,12 +161,11 @@ static void tick_do_broadcast(cpumask_t mask)
160 */ 161 */
161static void tick_do_periodic_broadcast(void) 162static void tick_do_periodic_broadcast(void)
162{ 163{
163 cpumask_t mask;
164
165 spin_lock(&tick_broadcast_lock); 164 spin_lock(&tick_broadcast_lock);
166 165
167 cpus_and(mask, cpu_online_map, tick_broadcast_mask); 166 cpumask_and(to_cpumask(tmpmask),
168 tick_do_broadcast(mask); 167 cpu_online_mask, tick_get_broadcast_mask());
168 tick_do_broadcast(to_cpumask(tmpmask));
169 169
170 spin_unlock(&tick_broadcast_lock); 170 spin_unlock(&tick_broadcast_lock);
171} 171}
@@ -228,13 +228,13 @@ static void tick_do_broadcast_on_off(void *why)
228 if (!tick_device_is_functional(dev)) 228 if (!tick_device_is_functional(dev))
229 goto out; 229 goto out;
230 230
231 bc_stopped = cpus_empty(tick_broadcast_mask); 231 bc_stopped = cpumask_empty(tick_get_broadcast_mask());
232 232
233 switch (*reason) { 233 switch (*reason) {
234 case CLOCK_EVT_NOTIFY_BROADCAST_ON: 234 case CLOCK_EVT_NOTIFY_BROADCAST_ON:
235 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE: 235 case CLOCK_EVT_NOTIFY_BROADCAST_FORCE:
236 if (!cpu_isset(cpu, tick_broadcast_mask)) { 236 if (!cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
237 cpu_set(cpu, tick_broadcast_mask); 237 cpumask_set_cpu(cpu, tick_get_broadcast_mask());
238 if (tick_broadcast_device.mode == 238 if (tick_broadcast_device.mode ==
239 TICKDEV_MODE_PERIODIC) 239 TICKDEV_MODE_PERIODIC)
240 clockevents_shutdown(dev); 240 clockevents_shutdown(dev);
@@ -244,8 +244,8 @@ static void tick_do_broadcast_on_off(void *why)
244 break; 244 break;
245 case CLOCK_EVT_NOTIFY_BROADCAST_OFF: 245 case CLOCK_EVT_NOTIFY_BROADCAST_OFF:
246 if (!tick_broadcast_force && 246 if (!tick_broadcast_force &&
247 cpu_isset(cpu, tick_broadcast_mask)) { 247 cpumask_test_cpu(cpu, tick_get_broadcast_mask())) {
248 cpu_clear(cpu, tick_broadcast_mask); 248 cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
249 if (tick_broadcast_device.mode == 249 if (tick_broadcast_device.mode ==
250 TICKDEV_MODE_PERIODIC) 250 TICKDEV_MODE_PERIODIC)
251 tick_setup_periodic(dev, 0); 251 tick_setup_periodic(dev, 0);
@@ -253,7 +253,7 @@ static void tick_do_broadcast_on_off(void *why)
253 break; 253 break;
254 } 254 }
255 255
256 if (cpus_empty(tick_broadcast_mask)) { 256 if (cpumask_empty(tick_get_broadcast_mask())) {
257 if (!bc_stopped) 257 if (!bc_stopped)
258 clockevents_shutdown(bc); 258 clockevents_shutdown(bc);
259 } else if (bc_stopped) { 259 } else if (bc_stopped) {
@@ -272,7 +272,7 @@ out:
272 */ 272 */
273void tick_broadcast_on_off(unsigned long reason, int *oncpu) 273void tick_broadcast_on_off(unsigned long reason, int *oncpu)
274{ 274{
275 if (!cpu_isset(*oncpu, cpu_online_map)) 275 if (!cpumask_test_cpu(*oncpu, cpu_online_mask))
276 printk(KERN_ERR "tick-broadcast: ignoring broadcast for " 276 printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
277 "offline CPU #%d\n", *oncpu); 277 "offline CPU #%d\n", *oncpu);
278 else 278 else
@@ -303,10 +303,10 @@ void tick_shutdown_broadcast(unsigned int *cpup)
303 spin_lock_irqsave(&tick_broadcast_lock, flags); 303 spin_lock_irqsave(&tick_broadcast_lock, flags);
304 304
305 bc = tick_broadcast_device.evtdev; 305 bc = tick_broadcast_device.evtdev;
306 cpu_clear(cpu, tick_broadcast_mask); 306 cpumask_clear_cpu(cpu, tick_get_broadcast_mask());
307 307
308 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) { 308 if (tick_broadcast_device.mode == TICKDEV_MODE_PERIODIC) {
309 if (bc && cpus_empty(tick_broadcast_mask)) 309 if (bc && cpumask_empty(tick_get_broadcast_mask()))
310 clockevents_shutdown(bc); 310 clockevents_shutdown(bc);
311 } 311 }
312 312
@@ -342,10 +342,10 @@ int tick_resume_broadcast(void)
342 342
343 switch (tick_broadcast_device.mode) { 343 switch (tick_broadcast_device.mode) {
344 case TICKDEV_MODE_PERIODIC: 344 case TICKDEV_MODE_PERIODIC:
345 if(!cpus_empty(tick_broadcast_mask)) 345 if (!cpumask_empty(tick_get_broadcast_mask()))
346 tick_broadcast_start_periodic(bc); 346 tick_broadcast_start_periodic(bc);
347 broadcast = cpu_isset(smp_processor_id(), 347 broadcast = cpumask_test_cpu(smp_processor_id(),
348 tick_broadcast_mask); 348 tick_get_broadcast_mask());
349 break; 349 break;
350 case TICKDEV_MODE_ONESHOT: 350 case TICKDEV_MODE_ONESHOT:
351 broadcast = tick_resume_broadcast_oneshot(bc); 351 broadcast = tick_resume_broadcast_oneshot(bc);
@@ -360,14 +360,15 @@ int tick_resume_broadcast(void)
360 360
361#ifdef CONFIG_TICK_ONESHOT 361#ifdef CONFIG_TICK_ONESHOT
362 362
363static cpumask_t tick_broadcast_oneshot_mask; 363/* FIXME: use cpumask_var_t. */
364static DECLARE_BITMAP(tick_broadcast_oneshot_mask, NR_CPUS);
364 365
365/* 366/*
366 * Debugging: see timer_list.c 367 * Exposed for debugging: see timer_list.c
367 */ 368 */
368cpumask_t *tick_get_broadcast_oneshot_mask(void) 369struct cpumask *tick_get_broadcast_oneshot_mask(void)
369{ 370{
370 return &tick_broadcast_oneshot_mask; 371 return to_cpumask(tick_broadcast_oneshot_mask);
371} 372}
372 373
373static int tick_broadcast_set_event(ktime_t expires, int force) 374static int tick_broadcast_set_event(ktime_t expires, int force)
@@ -389,7 +390,7 @@ int tick_resume_broadcast_oneshot(struct clock_event_device *bc)
389 */ 390 */
390void tick_check_oneshot_broadcast(int cpu) 391void tick_check_oneshot_broadcast(int cpu)
391{ 392{
392 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 393 if (cpumask_test_cpu(cpu, to_cpumask(tick_broadcast_oneshot_mask))) {
393 struct tick_device *td = &per_cpu(tick_cpu_device, cpu); 394 struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
394 395
395 clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT); 396 clockevents_set_mode(td->evtdev, CLOCK_EVT_MODE_ONESHOT);
@@ -402,7 +403,6 @@ void tick_check_oneshot_broadcast(int cpu)
402static void tick_handle_oneshot_broadcast(struct clock_event_device *dev) 403static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
403{ 404{
404 struct tick_device *td; 405 struct tick_device *td;
405 cpumask_t mask;
406 ktime_t now, next_event; 406 ktime_t now, next_event;
407 int cpu; 407 int cpu;
408 408
@@ -410,13 +410,13 @@ static void tick_handle_oneshot_broadcast(struct clock_event_device *dev)
410again: 410again:
411 dev->next_event.tv64 = KTIME_MAX; 411 dev->next_event.tv64 = KTIME_MAX;
412 next_event.tv64 = KTIME_MAX; 412 next_event.tv64 = KTIME_MAX;
413 mask = CPU_MASK_NONE; 413 cpumask_clear(to_cpumask(tmpmask));
414 now = ktime_get(); 414 now = ktime_get();
415 /* Find all expired events */ 415 /* Find all expired events */
416 for_each_cpu_mask_nr(cpu, tick_broadcast_oneshot_mask) { 416 for_each_cpu(cpu, tick_get_broadcast_oneshot_mask()) {
417 td = &per_cpu(tick_cpu_device, cpu); 417 td = &per_cpu(tick_cpu_device, cpu);
418 if (td->evtdev->next_event.tv64 <= now.tv64) 418 if (td->evtdev->next_event.tv64 <= now.tv64)
419 cpu_set(cpu, mask); 419 cpumask_set_cpu(cpu, to_cpumask(tmpmask));
420 else if (td->evtdev->next_event.tv64 < next_event.tv64) 420 else if (td->evtdev->next_event.tv64 < next_event.tv64)
421 next_event.tv64 = td->evtdev->next_event.tv64; 421 next_event.tv64 = td->evtdev->next_event.tv64;
422 } 422 }
@@ -424,7 +424,7 @@ again:
424 /* 424 /*
425 * Wakeup the cpus which have an expired event. 425 * Wakeup the cpus which have an expired event.
426 */ 426 */
427 tick_do_broadcast(mask); 427 tick_do_broadcast(to_cpumask(tmpmask));
428 428
429 /* 429 /*
430 * Two reasons for reprogram: 430 * Two reasons for reprogram:
@@ -476,15 +476,16 @@ void tick_broadcast_oneshot_control(unsigned long reason)
476 goto out; 476 goto out;
477 477
478 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) { 478 if (reason == CLOCK_EVT_NOTIFY_BROADCAST_ENTER) {
479 if (!cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 479 if (!cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
480 cpu_set(cpu, tick_broadcast_oneshot_mask); 480 cpumask_set_cpu(cpu, tick_get_broadcast_oneshot_mask());
481 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN); 481 clockevents_set_mode(dev, CLOCK_EVT_MODE_SHUTDOWN);
482 if (dev->next_event.tv64 < bc->next_event.tv64) 482 if (dev->next_event.tv64 < bc->next_event.tv64)
483 tick_broadcast_set_event(dev->next_event, 1); 483 tick_broadcast_set_event(dev->next_event, 1);
484 } 484 }
485 } else { 485 } else {
486 if (cpu_isset(cpu, tick_broadcast_oneshot_mask)) { 486 if (cpumask_test_cpu(cpu, tick_get_broadcast_oneshot_mask())) {
487 cpu_clear(cpu, tick_broadcast_oneshot_mask); 487 cpumask_clear_cpu(cpu,
488 tick_get_broadcast_oneshot_mask());
488 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT); 489 clockevents_set_mode(dev, CLOCK_EVT_MODE_ONESHOT);
489 if (dev->next_event.tv64 != KTIME_MAX) 490 if (dev->next_event.tv64 != KTIME_MAX)
490 tick_program_event(dev->next_event, 1); 491 tick_program_event(dev->next_event, 1);
@@ -502,15 +503,16 @@ out:
502 */ 503 */
503static void tick_broadcast_clear_oneshot(int cpu) 504static void tick_broadcast_clear_oneshot(int cpu)
504{ 505{
505 cpu_clear(cpu, tick_broadcast_oneshot_mask); 506 cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
506} 507}
507 508
508static void tick_broadcast_init_next_event(cpumask_t *mask, ktime_t expires) 509static void tick_broadcast_init_next_event(struct cpumask *mask,
510 ktime_t expires)
509{ 511{
510 struct tick_device *td; 512 struct tick_device *td;
511 int cpu; 513 int cpu;
512 514
513 for_each_cpu_mask_nr(cpu, *mask) { 515 for_each_cpu(cpu, mask) {
514 td = &per_cpu(tick_cpu_device, cpu); 516 td = &per_cpu(tick_cpu_device, cpu);
515 if (td->evtdev) 517 if (td->evtdev)
516 td->evtdev->next_event = expires; 518 td->evtdev->next_event = expires;
@@ -526,7 +528,6 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
526 if (bc->event_handler != tick_handle_oneshot_broadcast) { 528 if (bc->event_handler != tick_handle_oneshot_broadcast) {
527 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC; 529 int was_periodic = bc->mode == CLOCK_EVT_MODE_PERIODIC;
528 int cpu = smp_processor_id(); 530 int cpu = smp_processor_id();
529 cpumask_t mask;
530 531
531 bc->event_handler = tick_handle_oneshot_broadcast; 532 bc->event_handler = tick_handle_oneshot_broadcast;
532 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT); 533 clockevents_set_mode(bc, CLOCK_EVT_MODE_ONESHOT);
@@ -540,13 +541,15 @@ void tick_broadcast_setup_oneshot(struct clock_event_device *bc)
540 * oneshot_mask bits for those and program the 541 * oneshot_mask bits for those and program the
541 * broadcast device to fire. 542 * broadcast device to fire.
542 */ 543 */
543 mask = tick_broadcast_mask; 544 cpumask_copy(to_cpumask(tmpmask), tick_get_broadcast_mask());
544 cpu_clear(cpu, mask); 545 cpumask_clear_cpu(cpu, to_cpumask(tmpmask));
545 cpus_or(tick_broadcast_oneshot_mask, 546 cpumask_or(tick_get_broadcast_oneshot_mask(),
546 tick_broadcast_oneshot_mask, mask); 547 tick_get_broadcast_oneshot_mask(),
547 548 to_cpumask(tmpmask));
548 if (was_periodic && !cpus_empty(mask)) { 549
549 tick_broadcast_init_next_event(&mask, tick_next_period); 550 if (was_periodic && !cpumask_empty(to_cpumask(tmpmask))) {
551 tick_broadcast_init_next_event(to_cpumask(tmpmask),
552 tick_next_period);
550 tick_broadcast_set_event(tick_next_period, 1); 553 tick_broadcast_set_event(tick_next_period, 1);
551 } else 554 } else
552 bc->next_event.tv64 = KTIME_MAX; 555 bc->next_event.tv64 = KTIME_MAX;
@@ -585,7 +588,7 @@ void tick_shutdown_broadcast_oneshot(unsigned int *cpup)
585 * Clear the broadcast mask flag for the dead cpu, but do not 588 * Clear the broadcast mask flag for the dead cpu, but do not
586 * stop the broadcast device! 589 * stop the broadcast device!
587 */ 590 */
588 cpu_clear(cpu, tick_broadcast_oneshot_mask); 591 cpumask_clear_cpu(cpu, tick_get_broadcast_oneshot_mask());
589 592
590 spin_unlock_irqrestore(&tick_broadcast_lock, flags); 593 spin_unlock_irqrestore(&tick_broadcast_lock, flags);
591} 594}
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
index df12434b43ca..63e05d423a09 100644
--- a/kernel/time/tick-common.c
+++ b/kernel/time/tick-common.c
@@ -136,7 +136,7 @@ void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
136 */ 136 */
137static void tick_setup_device(struct tick_device *td, 137static void tick_setup_device(struct tick_device *td,
138 struct clock_event_device *newdev, int cpu, 138 struct clock_event_device *newdev, int cpu,
139 const cpumask_t *cpumask) 139 const struct cpumask *cpumask)
140{ 140{
141 ktime_t next_event; 141 ktime_t next_event;
142 void (*handler)(struct clock_event_device *) = NULL; 142 void (*handler)(struct clock_event_device *) = NULL;
@@ -171,8 +171,8 @@ static void tick_setup_device(struct tick_device *td,
171 * When the device is not per cpu, pin the interrupt to the 171 * When the device is not per cpu, pin the interrupt to the
172 * current cpu: 172 * current cpu:
173 */ 173 */
174 if (!cpus_equal(newdev->cpumask, *cpumask)) 174 if (!cpumask_equal(newdev->cpumask, cpumask))
175 irq_set_affinity(newdev->irq, *cpumask); 175 irq_set_affinity(newdev->irq, cpumask);
176 176
177 /* 177 /*
178 * When global broadcasting is active, check if the current 178 * When global broadcasting is active, check if the current
@@ -202,14 +202,14 @@ static int tick_check_new_device(struct clock_event_device *newdev)
202 spin_lock_irqsave(&tick_device_lock, flags); 202 spin_lock_irqsave(&tick_device_lock, flags);
203 203
204 cpu = smp_processor_id(); 204 cpu = smp_processor_id();
205 if (!cpu_isset(cpu, newdev->cpumask)) 205 if (!cpumask_test_cpu(cpu, newdev->cpumask))
206 goto out_bc; 206 goto out_bc;
207 207
208 td = &per_cpu(tick_cpu_device, cpu); 208 td = &per_cpu(tick_cpu_device, cpu);
209 curdev = td->evtdev; 209 curdev = td->evtdev;
210 210
211 /* cpu local device ? */ 211 /* cpu local device ? */
212 if (!cpus_equal(newdev->cpumask, cpumask_of_cpu(cpu))) { 212 if (!cpumask_equal(newdev->cpumask, cpumask_of(cpu))) {
213 213
214 /* 214 /*
215 * If the cpu affinity of the device interrupt can not 215 * If the cpu affinity of the device interrupt can not
@@ -222,7 +222,7 @@ static int tick_check_new_device(struct clock_event_device *newdev)
222 * If we have a cpu local device already, do not replace it 222 * If we have a cpu local device already, do not replace it
223 * by a non cpu local device 223 * by a non cpu local device
224 */ 224 */
225 if (curdev && cpus_equal(curdev->cpumask, cpumask_of_cpu(cpu))) 225 if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
226 goto out_bc; 226 goto out_bc;
227 } 227 }
228 228
@@ -254,7 +254,7 @@ static int tick_check_new_device(struct clock_event_device *newdev)
254 curdev = NULL; 254 curdev = NULL;
255 } 255 }
256 clockevents_exchange_device(curdev, newdev); 256 clockevents_exchange_device(curdev, newdev);
257 tick_setup_device(td, newdev, cpu, &cpumask_of_cpu(cpu)); 257 tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
258 if (newdev->features & CLOCK_EVT_FEAT_ONESHOT) 258 if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
259 tick_oneshot_notify(); 259 tick_oneshot_notify();
260 260
@@ -299,9 +299,9 @@ static void tick_shutdown(unsigned int *cpup)
299 } 299 }
300 /* Transfer the do_timer job away from this cpu */ 300 /* Transfer the do_timer job away from this cpu */
301 if (*cpup == tick_do_timer_cpu) { 301 if (*cpup == tick_do_timer_cpu) {
302 int cpu = first_cpu(cpu_online_map); 302 int cpu = cpumask_first(cpu_online_mask);
303 303
304 tick_do_timer_cpu = (cpu != NR_CPUS) ? cpu : 304 tick_do_timer_cpu = (cpu < nr_cpu_ids) ? cpu :
305 TICK_DO_TIMER_NONE; 305 TICK_DO_TIMER_NONE;
306 } 306 }
307 spin_unlock_irqrestore(&tick_device_lock, flags); 307 spin_unlock_irqrestore(&tick_device_lock, flags);
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 342fc9ccab46..1b6c05bd0d0a 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -144,7 +144,7 @@ void tick_nohz_update_jiffies(void)
144 if (!ts->tick_stopped) 144 if (!ts->tick_stopped)
145 return; 145 return;
146 146
147 cpu_clear(cpu, nohz_cpu_mask); 147 cpumask_clear_cpu(cpu, nohz_cpu_mask);
148 now = ktime_get(); 148 now = ktime_get();
149 ts->idle_waketime = now; 149 ts->idle_waketime = now;
150 150
@@ -247,7 +247,7 @@ void tick_nohz_stop_sched_tick(int inidle)
247 if (need_resched()) 247 if (need_resched())
248 goto end; 248 goto end;
249 249
250 if (unlikely(local_softirq_pending())) { 250 if (unlikely(local_softirq_pending() && cpu_online(cpu))) {
251 static int ratelimit; 251 static int ratelimit;
252 252
253 if (ratelimit < 10) { 253 if (ratelimit < 10) {
@@ -282,8 +282,31 @@ void tick_nohz_stop_sched_tick(int inidle)
282 /* Schedule the tick, if we are at least one jiffie off */ 282 /* Schedule the tick, if we are at least one jiffie off */
283 if ((long)delta_jiffies >= 1) { 283 if ((long)delta_jiffies >= 1) {
284 284
285 /*
286 * calculate the expiry time for the next timer wheel
287 * timer
288 */
289 expires = ktime_add_ns(last_update, tick_period.tv64 *
290 delta_jiffies);
291
292 /*
293 * If this cpu is the one which updates jiffies, then
294 * give up the assignment and let it be taken by the
295 * cpu which runs the tick timer next, which might be
296 * this cpu as well. If we don't drop this here the
297 * jiffies might be stale and do_timer() never
298 * invoked.
299 */
300 if (cpu == tick_do_timer_cpu)
301 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
302
285 if (delta_jiffies > 1) 303 if (delta_jiffies > 1)
286 cpu_set(cpu, nohz_cpu_mask); 304 cpumask_set_cpu(cpu, nohz_cpu_mask);
305
306 /* Skip reprogram of event if its not changed */
307 if (ts->tick_stopped && ktime_equal(expires, dev->next_event))
308 goto out;
309
287 /* 310 /*
288 * nohz_stop_sched_tick can be called several times before 311 * nohz_stop_sched_tick can be called several times before
289 * the nohz_restart_sched_tick is called. This happens when 312 * the nohz_restart_sched_tick is called. This happens when
@@ -296,7 +319,7 @@ void tick_nohz_stop_sched_tick(int inidle)
296 /* 319 /*
297 * sched tick not stopped! 320 * sched tick not stopped!
298 */ 321 */
299 cpu_clear(cpu, nohz_cpu_mask); 322 cpumask_clear_cpu(cpu, nohz_cpu_mask);
300 goto out; 323 goto out;
301 } 324 }
302 325
@@ -306,17 +329,6 @@ void tick_nohz_stop_sched_tick(int inidle)
306 rcu_enter_nohz(); 329 rcu_enter_nohz();
307 } 330 }
308 331
309 /*
310 * If this cpu is the one which updates jiffies, then
311 * give up the assignment and let it be taken by the
312 * cpu which runs the tick timer next, which might be
313 * this cpu as well. If we don't drop this here the
314 * jiffies might be stale and do_timer() never
315 * invoked.
316 */
317 if (cpu == tick_do_timer_cpu)
318 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
319
320 ts->idle_sleeps++; 332 ts->idle_sleeps++;
321 333
322 /* 334 /*
@@ -332,12 +344,7 @@ void tick_nohz_stop_sched_tick(int inidle)
332 goto out; 344 goto out;
333 } 345 }
334 346
335 /* 347 /* Mark expiries */
336 * calculate the expiry time for the next timer wheel
337 * timer
338 */
339 expires = ktime_add_ns(last_update, tick_period.tv64 *
340 delta_jiffies);
341 ts->idle_expires = expires; 348 ts->idle_expires = expires;
342 349
343 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) { 350 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
@@ -354,7 +361,7 @@ void tick_nohz_stop_sched_tick(int inidle)
354 * softirq. 361 * softirq.
355 */ 362 */
356 tick_do_update_jiffies64(ktime_get()); 363 tick_do_update_jiffies64(ktime_get());
357 cpu_clear(cpu, nohz_cpu_mask); 364 cpumask_clear_cpu(cpu, nohz_cpu_mask);
358 } 365 }
359 raise_softirq_irqoff(TIMER_SOFTIRQ); 366 raise_softirq_irqoff(TIMER_SOFTIRQ);
360out: 367out:
@@ -412,7 +419,9 @@ void tick_nohz_restart_sched_tick(void)
412{ 419{
413 int cpu = smp_processor_id(); 420 int cpu = smp_processor_id();
414 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 421 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
422#ifndef CONFIG_VIRT_CPU_ACCOUNTING
415 unsigned long ticks; 423 unsigned long ticks;
424#endif
416 ktime_t now; 425 ktime_t now;
417 426
418 local_irq_disable(); 427 local_irq_disable();
@@ -432,8 +441,9 @@ void tick_nohz_restart_sched_tick(void)
432 select_nohz_load_balancer(0); 441 select_nohz_load_balancer(0);
433 now = ktime_get(); 442 now = ktime_get();
434 tick_do_update_jiffies64(now); 443 tick_do_update_jiffies64(now);
435 cpu_clear(cpu, nohz_cpu_mask); 444 cpumask_clear_cpu(cpu, nohz_cpu_mask);
436 445
446#ifndef CONFIG_VIRT_CPU_ACCOUNTING
437 /* 447 /*
438 * We stopped the tick in idle. Update process times would miss the 448 * We stopped the tick in idle. Update process times would miss the
439 * time we slept as update_process_times does only a 1 tick 449 * time we slept as update_process_times does only a 1 tick
@@ -443,12 +453,9 @@ void tick_nohz_restart_sched_tick(void)
443 /* 453 /*
444 * We might be one off. Do not randomly account a huge number of ticks! 454 * We might be one off. Do not randomly account a huge number of ticks!
445 */ 455 */
446 if (ticks && ticks < LONG_MAX) { 456 if (ticks && ticks < LONG_MAX)
447 add_preempt_count(HARDIRQ_OFFSET); 457 account_idle_ticks(ticks);
448 account_system_time(current, HARDIRQ_OFFSET, 458#endif
449 jiffies_to_cputime(ticks));
450 sub_preempt_count(HARDIRQ_OFFSET);
451 }
452 459
453 touch_softlockup_watchdog(); 460 touch_softlockup_watchdog();
454 /* 461 /*
@@ -681,7 +688,6 @@ void tick_setup_sched_timer(void)
681 */ 688 */
682 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); 689 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
683 ts->sched_timer.function = tick_sched_timer; 690 ts->sched_timer.function = tick_sched_timer;
684 ts->sched_timer.cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
685 691
686 /* Get the next period (per cpu) */ 692 /* Get the next period (per cpu) */
687 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update()); 693 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c
index e7acfb482a68..fa05e88aa76f 100644
--- a/kernel/time/timekeeping.c
+++ b/kernel/time/timekeeping.c
@@ -518,6 +518,28 @@ void update_wall_time(void)
518 /* correct the clock when NTP error is too big */ 518 /* correct the clock when NTP error is too big */
519 clocksource_adjust(offset); 519 clocksource_adjust(offset);
520 520
521 /*
522 * Since in the loop above, we accumulate any amount of time
523 * in xtime_nsec over a second into xtime.tv_sec, its possible for
524 * xtime_nsec to be fairly small after the loop. Further, if we're
525 * slightly speeding the clocksource up in clocksource_adjust(),
526 * its possible the required corrective factor to xtime_nsec could
527 * cause it to underflow.
528 *
529 * Now, we cannot simply roll the accumulated second back, since
530 * the NTP subsystem has been notified via second_overflow. So
531 * instead we push xtime_nsec forward by the amount we underflowed,
532 * and add that amount into the error.
533 *
534 * We'll correct this error next time through this function, when
535 * xtime_nsec is not as small.
536 */
537 if (unlikely((s64)clock->xtime_nsec < 0)) {
538 s64 neg = -(s64)clock->xtime_nsec;
539 clock->xtime_nsec = 0;
540 clock->error += neg << (NTP_SCALE_SHIFT - clock->shift);
541 }
542
521 /* store full nanoseconds into xtime after rounding it up and 543 /* store full nanoseconds into xtime after rounding it up and
522 * add the remainder to the error difference. 544 * add the remainder to the error difference.
523 */ 545 */
diff --git a/kernel/timer.c b/kernel/timer.c
index dbd50fabe4c7..dee3f641a7a7 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1018,21 +1018,6 @@ unsigned long get_next_timer_interrupt(unsigned long now)
1018} 1018}
1019#endif 1019#endif
1020 1020
1021#ifndef CONFIG_VIRT_CPU_ACCOUNTING
1022void account_process_tick(struct task_struct *p, int user_tick)
1023{
1024 cputime_t one_jiffy = jiffies_to_cputime(1);
1025
1026 if (user_tick) {
1027 account_user_time(p, one_jiffy);
1028 account_user_time_scaled(p, cputime_to_scaled(one_jiffy));
1029 } else {
1030 account_system_time(p, HARDIRQ_OFFSET, one_jiffy);
1031 account_system_time_scaled(p, cputime_to_scaled(one_jiffy));
1032 }
1033}
1034#endif
1035
1036/* 1021/*
1037 * Called from the timer interrupt handler to charge one tick to the current 1022 * Called from the timer interrupt handler to charge one tick to the current
1038 * process. user_tick is 1 if the tick is user time, 0 for system. 1023 * process. user_tick is 1 if the tick is user time, 0 for system.
@@ -1192,25 +1177,25 @@ asmlinkage long sys_getppid(void)
1192asmlinkage long sys_getuid(void) 1177asmlinkage long sys_getuid(void)
1193{ 1178{
1194 /* Only we change this so SMP safe */ 1179 /* Only we change this so SMP safe */
1195 return current->uid; 1180 return current_uid();
1196} 1181}
1197 1182
1198asmlinkage long sys_geteuid(void) 1183asmlinkage long sys_geteuid(void)
1199{ 1184{
1200 /* Only we change this so SMP safe */ 1185 /* Only we change this so SMP safe */
1201 return current->euid; 1186 return current_euid();
1202} 1187}
1203 1188
1204asmlinkage long sys_getgid(void) 1189asmlinkage long sys_getgid(void)
1205{ 1190{
1206 /* Only we change this so SMP safe */ 1191 /* Only we change this so SMP safe */
1207 return current->gid; 1192 return current_gid();
1208} 1193}
1209 1194
1210asmlinkage long sys_getegid(void) 1195asmlinkage long sys_getegid(void)
1211{ 1196{
1212 /* Only we change this so SMP safe */ 1197 /* Only we change this so SMP safe */
1213 return current->egid; 1198 return current_egid();
1214} 1199}
1215 1200
1216#endif 1201#endif
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 33dbefd471e8..e2a4ff6fc3a6 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -3,18 +3,34 @@
3# select HAVE_FUNCTION_TRACER: 3# select HAVE_FUNCTION_TRACER:
4# 4#
5 5
6config USER_STACKTRACE_SUPPORT
7 bool
8
6config NOP_TRACER 9config NOP_TRACER
7 bool 10 bool
8 11
9config HAVE_FUNCTION_TRACER 12config HAVE_FUNCTION_TRACER
10 bool 13 bool
11 14
15config HAVE_FUNCTION_GRAPH_TRACER
16 bool
17
18config HAVE_FUNCTION_TRACE_MCOUNT_TEST
19 bool
20 help
21 This gets selected when the arch tests the function_trace_stop
22 variable at the mcount call site. Otherwise, this variable
23 is tested by the called function.
24
12config HAVE_DYNAMIC_FTRACE 25config HAVE_DYNAMIC_FTRACE
13 bool 26 bool
14 27
15config HAVE_FTRACE_MCOUNT_RECORD 28config HAVE_FTRACE_MCOUNT_RECORD
16 bool 29 bool
17 30
31config HAVE_HW_BRANCH_TRACER
32 bool
33
18config TRACER_MAX_TRACE 34config TRACER_MAX_TRACE
19 bool 35 bool
20 36
@@ -47,6 +63,20 @@ config FUNCTION_TRACER
47 (the bootup default), then the overhead of the instructions is very 63 (the bootup default), then the overhead of the instructions is very
48 small and not measurable even in micro-benchmarks. 64 small and not measurable even in micro-benchmarks.
49 65
66config FUNCTION_GRAPH_TRACER
67 bool "Kernel Function Graph Tracer"
68 depends on HAVE_FUNCTION_GRAPH_TRACER
69 depends on FUNCTION_TRACER
70 default y
71 help
72 Enable the kernel to trace a function at both its return
73 and its entry.
74 It's first purpose is to trace the duration of functions and
75 draw a call graph for each thread with some informations like
76 the return value.
77 This is done by setting the current return address on the current
78 task structure into a stack of calls.
79
50config IRQSOFF_TRACER 80config IRQSOFF_TRACER
51 bool "Interrupts-off Latency Tracer" 81 bool "Interrupts-off Latency Tracer"
52 default n 82 default n
@@ -138,6 +168,70 @@ config BOOT_TRACER
138 selected, because the self-tests are an initcall as well and that 168 selected, because the self-tests are an initcall as well and that
139 would invalidate the boot trace. ) 169 would invalidate the boot trace. )
140 170
171config TRACE_BRANCH_PROFILING
172 bool "Trace likely/unlikely profiler"
173 depends on DEBUG_KERNEL
174 select TRACING
175 help
176 This tracer profiles all the the likely and unlikely macros
177 in the kernel. It will display the results in:
178
179 /debugfs/tracing/profile_annotated_branch
180
181 Note: this will add a significant overhead, only turn this
182 on if you need to profile the system's use of these macros.
183
184 Say N if unsure.
185
186config PROFILE_ALL_BRANCHES
187 bool "Profile all if conditionals"
188 depends on TRACE_BRANCH_PROFILING
189 help
190 This tracer profiles all branch conditions. Every if ()
191 taken in the kernel is recorded whether it hit or miss.
192 The results will be displayed in:
193
194 /debugfs/tracing/profile_branch
195
196 This configuration, when enabled, will impose a great overhead
197 on the system. This should only be enabled when the system
198 is to be analyzed
199
200 Say N if unsure.
201
202config TRACING_BRANCHES
203 bool
204 help
205 Selected by tracers that will trace the likely and unlikely
206 conditions. This prevents the tracers themselves from being
207 profiled. Profiling the tracing infrastructure can only happen
208 when the likelys and unlikelys are not being traced.
209
210config BRANCH_TRACER
211 bool "Trace likely/unlikely instances"
212 depends on TRACE_BRANCH_PROFILING
213 select TRACING_BRANCHES
214 help
215 This traces the events of likely and unlikely condition
216 calls in the kernel. The difference between this and the
217 "Trace likely/unlikely profiler" is that this is not a
218 histogram of the callers, but actually places the calling
219 events into a running trace buffer to see when and where the
220 events happened, as well as their results.
221
222 Say N if unsure.
223
224config POWER_TRACER
225 bool "Trace power consumption behavior"
226 depends on DEBUG_KERNEL
227 depends on X86
228 select TRACING
229 help
230 This tracer helps developers to analyze and optimize the kernels
231 power management decisions, specifically the C-state and P-state
232 behavior.
233
234
141config STACK_TRACER 235config STACK_TRACER
142 bool "Trace max stack" 236 bool "Trace max stack"
143 depends on HAVE_FUNCTION_TRACER 237 depends on HAVE_FUNCTION_TRACER
@@ -150,13 +244,26 @@ config STACK_TRACER
150 244
151 This tracer works by hooking into every function call that the 245 This tracer works by hooking into every function call that the
152 kernel executes, and keeping a maximum stack depth value and 246 kernel executes, and keeping a maximum stack depth value and
153 stack-trace saved. Because this logic has to execute in every 247 stack-trace saved. If this is configured with DYNAMIC_FTRACE
154 kernel function, all the time, this option can slow down the 248 then it will not have any overhead while the stack tracer
155 kernel measurably and is generally intended for kernel 249 is disabled.
156 developers only. 250
251 To enable the stack tracer on bootup, pass in 'stacktrace'
252 on the kernel command line.
253
254 The stack tracer can also be enabled or disabled via the
255 sysctl kernel.stack_tracer_enabled
157 256
158 Say N if unsure. 257 Say N if unsure.
159 258
259config HW_BRANCH_TRACER
260 depends on HAVE_HW_BRANCH_TRACER
261 bool "Trace hw branches"
262 select TRACING
263 help
264 This tracer records all branches on the system in a circular
265 buffer giving access to the last N branches for each cpu.
266
160config DYNAMIC_FTRACE 267config DYNAMIC_FTRACE
161 bool "enable/disable ftrace tracepoints dynamically" 268 bool "enable/disable ftrace tracepoints dynamically"
162 depends on FUNCTION_TRACER 269 depends on FUNCTION_TRACER
diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile
index c8228b1a49e9..349d5a93653f 100644
--- a/kernel/trace/Makefile
+++ b/kernel/trace/Makefile
@@ -10,6 +10,11 @@ CFLAGS_trace_selftest_dynamic.o = -pg
10obj-y += trace_selftest_dynamic.o 10obj-y += trace_selftest_dynamic.o
11endif 11endif
12 12
13# If unlikely tracing is enabled, do not trace these files
14ifdef CONFIG_TRACING_BRANCHES
15KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING
16endif
17
13obj-$(CONFIG_FUNCTION_TRACER) += libftrace.o 18obj-$(CONFIG_FUNCTION_TRACER) += libftrace.o
14obj-$(CONFIG_RING_BUFFER) += ring_buffer.o 19obj-$(CONFIG_RING_BUFFER) += ring_buffer.o
15 20
@@ -24,5 +29,9 @@ obj-$(CONFIG_NOP_TRACER) += trace_nop.o
24obj-$(CONFIG_STACK_TRACER) += trace_stack.o 29obj-$(CONFIG_STACK_TRACER) += trace_stack.o
25obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o 30obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o
26obj-$(CONFIG_BOOT_TRACER) += trace_boot.o 31obj-$(CONFIG_BOOT_TRACER) += trace_boot.o
32obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += trace_functions_graph.o
33obj-$(CONFIG_TRACE_BRANCH_PROFILING) += trace_branch.o
34obj-$(CONFIG_HW_BRANCH_TRACER) += trace_hw_branches.o
35obj-$(CONFIG_POWER_TRACER) += trace_power.o
27 36
28libftrace-y := ftrace.o 37libftrace-y := ftrace.o
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 78db083390f0..2f32969c09df 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -47,6 +47,13 @@
47int ftrace_enabled __read_mostly; 47int ftrace_enabled __read_mostly;
48static int last_ftrace_enabled; 48static int last_ftrace_enabled;
49 49
50/* set when tracing only a pid */
51struct pid *ftrace_pid_trace;
52static struct pid * const ftrace_swapper_pid = &init_struct_pid;
53
54/* Quick disabling of function tracer. */
55int function_trace_stop;
56
50/* 57/*
51 * ftrace_disabled is set when an anomaly is discovered. 58 * ftrace_disabled is set when an anomaly is discovered.
52 * ftrace_disabled is much stronger than ftrace_enabled. 59 * ftrace_disabled is much stronger than ftrace_enabled.
@@ -55,6 +62,7 @@ static int ftrace_disabled __read_mostly;
55 62
56static DEFINE_SPINLOCK(ftrace_lock); 63static DEFINE_SPINLOCK(ftrace_lock);
57static DEFINE_MUTEX(ftrace_sysctl_lock); 64static DEFINE_MUTEX(ftrace_sysctl_lock);
65static DEFINE_MUTEX(ftrace_start_lock);
58 66
59static struct ftrace_ops ftrace_list_end __read_mostly = 67static struct ftrace_ops ftrace_list_end __read_mostly =
60{ 68{
@@ -63,6 +71,8 @@ static struct ftrace_ops ftrace_list_end __read_mostly =
63 71
64static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end; 72static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
65ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub; 73ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
74ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
75ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
66 76
67static void ftrace_list_func(unsigned long ip, unsigned long parent_ip) 77static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
68{ 78{
@@ -79,6 +89,21 @@ static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
79 }; 89 };
80} 90}
81 91
92static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
93{
94 if (!test_tsk_trace_trace(current))
95 return;
96
97 ftrace_pid_function(ip, parent_ip);
98}
99
100static void set_ftrace_pid_function(ftrace_func_t func)
101{
102 /* do not set ftrace_pid_function to itself! */
103 if (func != ftrace_pid_func)
104 ftrace_pid_function = func;
105}
106
82/** 107/**
83 * clear_ftrace_function - reset the ftrace function 108 * clear_ftrace_function - reset the ftrace function
84 * 109 *
@@ -88,7 +113,23 @@ static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
88void clear_ftrace_function(void) 113void clear_ftrace_function(void)
89{ 114{
90 ftrace_trace_function = ftrace_stub; 115 ftrace_trace_function = ftrace_stub;
116 __ftrace_trace_function = ftrace_stub;
117 ftrace_pid_function = ftrace_stub;
118}
119
120#ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
121/*
122 * For those archs that do not test ftrace_trace_stop in their
123 * mcount call site, we need to do it from C.
124 */
125static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
126{
127 if (function_trace_stop)
128 return;
129
130 __ftrace_trace_function(ip, parent_ip);
91} 131}
132#endif
92 133
93static int __register_ftrace_function(struct ftrace_ops *ops) 134static int __register_ftrace_function(struct ftrace_ops *ops)
94{ 135{
@@ -106,14 +147,28 @@ static int __register_ftrace_function(struct ftrace_ops *ops)
106 ftrace_list = ops; 147 ftrace_list = ops;
107 148
108 if (ftrace_enabled) { 149 if (ftrace_enabled) {
150 ftrace_func_t func;
151
152 if (ops->next == &ftrace_list_end)
153 func = ops->func;
154 else
155 func = ftrace_list_func;
156
157 if (ftrace_pid_trace) {
158 set_ftrace_pid_function(func);
159 func = ftrace_pid_func;
160 }
161
109 /* 162 /*
110 * For one func, simply call it directly. 163 * For one func, simply call it directly.
111 * For more than one func, call the chain. 164 * For more than one func, call the chain.
112 */ 165 */
113 if (ops->next == &ftrace_list_end) 166#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
114 ftrace_trace_function = ops->func; 167 ftrace_trace_function = func;
115 else 168#else
116 ftrace_trace_function = ftrace_list_func; 169 __ftrace_trace_function = func;
170 ftrace_trace_function = ftrace_test_stop_func;
171#endif
117 } 172 }
118 173
119 spin_unlock(&ftrace_lock); 174 spin_unlock(&ftrace_lock);
@@ -152,9 +207,19 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
152 207
153 if (ftrace_enabled) { 208 if (ftrace_enabled) {
154 /* If we only have one func left, then call that directly */ 209 /* If we only have one func left, then call that directly */
155 if (ftrace_list == &ftrace_list_end || 210 if (ftrace_list->next == &ftrace_list_end) {
156 ftrace_list->next == &ftrace_list_end) 211 ftrace_func_t func = ftrace_list->func;
157 ftrace_trace_function = ftrace_list->func; 212
213 if (ftrace_pid_trace) {
214 set_ftrace_pid_function(func);
215 func = ftrace_pid_func;
216 }
217#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
218 ftrace_trace_function = func;
219#else
220 __ftrace_trace_function = func;
221#endif
222 }
158 } 223 }
159 224
160 out: 225 out:
@@ -163,6 +228,36 @@ static int __unregister_ftrace_function(struct ftrace_ops *ops)
163 return ret; 228 return ret;
164} 229}
165 230
231static void ftrace_update_pid_func(void)
232{
233 ftrace_func_t func;
234
235 /* should not be called from interrupt context */
236 spin_lock(&ftrace_lock);
237
238 if (ftrace_trace_function == ftrace_stub)
239 goto out;
240
241 func = ftrace_trace_function;
242
243 if (ftrace_pid_trace) {
244 set_ftrace_pid_function(func);
245 func = ftrace_pid_func;
246 } else {
247 if (func == ftrace_pid_func)
248 func = ftrace_pid_function;
249 }
250
251#ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
252 ftrace_trace_function = func;
253#else
254 __ftrace_trace_function = func;
255#endif
256
257 out:
258 spin_unlock(&ftrace_lock);
259}
260
166#ifdef CONFIG_DYNAMIC_FTRACE 261#ifdef CONFIG_DYNAMIC_FTRACE
167#ifndef CONFIG_FTRACE_MCOUNT_RECORD 262#ifndef CONFIG_FTRACE_MCOUNT_RECORD
168# error Dynamic ftrace depends on MCOUNT_RECORD 263# error Dynamic ftrace depends on MCOUNT_RECORD
@@ -182,6 +277,8 @@ enum {
182 FTRACE_UPDATE_TRACE_FUNC = (1 << 2), 277 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
183 FTRACE_ENABLE_MCOUNT = (1 << 3), 278 FTRACE_ENABLE_MCOUNT = (1 << 3),
184 FTRACE_DISABLE_MCOUNT = (1 << 4), 279 FTRACE_DISABLE_MCOUNT = (1 << 4),
280 FTRACE_START_FUNC_RET = (1 << 5),
281 FTRACE_STOP_FUNC_RET = (1 << 6),
185}; 282};
186 283
187static int ftrace_filtered; 284static int ftrace_filtered;
@@ -308,7 +405,7 @@ ftrace_record_ip(unsigned long ip)
308{ 405{
309 struct dyn_ftrace *rec; 406 struct dyn_ftrace *rec;
310 407
311 if (!ftrace_enabled || ftrace_disabled) 408 if (ftrace_disabled)
312 return NULL; 409 return NULL;
313 410
314 rec = ftrace_alloc_dyn_node(ip); 411 rec = ftrace_alloc_dyn_node(ip);
@@ -322,14 +419,51 @@ ftrace_record_ip(unsigned long ip)
322 return rec; 419 return rec;
323} 420}
324 421
325#define FTRACE_ADDR ((long)(ftrace_caller)) 422static void print_ip_ins(const char *fmt, unsigned char *p)
423{
424 int i;
425
426 printk(KERN_CONT "%s", fmt);
427
428 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
429 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
430}
431
432static void ftrace_bug(int failed, unsigned long ip)
433{
434 switch (failed) {
435 case -EFAULT:
436 FTRACE_WARN_ON_ONCE(1);
437 pr_info("ftrace faulted on modifying ");
438 print_ip_sym(ip);
439 break;
440 case -EINVAL:
441 FTRACE_WARN_ON_ONCE(1);
442 pr_info("ftrace failed to modify ");
443 print_ip_sym(ip);
444 print_ip_ins(" actual: ", (unsigned char *)ip);
445 printk(KERN_CONT "\n");
446 break;
447 case -EPERM:
448 FTRACE_WARN_ON_ONCE(1);
449 pr_info("ftrace faulted on writing ");
450 print_ip_sym(ip);
451 break;
452 default:
453 FTRACE_WARN_ON_ONCE(1);
454 pr_info("ftrace faulted on unknown error ");
455 print_ip_sym(ip);
456 }
457}
458
326 459
327static int 460static int
328__ftrace_replace_code(struct dyn_ftrace *rec, 461__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
329 unsigned char *nop, int enable)
330{ 462{
331 unsigned long ip, fl; 463 unsigned long ip, fl;
332 unsigned char *call, *old, *new; 464 unsigned long ftrace_addr;
465
466 ftrace_addr = (unsigned long)ftrace_caller;
333 467
334 ip = rec->ip; 468 ip = rec->ip;
335 469
@@ -388,34 +522,28 @@ __ftrace_replace_code(struct dyn_ftrace *rec,
388 } 522 }
389 } 523 }
390 524
391 call = ftrace_call_replace(ip, FTRACE_ADDR); 525 if (rec->flags & FTRACE_FL_ENABLED)
392 526 return ftrace_make_call(rec, ftrace_addr);
393 if (rec->flags & FTRACE_FL_ENABLED) { 527 else
394 old = nop; 528 return ftrace_make_nop(NULL, rec, ftrace_addr);
395 new = call;
396 } else {
397 old = call;
398 new = nop;
399 }
400
401 return ftrace_modify_code(ip, old, new);
402} 529}
403 530
404static void ftrace_replace_code(int enable) 531static void ftrace_replace_code(int enable)
405{ 532{
406 int i, failed; 533 int i, failed;
407 unsigned char *nop = NULL;
408 struct dyn_ftrace *rec; 534 struct dyn_ftrace *rec;
409 struct ftrace_page *pg; 535 struct ftrace_page *pg;
410 536
411 nop = ftrace_nop_replace();
412
413 for (pg = ftrace_pages_start; pg; pg = pg->next) { 537 for (pg = ftrace_pages_start; pg; pg = pg->next) {
414 for (i = 0; i < pg->index; i++) { 538 for (i = 0; i < pg->index; i++) {
415 rec = &pg->records[i]; 539 rec = &pg->records[i];
416 540
417 /* don't modify code that has already faulted */ 541 /*
418 if (rec->flags & FTRACE_FL_FAILED) 542 * Skip over free records and records that have
543 * failed.
544 */
545 if (rec->flags & FTRACE_FL_FREE ||
546 rec->flags & FTRACE_FL_FAILED)
419 continue; 547 continue;
420 548
421 /* ignore updates to this record's mcount site */ 549 /* ignore updates to this record's mcount site */
@@ -426,68 +554,30 @@ static void ftrace_replace_code(int enable)
426 unfreeze_record(rec); 554 unfreeze_record(rec);
427 } 555 }
428 556
429 failed = __ftrace_replace_code(rec, nop, enable); 557 failed = __ftrace_replace_code(rec, enable);
430 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) { 558 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
431 rec->flags |= FTRACE_FL_FAILED; 559 rec->flags |= FTRACE_FL_FAILED;
432 if ((system_state == SYSTEM_BOOTING) || 560 if ((system_state == SYSTEM_BOOTING) ||
433 !core_kernel_text(rec->ip)) { 561 !core_kernel_text(rec->ip)) {
434 ftrace_free_rec(rec); 562 ftrace_free_rec(rec);
435 } 563 } else
564 ftrace_bug(failed, rec->ip);
436 } 565 }
437 } 566 }
438 } 567 }
439} 568}
440 569
441static void print_ip_ins(const char *fmt, unsigned char *p)
442{
443 int i;
444
445 printk(KERN_CONT "%s", fmt);
446
447 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
448 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
449}
450
451static int 570static int
452ftrace_code_disable(struct dyn_ftrace *rec) 571ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
453{ 572{
454 unsigned long ip; 573 unsigned long ip;
455 unsigned char *nop, *call;
456 int ret; 574 int ret;
457 575
458 ip = rec->ip; 576 ip = rec->ip;
459 577
460 nop = ftrace_nop_replace(); 578 ret = ftrace_make_nop(mod, rec, mcount_addr);
461 call = ftrace_call_replace(ip, mcount_addr);
462
463 ret = ftrace_modify_code(ip, call, nop);
464 if (ret) { 579 if (ret) {
465 switch (ret) { 580 ftrace_bug(ret, ip);
466 case -EFAULT:
467 FTRACE_WARN_ON_ONCE(1);
468 pr_info("ftrace faulted on modifying ");
469 print_ip_sym(ip);
470 break;
471 case -EINVAL:
472 FTRACE_WARN_ON_ONCE(1);
473 pr_info("ftrace failed to modify ");
474 print_ip_sym(ip);
475 print_ip_ins(" expected: ", call);
476 print_ip_ins(" actual: ", (unsigned char *)ip);
477 print_ip_ins(" replace: ", nop);
478 printk(KERN_CONT "\n");
479 break;
480 case -EPERM:
481 FTRACE_WARN_ON_ONCE(1);
482 pr_info("ftrace faulted on writing ");
483 print_ip_sym(ip);
484 break;
485 default:
486 FTRACE_WARN_ON_ONCE(1);
487 pr_info("ftrace faulted on unknown error ");
488 print_ip_sym(ip);
489 }
490
491 rec->flags |= FTRACE_FL_FAILED; 581 rec->flags |= FTRACE_FL_FAILED;
492 return 0; 582 return 0;
493 } 583 }
@@ -506,6 +596,11 @@ static int __ftrace_modify_code(void *data)
506 if (*command & FTRACE_UPDATE_TRACE_FUNC) 596 if (*command & FTRACE_UPDATE_TRACE_FUNC)
507 ftrace_update_ftrace_func(ftrace_trace_function); 597 ftrace_update_ftrace_func(ftrace_trace_function);
508 598
599 if (*command & FTRACE_START_FUNC_RET)
600 ftrace_enable_ftrace_graph_caller();
601 else if (*command & FTRACE_STOP_FUNC_RET)
602 ftrace_disable_ftrace_graph_caller();
603
509 return 0; 604 return 0;
510} 605}
511 606
@@ -515,43 +610,43 @@ static void ftrace_run_update_code(int command)
515} 610}
516 611
517static ftrace_func_t saved_ftrace_func; 612static ftrace_func_t saved_ftrace_func;
518static int ftrace_start; 613static int ftrace_start_up;
519static DEFINE_MUTEX(ftrace_start_lock);
520 614
521static void ftrace_startup(void) 615static void ftrace_startup_enable(int command)
522{ 616{
523 int command = 0;
524
525 if (unlikely(ftrace_disabled))
526 return;
527
528 mutex_lock(&ftrace_start_lock);
529 ftrace_start++;
530 command |= FTRACE_ENABLE_CALLS;
531
532 if (saved_ftrace_func != ftrace_trace_function) { 617 if (saved_ftrace_func != ftrace_trace_function) {
533 saved_ftrace_func = ftrace_trace_function; 618 saved_ftrace_func = ftrace_trace_function;
534 command |= FTRACE_UPDATE_TRACE_FUNC; 619 command |= FTRACE_UPDATE_TRACE_FUNC;
535 } 620 }
536 621
537 if (!command || !ftrace_enabled) 622 if (!command || !ftrace_enabled)
538 goto out; 623 return;
539 624
540 ftrace_run_update_code(command); 625 ftrace_run_update_code(command);
541 out:
542 mutex_unlock(&ftrace_start_lock);
543} 626}
544 627
545static void ftrace_shutdown(void) 628static void ftrace_startup(int command)
546{ 629{
547 int command = 0; 630 if (unlikely(ftrace_disabled))
631 return;
632
633 mutex_lock(&ftrace_start_lock);
634 ftrace_start_up++;
635 command |= FTRACE_ENABLE_CALLS;
548 636
637 ftrace_startup_enable(command);
638
639 mutex_unlock(&ftrace_start_lock);
640}
641
642static void ftrace_shutdown(int command)
643{
549 if (unlikely(ftrace_disabled)) 644 if (unlikely(ftrace_disabled))
550 return; 645 return;
551 646
552 mutex_lock(&ftrace_start_lock); 647 mutex_lock(&ftrace_start_lock);
553 ftrace_start--; 648 ftrace_start_up--;
554 if (!ftrace_start) 649 if (!ftrace_start_up)
555 command |= FTRACE_DISABLE_CALLS; 650 command |= FTRACE_DISABLE_CALLS;
556 651
557 if (saved_ftrace_func != ftrace_trace_function) { 652 if (saved_ftrace_func != ftrace_trace_function) {
@@ -577,8 +672,8 @@ static void ftrace_startup_sysctl(void)
577 mutex_lock(&ftrace_start_lock); 672 mutex_lock(&ftrace_start_lock);
578 /* Force update next time */ 673 /* Force update next time */
579 saved_ftrace_func = NULL; 674 saved_ftrace_func = NULL;
580 /* ftrace_start is true if we want ftrace running */ 675 /* ftrace_start_up is true if we want ftrace running */
581 if (ftrace_start) 676 if (ftrace_start_up)
582 command |= FTRACE_ENABLE_CALLS; 677 command |= FTRACE_ENABLE_CALLS;
583 678
584 ftrace_run_update_code(command); 679 ftrace_run_update_code(command);
@@ -593,8 +688,8 @@ static void ftrace_shutdown_sysctl(void)
593 return; 688 return;
594 689
595 mutex_lock(&ftrace_start_lock); 690 mutex_lock(&ftrace_start_lock);
596 /* ftrace_start is true if ftrace is running */ 691 /* ftrace_start_up is true if ftrace is running */
597 if (ftrace_start) 692 if (ftrace_start_up)
598 command |= FTRACE_DISABLE_CALLS; 693 command |= FTRACE_DISABLE_CALLS;
599 694
600 ftrace_run_update_code(command); 695 ftrace_run_update_code(command);
@@ -605,7 +700,7 @@ static cycle_t ftrace_update_time;
605static unsigned long ftrace_update_cnt; 700static unsigned long ftrace_update_cnt;
606unsigned long ftrace_update_tot_cnt; 701unsigned long ftrace_update_tot_cnt;
607 702
608static int ftrace_update_code(void) 703static int ftrace_update_code(struct module *mod)
609{ 704{
610 struct dyn_ftrace *p, *t; 705 struct dyn_ftrace *p, *t;
611 cycle_t start, stop; 706 cycle_t start, stop;
@@ -622,7 +717,7 @@ static int ftrace_update_code(void)
622 list_del_init(&p->list); 717 list_del_init(&p->list);
623 718
624 /* convert record (i.e, patch mcount-call with NOP) */ 719 /* convert record (i.e, patch mcount-call with NOP) */
625 if (ftrace_code_disable(p)) { 720 if (ftrace_code_disable(mod, p)) {
626 p->flags |= FTRACE_FL_CONVERTED; 721 p->flags |= FTRACE_FL_CONVERTED;
627 ftrace_update_cnt++; 722 ftrace_update_cnt++;
628 } else 723 } else
@@ -690,7 +785,6 @@ enum {
690#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */ 785#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
691 786
692struct ftrace_iterator { 787struct ftrace_iterator {
693 loff_t pos;
694 struct ftrace_page *pg; 788 struct ftrace_page *pg;
695 unsigned idx; 789 unsigned idx;
696 unsigned flags; 790 unsigned flags;
@@ -715,6 +809,8 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
715 iter->pg = iter->pg->next; 809 iter->pg = iter->pg->next;
716 iter->idx = 0; 810 iter->idx = 0;
717 goto retry; 811 goto retry;
812 } else {
813 iter->idx = -1;
718 } 814 }
719 } else { 815 } else {
720 rec = &iter->pg->records[iter->idx++]; 816 rec = &iter->pg->records[iter->idx++];
@@ -737,8 +833,6 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
737 } 833 }
738 spin_unlock(&ftrace_lock); 834 spin_unlock(&ftrace_lock);
739 835
740 iter->pos = *pos;
741
742 return rec; 836 return rec;
743} 837}
744 838
@@ -746,13 +840,15 @@ static void *t_start(struct seq_file *m, loff_t *pos)
746{ 840{
747 struct ftrace_iterator *iter = m->private; 841 struct ftrace_iterator *iter = m->private;
748 void *p = NULL; 842 void *p = NULL;
749 loff_t l = -1;
750 843
751 if (*pos > iter->pos) 844 if (*pos > 0) {
752 *pos = iter->pos; 845 if (iter->idx < 0)
846 return p;
847 (*pos)--;
848 iter->idx--;
849 }
753 850
754 l = *pos; 851 p = t_next(m, p, pos);
755 p = t_next(m, p, &l);
756 852
757 return p; 853 return p;
758} 854}
@@ -763,21 +859,15 @@ static void t_stop(struct seq_file *m, void *p)
763 859
764static int t_show(struct seq_file *m, void *v) 860static int t_show(struct seq_file *m, void *v)
765{ 861{
766 struct ftrace_iterator *iter = m->private;
767 struct dyn_ftrace *rec = v; 862 struct dyn_ftrace *rec = v;
768 char str[KSYM_SYMBOL_LEN]; 863 char str[KSYM_SYMBOL_LEN];
769 int ret = 0;
770 864
771 if (!rec) 865 if (!rec)
772 return 0; 866 return 0;
773 867
774 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); 868 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
775 869
776 ret = seq_printf(m, "%s\n", str); 870 seq_printf(m, "%s\n", str);
777 if (ret < 0) {
778 iter->pos--;
779 iter->idx--;
780 }
781 871
782 return 0; 872 return 0;
783} 873}
@@ -803,7 +893,6 @@ ftrace_avail_open(struct inode *inode, struct file *file)
803 return -ENOMEM; 893 return -ENOMEM;
804 894
805 iter->pg = ftrace_pages_start; 895 iter->pg = ftrace_pages_start;
806 iter->pos = 0;
807 896
808 ret = seq_open(file, &show_ftrace_seq_ops); 897 ret = seq_open(file, &show_ftrace_seq_ops);
809 if (!ret) { 898 if (!ret) {
@@ -890,7 +979,6 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable)
890 979
891 if (file->f_mode & FMODE_READ) { 980 if (file->f_mode & FMODE_READ) {
892 iter->pg = ftrace_pages_start; 981 iter->pg = ftrace_pages_start;
893 iter->pos = 0;
894 iter->flags = enable ? FTRACE_ITER_FILTER : 982 iter->flags = enable ? FTRACE_ITER_FILTER :
895 FTRACE_ITER_NOTRACE; 983 FTRACE_ITER_NOTRACE;
896 984
@@ -959,6 +1047,13 @@ ftrace_match(unsigned char *buff, int len, int enable)
959 int type = MATCH_FULL; 1047 int type = MATCH_FULL;
960 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE; 1048 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
961 unsigned i, match = 0, search_len = 0; 1049 unsigned i, match = 0, search_len = 0;
1050 int not = 0;
1051
1052 if (buff[0] == '!') {
1053 not = 1;
1054 buff++;
1055 len--;
1056 }
962 1057
963 for (i = 0; i < len; i++) { 1058 for (i = 0; i < len; i++) {
964 if (buff[i] == '*') { 1059 if (buff[i] == '*') {
@@ -1012,8 +1107,12 @@ ftrace_match(unsigned char *buff, int len, int enable)
1012 matched = 1; 1107 matched = 1;
1013 break; 1108 break;
1014 } 1109 }
1015 if (matched) 1110 if (matched) {
1016 rec->flags |= flag; 1111 if (not)
1112 rec->flags &= ~flag;
1113 else
1114 rec->flags |= flag;
1115 }
1017 } 1116 }
1018 pg = pg->next; 1117 pg = pg->next;
1019 } 1118 }
@@ -1181,7 +1280,7 @@ ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1181 1280
1182 mutex_lock(&ftrace_sysctl_lock); 1281 mutex_lock(&ftrace_sysctl_lock);
1183 mutex_lock(&ftrace_start_lock); 1282 mutex_lock(&ftrace_start_lock);
1184 if (ftrace_start && ftrace_enabled) 1283 if (ftrace_start_up && ftrace_enabled)
1185 ftrace_run_update_code(FTRACE_ENABLE_CALLS); 1284 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1186 mutex_unlock(&ftrace_start_lock); 1285 mutex_unlock(&ftrace_start_lock);
1187 mutex_unlock(&ftrace_sysctl_lock); 1286 mutex_unlock(&ftrace_sysctl_lock);
@@ -1233,12 +1332,233 @@ static struct file_operations ftrace_notrace_fops = {
1233 .release = ftrace_notrace_release, 1332 .release = ftrace_notrace_release,
1234}; 1333};
1235 1334
1236static __init int ftrace_init_debugfs(void) 1335#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1336
1337static DEFINE_MUTEX(graph_lock);
1338
1339int ftrace_graph_count;
1340unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1341
1342static void *
1343g_next(struct seq_file *m, void *v, loff_t *pos)
1237{ 1344{
1238 struct dentry *d_tracer; 1345 unsigned long *array = m->private;
1239 struct dentry *entry; 1346 int index = *pos;
1240 1347
1241 d_tracer = tracing_init_dentry(); 1348 (*pos)++;
1349
1350 if (index >= ftrace_graph_count)
1351 return NULL;
1352
1353 return &array[index];
1354}
1355
1356static void *g_start(struct seq_file *m, loff_t *pos)
1357{
1358 void *p = NULL;
1359
1360 mutex_lock(&graph_lock);
1361
1362 p = g_next(m, p, pos);
1363
1364 return p;
1365}
1366
1367static void g_stop(struct seq_file *m, void *p)
1368{
1369 mutex_unlock(&graph_lock);
1370}
1371
1372static int g_show(struct seq_file *m, void *v)
1373{
1374 unsigned long *ptr = v;
1375 char str[KSYM_SYMBOL_LEN];
1376
1377 if (!ptr)
1378 return 0;
1379
1380 kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1381
1382 seq_printf(m, "%s\n", str);
1383
1384 return 0;
1385}
1386
1387static struct seq_operations ftrace_graph_seq_ops = {
1388 .start = g_start,
1389 .next = g_next,
1390 .stop = g_stop,
1391 .show = g_show,
1392};
1393
1394static int
1395ftrace_graph_open(struct inode *inode, struct file *file)
1396{
1397 int ret = 0;
1398
1399 if (unlikely(ftrace_disabled))
1400 return -ENODEV;
1401
1402 mutex_lock(&graph_lock);
1403 if ((file->f_mode & FMODE_WRITE) &&
1404 !(file->f_flags & O_APPEND)) {
1405 ftrace_graph_count = 0;
1406 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1407 }
1408
1409 if (file->f_mode & FMODE_READ) {
1410 ret = seq_open(file, &ftrace_graph_seq_ops);
1411 if (!ret) {
1412 struct seq_file *m = file->private_data;
1413 m->private = ftrace_graph_funcs;
1414 }
1415 } else
1416 file->private_data = ftrace_graph_funcs;
1417 mutex_unlock(&graph_lock);
1418
1419 return ret;
1420}
1421
1422static ssize_t
1423ftrace_graph_read(struct file *file, char __user *ubuf,
1424 size_t cnt, loff_t *ppos)
1425{
1426 if (file->f_mode & FMODE_READ)
1427 return seq_read(file, ubuf, cnt, ppos);
1428 else
1429 return -EPERM;
1430}
1431
1432static int
1433ftrace_set_func(unsigned long *array, int idx, char *buffer)
1434{
1435 char str[KSYM_SYMBOL_LEN];
1436 struct dyn_ftrace *rec;
1437 struct ftrace_page *pg;
1438 int found = 0;
1439 int i, j;
1440
1441 if (ftrace_disabled)
1442 return -ENODEV;
1443
1444 /* should not be called from interrupt context */
1445 spin_lock(&ftrace_lock);
1446
1447 for (pg = ftrace_pages_start; pg; pg = pg->next) {
1448 for (i = 0; i < pg->index; i++) {
1449 rec = &pg->records[i];
1450
1451 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
1452 continue;
1453
1454 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1455 if (strcmp(str, buffer) == 0) {
1456 found = 1;
1457 for (j = 0; j < idx; j++)
1458 if (array[j] == rec->ip) {
1459 found = 0;
1460 break;
1461 }
1462 if (found)
1463 array[idx] = rec->ip;
1464 break;
1465 }
1466 }
1467 }
1468 spin_unlock(&ftrace_lock);
1469
1470 return found ? 0 : -EINVAL;
1471}
1472
1473static ssize_t
1474ftrace_graph_write(struct file *file, const char __user *ubuf,
1475 size_t cnt, loff_t *ppos)
1476{
1477 unsigned char buffer[FTRACE_BUFF_MAX+1];
1478 unsigned long *array;
1479 size_t read = 0;
1480 ssize_t ret;
1481 int index = 0;
1482 char ch;
1483
1484 if (!cnt || cnt < 0)
1485 return 0;
1486
1487 mutex_lock(&graph_lock);
1488
1489 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
1490 ret = -EBUSY;
1491 goto out;
1492 }
1493
1494 if (file->f_mode & FMODE_READ) {
1495 struct seq_file *m = file->private_data;
1496 array = m->private;
1497 } else
1498 array = file->private_data;
1499
1500 ret = get_user(ch, ubuf++);
1501 if (ret)
1502 goto out;
1503 read++;
1504 cnt--;
1505
1506 /* skip white space */
1507 while (cnt && isspace(ch)) {
1508 ret = get_user(ch, ubuf++);
1509 if (ret)
1510 goto out;
1511 read++;
1512 cnt--;
1513 }
1514
1515 if (isspace(ch)) {
1516 *ppos += read;
1517 ret = read;
1518 goto out;
1519 }
1520
1521 while (cnt && !isspace(ch)) {
1522 if (index < FTRACE_BUFF_MAX)
1523 buffer[index++] = ch;
1524 else {
1525 ret = -EINVAL;
1526 goto out;
1527 }
1528 ret = get_user(ch, ubuf++);
1529 if (ret)
1530 goto out;
1531 read++;
1532 cnt--;
1533 }
1534 buffer[index] = 0;
1535
1536 /* we allow only one at a time */
1537 ret = ftrace_set_func(array, ftrace_graph_count, buffer);
1538 if (ret)
1539 goto out;
1540
1541 ftrace_graph_count++;
1542
1543 file->f_pos += read;
1544
1545 ret = read;
1546 out:
1547 mutex_unlock(&graph_lock);
1548
1549 return ret;
1550}
1551
1552static const struct file_operations ftrace_graph_fops = {
1553 .open = ftrace_graph_open,
1554 .read = ftrace_graph_read,
1555 .write = ftrace_graph_write,
1556};
1557#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1558
1559static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
1560{
1561 struct dentry *entry;
1242 1562
1243 entry = debugfs_create_file("available_filter_functions", 0444, 1563 entry = debugfs_create_file("available_filter_functions", 0444,
1244 d_tracer, NULL, &ftrace_avail_fops); 1564 d_tracer, NULL, &ftrace_avail_fops);
@@ -1263,12 +1583,20 @@ static __init int ftrace_init_debugfs(void)
1263 pr_warning("Could not create debugfs " 1583 pr_warning("Could not create debugfs "
1264 "'set_ftrace_notrace' entry\n"); 1584 "'set_ftrace_notrace' entry\n");
1265 1585
1586#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1587 entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
1588 NULL,
1589 &ftrace_graph_fops);
1590 if (!entry)
1591 pr_warning("Could not create debugfs "
1592 "'set_graph_function' entry\n");
1593#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1594
1266 return 0; 1595 return 0;
1267} 1596}
1268 1597
1269fs_initcall(ftrace_init_debugfs); 1598static int ftrace_convert_nops(struct module *mod,
1270 1599 unsigned long *start,
1271static int ftrace_convert_nops(unsigned long *start,
1272 unsigned long *end) 1600 unsigned long *end)
1273{ 1601{
1274 unsigned long *p; 1602 unsigned long *p;
@@ -1279,23 +1607,32 @@ static int ftrace_convert_nops(unsigned long *start,
1279 p = start; 1607 p = start;
1280 while (p < end) { 1608 while (p < end) {
1281 addr = ftrace_call_adjust(*p++); 1609 addr = ftrace_call_adjust(*p++);
1610 /*
1611 * Some architecture linkers will pad between
1612 * the different mcount_loc sections of different
1613 * object files to satisfy alignments.
1614 * Skip any NULL pointers.
1615 */
1616 if (!addr)
1617 continue;
1282 ftrace_record_ip(addr); 1618 ftrace_record_ip(addr);
1283 } 1619 }
1284 1620
1285 /* disable interrupts to prevent kstop machine */ 1621 /* disable interrupts to prevent kstop machine */
1286 local_irq_save(flags); 1622 local_irq_save(flags);
1287 ftrace_update_code(); 1623 ftrace_update_code(mod);
1288 local_irq_restore(flags); 1624 local_irq_restore(flags);
1289 mutex_unlock(&ftrace_start_lock); 1625 mutex_unlock(&ftrace_start_lock);
1290 1626
1291 return 0; 1627 return 0;
1292} 1628}
1293 1629
1294void ftrace_init_module(unsigned long *start, unsigned long *end) 1630void ftrace_init_module(struct module *mod,
1631 unsigned long *start, unsigned long *end)
1295{ 1632{
1296 if (ftrace_disabled || start == end) 1633 if (ftrace_disabled || start == end)
1297 return; 1634 return;
1298 ftrace_convert_nops(start, end); 1635 ftrace_convert_nops(mod, start, end);
1299} 1636}
1300 1637
1301extern unsigned long __start_mcount_loc[]; 1638extern unsigned long __start_mcount_loc[];
@@ -1325,7 +1662,8 @@ void __init ftrace_init(void)
1325 1662
1326 last_ftrace_enabled = ftrace_enabled = 1; 1663 last_ftrace_enabled = ftrace_enabled = 1;
1327 1664
1328 ret = ftrace_convert_nops(__start_mcount_loc, 1665 ret = ftrace_convert_nops(NULL,
1666 __start_mcount_loc,
1329 __stop_mcount_loc); 1667 __stop_mcount_loc);
1330 1668
1331 return; 1669 return;
@@ -1342,12 +1680,186 @@ static int __init ftrace_nodyn_init(void)
1342} 1680}
1343device_initcall(ftrace_nodyn_init); 1681device_initcall(ftrace_nodyn_init);
1344 1682
1345# define ftrace_startup() do { } while (0) 1683static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
1346# define ftrace_shutdown() do { } while (0) 1684static inline void ftrace_startup_enable(int command) { }
1685/* Keep as macros so we do not need to define the commands */
1686# define ftrace_startup(command) do { } while (0)
1687# define ftrace_shutdown(command) do { } while (0)
1347# define ftrace_startup_sysctl() do { } while (0) 1688# define ftrace_startup_sysctl() do { } while (0)
1348# define ftrace_shutdown_sysctl() do { } while (0) 1689# define ftrace_shutdown_sysctl() do { } while (0)
1349#endif /* CONFIG_DYNAMIC_FTRACE */ 1690#endif /* CONFIG_DYNAMIC_FTRACE */
1350 1691
1692static ssize_t
1693ftrace_pid_read(struct file *file, char __user *ubuf,
1694 size_t cnt, loff_t *ppos)
1695{
1696 char buf[64];
1697 int r;
1698
1699 if (ftrace_pid_trace == ftrace_swapper_pid)
1700 r = sprintf(buf, "swapper tasks\n");
1701 else if (ftrace_pid_trace)
1702 r = sprintf(buf, "%u\n", pid_nr(ftrace_pid_trace));
1703 else
1704 r = sprintf(buf, "no pid\n");
1705
1706 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1707}
1708
1709static void clear_ftrace_swapper(void)
1710{
1711 struct task_struct *p;
1712 int cpu;
1713
1714 get_online_cpus();
1715 for_each_online_cpu(cpu) {
1716 p = idle_task(cpu);
1717 clear_tsk_trace_trace(p);
1718 }
1719 put_online_cpus();
1720}
1721
1722static void set_ftrace_swapper(void)
1723{
1724 struct task_struct *p;
1725 int cpu;
1726
1727 get_online_cpus();
1728 for_each_online_cpu(cpu) {
1729 p = idle_task(cpu);
1730 set_tsk_trace_trace(p);
1731 }
1732 put_online_cpus();
1733}
1734
1735static void clear_ftrace_pid(struct pid *pid)
1736{
1737 struct task_struct *p;
1738
1739 do_each_pid_task(pid, PIDTYPE_PID, p) {
1740 clear_tsk_trace_trace(p);
1741 } while_each_pid_task(pid, PIDTYPE_PID, p);
1742 put_pid(pid);
1743}
1744
1745static void set_ftrace_pid(struct pid *pid)
1746{
1747 struct task_struct *p;
1748
1749 do_each_pid_task(pid, PIDTYPE_PID, p) {
1750 set_tsk_trace_trace(p);
1751 } while_each_pid_task(pid, PIDTYPE_PID, p);
1752}
1753
1754static void clear_ftrace_pid_task(struct pid **pid)
1755{
1756 if (*pid == ftrace_swapper_pid)
1757 clear_ftrace_swapper();
1758 else
1759 clear_ftrace_pid(*pid);
1760
1761 *pid = NULL;
1762}
1763
1764static void set_ftrace_pid_task(struct pid *pid)
1765{
1766 if (pid == ftrace_swapper_pid)
1767 set_ftrace_swapper();
1768 else
1769 set_ftrace_pid(pid);
1770}
1771
1772static ssize_t
1773ftrace_pid_write(struct file *filp, const char __user *ubuf,
1774 size_t cnt, loff_t *ppos)
1775{
1776 struct pid *pid;
1777 char buf[64];
1778 long val;
1779 int ret;
1780
1781 if (cnt >= sizeof(buf))
1782 return -EINVAL;
1783
1784 if (copy_from_user(&buf, ubuf, cnt))
1785 return -EFAULT;
1786
1787 buf[cnt] = 0;
1788
1789 ret = strict_strtol(buf, 10, &val);
1790 if (ret < 0)
1791 return ret;
1792
1793 mutex_lock(&ftrace_start_lock);
1794 if (val < 0) {
1795 /* disable pid tracing */
1796 if (!ftrace_pid_trace)
1797 goto out;
1798
1799 clear_ftrace_pid_task(&ftrace_pid_trace);
1800
1801 } else {
1802 /* swapper task is special */
1803 if (!val) {
1804 pid = ftrace_swapper_pid;
1805 if (pid == ftrace_pid_trace)
1806 goto out;
1807 } else {
1808 pid = find_get_pid(val);
1809
1810 if (pid == ftrace_pid_trace) {
1811 put_pid(pid);
1812 goto out;
1813 }
1814 }
1815
1816 if (ftrace_pid_trace)
1817 clear_ftrace_pid_task(&ftrace_pid_trace);
1818
1819 if (!pid)
1820 goto out;
1821
1822 ftrace_pid_trace = pid;
1823
1824 set_ftrace_pid_task(ftrace_pid_trace);
1825 }
1826
1827 /* update the function call */
1828 ftrace_update_pid_func();
1829 ftrace_startup_enable(0);
1830
1831 out:
1832 mutex_unlock(&ftrace_start_lock);
1833
1834 return cnt;
1835}
1836
1837static struct file_operations ftrace_pid_fops = {
1838 .read = ftrace_pid_read,
1839 .write = ftrace_pid_write,
1840};
1841
1842static __init int ftrace_init_debugfs(void)
1843{
1844 struct dentry *d_tracer;
1845 struct dentry *entry;
1846
1847 d_tracer = tracing_init_dentry();
1848 if (!d_tracer)
1849 return 0;
1850
1851 ftrace_init_dyn_debugfs(d_tracer);
1852
1853 entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
1854 NULL, &ftrace_pid_fops);
1855 if (!entry)
1856 pr_warning("Could not create debugfs "
1857 "'set_ftrace_pid' entry\n");
1858 return 0;
1859}
1860
1861fs_initcall(ftrace_init_debugfs);
1862
1351/** 1863/**
1352 * ftrace_kill - kill ftrace 1864 * ftrace_kill - kill ftrace
1353 * 1865 *
@@ -1381,10 +1893,11 @@ int register_ftrace_function(struct ftrace_ops *ops)
1381 return -1; 1893 return -1;
1382 1894
1383 mutex_lock(&ftrace_sysctl_lock); 1895 mutex_lock(&ftrace_sysctl_lock);
1896
1384 ret = __register_ftrace_function(ops); 1897 ret = __register_ftrace_function(ops);
1385 ftrace_startup(); 1898 ftrace_startup(0);
1386 mutex_unlock(&ftrace_sysctl_lock);
1387 1899
1900 mutex_unlock(&ftrace_sysctl_lock);
1388 return ret; 1901 return ret;
1389} 1902}
1390 1903
@@ -1400,7 +1913,7 @@ int unregister_ftrace_function(struct ftrace_ops *ops)
1400 1913
1401 mutex_lock(&ftrace_sysctl_lock); 1914 mutex_lock(&ftrace_sysctl_lock);
1402 ret = __unregister_ftrace_function(ops); 1915 ret = __unregister_ftrace_function(ops);
1403 ftrace_shutdown(); 1916 ftrace_shutdown(0);
1404 mutex_unlock(&ftrace_sysctl_lock); 1917 mutex_unlock(&ftrace_sysctl_lock);
1405 1918
1406 return ret; 1919 return ret;
@@ -1449,3 +1962,153 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
1449 return ret; 1962 return ret;
1450} 1963}
1451 1964
1965#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1966
1967static atomic_t ftrace_graph_active;
1968
1969int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
1970{
1971 return 0;
1972}
1973
1974/* The callbacks that hook a function */
1975trace_func_graph_ret_t ftrace_graph_return =
1976 (trace_func_graph_ret_t)ftrace_stub;
1977trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
1978
1979/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1980static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
1981{
1982 int i;
1983 int ret = 0;
1984 unsigned long flags;
1985 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1986 struct task_struct *g, *t;
1987
1988 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1989 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
1990 * sizeof(struct ftrace_ret_stack),
1991 GFP_KERNEL);
1992 if (!ret_stack_list[i]) {
1993 start = 0;
1994 end = i;
1995 ret = -ENOMEM;
1996 goto free;
1997 }
1998 }
1999
2000 read_lock_irqsave(&tasklist_lock, flags);
2001 do_each_thread(g, t) {
2002 if (start == end) {
2003 ret = -EAGAIN;
2004 goto unlock;
2005 }
2006
2007 if (t->ret_stack == NULL) {
2008 t->curr_ret_stack = -1;
2009 /* Make sure IRQs see the -1 first: */
2010 barrier();
2011 t->ret_stack = ret_stack_list[start++];
2012 atomic_set(&t->tracing_graph_pause, 0);
2013 atomic_set(&t->trace_overrun, 0);
2014 }
2015 } while_each_thread(g, t);
2016
2017unlock:
2018 read_unlock_irqrestore(&tasklist_lock, flags);
2019free:
2020 for (i = start; i < end; i++)
2021 kfree(ret_stack_list[i]);
2022 return ret;
2023}
2024
2025/* Allocate a return stack for each task */
2026static int start_graph_tracing(void)
2027{
2028 struct ftrace_ret_stack **ret_stack_list;
2029 int ret;
2030
2031 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2032 sizeof(struct ftrace_ret_stack *),
2033 GFP_KERNEL);
2034
2035 if (!ret_stack_list)
2036 return -ENOMEM;
2037
2038 do {
2039 ret = alloc_retstack_tasklist(ret_stack_list);
2040 } while (ret == -EAGAIN);
2041
2042 kfree(ret_stack_list);
2043 return ret;
2044}
2045
2046int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2047 trace_func_graph_ent_t entryfunc)
2048{
2049 int ret = 0;
2050
2051 mutex_lock(&ftrace_sysctl_lock);
2052
2053 atomic_inc(&ftrace_graph_active);
2054 ret = start_graph_tracing();
2055 if (ret) {
2056 atomic_dec(&ftrace_graph_active);
2057 goto out;
2058 }
2059
2060 ftrace_graph_return = retfunc;
2061 ftrace_graph_entry = entryfunc;
2062
2063 ftrace_startup(FTRACE_START_FUNC_RET);
2064
2065out:
2066 mutex_unlock(&ftrace_sysctl_lock);
2067 return ret;
2068}
2069
2070void unregister_ftrace_graph(void)
2071{
2072 mutex_lock(&ftrace_sysctl_lock);
2073
2074 atomic_dec(&ftrace_graph_active);
2075 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2076 ftrace_graph_entry = ftrace_graph_entry_stub;
2077 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2078
2079 mutex_unlock(&ftrace_sysctl_lock);
2080}
2081
2082/* Allocate a return stack for newly created task */
2083void ftrace_graph_init_task(struct task_struct *t)
2084{
2085 if (atomic_read(&ftrace_graph_active)) {
2086 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2087 * sizeof(struct ftrace_ret_stack),
2088 GFP_KERNEL);
2089 if (!t->ret_stack)
2090 return;
2091 t->curr_ret_stack = -1;
2092 atomic_set(&t->tracing_graph_pause, 0);
2093 atomic_set(&t->trace_overrun, 0);
2094 } else
2095 t->ret_stack = NULL;
2096}
2097
2098void ftrace_graph_exit_task(struct task_struct *t)
2099{
2100 struct ftrace_ret_stack *ret_stack = t->ret_stack;
2101
2102 t->ret_stack = NULL;
2103 /* NULL must become visible to IRQs before we free it: */
2104 barrier();
2105
2106 kfree(ret_stack);
2107}
2108
2109void ftrace_graph_stop(void)
2110{
2111 ftrace_stop();
2112}
2113#endif
2114
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index f780e9552f91..a9d9760dc7b6 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -18,8 +18,46 @@
18 18
19#include "trace.h" 19#include "trace.h"
20 20
21/* Global flag to disable all recording to ring buffers */ 21/*
22static int ring_buffers_off __read_mostly; 22 * A fast way to enable or disable all ring buffers is to
23 * call tracing_on or tracing_off. Turning off the ring buffers
24 * prevents all ring buffers from being recorded to.
25 * Turning this switch on, makes it OK to write to the
26 * ring buffer, if the ring buffer is enabled itself.
27 *
28 * There's three layers that must be on in order to write
29 * to the ring buffer.
30 *
31 * 1) This global flag must be set.
32 * 2) The ring buffer must be enabled for recording.
33 * 3) The per cpu buffer must be enabled for recording.
34 *
35 * In case of an anomaly, this global flag has a bit set that
36 * will permantly disable all ring buffers.
37 */
38
39/*
40 * Global flag to disable all recording to ring buffers
41 * This has two bits: ON, DISABLED
42 *
43 * ON DISABLED
44 * ---- ----------
45 * 0 0 : ring buffers are off
46 * 1 0 : ring buffers are on
47 * X 1 : ring buffers are permanently disabled
48 */
49
50enum {
51 RB_BUFFERS_ON_BIT = 0,
52 RB_BUFFERS_DISABLED_BIT = 1,
53};
54
55enum {
56 RB_BUFFERS_ON = 1 << RB_BUFFERS_ON_BIT,
57 RB_BUFFERS_DISABLED = 1 << RB_BUFFERS_DISABLED_BIT,
58};
59
60static long ring_buffer_flags __read_mostly = RB_BUFFERS_ON;
23 61
24/** 62/**
25 * tracing_on - enable all tracing buffers 63 * tracing_on - enable all tracing buffers
@@ -29,8 +67,9 @@ static int ring_buffers_off __read_mostly;
29 */ 67 */
30void tracing_on(void) 68void tracing_on(void)
31{ 69{
32 ring_buffers_off = 0; 70 set_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
33} 71}
72EXPORT_SYMBOL_GPL(tracing_on);
34 73
35/** 74/**
36 * tracing_off - turn off all tracing buffers 75 * tracing_off - turn off all tracing buffers
@@ -42,8 +81,22 @@ void tracing_on(void)
42 */ 81 */
43void tracing_off(void) 82void tracing_off(void)
44{ 83{
45 ring_buffers_off = 1; 84 clear_bit(RB_BUFFERS_ON_BIT, &ring_buffer_flags);
46} 85}
86EXPORT_SYMBOL_GPL(tracing_off);
87
88/**
89 * tracing_off_permanent - permanently disable ring buffers
90 *
91 * This function, once called, will disable all ring buffers
92 * permanenty.
93 */
94void tracing_off_permanent(void)
95{
96 set_bit(RB_BUFFERS_DISABLED_BIT, &ring_buffer_flags);
97}
98
99#include "trace.h"
47 100
48/* Up this if you want to test the TIME_EXTENTS and normalization */ 101/* Up this if you want to test the TIME_EXTENTS and normalization */
49#define DEBUG_SHIFT 0 102#define DEBUG_SHIFT 0
@@ -56,16 +109,18 @@ u64 ring_buffer_time_stamp(int cpu)
56 preempt_disable_notrace(); 109 preempt_disable_notrace();
57 /* shift to debug/test normalization and TIME_EXTENTS */ 110 /* shift to debug/test normalization and TIME_EXTENTS */
58 time = sched_clock() << DEBUG_SHIFT; 111 time = sched_clock() << DEBUG_SHIFT;
59 preempt_enable_notrace(); 112 preempt_enable_no_resched_notrace();
60 113
61 return time; 114 return time;
62} 115}
116EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
63 117
64void ring_buffer_normalize_time_stamp(int cpu, u64 *ts) 118void ring_buffer_normalize_time_stamp(int cpu, u64 *ts)
65{ 119{
66 /* Just stupid testing the normalize function and deltas */ 120 /* Just stupid testing the normalize function and deltas */
67 *ts >>= DEBUG_SHIFT; 121 *ts >>= DEBUG_SHIFT;
68} 122}
123EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
69 124
70#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event)) 125#define RB_EVNT_HDR_SIZE (sizeof(struct ring_buffer_event))
71#define RB_ALIGNMENT_SHIFT 2 126#define RB_ALIGNMENT_SHIFT 2
@@ -115,6 +170,7 @@ unsigned ring_buffer_event_length(struct ring_buffer_event *event)
115{ 170{
116 return rb_event_length(event); 171 return rb_event_length(event);
117} 172}
173EXPORT_SYMBOL_GPL(ring_buffer_event_length);
118 174
119/* inline for ring buffer fast paths */ 175/* inline for ring buffer fast paths */
120static inline void * 176static inline void *
@@ -136,28 +192,33 @@ void *ring_buffer_event_data(struct ring_buffer_event *event)
136{ 192{
137 return rb_event_data(event); 193 return rb_event_data(event);
138} 194}
195EXPORT_SYMBOL_GPL(ring_buffer_event_data);
139 196
140#define for_each_buffer_cpu(buffer, cpu) \ 197#define for_each_buffer_cpu(buffer, cpu) \
141 for_each_cpu_mask(cpu, buffer->cpumask) 198 for_each_cpu(cpu, buffer->cpumask)
142 199
143#define TS_SHIFT 27 200#define TS_SHIFT 27
144#define TS_MASK ((1ULL << TS_SHIFT) - 1) 201#define TS_MASK ((1ULL << TS_SHIFT) - 1)
145#define TS_DELTA_TEST (~TS_MASK) 202#define TS_DELTA_TEST (~TS_MASK)
146 203
147/* 204struct buffer_data_page {
148 * This hack stolen from mm/slob.c.
149 * We can store per page timing information in the page frame of the page.
150 * Thanks to Peter Zijlstra for suggesting this idea.
151 */
152struct buffer_page {
153 u64 time_stamp; /* page time stamp */ 205 u64 time_stamp; /* page time stamp */
154 local_t write; /* index for next write */
155 local_t commit; /* write commited index */ 206 local_t commit; /* write commited index */
207 unsigned char data[]; /* data of buffer page */
208};
209
210struct buffer_page {
211 local_t write; /* index for next write */
156 unsigned read; /* index for next read */ 212 unsigned read; /* index for next read */
157 struct list_head list; /* list of free pages */ 213 struct list_head list; /* list of free pages */
158 void *page; /* Actual data page */ 214 struct buffer_data_page *page; /* Actual data page */
159}; 215};
160 216
217static void rb_init_page(struct buffer_data_page *bpage)
218{
219 local_set(&bpage->commit, 0);
220}
221
161/* 222/*
162 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing 223 * Also stolen from mm/slob.c. Thanks to Mathieu Desnoyers for pointing
163 * this issue out. 224 * this issue out.
@@ -179,7 +240,7 @@ static inline int test_time_stamp(u64 delta)
179 return 0; 240 return 0;
180} 241}
181 242
182#define BUF_PAGE_SIZE PAGE_SIZE 243#define BUF_PAGE_SIZE (PAGE_SIZE - sizeof(struct buffer_data_page))
183 244
184/* 245/*
185 * head_page == tail_page && head == tail then buffer is empty. 246 * head_page == tail_page && head == tail then buffer is empty.
@@ -187,7 +248,8 @@ static inline int test_time_stamp(u64 delta)
187struct ring_buffer_per_cpu { 248struct ring_buffer_per_cpu {
188 int cpu; 249 int cpu;
189 struct ring_buffer *buffer; 250 struct ring_buffer *buffer;
190 spinlock_t lock; 251 spinlock_t reader_lock; /* serialize readers */
252 raw_spinlock_t lock;
191 struct lock_class_key lock_key; 253 struct lock_class_key lock_key;
192 struct list_head pages; 254 struct list_head pages;
193 struct buffer_page *head_page; /* read from head */ 255 struct buffer_page *head_page; /* read from head */
@@ -202,11 +264,10 @@ struct ring_buffer_per_cpu {
202}; 264};
203 265
204struct ring_buffer { 266struct ring_buffer {
205 unsigned long size;
206 unsigned pages; 267 unsigned pages;
207 unsigned flags; 268 unsigned flags;
208 int cpus; 269 int cpus;
209 cpumask_t cpumask; 270 cpumask_var_t cpumask;
210 atomic_t record_disabled; 271 atomic_t record_disabled;
211 272
212 struct mutex mutex; 273 struct mutex mutex;
@@ -221,32 +282,16 @@ struct ring_buffer_iter {
221 u64 read_stamp; 282 u64 read_stamp;
222}; 283};
223 284
285/* buffer may be either ring_buffer or ring_buffer_per_cpu */
224#define RB_WARN_ON(buffer, cond) \ 286#define RB_WARN_ON(buffer, cond) \
225 do { \ 287 ({ \
226 if (unlikely(cond)) { \ 288 int _____ret = unlikely(cond); \
227 atomic_inc(&buffer->record_disabled); \ 289 if (_____ret) { \
228 WARN_ON(1); \
229 } \
230 } while (0)
231
232#define RB_WARN_ON_RET(buffer, cond) \
233 do { \
234 if (unlikely(cond)) { \
235 atomic_inc(&buffer->record_disabled); \
236 WARN_ON(1); \
237 return -1; \
238 } \
239 } while (0)
240
241#define RB_WARN_ON_ONCE(buffer, cond) \
242 do { \
243 static int once; \
244 if (unlikely(cond) && !once) { \
245 once++; \
246 atomic_inc(&buffer->record_disabled); \ 290 atomic_inc(&buffer->record_disabled); \
247 WARN_ON(1); \ 291 WARN_ON(1); \
248 } \ 292 } \
249 } while (0) 293 _____ret; \
294 })
250 295
251/** 296/**
252 * check_pages - integrity check of buffer pages 297 * check_pages - integrity check of buffer pages
@@ -258,16 +303,20 @@ struct ring_buffer_iter {
258static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer) 303static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
259{ 304{
260 struct list_head *head = &cpu_buffer->pages; 305 struct list_head *head = &cpu_buffer->pages;
261 struct buffer_page *page, *tmp; 306 struct buffer_page *bpage, *tmp;
262 307
263 RB_WARN_ON_RET(cpu_buffer, head->next->prev != head); 308 if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
264 RB_WARN_ON_RET(cpu_buffer, head->prev->next != head); 309 return -1;
310 if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
311 return -1;
265 312
266 list_for_each_entry_safe(page, tmp, head, list) { 313 list_for_each_entry_safe(bpage, tmp, head, list) {
267 RB_WARN_ON_RET(cpu_buffer, 314 if (RB_WARN_ON(cpu_buffer,
268 page->list.next->prev != &page->list); 315 bpage->list.next->prev != &bpage->list))
269 RB_WARN_ON_RET(cpu_buffer, 316 return -1;
270 page->list.prev->next != &page->list); 317 if (RB_WARN_ON(cpu_buffer,
318 bpage->list.prev->next != &bpage->list))
319 return -1;
271 } 320 }
272 321
273 return 0; 322 return 0;
@@ -277,22 +326,23 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
277 unsigned nr_pages) 326 unsigned nr_pages)
278{ 327{
279 struct list_head *head = &cpu_buffer->pages; 328 struct list_head *head = &cpu_buffer->pages;
280 struct buffer_page *page, *tmp; 329 struct buffer_page *bpage, *tmp;
281 unsigned long addr; 330 unsigned long addr;
282 LIST_HEAD(pages); 331 LIST_HEAD(pages);
283 unsigned i; 332 unsigned i;
284 333
285 for (i = 0; i < nr_pages; i++) { 334 for (i = 0; i < nr_pages; i++) {
286 page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()), 335 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
287 GFP_KERNEL, cpu_to_node(cpu_buffer->cpu)); 336 GFP_KERNEL, cpu_to_node(cpu_buffer->cpu));
288 if (!page) 337 if (!bpage)
289 goto free_pages; 338 goto free_pages;
290 list_add(&page->list, &pages); 339 list_add(&bpage->list, &pages);
291 340
292 addr = __get_free_page(GFP_KERNEL); 341 addr = __get_free_page(GFP_KERNEL);
293 if (!addr) 342 if (!addr)
294 goto free_pages; 343 goto free_pages;
295 page->page = (void *)addr; 344 bpage->page = (void *)addr;
345 rb_init_page(bpage->page);
296 } 346 }
297 347
298 list_splice(&pages, head); 348 list_splice(&pages, head);
@@ -302,9 +352,9 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
302 return 0; 352 return 0;
303 353
304 free_pages: 354 free_pages:
305 list_for_each_entry_safe(page, tmp, &pages, list) { 355 list_for_each_entry_safe(bpage, tmp, &pages, list) {
306 list_del_init(&page->list); 356 list_del_init(&bpage->list);
307 free_buffer_page(page); 357 free_buffer_page(bpage);
308 } 358 }
309 return -ENOMEM; 359 return -ENOMEM;
310} 360}
@@ -313,7 +363,7 @@ static struct ring_buffer_per_cpu *
313rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) 363rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
314{ 364{
315 struct ring_buffer_per_cpu *cpu_buffer; 365 struct ring_buffer_per_cpu *cpu_buffer;
316 struct buffer_page *page; 366 struct buffer_page *bpage;
317 unsigned long addr; 367 unsigned long addr;
318 int ret; 368 int ret;
319 369
@@ -324,19 +374,21 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
324 374
325 cpu_buffer->cpu = cpu; 375 cpu_buffer->cpu = cpu;
326 cpu_buffer->buffer = buffer; 376 cpu_buffer->buffer = buffer;
327 spin_lock_init(&cpu_buffer->lock); 377 spin_lock_init(&cpu_buffer->reader_lock);
378 cpu_buffer->lock = (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
328 INIT_LIST_HEAD(&cpu_buffer->pages); 379 INIT_LIST_HEAD(&cpu_buffer->pages);
329 380
330 page = kzalloc_node(ALIGN(sizeof(*page), cache_line_size()), 381 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
331 GFP_KERNEL, cpu_to_node(cpu)); 382 GFP_KERNEL, cpu_to_node(cpu));
332 if (!page) 383 if (!bpage)
333 goto fail_free_buffer; 384 goto fail_free_buffer;
334 385
335 cpu_buffer->reader_page = page; 386 cpu_buffer->reader_page = bpage;
336 addr = __get_free_page(GFP_KERNEL); 387 addr = __get_free_page(GFP_KERNEL);
337 if (!addr) 388 if (!addr)
338 goto fail_free_reader; 389 goto fail_free_reader;
339 page->page = (void *)addr; 390 bpage->page = (void *)addr;
391 rb_init_page(bpage->page);
340 392
341 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 393 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
342 394
@@ -361,14 +413,14 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
361static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) 413static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
362{ 414{
363 struct list_head *head = &cpu_buffer->pages; 415 struct list_head *head = &cpu_buffer->pages;
364 struct buffer_page *page, *tmp; 416 struct buffer_page *bpage, *tmp;
365 417
366 list_del_init(&cpu_buffer->reader_page->list); 418 list_del_init(&cpu_buffer->reader_page->list);
367 free_buffer_page(cpu_buffer->reader_page); 419 free_buffer_page(cpu_buffer->reader_page);
368 420
369 list_for_each_entry_safe(page, tmp, head, list) { 421 list_for_each_entry_safe(bpage, tmp, head, list) {
370 list_del_init(&page->list); 422 list_del_init(&bpage->list);
371 free_buffer_page(page); 423 free_buffer_page(bpage);
372 } 424 }
373 kfree(cpu_buffer); 425 kfree(cpu_buffer);
374} 426}
@@ -381,7 +433,7 @@ extern int ring_buffer_page_too_big(void);
381 433
382/** 434/**
383 * ring_buffer_alloc - allocate a new ring_buffer 435 * ring_buffer_alloc - allocate a new ring_buffer
384 * @size: the size in bytes that is needed. 436 * @size: the size in bytes per cpu that is needed.
385 * @flags: attributes to set for the ring buffer. 437 * @flags: attributes to set for the ring buffer.
386 * 438 *
387 * Currently the only flag that is available is the RB_FL_OVERWRITE 439 * Currently the only flag that is available is the RB_FL_OVERWRITE
@@ -406,6 +458,9 @@ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
406 if (!buffer) 458 if (!buffer)
407 return NULL; 459 return NULL;
408 460
461 if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
462 goto fail_free_buffer;
463
409 buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 464 buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
410 buffer->flags = flags; 465 buffer->flags = flags;
411 466
@@ -413,14 +468,14 @@ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
413 if (buffer->pages == 1) 468 if (buffer->pages == 1)
414 buffer->pages++; 469 buffer->pages++;
415 470
416 buffer->cpumask = cpu_possible_map; 471 cpumask_copy(buffer->cpumask, cpu_possible_mask);
417 buffer->cpus = nr_cpu_ids; 472 buffer->cpus = nr_cpu_ids;
418 473
419 bsize = sizeof(void *) * nr_cpu_ids; 474 bsize = sizeof(void *) * nr_cpu_ids;
420 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), 475 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
421 GFP_KERNEL); 476 GFP_KERNEL);
422 if (!buffer->buffers) 477 if (!buffer->buffers)
423 goto fail_free_buffer; 478 goto fail_free_cpumask;
424 479
425 for_each_buffer_cpu(buffer, cpu) { 480 for_each_buffer_cpu(buffer, cpu) {
426 buffer->buffers[cpu] = 481 buffer->buffers[cpu] =
@@ -440,10 +495,14 @@ struct ring_buffer *ring_buffer_alloc(unsigned long size, unsigned flags)
440 } 495 }
441 kfree(buffer->buffers); 496 kfree(buffer->buffers);
442 497
498 fail_free_cpumask:
499 free_cpumask_var(buffer->cpumask);
500
443 fail_free_buffer: 501 fail_free_buffer:
444 kfree(buffer); 502 kfree(buffer);
445 return NULL; 503 return NULL;
446} 504}
505EXPORT_SYMBOL_GPL(ring_buffer_alloc);
447 506
448/** 507/**
449 * ring_buffer_free - free a ring buffer. 508 * ring_buffer_free - free a ring buffer.
@@ -457,15 +516,18 @@ ring_buffer_free(struct ring_buffer *buffer)
457 for_each_buffer_cpu(buffer, cpu) 516 for_each_buffer_cpu(buffer, cpu)
458 rb_free_cpu_buffer(buffer->buffers[cpu]); 517 rb_free_cpu_buffer(buffer->buffers[cpu]);
459 518
519 free_cpumask_var(buffer->cpumask);
520
460 kfree(buffer); 521 kfree(buffer);
461} 522}
523EXPORT_SYMBOL_GPL(ring_buffer_free);
462 524
463static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); 525static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
464 526
465static void 527static void
466rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages) 528rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
467{ 529{
468 struct buffer_page *page; 530 struct buffer_page *bpage;
469 struct list_head *p; 531 struct list_head *p;
470 unsigned i; 532 unsigned i;
471 533
@@ -473,13 +535,15 @@ rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
473 synchronize_sched(); 535 synchronize_sched();
474 536
475 for (i = 0; i < nr_pages; i++) { 537 for (i = 0; i < nr_pages; i++) {
476 BUG_ON(list_empty(&cpu_buffer->pages)); 538 if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
539 return;
477 p = cpu_buffer->pages.next; 540 p = cpu_buffer->pages.next;
478 page = list_entry(p, struct buffer_page, list); 541 bpage = list_entry(p, struct buffer_page, list);
479 list_del_init(&page->list); 542 list_del_init(&bpage->list);
480 free_buffer_page(page); 543 free_buffer_page(bpage);
481 } 544 }
482 BUG_ON(list_empty(&cpu_buffer->pages)); 545 if (RB_WARN_ON(cpu_buffer, list_empty(&cpu_buffer->pages)))
546 return;
483 547
484 rb_reset_cpu(cpu_buffer); 548 rb_reset_cpu(cpu_buffer);
485 549
@@ -493,7 +557,7 @@ static void
493rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, 557rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
494 struct list_head *pages, unsigned nr_pages) 558 struct list_head *pages, unsigned nr_pages)
495{ 559{
496 struct buffer_page *page; 560 struct buffer_page *bpage;
497 struct list_head *p; 561 struct list_head *p;
498 unsigned i; 562 unsigned i;
499 563
@@ -501,11 +565,12 @@ rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer,
501 synchronize_sched(); 565 synchronize_sched();
502 566
503 for (i = 0; i < nr_pages; i++) { 567 for (i = 0; i < nr_pages; i++) {
504 BUG_ON(list_empty(pages)); 568 if (RB_WARN_ON(cpu_buffer, list_empty(pages)))
569 return;
505 p = pages->next; 570 p = pages->next;
506 page = list_entry(p, struct buffer_page, list); 571 bpage = list_entry(p, struct buffer_page, list);
507 list_del_init(&page->list); 572 list_del_init(&bpage->list);
508 list_add_tail(&page->list, &cpu_buffer->pages); 573 list_add_tail(&bpage->list, &cpu_buffer->pages);
509 } 574 }
510 rb_reset_cpu(cpu_buffer); 575 rb_reset_cpu(cpu_buffer);
511 576
@@ -532,7 +597,7 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
532{ 597{
533 struct ring_buffer_per_cpu *cpu_buffer; 598 struct ring_buffer_per_cpu *cpu_buffer;
534 unsigned nr_pages, rm_pages, new_pages; 599 unsigned nr_pages, rm_pages, new_pages;
535 struct buffer_page *page, *tmp; 600 struct buffer_page *bpage, *tmp;
536 unsigned long buffer_size; 601 unsigned long buffer_size;
537 unsigned long addr; 602 unsigned long addr;
538 LIST_HEAD(pages); 603 LIST_HEAD(pages);
@@ -562,7 +627,10 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
562 if (size < buffer_size) { 627 if (size < buffer_size) {
563 628
564 /* easy case, just free pages */ 629 /* easy case, just free pages */
565 BUG_ON(nr_pages >= buffer->pages); 630 if (RB_WARN_ON(buffer, nr_pages >= buffer->pages)) {
631 mutex_unlock(&buffer->mutex);
632 return -1;
633 }
566 634
567 rm_pages = buffer->pages - nr_pages; 635 rm_pages = buffer->pages - nr_pages;
568 636
@@ -581,21 +649,26 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
581 * add these pages to the cpu_buffers. Otherwise we just free 649 * add these pages to the cpu_buffers. Otherwise we just free
582 * them all and return -ENOMEM; 650 * them all and return -ENOMEM;
583 */ 651 */
584 BUG_ON(nr_pages <= buffer->pages); 652 if (RB_WARN_ON(buffer, nr_pages <= buffer->pages)) {
653 mutex_unlock(&buffer->mutex);
654 return -1;
655 }
656
585 new_pages = nr_pages - buffer->pages; 657 new_pages = nr_pages - buffer->pages;
586 658
587 for_each_buffer_cpu(buffer, cpu) { 659 for_each_buffer_cpu(buffer, cpu) {
588 for (i = 0; i < new_pages; i++) { 660 for (i = 0; i < new_pages; i++) {
589 page = kzalloc_node(ALIGN(sizeof(*page), 661 bpage = kzalloc_node(ALIGN(sizeof(*bpage),
590 cache_line_size()), 662 cache_line_size()),
591 GFP_KERNEL, cpu_to_node(cpu)); 663 GFP_KERNEL, cpu_to_node(cpu));
592 if (!page) 664 if (!bpage)
593 goto free_pages; 665 goto free_pages;
594 list_add(&page->list, &pages); 666 list_add(&bpage->list, &pages);
595 addr = __get_free_page(GFP_KERNEL); 667 addr = __get_free_page(GFP_KERNEL);
596 if (!addr) 668 if (!addr)
597 goto free_pages; 669 goto free_pages;
598 page->page = (void *)addr; 670 bpage->page = (void *)addr;
671 rb_init_page(bpage->page);
599 } 672 }
600 } 673 }
601 674
@@ -604,7 +677,10 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
604 rb_insert_pages(cpu_buffer, &pages, new_pages); 677 rb_insert_pages(cpu_buffer, &pages, new_pages);
605 } 678 }
606 679
607 BUG_ON(!list_empty(&pages)); 680 if (RB_WARN_ON(buffer, !list_empty(&pages))) {
681 mutex_unlock(&buffer->mutex);
682 return -1;
683 }
608 684
609 out: 685 out:
610 buffer->pages = nr_pages; 686 buffer->pages = nr_pages;
@@ -613,22 +689,29 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
613 return size; 689 return size;
614 690
615 free_pages: 691 free_pages:
616 list_for_each_entry_safe(page, tmp, &pages, list) { 692 list_for_each_entry_safe(bpage, tmp, &pages, list) {
617 list_del_init(&page->list); 693 list_del_init(&bpage->list);
618 free_buffer_page(page); 694 free_buffer_page(bpage);
619 } 695 }
620 mutex_unlock(&buffer->mutex); 696 mutex_unlock(&buffer->mutex);
621 return -ENOMEM; 697 return -ENOMEM;
622} 698}
699EXPORT_SYMBOL_GPL(ring_buffer_resize);
623 700
624static inline int rb_null_event(struct ring_buffer_event *event) 701static inline int rb_null_event(struct ring_buffer_event *event)
625{ 702{
626 return event->type == RINGBUF_TYPE_PADDING; 703 return event->type == RINGBUF_TYPE_PADDING;
627} 704}
628 705
629static inline void *__rb_page_index(struct buffer_page *page, unsigned index) 706static inline void *
707__rb_data_page_index(struct buffer_data_page *bpage, unsigned index)
708{
709 return bpage->data + index;
710}
711
712static inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
630{ 713{
631 return page->page + index; 714 return bpage->page->data + index;
632} 715}
633 716
634static inline struct ring_buffer_event * 717static inline struct ring_buffer_event *
@@ -658,7 +741,7 @@ static inline unsigned rb_page_write(struct buffer_page *bpage)
658 741
659static inline unsigned rb_page_commit(struct buffer_page *bpage) 742static inline unsigned rb_page_commit(struct buffer_page *bpage)
660{ 743{
661 return local_read(&bpage->commit); 744 return local_read(&bpage->page->commit);
662} 745}
663 746
664/* Size is determined by what has been commited */ 747/* Size is determined by what has been commited */
@@ -693,7 +776,8 @@ static void rb_update_overflow(struct ring_buffer_per_cpu *cpu_buffer)
693 head += rb_event_length(event)) { 776 head += rb_event_length(event)) {
694 777
695 event = __rb_page_index(cpu_buffer->head_page, head); 778 event = __rb_page_index(cpu_buffer->head_page, head);
696 BUG_ON(rb_null_event(event)); 779 if (RB_WARN_ON(cpu_buffer, rb_null_event(event)))
780 return;
697 /* Only count data entries */ 781 /* Only count data entries */
698 if (event->type != RINGBUF_TYPE_DATA) 782 if (event->type != RINGBUF_TYPE_DATA)
699 continue; 783 continue;
@@ -703,14 +787,14 @@ static void rb_update_overflow(struct ring_buffer_per_cpu *cpu_buffer)
703} 787}
704 788
705static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer, 789static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
706 struct buffer_page **page) 790 struct buffer_page **bpage)
707{ 791{
708 struct list_head *p = (*page)->list.next; 792 struct list_head *p = (*bpage)->list.next;
709 793
710 if (p == &cpu_buffer->pages) 794 if (p == &cpu_buffer->pages)
711 p = p->next; 795 p = p->next;
712 796
713 *page = list_entry(p, struct buffer_page, list); 797 *bpage = list_entry(p, struct buffer_page, list);
714} 798}
715 799
716static inline unsigned 800static inline unsigned
@@ -746,16 +830,18 @@ rb_set_commit_event(struct ring_buffer_per_cpu *cpu_buffer,
746 addr &= PAGE_MASK; 830 addr &= PAGE_MASK;
747 831
748 while (cpu_buffer->commit_page->page != (void *)addr) { 832 while (cpu_buffer->commit_page->page != (void *)addr) {
749 RB_WARN_ON(cpu_buffer, 833 if (RB_WARN_ON(cpu_buffer,
750 cpu_buffer->commit_page == cpu_buffer->tail_page); 834 cpu_buffer->commit_page == cpu_buffer->tail_page))
751 cpu_buffer->commit_page->commit = 835 return;
836 cpu_buffer->commit_page->page->commit =
752 cpu_buffer->commit_page->write; 837 cpu_buffer->commit_page->write;
753 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page); 838 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
754 cpu_buffer->write_stamp = cpu_buffer->commit_page->time_stamp; 839 cpu_buffer->write_stamp =
840 cpu_buffer->commit_page->page->time_stamp;
755 } 841 }
756 842
757 /* Now set the commit to the event's index */ 843 /* Now set the commit to the event's index */
758 local_set(&cpu_buffer->commit_page->commit, index); 844 local_set(&cpu_buffer->commit_page->page->commit, index);
759} 845}
760 846
761static inline void 847static inline void
@@ -769,25 +855,38 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
769 * back to us). This allows us to do a simple loop to 855 * back to us). This allows us to do a simple loop to
770 * assign the commit to the tail. 856 * assign the commit to the tail.
771 */ 857 */
858 again:
772 while (cpu_buffer->commit_page != cpu_buffer->tail_page) { 859 while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
773 cpu_buffer->commit_page->commit = 860 cpu_buffer->commit_page->page->commit =
774 cpu_buffer->commit_page->write; 861 cpu_buffer->commit_page->write;
775 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page); 862 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
776 cpu_buffer->write_stamp = cpu_buffer->commit_page->time_stamp; 863 cpu_buffer->write_stamp =
864 cpu_buffer->commit_page->page->time_stamp;
777 /* add barrier to keep gcc from optimizing too much */ 865 /* add barrier to keep gcc from optimizing too much */
778 barrier(); 866 barrier();
779 } 867 }
780 while (rb_commit_index(cpu_buffer) != 868 while (rb_commit_index(cpu_buffer) !=
781 rb_page_write(cpu_buffer->commit_page)) { 869 rb_page_write(cpu_buffer->commit_page)) {
782 cpu_buffer->commit_page->commit = 870 cpu_buffer->commit_page->page->commit =
783 cpu_buffer->commit_page->write; 871 cpu_buffer->commit_page->write;
784 barrier(); 872 barrier();
785 } 873 }
874
875 /* again, keep gcc from optimizing */
876 barrier();
877
878 /*
879 * If an interrupt came in just after the first while loop
880 * and pushed the tail page forward, we will be left with
881 * a dangling commit that will never go forward.
882 */
883 if (unlikely(cpu_buffer->commit_page != cpu_buffer->tail_page))
884 goto again;
786} 885}
787 886
788static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer) 887static void rb_reset_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
789{ 888{
790 cpu_buffer->read_stamp = cpu_buffer->reader_page->time_stamp; 889 cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;
791 cpu_buffer->reader_page->read = 0; 890 cpu_buffer->reader_page->read = 0;
792} 891}
793 892
@@ -806,7 +905,7 @@ static inline void rb_inc_iter(struct ring_buffer_iter *iter)
806 else 905 else
807 rb_inc_page(cpu_buffer, &iter->head_page); 906 rb_inc_page(cpu_buffer, &iter->head_page);
808 907
809 iter->read_stamp = iter->head_page->time_stamp; 908 iter->read_stamp = iter->head_page->page->time_stamp;
810 iter->head = 0; 909 iter->head = 0;
811} 910}
812 911
@@ -880,12 +979,15 @@ static struct ring_buffer_event *
880__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, 979__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
881 unsigned type, unsigned long length, u64 *ts) 980 unsigned type, unsigned long length, u64 *ts)
882{ 981{
883 struct buffer_page *tail_page, *head_page, *reader_page; 982 struct buffer_page *tail_page, *head_page, *reader_page, *commit_page;
884 unsigned long tail, write; 983 unsigned long tail, write;
885 struct ring_buffer *buffer = cpu_buffer->buffer; 984 struct ring_buffer *buffer = cpu_buffer->buffer;
886 struct ring_buffer_event *event; 985 struct ring_buffer_event *event;
887 unsigned long flags; 986 unsigned long flags;
888 987
988 commit_page = cpu_buffer->commit_page;
989 /* we just need to protect against interrupts */
990 barrier();
889 tail_page = cpu_buffer->tail_page; 991 tail_page = cpu_buffer->tail_page;
890 write = local_add_return(length, &tail_page->write); 992 write = local_add_return(length, &tail_page->write);
891 tail = write - length; 993 tail = write - length;
@@ -894,7 +996,8 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
894 if (write > BUF_PAGE_SIZE) { 996 if (write > BUF_PAGE_SIZE) {
895 struct buffer_page *next_page = tail_page; 997 struct buffer_page *next_page = tail_page;
896 998
897 spin_lock_irqsave(&cpu_buffer->lock, flags); 999 local_irq_save(flags);
1000 __raw_spin_lock(&cpu_buffer->lock);
898 1001
899 rb_inc_page(cpu_buffer, &next_page); 1002 rb_inc_page(cpu_buffer, &next_page);
900 1003
@@ -902,14 +1005,15 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
902 reader_page = cpu_buffer->reader_page; 1005 reader_page = cpu_buffer->reader_page;
903 1006
904 /* we grabbed the lock before incrementing */ 1007 /* we grabbed the lock before incrementing */
905 RB_WARN_ON(cpu_buffer, next_page == reader_page); 1008 if (RB_WARN_ON(cpu_buffer, next_page == reader_page))
1009 goto out_unlock;
906 1010
907 /* 1011 /*
908 * If for some reason, we had an interrupt storm that made 1012 * If for some reason, we had an interrupt storm that made
909 * it all the way around the buffer, bail, and warn 1013 * it all the way around the buffer, bail, and warn
910 * about it. 1014 * about it.
911 */ 1015 */
912 if (unlikely(next_page == cpu_buffer->commit_page)) { 1016 if (unlikely(next_page == commit_page)) {
913 WARN_ON_ONCE(1); 1017 WARN_ON_ONCE(1);
914 goto out_unlock; 1018 goto out_unlock;
915 } 1019 }
@@ -940,12 +1044,12 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
940 */ 1044 */
941 if (tail_page == cpu_buffer->tail_page) { 1045 if (tail_page == cpu_buffer->tail_page) {
942 local_set(&next_page->write, 0); 1046 local_set(&next_page->write, 0);
943 local_set(&next_page->commit, 0); 1047 local_set(&next_page->page->commit, 0);
944 cpu_buffer->tail_page = next_page; 1048 cpu_buffer->tail_page = next_page;
945 1049
946 /* reread the time stamp */ 1050 /* reread the time stamp */
947 *ts = ring_buffer_time_stamp(cpu_buffer->cpu); 1051 *ts = ring_buffer_time_stamp(cpu_buffer->cpu);
948 cpu_buffer->tail_page->time_stamp = *ts; 1052 cpu_buffer->tail_page->page->time_stamp = *ts;
949 } 1053 }
950 1054
951 /* 1055 /*
@@ -970,7 +1074,8 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
970 rb_set_commit_to_write(cpu_buffer); 1074 rb_set_commit_to_write(cpu_buffer);
971 } 1075 }
972 1076
973 spin_unlock_irqrestore(&cpu_buffer->lock, flags); 1077 __raw_spin_unlock(&cpu_buffer->lock);
1078 local_irq_restore(flags);
974 1079
975 /* fail and let the caller try again */ 1080 /* fail and let the caller try again */
976 return ERR_PTR(-EAGAIN); 1081 return ERR_PTR(-EAGAIN);
@@ -978,7 +1083,8 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
978 1083
979 /* We reserved something on the buffer */ 1084 /* We reserved something on the buffer */
980 1085
981 BUG_ON(write > BUF_PAGE_SIZE); 1086 if (RB_WARN_ON(cpu_buffer, write > BUF_PAGE_SIZE))
1087 return NULL;
982 1088
983 event = __rb_page_index(tail_page, tail); 1089 event = __rb_page_index(tail_page, tail);
984 rb_update_event(event, type, length); 1090 rb_update_event(event, type, length);
@@ -988,12 +1094,13 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
988 * this page's time stamp. 1094 * this page's time stamp.
989 */ 1095 */
990 if (!tail && rb_is_commit(cpu_buffer, event)) 1096 if (!tail && rb_is_commit(cpu_buffer, event))
991 cpu_buffer->commit_page->time_stamp = *ts; 1097 cpu_buffer->commit_page->page->time_stamp = *ts;
992 1098
993 return event; 1099 return event;
994 1100
995 out_unlock: 1101 out_unlock:
996 spin_unlock_irqrestore(&cpu_buffer->lock, flags); 1102 __raw_spin_unlock(&cpu_buffer->lock);
1103 local_irq_restore(flags);
997 return NULL; 1104 return NULL;
998} 1105}
999 1106
@@ -1038,7 +1145,7 @@ rb_add_time_stamp(struct ring_buffer_per_cpu *cpu_buffer,
1038 event->time_delta = *delta & TS_MASK; 1145 event->time_delta = *delta & TS_MASK;
1039 event->array[0] = *delta >> TS_SHIFT; 1146 event->array[0] = *delta >> TS_SHIFT;
1040 } else { 1147 } else {
1041 cpu_buffer->commit_page->time_stamp = *ts; 1148 cpu_buffer->commit_page->page->time_stamp = *ts;
1042 event->time_delta = 0; 1149 event->time_delta = 0;
1043 event->array[0] = 0; 1150 event->array[0] = 0;
1044 } 1151 }
@@ -1076,10 +1183,8 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
1076 * storm or we have something buggy. 1183 * storm or we have something buggy.
1077 * Bail! 1184 * Bail!
1078 */ 1185 */
1079 if (unlikely(++nr_loops > 1000)) { 1186 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
1080 RB_WARN_ON(cpu_buffer, 1);
1081 return NULL; 1187 return NULL;
1082 }
1083 1188
1084 ts = ring_buffer_time_stamp(cpu_buffer->cpu); 1189 ts = ring_buffer_time_stamp(cpu_buffer->cpu);
1085 1190
@@ -1175,19 +1280,18 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer,
1175 struct ring_buffer_event *event; 1280 struct ring_buffer_event *event;
1176 int cpu, resched; 1281 int cpu, resched;
1177 1282
1178 if (ring_buffers_off) 1283 if (ring_buffer_flags != RB_BUFFERS_ON)
1179 return NULL; 1284 return NULL;
1180 1285
1181 if (atomic_read(&buffer->record_disabled)) 1286 if (atomic_read(&buffer->record_disabled))
1182 return NULL; 1287 return NULL;
1183 1288
1184 /* If we are tracing schedule, we don't want to recurse */ 1289 /* If we are tracing schedule, we don't want to recurse */
1185 resched = need_resched(); 1290 resched = ftrace_preempt_disable();
1186 preempt_disable_notrace();
1187 1291
1188 cpu = raw_smp_processor_id(); 1292 cpu = raw_smp_processor_id();
1189 1293
1190 if (!cpu_isset(cpu, buffer->cpumask)) 1294 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1191 goto out; 1295 goto out;
1192 1296
1193 cpu_buffer = buffer->buffers[cpu]; 1297 cpu_buffer = buffer->buffers[cpu];
@@ -1214,12 +1318,10 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer,
1214 return event; 1318 return event;
1215 1319
1216 out: 1320 out:
1217 if (resched) 1321 ftrace_preempt_enable(resched);
1218 preempt_enable_notrace();
1219 else
1220 preempt_enable_notrace();
1221 return NULL; 1322 return NULL;
1222} 1323}
1324EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
1223 1325
1224static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer, 1326static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
1225 struct ring_buffer_event *event) 1327 struct ring_buffer_event *event)
@@ -1259,16 +1361,14 @@ int ring_buffer_unlock_commit(struct ring_buffer *buffer,
1259 /* 1361 /*
1260 * Only the last preempt count needs to restore preemption. 1362 * Only the last preempt count needs to restore preemption.
1261 */ 1363 */
1262 if (preempt_count() == 1) { 1364 if (preempt_count() == 1)
1263 if (per_cpu(rb_need_resched, cpu)) 1365 ftrace_preempt_enable(per_cpu(rb_need_resched, cpu));
1264 preempt_enable_no_resched_notrace(); 1366 else
1265 else
1266 preempt_enable_notrace();
1267 } else
1268 preempt_enable_no_resched_notrace(); 1367 preempt_enable_no_resched_notrace();
1269 1368
1270 return 0; 1369 return 0;
1271} 1370}
1371EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
1272 1372
1273/** 1373/**
1274 * ring_buffer_write - write data to the buffer without reserving 1374 * ring_buffer_write - write data to the buffer without reserving
@@ -1294,18 +1394,17 @@ int ring_buffer_write(struct ring_buffer *buffer,
1294 int ret = -EBUSY; 1394 int ret = -EBUSY;
1295 int cpu, resched; 1395 int cpu, resched;
1296 1396
1297 if (ring_buffers_off) 1397 if (ring_buffer_flags != RB_BUFFERS_ON)
1298 return -EBUSY; 1398 return -EBUSY;
1299 1399
1300 if (atomic_read(&buffer->record_disabled)) 1400 if (atomic_read(&buffer->record_disabled))
1301 return -EBUSY; 1401 return -EBUSY;
1302 1402
1303 resched = need_resched(); 1403 resched = ftrace_preempt_disable();
1304 preempt_disable_notrace();
1305 1404
1306 cpu = raw_smp_processor_id(); 1405 cpu = raw_smp_processor_id();
1307 1406
1308 if (!cpu_isset(cpu, buffer->cpumask)) 1407 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1309 goto out; 1408 goto out;
1310 1409
1311 cpu_buffer = buffer->buffers[cpu]; 1410 cpu_buffer = buffer->buffers[cpu];
@@ -1327,13 +1426,11 @@ int ring_buffer_write(struct ring_buffer *buffer,
1327 1426
1328 ret = 0; 1427 ret = 0;
1329 out: 1428 out:
1330 if (resched) 1429 ftrace_preempt_enable(resched);
1331 preempt_enable_no_resched_notrace();
1332 else
1333 preempt_enable_notrace();
1334 1430
1335 return ret; 1431 return ret;
1336} 1432}
1433EXPORT_SYMBOL_GPL(ring_buffer_write);
1337 1434
1338static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer) 1435static inline int rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
1339{ 1436{
@@ -1360,6 +1457,7 @@ void ring_buffer_record_disable(struct ring_buffer *buffer)
1360{ 1457{
1361 atomic_inc(&buffer->record_disabled); 1458 atomic_inc(&buffer->record_disabled);
1362} 1459}
1460EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
1363 1461
1364/** 1462/**
1365 * ring_buffer_record_enable - enable writes to the buffer 1463 * ring_buffer_record_enable - enable writes to the buffer
@@ -1372,6 +1470,7 @@ void ring_buffer_record_enable(struct ring_buffer *buffer)
1372{ 1470{
1373 atomic_dec(&buffer->record_disabled); 1471 atomic_dec(&buffer->record_disabled);
1374} 1472}
1473EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
1375 1474
1376/** 1475/**
1377 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer 1476 * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
@@ -1387,12 +1486,13 @@ void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
1387{ 1486{
1388 struct ring_buffer_per_cpu *cpu_buffer; 1487 struct ring_buffer_per_cpu *cpu_buffer;
1389 1488
1390 if (!cpu_isset(cpu, buffer->cpumask)) 1489 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1391 return; 1490 return;
1392 1491
1393 cpu_buffer = buffer->buffers[cpu]; 1492 cpu_buffer = buffer->buffers[cpu];
1394 atomic_inc(&cpu_buffer->record_disabled); 1493 atomic_inc(&cpu_buffer->record_disabled);
1395} 1494}
1495EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
1396 1496
1397/** 1497/**
1398 * ring_buffer_record_enable_cpu - enable writes to the buffer 1498 * ring_buffer_record_enable_cpu - enable writes to the buffer
@@ -1406,12 +1506,13 @@ void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
1406{ 1506{
1407 struct ring_buffer_per_cpu *cpu_buffer; 1507 struct ring_buffer_per_cpu *cpu_buffer;
1408 1508
1409 if (!cpu_isset(cpu, buffer->cpumask)) 1509 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1410 return; 1510 return;
1411 1511
1412 cpu_buffer = buffer->buffers[cpu]; 1512 cpu_buffer = buffer->buffers[cpu];
1413 atomic_dec(&cpu_buffer->record_disabled); 1513 atomic_dec(&cpu_buffer->record_disabled);
1414} 1514}
1515EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
1415 1516
1416/** 1517/**
1417 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer 1518 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
@@ -1422,12 +1523,13 @@ unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
1422{ 1523{
1423 struct ring_buffer_per_cpu *cpu_buffer; 1524 struct ring_buffer_per_cpu *cpu_buffer;
1424 1525
1425 if (!cpu_isset(cpu, buffer->cpumask)) 1526 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1426 return 0; 1527 return 0;
1427 1528
1428 cpu_buffer = buffer->buffers[cpu]; 1529 cpu_buffer = buffer->buffers[cpu];
1429 return cpu_buffer->entries; 1530 return cpu_buffer->entries;
1430} 1531}
1532EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
1431 1533
1432/** 1534/**
1433 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer 1535 * ring_buffer_overrun_cpu - get the number of overruns in a cpu_buffer
@@ -1438,12 +1540,13 @@ unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
1438{ 1540{
1439 struct ring_buffer_per_cpu *cpu_buffer; 1541 struct ring_buffer_per_cpu *cpu_buffer;
1440 1542
1441 if (!cpu_isset(cpu, buffer->cpumask)) 1543 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1442 return 0; 1544 return 0;
1443 1545
1444 cpu_buffer = buffer->buffers[cpu]; 1546 cpu_buffer = buffer->buffers[cpu];
1445 return cpu_buffer->overrun; 1547 return cpu_buffer->overrun;
1446} 1548}
1549EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
1447 1550
1448/** 1551/**
1449 * ring_buffer_entries - get the number of entries in a buffer 1552 * ring_buffer_entries - get the number of entries in a buffer
@@ -1466,6 +1569,7 @@ unsigned long ring_buffer_entries(struct ring_buffer *buffer)
1466 1569
1467 return entries; 1570 return entries;
1468} 1571}
1572EXPORT_SYMBOL_GPL(ring_buffer_entries);
1469 1573
1470/** 1574/**
1471 * ring_buffer_overrun_cpu - get the number of overruns in buffer 1575 * ring_buffer_overrun_cpu - get the number of overruns in buffer
@@ -1488,15 +1592,9 @@ unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
1488 1592
1489 return overruns; 1593 return overruns;
1490} 1594}
1595EXPORT_SYMBOL_GPL(ring_buffer_overruns);
1491 1596
1492/** 1597static void rb_iter_reset(struct ring_buffer_iter *iter)
1493 * ring_buffer_iter_reset - reset an iterator
1494 * @iter: The iterator to reset
1495 *
1496 * Resets the iterator, so that it will start from the beginning
1497 * again.
1498 */
1499void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
1500{ 1598{
1501 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 1599 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
1502 1600
@@ -1511,10 +1609,28 @@ void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
1511 if (iter->head) 1609 if (iter->head)
1512 iter->read_stamp = cpu_buffer->read_stamp; 1610 iter->read_stamp = cpu_buffer->read_stamp;
1513 else 1611 else
1514 iter->read_stamp = iter->head_page->time_stamp; 1612 iter->read_stamp = iter->head_page->page->time_stamp;
1515} 1613}
1516 1614
1517/** 1615/**
1616 * ring_buffer_iter_reset - reset an iterator
1617 * @iter: The iterator to reset
1618 *
1619 * Resets the iterator, so that it will start from the beginning
1620 * again.
1621 */
1622void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
1623{
1624 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
1625 unsigned long flags;
1626
1627 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
1628 rb_iter_reset(iter);
1629 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1630}
1631EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
1632
1633/**
1518 * ring_buffer_iter_empty - check if an iterator has no more to read 1634 * ring_buffer_iter_empty - check if an iterator has no more to read
1519 * @iter: The iterator to check 1635 * @iter: The iterator to check
1520 */ 1636 */
@@ -1527,6 +1643,7 @@ int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
1527 return iter->head_page == cpu_buffer->commit_page && 1643 return iter->head_page == cpu_buffer->commit_page &&
1528 iter->head == rb_commit_index(cpu_buffer); 1644 iter->head == rb_commit_index(cpu_buffer);
1529} 1645}
1646EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
1530 1647
1531static void 1648static void
1532rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer, 1649rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
@@ -1597,7 +1714,8 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1597 unsigned long flags; 1714 unsigned long flags;
1598 int nr_loops = 0; 1715 int nr_loops = 0;
1599 1716
1600 spin_lock_irqsave(&cpu_buffer->lock, flags); 1717 local_irq_save(flags);
1718 __raw_spin_lock(&cpu_buffer->lock);
1601 1719
1602 again: 1720 again:
1603 /* 1721 /*
@@ -1606,8 +1724,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1606 * a case where we will loop three times. There should be no 1724 * a case where we will loop three times. There should be no
1607 * reason to loop four times (that I know of). 1725 * reason to loop four times (that I know of).
1608 */ 1726 */
1609 if (unlikely(++nr_loops > 3)) { 1727 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
1610 RB_WARN_ON(cpu_buffer, 1);
1611 reader = NULL; 1728 reader = NULL;
1612 goto out; 1729 goto out;
1613 } 1730 }
@@ -1619,8 +1736,9 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1619 goto out; 1736 goto out;
1620 1737
1621 /* Never should we have an index greater than the size */ 1738 /* Never should we have an index greater than the size */
1622 RB_WARN_ON(cpu_buffer, 1739 if (RB_WARN_ON(cpu_buffer,
1623 cpu_buffer->reader_page->read > rb_page_size(reader)); 1740 cpu_buffer->reader_page->read > rb_page_size(reader)))
1741 goto out;
1624 1742
1625 /* check if we caught up to the tail */ 1743 /* check if we caught up to the tail */
1626 reader = NULL; 1744 reader = NULL;
@@ -1637,7 +1755,7 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1637 cpu_buffer->reader_page->list.prev = reader->list.prev; 1755 cpu_buffer->reader_page->list.prev = reader->list.prev;
1638 1756
1639 local_set(&cpu_buffer->reader_page->write, 0); 1757 local_set(&cpu_buffer->reader_page->write, 0);
1640 local_set(&cpu_buffer->reader_page->commit, 0); 1758 local_set(&cpu_buffer->reader_page->page->commit, 0);
1641 1759
1642 /* Make the reader page now replace the head */ 1760 /* Make the reader page now replace the head */
1643 reader->list.prev->next = &cpu_buffer->reader_page->list; 1761 reader->list.prev->next = &cpu_buffer->reader_page->list;
@@ -1659,7 +1777,8 @@ rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
1659 goto again; 1777 goto again;
1660 1778
1661 out: 1779 out:
1662 spin_unlock_irqrestore(&cpu_buffer->lock, flags); 1780 __raw_spin_unlock(&cpu_buffer->lock);
1781 local_irq_restore(flags);
1663 1782
1664 return reader; 1783 return reader;
1665} 1784}
@@ -1673,7 +1792,8 @@ static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
1673 reader = rb_get_reader_page(cpu_buffer); 1792 reader = rb_get_reader_page(cpu_buffer);
1674 1793
1675 /* This function should not be called when buffer is empty */ 1794 /* This function should not be called when buffer is empty */
1676 BUG_ON(!reader); 1795 if (RB_WARN_ON(cpu_buffer, !reader))
1796 return;
1677 1797
1678 event = rb_reader_event(cpu_buffer); 1798 event = rb_reader_event(cpu_buffer);
1679 1799
@@ -1700,7 +1820,9 @@ static void rb_advance_iter(struct ring_buffer_iter *iter)
1700 * Check if we are at the end of the buffer. 1820 * Check if we are at the end of the buffer.
1701 */ 1821 */
1702 if (iter->head >= rb_page_size(iter->head_page)) { 1822 if (iter->head >= rb_page_size(iter->head_page)) {
1703 BUG_ON(iter->head_page == cpu_buffer->commit_page); 1823 if (RB_WARN_ON(buffer,
1824 iter->head_page == cpu_buffer->commit_page))
1825 return;
1704 rb_inc_iter(iter); 1826 rb_inc_iter(iter);
1705 return; 1827 return;
1706 } 1828 }
@@ -1713,8 +1835,10 @@ static void rb_advance_iter(struct ring_buffer_iter *iter)
1713 * This should not be called to advance the header if we are 1835 * This should not be called to advance the header if we are
1714 * at the tail of the buffer. 1836 * at the tail of the buffer.
1715 */ 1837 */
1716 BUG_ON((iter->head_page == cpu_buffer->commit_page) && 1838 if (RB_WARN_ON(cpu_buffer,
1717 (iter->head + length > rb_commit_index(cpu_buffer))); 1839 (iter->head_page == cpu_buffer->commit_page) &&
1840 (iter->head + length > rb_commit_index(cpu_buffer))))
1841 return;
1718 1842
1719 rb_update_iter_read_stamp(iter, event); 1843 rb_update_iter_read_stamp(iter, event);
1720 1844
@@ -1726,24 +1850,15 @@ static void rb_advance_iter(struct ring_buffer_iter *iter)
1726 rb_advance_iter(iter); 1850 rb_advance_iter(iter);
1727} 1851}
1728 1852
1729/** 1853static struct ring_buffer_event *
1730 * ring_buffer_peek - peek at the next event to be read 1854rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
1731 * @buffer: The ring buffer to read
1732 * @cpu: The cpu to peak at
1733 * @ts: The timestamp counter of this event.
1734 *
1735 * This will return the event that will be read next, but does
1736 * not consume the data.
1737 */
1738struct ring_buffer_event *
1739ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
1740{ 1855{
1741 struct ring_buffer_per_cpu *cpu_buffer; 1856 struct ring_buffer_per_cpu *cpu_buffer;
1742 struct ring_buffer_event *event; 1857 struct ring_buffer_event *event;
1743 struct buffer_page *reader; 1858 struct buffer_page *reader;
1744 int nr_loops = 0; 1859 int nr_loops = 0;
1745 1860
1746 if (!cpu_isset(cpu, buffer->cpumask)) 1861 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1747 return NULL; 1862 return NULL;
1748 1863
1749 cpu_buffer = buffer->buffers[cpu]; 1864 cpu_buffer = buffer->buffers[cpu];
@@ -1757,10 +1872,8 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
1757 * can have. Nesting 10 deep of interrupts is clearly 1872 * can have. Nesting 10 deep of interrupts is clearly
1758 * an anomaly. 1873 * an anomaly.
1759 */ 1874 */
1760 if (unlikely(++nr_loops > 10)) { 1875 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 10))
1761 RB_WARN_ON(cpu_buffer, 1);
1762 return NULL; 1876 return NULL;
1763 }
1764 1877
1765 reader = rb_get_reader_page(cpu_buffer); 1878 reader = rb_get_reader_page(cpu_buffer);
1766 if (!reader) 1879 if (!reader)
@@ -1797,17 +1910,10 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
1797 1910
1798 return NULL; 1911 return NULL;
1799} 1912}
1913EXPORT_SYMBOL_GPL(ring_buffer_peek);
1800 1914
1801/** 1915static struct ring_buffer_event *
1802 * ring_buffer_iter_peek - peek at the next event to be read 1916rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
1803 * @iter: The ring buffer iterator
1804 * @ts: The timestamp counter of this event.
1805 *
1806 * This will return the event that will be read next, but does
1807 * not increment the iterator.
1808 */
1809struct ring_buffer_event *
1810ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
1811{ 1917{
1812 struct ring_buffer *buffer; 1918 struct ring_buffer *buffer;
1813 struct ring_buffer_per_cpu *cpu_buffer; 1919 struct ring_buffer_per_cpu *cpu_buffer;
@@ -1829,10 +1935,8 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
1829 * can have. Nesting 10 deep of interrupts is clearly 1935 * can have. Nesting 10 deep of interrupts is clearly
1830 * an anomaly. 1936 * an anomaly.
1831 */ 1937 */
1832 if (unlikely(++nr_loops > 10)) { 1938 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 10))
1833 RB_WARN_ON(cpu_buffer, 1);
1834 return NULL; 1939 return NULL;
1835 }
1836 1940
1837 if (rb_per_cpu_empty(cpu_buffer)) 1941 if (rb_per_cpu_empty(cpu_buffer))
1838 return NULL; 1942 return NULL;
@@ -1867,6 +1971,52 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
1867 1971
1868 return NULL; 1972 return NULL;
1869} 1973}
1974EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
1975
1976/**
1977 * ring_buffer_peek - peek at the next event to be read
1978 * @buffer: The ring buffer to read
1979 * @cpu: The cpu to peak at
1980 * @ts: The timestamp counter of this event.
1981 *
1982 * This will return the event that will be read next, but does
1983 * not consume the data.
1984 */
1985struct ring_buffer_event *
1986ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
1987{
1988 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
1989 struct ring_buffer_event *event;
1990 unsigned long flags;
1991
1992 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
1993 event = rb_buffer_peek(buffer, cpu, ts);
1994 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1995
1996 return event;
1997}
1998
1999/**
2000 * ring_buffer_iter_peek - peek at the next event to be read
2001 * @iter: The ring buffer iterator
2002 * @ts: The timestamp counter of this event.
2003 *
2004 * This will return the event that will be read next, but does
2005 * not increment the iterator.
2006 */
2007struct ring_buffer_event *
2008ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
2009{
2010 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
2011 struct ring_buffer_event *event;
2012 unsigned long flags;
2013
2014 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2015 event = rb_iter_peek(iter, ts);
2016 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2017
2018 return event;
2019}
1870 2020
1871/** 2021/**
1872 * ring_buffer_consume - return an event and consume it 2022 * ring_buffer_consume - return an event and consume it
@@ -1879,21 +2029,27 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
1879struct ring_buffer_event * 2029struct ring_buffer_event *
1880ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts) 2030ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
1881{ 2031{
1882 struct ring_buffer_per_cpu *cpu_buffer; 2032 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
1883 struct ring_buffer_event *event; 2033 struct ring_buffer_event *event;
2034 unsigned long flags;
1884 2035
1885 if (!cpu_isset(cpu, buffer->cpumask)) 2036 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1886 return NULL; 2037 return NULL;
1887 2038
1888 event = ring_buffer_peek(buffer, cpu, ts); 2039 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2040
2041 event = rb_buffer_peek(buffer, cpu, ts);
1889 if (!event) 2042 if (!event)
1890 return NULL; 2043 goto out;
1891 2044
1892 cpu_buffer = buffer->buffers[cpu];
1893 rb_advance_reader(cpu_buffer); 2045 rb_advance_reader(cpu_buffer);
1894 2046
2047 out:
2048 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2049
1895 return event; 2050 return event;
1896} 2051}
2052EXPORT_SYMBOL_GPL(ring_buffer_consume);
1897 2053
1898/** 2054/**
1899 * ring_buffer_read_start - start a non consuming read of the buffer 2055 * ring_buffer_read_start - start a non consuming read of the buffer
@@ -1914,7 +2070,7 @@ ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
1914 struct ring_buffer_iter *iter; 2070 struct ring_buffer_iter *iter;
1915 unsigned long flags; 2071 unsigned long flags;
1916 2072
1917 if (!cpu_isset(cpu, buffer->cpumask)) 2073 if (!cpumask_test_cpu(cpu, buffer->cpumask))
1918 return NULL; 2074 return NULL;
1919 2075
1920 iter = kmalloc(sizeof(*iter), GFP_KERNEL); 2076 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
@@ -1928,12 +2084,15 @@ ring_buffer_read_start(struct ring_buffer *buffer, int cpu)
1928 atomic_inc(&cpu_buffer->record_disabled); 2084 atomic_inc(&cpu_buffer->record_disabled);
1929 synchronize_sched(); 2085 synchronize_sched();
1930 2086
1931 spin_lock_irqsave(&cpu_buffer->lock, flags); 2087 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
1932 ring_buffer_iter_reset(iter); 2088 __raw_spin_lock(&cpu_buffer->lock);
1933 spin_unlock_irqrestore(&cpu_buffer->lock, flags); 2089 rb_iter_reset(iter);
2090 __raw_spin_unlock(&cpu_buffer->lock);
2091 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1934 2092
1935 return iter; 2093 return iter;
1936} 2094}
2095EXPORT_SYMBOL_GPL(ring_buffer_read_start);
1937 2096
1938/** 2097/**
1939 * ring_buffer_finish - finish reading the iterator of the buffer 2098 * ring_buffer_finish - finish reading the iterator of the buffer
@@ -1950,6 +2109,7 @@ ring_buffer_read_finish(struct ring_buffer_iter *iter)
1950 atomic_dec(&cpu_buffer->record_disabled); 2109 atomic_dec(&cpu_buffer->record_disabled);
1951 kfree(iter); 2110 kfree(iter);
1952} 2111}
2112EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
1953 2113
1954/** 2114/**
1955 * ring_buffer_read - read the next item in the ring buffer by the iterator 2115 * ring_buffer_read - read the next item in the ring buffer by the iterator
@@ -1962,15 +2122,21 @@ struct ring_buffer_event *
1962ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts) 2122ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
1963{ 2123{
1964 struct ring_buffer_event *event; 2124 struct ring_buffer_event *event;
2125 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
2126 unsigned long flags;
1965 2127
1966 event = ring_buffer_iter_peek(iter, ts); 2128 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2129 event = rb_iter_peek(iter, ts);
1967 if (!event) 2130 if (!event)
1968 return NULL; 2131 goto out;
1969 2132
1970 rb_advance_iter(iter); 2133 rb_advance_iter(iter);
2134 out:
2135 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
1971 2136
1972 return event; 2137 return event;
1973} 2138}
2139EXPORT_SYMBOL_GPL(ring_buffer_read);
1974 2140
1975/** 2141/**
1976 * ring_buffer_size - return the size of the ring buffer (in bytes) 2142 * ring_buffer_size - return the size of the ring buffer (in bytes)
@@ -1980,6 +2146,7 @@ unsigned long ring_buffer_size(struct ring_buffer *buffer)
1980{ 2146{
1981 return BUF_PAGE_SIZE * buffer->pages; 2147 return BUF_PAGE_SIZE * buffer->pages;
1982} 2148}
2149EXPORT_SYMBOL_GPL(ring_buffer_size);
1983 2150
1984static void 2151static void
1985rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) 2152rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
@@ -1987,7 +2154,7 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
1987 cpu_buffer->head_page 2154 cpu_buffer->head_page
1988 = list_entry(cpu_buffer->pages.next, struct buffer_page, list); 2155 = list_entry(cpu_buffer->pages.next, struct buffer_page, list);
1989 local_set(&cpu_buffer->head_page->write, 0); 2156 local_set(&cpu_buffer->head_page->write, 0);
1990 local_set(&cpu_buffer->head_page->commit, 0); 2157 local_set(&cpu_buffer->head_page->page->commit, 0);
1991 2158
1992 cpu_buffer->head_page->read = 0; 2159 cpu_buffer->head_page->read = 0;
1993 2160
@@ -1996,7 +2163,7 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
1996 2163
1997 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 2164 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
1998 local_set(&cpu_buffer->reader_page->write, 0); 2165 local_set(&cpu_buffer->reader_page->write, 0);
1999 local_set(&cpu_buffer->reader_page->commit, 0); 2166 local_set(&cpu_buffer->reader_page->page->commit, 0);
2000 cpu_buffer->reader_page->read = 0; 2167 cpu_buffer->reader_page->read = 0;
2001 2168
2002 cpu_buffer->overrun = 0; 2169 cpu_buffer->overrun = 0;
@@ -2013,15 +2180,20 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
2013 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; 2180 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2014 unsigned long flags; 2181 unsigned long flags;
2015 2182
2016 if (!cpu_isset(cpu, buffer->cpumask)) 2183 if (!cpumask_test_cpu(cpu, buffer->cpumask))
2017 return; 2184 return;
2018 2185
2019 spin_lock_irqsave(&cpu_buffer->lock, flags); 2186 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2187
2188 __raw_spin_lock(&cpu_buffer->lock);
2020 2189
2021 rb_reset_cpu(cpu_buffer); 2190 rb_reset_cpu(cpu_buffer);
2022 2191
2023 spin_unlock_irqrestore(&cpu_buffer->lock, flags); 2192 __raw_spin_unlock(&cpu_buffer->lock);
2193
2194 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2024} 2195}
2196EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
2025 2197
2026/** 2198/**
2027 * ring_buffer_reset - reset a ring buffer 2199 * ring_buffer_reset - reset a ring buffer
@@ -2034,6 +2206,7 @@ void ring_buffer_reset(struct ring_buffer *buffer)
2034 for_each_buffer_cpu(buffer, cpu) 2206 for_each_buffer_cpu(buffer, cpu)
2035 ring_buffer_reset_cpu(buffer, cpu); 2207 ring_buffer_reset_cpu(buffer, cpu);
2036} 2208}
2209EXPORT_SYMBOL_GPL(ring_buffer_reset);
2037 2210
2038/** 2211/**
2039 * rind_buffer_empty - is the ring buffer empty? 2212 * rind_buffer_empty - is the ring buffer empty?
@@ -2052,6 +2225,7 @@ int ring_buffer_empty(struct ring_buffer *buffer)
2052 } 2225 }
2053 return 1; 2226 return 1;
2054} 2227}
2228EXPORT_SYMBOL_GPL(ring_buffer_empty);
2055 2229
2056/** 2230/**
2057 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty? 2231 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
@@ -2062,12 +2236,13 @@ int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
2062{ 2236{
2063 struct ring_buffer_per_cpu *cpu_buffer; 2237 struct ring_buffer_per_cpu *cpu_buffer;
2064 2238
2065 if (!cpu_isset(cpu, buffer->cpumask)) 2239 if (!cpumask_test_cpu(cpu, buffer->cpumask))
2066 return 1; 2240 return 1;
2067 2241
2068 cpu_buffer = buffer->buffers[cpu]; 2242 cpu_buffer = buffer->buffers[cpu];
2069 return rb_per_cpu_empty(cpu_buffer); 2243 return rb_per_cpu_empty(cpu_buffer);
2070} 2244}
2245EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
2071 2246
2072/** 2247/**
2073 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers 2248 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
@@ -2085,13 +2260,12 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2085 struct ring_buffer_per_cpu *cpu_buffer_a; 2260 struct ring_buffer_per_cpu *cpu_buffer_a;
2086 struct ring_buffer_per_cpu *cpu_buffer_b; 2261 struct ring_buffer_per_cpu *cpu_buffer_b;
2087 2262
2088 if (!cpu_isset(cpu, buffer_a->cpumask) || 2263 if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
2089 !cpu_isset(cpu, buffer_b->cpumask)) 2264 !cpumask_test_cpu(cpu, buffer_b->cpumask))
2090 return -EINVAL; 2265 return -EINVAL;
2091 2266
2092 /* At least make sure the two buffers are somewhat the same */ 2267 /* At least make sure the two buffers are somewhat the same */
2093 if (buffer_a->size != buffer_b->size || 2268 if (buffer_a->pages != buffer_b->pages)
2094 buffer_a->pages != buffer_b->pages)
2095 return -EINVAL; 2269 return -EINVAL;
2096 2270
2097 cpu_buffer_a = buffer_a->buffers[cpu]; 2271 cpu_buffer_a = buffer_a->buffers[cpu];
@@ -2117,17 +2291,180 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
2117 2291
2118 return 0; 2292 return 0;
2119} 2293}
2294EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
2295
2296static void rb_remove_entries(struct ring_buffer_per_cpu *cpu_buffer,
2297 struct buffer_data_page *bpage)
2298{
2299 struct ring_buffer_event *event;
2300 unsigned long head;
2301
2302 __raw_spin_lock(&cpu_buffer->lock);
2303 for (head = 0; head < local_read(&bpage->commit);
2304 head += rb_event_length(event)) {
2305
2306 event = __rb_data_page_index(bpage, head);
2307 if (RB_WARN_ON(cpu_buffer, rb_null_event(event)))
2308 return;
2309 /* Only count data entries */
2310 if (event->type != RINGBUF_TYPE_DATA)
2311 continue;
2312 cpu_buffer->entries--;
2313 }
2314 __raw_spin_unlock(&cpu_buffer->lock);
2315}
2316
2317/**
2318 * ring_buffer_alloc_read_page - allocate a page to read from buffer
2319 * @buffer: the buffer to allocate for.
2320 *
2321 * This function is used in conjunction with ring_buffer_read_page.
2322 * When reading a full page from the ring buffer, these functions
2323 * can be used to speed up the process. The calling function should
2324 * allocate a few pages first with this function. Then when it
2325 * needs to get pages from the ring buffer, it passes the result
2326 * of this function into ring_buffer_read_page, which will swap
2327 * the page that was allocated, with the read page of the buffer.
2328 *
2329 * Returns:
2330 * The page allocated, or NULL on error.
2331 */
2332void *ring_buffer_alloc_read_page(struct ring_buffer *buffer)
2333{
2334 unsigned long addr;
2335 struct buffer_data_page *bpage;
2336
2337 addr = __get_free_page(GFP_KERNEL);
2338 if (!addr)
2339 return NULL;
2340
2341 bpage = (void *)addr;
2342
2343 return bpage;
2344}
2345
2346/**
2347 * ring_buffer_free_read_page - free an allocated read page
2348 * @buffer: the buffer the page was allocate for
2349 * @data: the page to free
2350 *
2351 * Free a page allocated from ring_buffer_alloc_read_page.
2352 */
2353void ring_buffer_free_read_page(struct ring_buffer *buffer, void *data)
2354{
2355 free_page((unsigned long)data);
2356}
2357
2358/**
2359 * ring_buffer_read_page - extract a page from the ring buffer
2360 * @buffer: buffer to extract from
2361 * @data_page: the page to use allocated from ring_buffer_alloc_read_page
2362 * @cpu: the cpu of the buffer to extract
2363 * @full: should the extraction only happen when the page is full.
2364 *
2365 * This function will pull out a page from the ring buffer and consume it.
2366 * @data_page must be the address of the variable that was returned
2367 * from ring_buffer_alloc_read_page. This is because the page might be used
2368 * to swap with a page in the ring buffer.
2369 *
2370 * for example:
2371 * rpage = ring_buffer_alloc_page(buffer);
2372 * if (!rpage)
2373 * return error;
2374 * ret = ring_buffer_read_page(buffer, &rpage, cpu, 0);
2375 * if (ret)
2376 * process_page(rpage);
2377 *
2378 * When @full is set, the function will not return true unless
2379 * the writer is off the reader page.
2380 *
2381 * Note: it is up to the calling functions to handle sleeps and wakeups.
2382 * The ring buffer can be used anywhere in the kernel and can not
2383 * blindly call wake_up. The layer that uses the ring buffer must be
2384 * responsible for that.
2385 *
2386 * Returns:
2387 * 1 if data has been transferred
2388 * 0 if no data has been transferred.
2389 */
2390int ring_buffer_read_page(struct ring_buffer *buffer,
2391 void **data_page, int cpu, int full)
2392{
2393 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
2394 struct ring_buffer_event *event;
2395 struct buffer_data_page *bpage;
2396 unsigned long flags;
2397 int ret = 0;
2398
2399 if (!data_page)
2400 return 0;
2401
2402 bpage = *data_page;
2403 if (!bpage)
2404 return 0;
2405
2406 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
2407
2408 /*
2409 * rb_buffer_peek will get the next ring buffer if
2410 * the current reader page is empty.
2411 */
2412 event = rb_buffer_peek(buffer, cpu, NULL);
2413 if (!event)
2414 goto out;
2415
2416 /* check for data */
2417 if (!local_read(&cpu_buffer->reader_page->page->commit))
2418 goto out;
2419 /*
2420 * If the writer is already off of the read page, then simply
2421 * switch the read page with the given page. Otherwise
2422 * we need to copy the data from the reader to the writer.
2423 */
2424 if (cpu_buffer->reader_page == cpu_buffer->commit_page) {
2425 unsigned int read = cpu_buffer->reader_page->read;
2426
2427 if (full)
2428 goto out;
2429 /* The writer is still on the reader page, we must copy */
2430 bpage = cpu_buffer->reader_page->page;
2431 memcpy(bpage->data,
2432 cpu_buffer->reader_page->page->data + read,
2433 local_read(&bpage->commit) - read);
2434
2435 /* consume what was read */
2436 cpu_buffer->reader_page += read;
2437
2438 } else {
2439 /* swap the pages */
2440 rb_init_page(bpage);
2441 bpage = cpu_buffer->reader_page->page;
2442 cpu_buffer->reader_page->page = *data_page;
2443 cpu_buffer->reader_page->read = 0;
2444 *data_page = bpage;
2445 }
2446 ret = 1;
2447
2448 /* update the entry counter */
2449 rb_remove_entries(cpu_buffer, bpage);
2450 out:
2451 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
2452
2453 return ret;
2454}
2120 2455
2121static ssize_t 2456static ssize_t
2122rb_simple_read(struct file *filp, char __user *ubuf, 2457rb_simple_read(struct file *filp, char __user *ubuf,
2123 size_t cnt, loff_t *ppos) 2458 size_t cnt, loff_t *ppos)
2124{ 2459{
2125 int *p = filp->private_data; 2460 long *p = filp->private_data;
2126 char buf[64]; 2461 char buf[64];
2127 int r; 2462 int r;
2128 2463
2129 /* !ring_buffers_off == tracing_on */ 2464 if (test_bit(RB_BUFFERS_DISABLED_BIT, p))
2130 r = sprintf(buf, "%d\n", !*p); 2465 r = sprintf(buf, "permanently disabled\n");
2466 else
2467 r = sprintf(buf, "%d\n", test_bit(RB_BUFFERS_ON_BIT, p));
2131 2468
2132 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2469 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2133} 2470}
@@ -2136,7 +2473,7 @@ static ssize_t
2136rb_simple_write(struct file *filp, const char __user *ubuf, 2473rb_simple_write(struct file *filp, const char __user *ubuf,
2137 size_t cnt, loff_t *ppos) 2474 size_t cnt, loff_t *ppos)
2138{ 2475{
2139 int *p = filp->private_data; 2476 long *p = filp->private_data;
2140 char buf[64]; 2477 char buf[64];
2141 long val; 2478 long val;
2142 int ret; 2479 int ret;
@@ -2153,8 +2490,10 @@ rb_simple_write(struct file *filp, const char __user *ubuf,
2153 if (ret < 0) 2490 if (ret < 0)
2154 return ret; 2491 return ret;
2155 2492
2156 /* !ring_buffers_off == tracing_on */ 2493 if (val)
2157 *p = !val; 2494 set_bit(RB_BUFFERS_ON_BIT, p);
2495 else
2496 clear_bit(RB_BUFFERS_ON_BIT, p);
2158 2497
2159 (*ppos)++; 2498 (*ppos)++;
2160 2499
@@ -2176,7 +2515,7 @@ static __init int rb_init_debugfs(void)
2176 d_tracer = tracing_init_dentry(); 2515 d_tracer = tracing_init_dentry();
2177 2516
2178 entry = debugfs_create_file("tracing_on", 0644, d_tracer, 2517 entry = debugfs_create_file("tracing_on", 0644, d_tracer,
2179 &ring_buffers_off, &rb_simple_fops); 2518 &ring_buffer_flags, &rb_simple_fops);
2180 if (!entry) 2519 if (!entry)
2181 pr_warning("Could not create debugfs 'tracing_on' entry\n"); 2520 pr_warning("Could not create debugfs 'tracing_on' entry\n");
2182 2521
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index d86e3252f300..c580233add95 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -43,6 +43,38 @@
43unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX; 43unsigned long __read_mostly tracing_max_latency = (cycle_t)ULONG_MAX;
44unsigned long __read_mostly tracing_thresh; 44unsigned long __read_mostly tracing_thresh;
45 45
46/*
47 * We need to change this state when a selftest is running.
48 * A selftest will lurk into the ring-buffer to count the
49 * entries inserted during the selftest although some concurrent
50 * insertions into the ring-buffer such as ftrace_printk could occurred
51 * at the same time, giving false positive or negative results.
52 */
53static bool __read_mostly tracing_selftest_running;
54
55/* For tracers that don't implement custom flags */
56static struct tracer_opt dummy_tracer_opt[] = {
57 { }
58};
59
60static struct tracer_flags dummy_tracer_flags = {
61 .val = 0,
62 .opts = dummy_tracer_opt
63};
64
65static int dummy_set_flag(u32 old_flags, u32 bit, int set)
66{
67 return 0;
68}
69
70/*
71 * Kill all tracing for good (never come back).
72 * It is initialized to 1 but will turn to zero if the initialization
73 * of the tracer is successful. But that is the only place that sets
74 * this back to zero.
75 */
76int tracing_disabled = 1;
77
46static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled); 78static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
47 79
48static inline void ftrace_disable_cpu(void) 80static inline void ftrace_disable_cpu(void)
@@ -57,12 +89,41 @@ static inline void ftrace_enable_cpu(void)
57 preempt_enable(); 89 preempt_enable();
58} 90}
59 91
60static cpumask_t __read_mostly tracing_buffer_mask; 92static cpumask_var_t __read_mostly tracing_buffer_mask;
61 93
62#define for_each_tracing_cpu(cpu) \ 94#define for_each_tracing_cpu(cpu) \
63 for_each_cpu_mask(cpu, tracing_buffer_mask) 95 for_each_cpu(cpu, tracing_buffer_mask)
96
97/*
98 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
99 *
100 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
101 * is set, then ftrace_dump is called. This will output the contents
102 * of the ftrace buffers to the console. This is very useful for
103 * capturing traces that lead to crashes and outputing it to a
104 * serial console.
105 *
106 * It is default off, but you can enable it with either specifying
107 * "ftrace_dump_on_oops" in the kernel command line, or setting
108 * /proc/sys/kernel/ftrace_dump_on_oops to true.
109 */
110int ftrace_dump_on_oops;
64 111
65static int tracing_disabled = 1; 112static int tracing_set_tracer(char *buf);
113
114static int __init set_ftrace(char *str)
115{
116 tracing_set_tracer(str);
117 return 1;
118}
119__setup("ftrace", set_ftrace);
120
121static int __init set_ftrace_dump_on_oops(char *str)
122{
123 ftrace_dump_on_oops = 1;
124 return 1;
125}
126__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
66 127
67long 128long
68ns2usecs(cycle_t nsec) 129ns2usecs(cycle_t nsec)
@@ -112,6 +173,19 @@ static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
112/* tracer_enabled is used to toggle activation of a tracer */ 173/* tracer_enabled is used to toggle activation of a tracer */
113static int tracer_enabled = 1; 174static int tracer_enabled = 1;
114 175
176/**
177 * tracing_is_enabled - return tracer_enabled status
178 *
179 * This function is used by other tracers to know the status
180 * of the tracer_enabled flag. Tracers may use this function
181 * to know if it should enable their features when starting
182 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
183 */
184int tracing_is_enabled(void)
185{
186 return tracer_enabled;
187}
188
115/* function tracing enabled */ 189/* function tracing enabled */
116int ftrace_function_enabled; 190int ftrace_function_enabled;
117 191
@@ -153,8 +227,9 @@ static DEFINE_MUTEX(trace_types_lock);
153/* trace_wait is a waitqueue for tasks blocked on trace_poll */ 227/* trace_wait is a waitqueue for tasks blocked on trace_poll */
154static DECLARE_WAIT_QUEUE_HEAD(trace_wait); 228static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
155 229
156/* trace_flags holds iter_ctrl options */ 230/* trace_flags holds trace_options default values */
157unsigned long trace_flags = TRACE_ITER_PRINT_PARENT; 231unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
232 TRACE_ITER_ANNOTATE;
158 233
159/** 234/**
160 * trace_wake_up - wake up tasks waiting for trace input 235 * trace_wake_up - wake up tasks waiting for trace input
@@ -193,13 +268,6 @@ unsigned long nsecs_to_usecs(unsigned long nsecs)
193 return nsecs / 1000; 268 return nsecs / 1000;
194} 269}
195 270
196/*
197 * TRACE_ITER_SYM_MASK masks the options in trace_flags that
198 * control the output of kernel symbols.
199 */
200#define TRACE_ITER_SYM_MASK \
201 (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
202
203/* These must match the bit postions in trace_iterator_flags */ 271/* These must match the bit postions in trace_iterator_flags */
204static const char *trace_options[] = { 272static const char *trace_options[] = {
205 "print-parent", 273 "print-parent",
@@ -213,6 +281,12 @@ static const char *trace_options[] = {
213 "stacktrace", 281 "stacktrace",
214 "sched-tree", 282 "sched-tree",
215 "ftrace_printk", 283 "ftrace_printk",
284 "ftrace_preempt",
285 "branch",
286 "annotate",
287 "userstacktrace",
288 "sym-userobj",
289 "printk-msg-only",
216 NULL 290 NULL
217}; 291};
218 292
@@ -246,7 +320,7 @@ __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
246 320
247 memcpy(data->comm, tsk->comm, TASK_COMM_LEN); 321 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
248 data->pid = tsk->pid; 322 data->pid = tsk->pid;
249 data->uid = tsk->uid; 323 data->uid = task_uid(tsk);
250 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 324 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
251 data->policy = tsk->policy; 325 data->policy = tsk->policy;
252 data->rt_priority = tsk->rt_priority; 326 data->rt_priority = tsk->rt_priority;
@@ -359,6 +433,28 @@ trace_seq_putmem_hex(struct trace_seq *s, void *mem, size_t len)
359 return trace_seq_putmem(s, hex, j); 433 return trace_seq_putmem(s, hex, j);
360} 434}
361 435
436static int
437trace_seq_path(struct trace_seq *s, struct path *path)
438{
439 unsigned char *p;
440
441 if (s->len >= (PAGE_SIZE - 1))
442 return 0;
443 p = d_path(path, s->buffer + s->len, PAGE_SIZE - s->len);
444 if (!IS_ERR(p)) {
445 p = mangle_path(s->buffer + s->len, p, "\n");
446 if (p) {
447 s->len = p - s->buffer;
448 return 1;
449 }
450 } else {
451 s->buffer[s->len++] = '?';
452 return 1;
453 }
454
455 return 0;
456}
457
362static void 458static void
363trace_seq_reset(struct trace_seq *s) 459trace_seq_reset(struct trace_seq *s)
364{ 460{
@@ -470,7 +566,17 @@ int register_tracer(struct tracer *type)
470 return -1; 566 return -1;
471 } 567 }
472 568
569 /*
570 * When this gets called we hold the BKL which means that
571 * preemption is disabled. Various trace selftests however
572 * need to disable and enable preemption for successful tests.
573 * So we drop the BKL here and grab it after the tests again.
574 */
575 unlock_kernel();
473 mutex_lock(&trace_types_lock); 576 mutex_lock(&trace_types_lock);
577
578 tracing_selftest_running = true;
579
474 for (t = trace_types; t; t = t->next) { 580 for (t = trace_types; t; t = t->next) {
475 if (strcmp(type->name, t->name) == 0) { 581 if (strcmp(type->name, t->name) == 0) {
476 /* already found */ 582 /* already found */
@@ -481,12 +587,20 @@ int register_tracer(struct tracer *type)
481 } 587 }
482 } 588 }
483 589
590 if (!type->set_flag)
591 type->set_flag = &dummy_set_flag;
592 if (!type->flags)
593 type->flags = &dummy_tracer_flags;
594 else
595 if (!type->flags->opts)
596 type->flags->opts = dummy_tracer_opt;
597
484#ifdef CONFIG_FTRACE_STARTUP_TEST 598#ifdef CONFIG_FTRACE_STARTUP_TEST
485 if (type->selftest) { 599 if (type->selftest) {
486 struct tracer *saved_tracer = current_trace; 600 struct tracer *saved_tracer = current_trace;
487 struct trace_array *tr = &global_trace; 601 struct trace_array *tr = &global_trace;
488 int saved_ctrl = tr->ctrl;
489 int i; 602 int i;
603
490 /* 604 /*
491 * Run a selftest on this tracer. 605 * Run a selftest on this tracer.
492 * Here we reset the trace buffer, and set the current 606 * Here we reset the trace buffer, and set the current
@@ -494,25 +608,23 @@ int register_tracer(struct tracer *type)
494 * internal tracing to verify that everything is in order. 608 * internal tracing to verify that everything is in order.
495 * If we fail, we do not register this tracer. 609 * If we fail, we do not register this tracer.
496 */ 610 */
497 for_each_tracing_cpu(i) { 611 for_each_tracing_cpu(i)
498 tracing_reset(tr, i); 612 tracing_reset(tr, i);
499 } 613
500 current_trace = type; 614 current_trace = type;
501 tr->ctrl = 0;
502 /* the test is responsible for initializing and enabling */ 615 /* the test is responsible for initializing and enabling */
503 pr_info("Testing tracer %s: ", type->name); 616 pr_info("Testing tracer %s: ", type->name);
504 ret = type->selftest(type, tr); 617 ret = type->selftest(type, tr);
505 /* the test is responsible for resetting too */ 618 /* the test is responsible for resetting too */
506 current_trace = saved_tracer; 619 current_trace = saved_tracer;
507 tr->ctrl = saved_ctrl;
508 if (ret) { 620 if (ret) {
509 printk(KERN_CONT "FAILED!\n"); 621 printk(KERN_CONT "FAILED!\n");
510 goto out; 622 goto out;
511 } 623 }
512 /* Only reset on passing, to avoid touching corrupted buffers */ 624 /* Only reset on passing, to avoid touching corrupted buffers */
513 for_each_tracing_cpu(i) { 625 for_each_tracing_cpu(i)
514 tracing_reset(tr, i); 626 tracing_reset(tr, i);
515 } 627
516 printk(KERN_CONT "PASSED\n"); 628 printk(KERN_CONT "PASSED\n");
517 } 629 }
518#endif 630#endif
@@ -524,7 +636,9 @@ int register_tracer(struct tracer *type)
524 max_tracer_type_len = len; 636 max_tracer_type_len = len;
525 637
526 out: 638 out:
639 tracing_selftest_running = false;
527 mutex_unlock(&trace_types_lock); 640 mutex_unlock(&trace_types_lock);
641 lock_kernel();
528 642
529 return ret; 643 return ret;
530} 644}
@@ -564,6 +678,16 @@ void tracing_reset(struct trace_array *tr, int cpu)
564 ftrace_enable_cpu(); 678 ftrace_enable_cpu();
565} 679}
566 680
681void tracing_reset_online_cpus(struct trace_array *tr)
682{
683 int cpu;
684
685 tr->time_start = ftrace_now(tr->cpu);
686
687 for_each_online_cpu(cpu)
688 tracing_reset(tr, cpu);
689}
690
567#define SAVED_CMDLINES 128 691#define SAVED_CMDLINES 128
568static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1]; 692static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
569static unsigned map_cmdline_to_pid[SAVED_CMDLINES]; 693static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
@@ -581,6 +705,91 @@ static void trace_init_cmdlines(void)
581 cmdline_idx = 0; 705 cmdline_idx = 0;
582} 706}
583 707
708static int trace_stop_count;
709static DEFINE_SPINLOCK(tracing_start_lock);
710
711/**
712 * ftrace_off_permanent - disable all ftrace code permanently
713 *
714 * This should only be called when a serious anomally has
715 * been detected. This will turn off the function tracing,
716 * ring buffers, and other tracing utilites. It takes no
717 * locks and can be called from any context.
718 */
719void ftrace_off_permanent(void)
720{
721 tracing_disabled = 1;
722 ftrace_stop();
723 tracing_off_permanent();
724}
725
726/**
727 * tracing_start - quick start of the tracer
728 *
729 * If tracing is enabled but was stopped by tracing_stop,
730 * this will start the tracer back up.
731 */
732void tracing_start(void)
733{
734 struct ring_buffer *buffer;
735 unsigned long flags;
736
737 if (tracing_disabled)
738 return;
739
740 spin_lock_irqsave(&tracing_start_lock, flags);
741 if (--trace_stop_count)
742 goto out;
743
744 if (trace_stop_count < 0) {
745 /* Someone screwed up their debugging */
746 WARN_ON_ONCE(1);
747 trace_stop_count = 0;
748 goto out;
749 }
750
751
752 buffer = global_trace.buffer;
753 if (buffer)
754 ring_buffer_record_enable(buffer);
755
756 buffer = max_tr.buffer;
757 if (buffer)
758 ring_buffer_record_enable(buffer);
759
760 ftrace_start();
761 out:
762 spin_unlock_irqrestore(&tracing_start_lock, flags);
763}
764
765/**
766 * tracing_stop - quick stop of the tracer
767 *
768 * Light weight way to stop tracing. Use in conjunction with
769 * tracing_start.
770 */
771void tracing_stop(void)
772{
773 struct ring_buffer *buffer;
774 unsigned long flags;
775
776 ftrace_stop();
777 spin_lock_irqsave(&tracing_start_lock, flags);
778 if (trace_stop_count++)
779 goto out;
780
781 buffer = global_trace.buffer;
782 if (buffer)
783 ring_buffer_record_disable(buffer);
784
785 buffer = max_tr.buffer;
786 if (buffer)
787 ring_buffer_record_disable(buffer);
788
789 out:
790 spin_unlock_irqrestore(&tracing_start_lock, flags);
791}
792
584void trace_stop_cmdline_recording(void); 793void trace_stop_cmdline_recording(void);
585 794
586static void trace_save_cmdline(struct task_struct *tsk) 795static void trace_save_cmdline(struct task_struct *tsk)
@@ -618,7 +827,7 @@ static void trace_save_cmdline(struct task_struct *tsk)
618 spin_unlock(&trace_cmdline_lock); 827 spin_unlock(&trace_cmdline_lock);
619} 828}
620 829
621static char *trace_find_cmdline(int pid) 830char *trace_find_cmdline(int pid)
622{ 831{
623 char *cmdline = "<...>"; 832 char *cmdline = "<...>";
624 unsigned map; 833 unsigned map;
@@ -655,6 +864,7 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
655 864
656 entry->preempt_count = pc & 0xff; 865 entry->preempt_count = pc & 0xff;
657 entry->pid = (tsk) ? tsk->pid : 0; 866 entry->pid = (tsk) ? tsk->pid : 0;
867 entry->tgid = (tsk) ? tsk->tgid : 0;
658 entry->flags = 868 entry->flags =
659#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT 869#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
660 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) | 870 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
@@ -691,6 +901,56 @@ trace_function(struct trace_array *tr, struct trace_array_cpu *data,
691 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 901 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
692} 902}
693 903
904#ifdef CONFIG_FUNCTION_GRAPH_TRACER
905static void __trace_graph_entry(struct trace_array *tr,
906 struct trace_array_cpu *data,
907 struct ftrace_graph_ent *trace,
908 unsigned long flags,
909 int pc)
910{
911 struct ring_buffer_event *event;
912 struct ftrace_graph_ent_entry *entry;
913 unsigned long irq_flags;
914
915 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
916 return;
917
918 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry),
919 &irq_flags);
920 if (!event)
921 return;
922 entry = ring_buffer_event_data(event);
923 tracing_generic_entry_update(&entry->ent, flags, pc);
924 entry->ent.type = TRACE_GRAPH_ENT;
925 entry->graph_ent = *trace;
926 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags);
927}
928
929static void __trace_graph_return(struct trace_array *tr,
930 struct trace_array_cpu *data,
931 struct ftrace_graph_ret *trace,
932 unsigned long flags,
933 int pc)
934{
935 struct ring_buffer_event *event;
936 struct ftrace_graph_ret_entry *entry;
937 unsigned long irq_flags;
938
939 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
940 return;
941
942 event = ring_buffer_lock_reserve(global_trace.buffer, sizeof(*entry),
943 &irq_flags);
944 if (!event)
945 return;
946 entry = ring_buffer_event_data(event);
947 tracing_generic_entry_update(&entry->ent, flags, pc);
948 entry->ent.type = TRACE_GRAPH_RET;
949 entry->ret = *trace;
950 ring_buffer_unlock_commit(global_trace.buffer, event, irq_flags);
951}
952#endif
953
694void 954void
695ftrace(struct trace_array *tr, struct trace_array_cpu *data, 955ftrace(struct trace_array *tr, struct trace_array_cpu *data,
696 unsigned long ip, unsigned long parent_ip, unsigned long flags, 956 unsigned long ip, unsigned long parent_ip, unsigned long flags,
@@ -742,6 +1002,46 @@ void __trace_stack(struct trace_array *tr,
742 ftrace_trace_stack(tr, data, flags, skip, preempt_count()); 1002 ftrace_trace_stack(tr, data, flags, skip, preempt_count());
743} 1003}
744 1004
1005static void ftrace_trace_userstack(struct trace_array *tr,
1006 struct trace_array_cpu *data,
1007 unsigned long flags, int pc)
1008{
1009#ifdef CONFIG_STACKTRACE
1010 struct ring_buffer_event *event;
1011 struct userstack_entry *entry;
1012 struct stack_trace trace;
1013 unsigned long irq_flags;
1014
1015 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1016 return;
1017
1018 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
1019 &irq_flags);
1020 if (!event)
1021 return;
1022 entry = ring_buffer_event_data(event);
1023 tracing_generic_entry_update(&entry->ent, flags, pc);
1024 entry->ent.type = TRACE_USER_STACK;
1025
1026 memset(&entry->caller, 0, sizeof(entry->caller));
1027
1028 trace.nr_entries = 0;
1029 trace.max_entries = FTRACE_STACK_ENTRIES;
1030 trace.skip = 0;
1031 trace.entries = entry->caller;
1032
1033 save_stack_trace_user(&trace);
1034 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
1035#endif
1036}
1037
1038void __trace_userstack(struct trace_array *tr,
1039 struct trace_array_cpu *data,
1040 unsigned long flags)
1041{
1042 ftrace_trace_userstack(tr, data, flags, preempt_count());
1043}
1044
745static void 1045static void
746ftrace_trace_special(void *__tr, void *__data, 1046ftrace_trace_special(void *__tr, void *__data,
747 unsigned long arg1, unsigned long arg2, unsigned long arg3, 1047 unsigned long arg1, unsigned long arg2, unsigned long arg3,
@@ -765,6 +1065,7 @@ ftrace_trace_special(void *__tr, void *__data,
765 entry->arg3 = arg3; 1065 entry->arg3 = arg3;
766 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1066 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
767 ftrace_trace_stack(tr, data, irq_flags, 4, pc); 1067 ftrace_trace_stack(tr, data, irq_flags, 4, pc);
1068 ftrace_trace_userstack(tr, data, irq_flags, pc);
768 1069
769 trace_wake_up(); 1070 trace_wake_up();
770} 1071}
@@ -803,6 +1104,7 @@ tracing_sched_switch_trace(struct trace_array *tr,
803 entry->next_cpu = task_cpu(next); 1104 entry->next_cpu = task_cpu(next);
804 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1105 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
805 ftrace_trace_stack(tr, data, flags, 5, pc); 1106 ftrace_trace_stack(tr, data, flags, 5, pc);
1107 ftrace_trace_userstack(tr, data, flags, pc);
806} 1108}
807 1109
808void 1110void
@@ -832,6 +1134,7 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
832 entry->next_cpu = task_cpu(wakee); 1134 entry->next_cpu = task_cpu(wakee);
833 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 1135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
834 ftrace_trace_stack(tr, data, flags, 6, pc); 1136 ftrace_trace_stack(tr, data, flags, 6, pc);
1137 ftrace_trace_userstack(tr, data, flags, pc);
835 1138
836 trace_wake_up(); 1139 trace_wake_up();
837} 1140}
@@ -841,26 +1144,28 @@ ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
841{ 1144{
842 struct trace_array *tr = &global_trace; 1145 struct trace_array *tr = &global_trace;
843 struct trace_array_cpu *data; 1146 struct trace_array_cpu *data;
1147 unsigned long flags;
844 int cpu; 1148 int cpu;
845 int pc; 1149 int pc;
846 1150
847 if (tracing_disabled || !tr->ctrl) 1151 if (tracing_disabled)
848 return; 1152 return;
849 1153
850 pc = preempt_count(); 1154 pc = preempt_count();
851 preempt_disable_notrace(); 1155 local_irq_save(flags);
852 cpu = raw_smp_processor_id(); 1156 cpu = raw_smp_processor_id();
853 data = tr->data[cpu]; 1157 data = tr->data[cpu];
854 1158
855 if (likely(!atomic_read(&data->disabled))) 1159 if (likely(atomic_inc_return(&data->disabled) == 1))
856 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc); 1160 ftrace_trace_special(tr, data, arg1, arg2, arg3, pc);
857 1161
858 preempt_enable_notrace(); 1162 atomic_dec(&data->disabled);
1163 local_irq_restore(flags);
859} 1164}
860 1165
861#ifdef CONFIG_FUNCTION_TRACER 1166#ifdef CONFIG_FUNCTION_TRACER
862static void 1167static void
863function_trace_call(unsigned long ip, unsigned long parent_ip) 1168function_trace_call_preempt_only(unsigned long ip, unsigned long parent_ip)
864{ 1169{
865 struct trace_array *tr = &global_trace; 1170 struct trace_array *tr = &global_trace;
866 struct trace_array_cpu *data; 1171 struct trace_array_cpu *data;
@@ -873,8 +1178,7 @@ function_trace_call(unsigned long ip, unsigned long parent_ip)
873 return; 1178 return;
874 1179
875 pc = preempt_count(); 1180 pc = preempt_count();
876 resched = need_resched(); 1181 resched = ftrace_preempt_disable();
877 preempt_disable_notrace();
878 local_save_flags(flags); 1182 local_save_flags(flags);
879 cpu = raw_smp_processor_id(); 1183 cpu = raw_smp_processor_id();
880 data = tr->data[cpu]; 1184 data = tr->data[cpu];
@@ -884,11 +1188,96 @@ function_trace_call(unsigned long ip, unsigned long parent_ip)
884 trace_function(tr, data, ip, parent_ip, flags, pc); 1188 trace_function(tr, data, ip, parent_ip, flags, pc);
885 1189
886 atomic_dec(&data->disabled); 1190 atomic_dec(&data->disabled);
887 if (resched) 1191 ftrace_preempt_enable(resched);
888 preempt_enable_no_resched_notrace(); 1192}
889 else 1193
890 preempt_enable_notrace(); 1194static void
1195function_trace_call(unsigned long ip, unsigned long parent_ip)
1196{
1197 struct trace_array *tr = &global_trace;
1198 struct trace_array_cpu *data;
1199 unsigned long flags;
1200 long disabled;
1201 int cpu;
1202 int pc;
1203
1204 if (unlikely(!ftrace_function_enabled))
1205 return;
1206
1207 /*
1208 * Need to use raw, since this must be called before the
1209 * recursive protection is performed.
1210 */
1211 local_irq_save(flags);
1212 cpu = raw_smp_processor_id();
1213 data = tr->data[cpu];
1214 disabled = atomic_inc_return(&data->disabled);
1215
1216 if (likely(disabled == 1)) {
1217 pc = preempt_count();
1218 trace_function(tr, data, ip, parent_ip, flags, pc);
1219 }
1220
1221 atomic_dec(&data->disabled);
1222 local_irq_restore(flags);
1223}
1224
1225#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1226int trace_graph_entry(struct ftrace_graph_ent *trace)
1227{
1228 struct trace_array *tr = &global_trace;
1229 struct trace_array_cpu *data;
1230 unsigned long flags;
1231 long disabled;
1232 int cpu;
1233 int pc;
1234
1235 if (!ftrace_trace_task(current))
1236 return 0;
1237
1238 if (!ftrace_graph_addr(trace->func))
1239 return 0;
1240
1241 local_irq_save(flags);
1242 cpu = raw_smp_processor_id();
1243 data = tr->data[cpu];
1244 disabled = atomic_inc_return(&data->disabled);
1245 if (likely(disabled == 1)) {
1246 pc = preempt_count();
1247 __trace_graph_entry(tr, data, trace, flags, pc);
1248 }
1249 /* Only do the atomic if it is not already set */
1250 if (!test_tsk_trace_graph(current))
1251 set_tsk_trace_graph(current);
1252 atomic_dec(&data->disabled);
1253 local_irq_restore(flags);
1254
1255 return 1;
1256}
1257
1258void trace_graph_return(struct ftrace_graph_ret *trace)
1259{
1260 struct trace_array *tr = &global_trace;
1261 struct trace_array_cpu *data;
1262 unsigned long flags;
1263 long disabled;
1264 int cpu;
1265 int pc;
1266
1267 local_irq_save(flags);
1268 cpu = raw_smp_processor_id();
1269 data = tr->data[cpu];
1270 disabled = atomic_inc_return(&data->disabled);
1271 if (likely(disabled == 1)) {
1272 pc = preempt_count();
1273 __trace_graph_return(tr, data, trace, flags, pc);
1274 }
1275 if (!trace->depth)
1276 clear_tsk_trace_graph(current);
1277 atomic_dec(&data->disabled);
1278 local_irq_restore(flags);
891} 1279}
1280#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
892 1281
893static struct ftrace_ops trace_ops __read_mostly = 1282static struct ftrace_ops trace_ops __read_mostly =
894{ 1283{
@@ -898,9 +1287,14 @@ static struct ftrace_ops trace_ops __read_mostly =
898void tracing_start_function_trace(void) 1287void tracing_start_function_trace(void)
899{ 1288{
900 ftrace_function_enabled = 0; 1289 ftrace_function_enabled = 0;
1290
1291 if (trace_flags & TRACE_ITER_PREEMPTONLY)
1292 trace_ops.func = function_trace_call_preempt_only;
1293 else
1294 trace_ops.func = function_trace_call;
1295
901 register_ftrace_function(&trace_ops); 1296 register_ftrace_function(&trace_ops);
902 if (tracer_enabled) 1297 ftrace_function_enabled = 1;
903 ftrace_function_enabled = 1;
904} 1298}
905 1299
906void tracing_stop_function_trace(void) 1300void tracing_stop_function_trace(void)
@@ -912,9 +1306,10 @@ void tracing_stop_function_trace(void)
912 1306
913enum trace_file_type { 1307enum trace_file_type {
914 TRACE_FILE_LAT_FMT = 1, 1308 TRACE_FILE_LAT_FMT = 1,
1309 TRACE_FILE_ANNOTATE = 2,
915}; 1310};
916 1311
917static void trace_iterator_increment(struct trace_iterator *iter, int cpu) 1312static void trace_iterator_increment(struct trace_iterator *iter)
918{ 1313{
919 /* Don't allow ftrace to trace into the ring buffers */ 1314 /* Don't allow ftrace to trace into the ring buffers */
920 ftrace_disable_cpu(); 1315 ftrace_disable_cpu();
@@ -993,7 +1388,7 @@ static void *find_next_entry_inc(struct trace_iterator *iter)
993 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts); 1388 iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
994 1389
995 if (iter->ent) 1390 if (iter->ent)
996 trace_iterator_increment(iter, iter->cpu); 1391 trace_iterator_increment(iter);
997 1392
998 return iter->ent ? iter : NULL; 1393 return iter->ent ? iter : NULL;
999} 1394}
@@ -1047,10 +1442,6 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1047 1442
1048 atomic_inc(&trace_record_cmdline_disabled); 1443 atomic_inc(&trace_record_cmdline_disabled);
1049 1444
1050 /* let the tracer grab locks here if needed */
1051 if (current_trace->start)
1052 current_trace->start(iter);
1053
1054 if (*pos != iter->pos) { 1445 if (*pos != iter->pos) {
1055 iter->ent = NULL; 1446 iter->ent = NULL;
1056 iter->cpu = 0; 1447 iter->cpu = 0;
@@ -1077,14 +1468,7 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1077 1468
1078static void s_stop(struct seq_file *m, void *p) 1469static void s_stop(struct seq_file *m, void *p)
1079{ 1470{
1080 struct trace_iterator *iter = m->private;
1081
1082 atomic_dec(&trace_record_cmdline_disabled); 1471 atomic_dec(&trace_record_cmdline_disabled);
1083
1084 /* let the tracer release locks here if needed */
1085 if (current_trace && current_trace == iter->trace && iter->trace->stop)
1086 iter->trace->stop(iter);
1087
1088 mutex_unlock(&trace_types_lock); 1472 mutex_unlock(&trace_types_lock);
1089} 1473}
1090 1474
@@ -1143,7 +1527,7 @@ seq_print_sym_offset(struct trace_seq *s, const char *fmt,
1143# define IP_FMT "%016lx" 1527# define IP_FMT "%016lx"
1144#endif 1528#endif
1145 1529
1146static int 1530int
1147seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags) 1531seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1148{ 1532{
1149 int ret; 1533 int ret;
@@ -1164,6 +1548,78 @@ seq_print_ip_sym(struct trace_seq *s, unsigned long ip, unsigned long sym_flags)
1164 return ret; 1548 return ret;
1165} 1549}
1166 1550
1551static inline int seq_print_user_ip(struct trace_seq *s, struct mm_struct *mm,
1552 unsigned long ip, unsigned long sym_flags)
1553{
1554 struct file *file = NULL;
1555 unsigned long vmstart = 0;
1556 int ret = 1;
1557
1558 if (mm) {
1559 const struct vm_area_struct *vma;
1560
1561 down_read(&mm->mmap_sem);
1562 vma = find_vma(mm, ip);
1563 if (vma) {
1564 file = vma->vm_file;
1565 vmstart = vma->vm_start;
1566 }
1567 if (file) {
1568 ret = trace_seq_path(s, &file->f_path);
1569 if (ret)
1570 ret = trace_seq_printf(s, "[+0x%lx]", ip - vmstart);
1571 }
1572 up_read(&mm->mmap_sem);
1573 }
1574 if (ret && ((sym_flags & TRACE_ITER_SYM_ADDR) || !file))
1575 ret = trace_seq_printf(s, " <" IP_FMT ">", ip);
1576 return ret;
1577}
1578
1579static int
1580seq_print_userip_objs(const struct userstack_entry *entry, struct trace_seq *s,
1581 unsigned long sym_flags)
1582{
1583 struct mm_struct *mm = NULL;
1584 int ret = 1;
1585 unsigned int i;
1586
1587 if (trace_flags & TRACE_ITER_SYM_USEROBJ) {
1588 struct task_struct *task;
1589 /*
1590 * we do the lookup on the thread group leader,
1591 * since individual threads might have already quit!
1592 */
1593 rcu_read_lock();
1594 task = find_task_by_vpid(entry->ent.tgid);
1595 if (task)
1596 mm = get_task_mm(task);
1597 rcu_read_unlock();
1598 }
1599
1600 for (i = 0; i < FTRACE_STACK_ENTRIES; i++) {
1601 unsigned long ip = entry->caller[i];
1602
1603 if (ip == ULONG_MAX || !ret)
1604 break;
1605 if (i && ret)
1606 ret = trace_seq_puts(s, " <- ");
1607 if (!ip) {
1608 if (ret)
1609 ret = trace_seq_puts(s, "??");
1610 continue;
1611 }
1612 if (!ret)
1613 break;
1614 if (ret)
1615 ret = seq_print_user_ip(s, mm, ip, sym_flags);
1616 }
1617
1618 if (mm)
1619 mmput(mm);
1620 return ret;
1621}
1622
1167static void print_lat_help_header(struct seq_file *m) 1623static void print_lat_help_header(struct seq_file *m)
1168{ 1624{
1169 seq_puts(m, "# _------=> CPU# \n"); 1625 seq_puts(m, "# _------=> CPU# \n");
@@ -1301,6 +1757,13 @@ lat_print_timestamp(struct trace_seq *s, u64 abs_usecs,
1301 1757
1302static const char state_to_char[] = TASK_STATE_TO_CHAR_STR; 1758static const char state_to_char[] = TASK_STATE_TO_CHAR_STR;
1303 1759
1760static int task_state_char(unsigned long state)
1761{
1762 int bit = state ? __ffs(state) + 1 : 0;
1763
1764 return bit < sizeof(state_to_char) - 1 ? state_to_char[bit] : '?';
1765}
1766
1304/* 1767/*
1305 * The message is supposed to contain an ending newline. 1768 * The message is supposed to contain an ending newline.
1306 * If the printing stops prematurely, try to add a newline of our own. 1769 * If the printing stops prematurely, try to add a newline of our own.
@@ -1338,6 +1801,23 @@ void trace_seq_print_cont(struct trace_seq *s, struct trace_iterator *iter)
1338 trace_seq_putc(s, '\n'); 1801 trace_seq_putc(s, '\n');
1339} 1802}
1340 1803
1804static void test_cpu_buff_start(struct trace_iterator *iter)
1805{
1806 struct trace_seq *s = &iter->seq;
1807
1808 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1809 return;
1810
1811 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1812 return;
1813
1814 if (cpumask_test_cpu(iter->cpu, iter->started))
1815 return;
1816
1817 cpumask_set_cpu(iter->cpu, iter->started);
1818 trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1819}
1820
1341static enum print_line_t 1821static enum print_line_t
1342print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu) 1822print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1343{ 1823{
@@ -1352,11 +1832,12 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1352 char *comm; 1832 char *comm;
1353 int S, T; 1833 int S, T;
1354 int i; 1834 int i;
1355 unsigned state;
1356 1835
1357 if (entry->type == TRACE_CONT) 1836 if (entry->type == TRACE_CONT)
1358 return TRACE_TYPE_HANDLED; 1837 return TRACE_TYPE_HANDLED;
1359 1838
1839 test_cpu_buff_start(iter);
1840
1360 next_entry = find_next_entry(iter, NULL, &next_ts); 1841 next_entry = find_next_entry(iter, NULL, &next_ts);
1361 if (!next_entry) 1842 if (!next_entry)
1362 next_ts = iter->ts; 1843 next_ts = iter->ts;
@@ -1396,12 +1877,8 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1396 1877
1397 trace_assign_type(field, entry); 1878 trace_assign_type(field, entry);
1398 1879
1399 T = field->next_state < sizeof(state_to_char) ? 1880 T = task_state_char(field->next_state);
1400 state_to_char[field->next_state] : 'X'; 1881 S = task_state_char(field->prev_state);
1401
1402 state = field->prev_state ?
1403 __ffs(field->prev_state) + 1 : 0;
1404 S = state < sizeof(state_to_char) - 1 ? state_to_char[state] : 'X';
1405 comm = trace_find_cmdline(field->next_pid); 1882 comm = trace_find_cmdline(field->next_pid);
1406 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n", 1883 trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c %s\n",
1407 field->prev_pid, 1884 field->prev_pid,
@@ -1448,6 +1925,27 @@ print_lat_fmt(struct trace_iterator *iter, unsigned int trace_idx, int cpu)
1448 trace_seq_print_cont(s, iter); 1925 trace_seq_print_cont(s, iter);
1449 break; 1926 break;
1450 } 1927 }
1928 case TRACE_BRANCH: {
1929 struct trace_branch *field;
1930
1931 trace_assign_type(field, entry);
1932
1933 trace_seq_printf(s, "[%s] %s:%s:%d\n",
1934 field->correct ? " ok " : " MISS ",
1935 field->func,
1936 field->file,
1937 field->line);
1938 break;
1939 }
1940 case TRACE_USER_STACK: {
1941 struct userstack_entry *field;
1942
1943 trace_assign_type(field, entry);
1944
1945 seq_print_userip_objs(field, s, sym_flags);
1946 trace_seq_putc(s, '\n');
1947 break;
1948 }
1451 default: 1949 default:
1452 trace_seq_printf(s, "Unknown type %d\n", entry->type); 1950 trace_seq_printf(s, "Unknown type %d\n", entry->type);
1453 } 1951 }
@@ -1472,6 +1970,8 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1472 if (entry->type == TRACE_CONT) 1970 if (entry->type == TRACE_CONT)
1473 return TRACE_TYPE_HANDLED; 1971 return TRACE_TYPE_HANDLED;
1474 1972
1973 test_cpu_buff_start(iter);
1974
1475 comm = trace_find_cmdline(iter->ent->pid); 1975 comm = trace_find_cmdline(iter->ent->pid);
1476 1976
1477 t = ns2usecs(iter->ts); 1977 t = ns2usecs(iter->ts);
@@ -1519,10 +2019,8 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1519 2019
1520 trace_assign_type(field, entry); 2020 trace_assign_type(field, entry);
1521 2021
1522 S = field->prev_state < sizeof(state_to_char) ? 2022 T = task_state_char(field->next_state);
1523 state_to_char[field->prev_state] : 'X'; 2023 S = task_state_char(field->prev_state);
1524 T = field->next_state < sizeof(state_to_char) ?
1525 state_to_char[field->next_state] : 'X';
1526 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n", 2024 ret = trace_seq_printf(s, " %5d:%3d:%c %s [%03d] %5d:%3d:%c\n",
1527 field->prev_pid, 2025 field->prev_pid,
1528 field->prev_prio, 2026 field->prev_prio,
@@ -1581,6 +2079,37 @@ static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1581 trace_seq_print_cont(s, iter); 2079 trace_seq_print_cont(s, iter);
1582 break; 2080 break;
1583 } 2081 }
2082 case TRACE_GRAPH_RET: {
2083 return print_graph_function(iter);
2084 }
2085 case TRACE_GRAPH_ENT: {
2086 return print_graph_function(iter);
2087 }
2088 case TRACE_BRANCH: {
2089 struct trace_branch *field;
2090
2091 trace_assign_type(field, entry);
2092
2093 trace_seq_printf(s, "[%s] %s:%s:%d\n",
2094 field->correct ? " ok " : " MISS ",
2095 field->func,
2096 field->file,
2097 field->line);
2098 break;
2099 }
2100 case TRACE_USER_STACK: {
2101 struct userstack_entry *field;
2102
2103 trace_assign_type(field, entry);
2104
2105 ret = seq_print_userip_objs(field, s, sym_flags);
2106 if (!ret)
2107 return TRACE_TYPE_PARTIAL_LINE;
2108 ret = trace_seq_putc(s, '\n');
2109 if (!ret)
2110 return TRACE_TYPE_PARTIAL_LINE;
2111 break;
2112 }
1584 } 2113 }
1585 return TRACE_TYPE_HANDLED; 2114 return TRACE_TYPE_HANDLED;
1586} 2115}
@@ -1621,12 +2150,9 @@ static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1621 2150
1622 trace_assign_type(field, entry); 2151 trace_assign_type(field, entry);
1623 2152
1624 S = field->prev_state < sizeof(state_to_char) ? 2153 T = task_state_char(field->next_state);
1625 state_to_char[field->prev_state] : 'X'; 2154 S = entry->type == TRACE_WAKE ? '+' :
1626 T = field->next_state < sizeof(state_to_char) ? 2155 task_state_char(field->prev_state);
1627 state_to_char[field->next_state] : 'X';
1628 if (entry->type == TRACE_WAKE)
1629 S = '+';
1630 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n", 2156 ret = trace_seq_printf(s, "%d %d %c %d %d %d %c\n",
1631 field->prev_pid, 2157 field->prev_pid,
1632 field->prev_prio, 2158 field->prev_prio,
@@ -1640,6 +2166,7 @@ static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1640 break; 2166 break;
1641 } 2167 }
1642 case TRACE_SPECIAL: 2168 case TRACE_SPECIAL:
2169 case TRACE_USER_STACK:
1643 case TRACE_STACK: { 2170 case TRACE_STACK: {
1644 struct special_entry *field; 2171 struct special_entry *field;
1645 2172
@@ -1712,12 +2239,9 @@ static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1712 2239
1713 trace_assign_type(field, entry); 2240 trace_assign_type(field, entry);
1714 2241
1715 S = field->prev_state < sizeof(state_to_char) ? 2242 T = task_state_char(field->next_state);
1716 state_to_char[field->prev_state] : 'X'; 2243 S = entry->type == TRACE_WAKE ? '+' :
1717 T = field->next_state < sizeof(state_to_char) ? 2244 task_state_char(field->prev_state);
1718 state_to_char[field->next_state] : 'X';
1719 if (entry->type == TRACE_WAKE)
1720 S = '+';
1721 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid); 2245 SEQ_PUT_HEX_FIELD_RET(s, field->prev_pid);
1722 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio); 2246 SEQ_PUT_HEX_FIELD_RET(s, field->prev_prio);
1723 SEQ_PUT_HEX_FIELD_RET(s, S); 2247 SEQ_PUT_HEX_FIELD_RET(s, S);
@@ -1728,6 +2252,7 @@ static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1728 break; 2252 break;
1729 } 2253 }
1730 case TRACE_SPECIAL: 2254 case TRACE_SPECIAL:
2255 case TRACE_USER_STACK:
1731 case TRACE_STACK: { 2256 case TRACE_STACK: {
1732 struct special_entry *field; 2257 struct special_entry *field;
1733 2258
@@ -1744,6 +2269,25 @@ static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1744 return TRACE_TYPE_HANDLED; 2269 return TRACE_TYPE_HANDLED;
1745} 2270}
1746 2271
2272static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
2273{
2274 struct trace_seq *s = &iter->seq;
2275 struct trace_entry *entry = iter->ent;
2276 struct print_entry *field;
2277 int ret;
2278
2279 trace_assign_type(field, entry);
2280
2281 ret = trace_seq_printf(s, field->buf);
2282 if (!ret)
2283 return TRACE_TYPE_PARTIAL_LINE;
2284
2285 if (entry->flags & TRACE_FLAG_CONT)
2286 trace_seq_print_cont(s, iter);
2287
2288 return TRACE_TYPE_HANDLED;
2289}
2290
1747static enum print_line_t print_bin_fmt(struct trace_iterator *iter) 2291static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1748{ 2292{
1749 struct trace_seq *s = &iter->seq; 2293 struct trace_seq *s = &iter->seq;
@@ -1782,6 +2326,7 @@ static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1782 break; 2326 break;
1783 } 2327 }
1784 case TRACE_SPECIAL: 2328 case TRACE_SPECIAL:
2329 case TRACE_USER_STACK:
1785 case TRACE_STACK: { 2330 case TRACE_STACK: {
1786 struct special_entry *field; 2331 struct special_entry *field;
1787 2332
@@ -1823,6 +2368,11 @@ static enum print_line_t print_trace_line(struct trace_iterator *iter)
1823 return ret; 2368 return ret;
1824 } 2369 }
1825 2370
2371 if (iter->ent->type == TRACE_PRINT &&
2372 trace_flags & TRACE_ITER_PRINTK &&
2373 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2374 return print_printk_msg_only(iter);
2375
1826 if (trace_flags & TRACE_ITER_BIN) 2376 if (trace_flags & TRACE_ITER_BIN)
1827 return print_bin_fmt(iter); 2377 return print_bin_fmt(iter);
1828 2378
@@ -1847,7 +2397,9 @@ static int s_show(struct seq_file *m, void *v)
1847 seq_printf(m, "# tracer: %s\n", iter->trace->name); 2397 seq_printf(m, "# tracer: %s\n", iter->trace->name);
1848 seq_puts(m, "#\n"); 2398 seq_puts(m, "#\n");
1849 } 2399 }
1850 if (iter->iter_flags & TRACE_FILE_LAT_FMT) { 2400 if (iter->trace && iter->trace->print_header)
2401 iter->trace->print_header(m);
2402 else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1851 /* print nothing if the buffers are empty */ 2403 /* print nothing if the buffers are empty */
1852 if (trace_empty(iter)) 2404 if (trace_empty(iter))
1853 return 0; 2405 return 0;
@@ -1899,6 +2451,15 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
1899 iter->trace = current_trace; 2451 iter->trace = current_trace;
1900 iter->pos = -1; 2452 iter->pos = -1;
1901 2453
2454 /* Notify the tracer early; before we stop tracing. */
2455 if (iter->trace && iter->trace->open)
2456 iter->trace->open(iter);
2457
2458 /* Annotate start of buffers if we had overruns */
2459 if (ring_buffer_overruns(iter->tr->buffer))
2460 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2461
2462
1902 for_each_tracing_cpu(cpu) { 2463 for_each_tracing_cpu(cpu) {
1903 2464
1904 iter->buffer_iter[cpu] = 2465 iter->buffer_iter[cpu] =
@@ -1917,13 +2478,7 @@ __tracing_open(struct inode *inode, struct file *file, int *ret)
1917 m->private = iter; 2478 m->private = iter;
1918 2479
1919 /* stop the trace while dumping */ 2480 /* stop the trace while dumping */
1920 if (iter->tr->ctrl) { 2481 tracing_stop();
1921 tracer_enabled = 0;
1922 ftrace_function_enabled = 0;
1923 }
1924
1925 if (iter->trace && iter->trace->open)
1926 iter->trace->open(iter);
1927 2482
1928 mutex_unlock(&trace_types_lock); 2483 mutex_unlock(&trace_types_lock);
1929 2484
@@ -1966,14 +2521,7 @@ int tracing_release(struct inode *inode, struct file *file)
1966 iter->trace->close(iter); 2521 iter->trace->close(iter);
1967 2522
1968 /* reenable tracing if it was previously enabled */ 2523 /* reenable tracing if it was previously enabled */
1969 if (iter->tr->ctrl) { 2524 tracing_start();
1970 tracer_enabled = 1;
1971 /*
1972 * It is safe to enable function tracing even if it
1973 * isn't used
1974 */
1975 ftrace_function_enabled = 1;
1976 }
1977 mutex_unlock(&trace_types_lock); 2525 mutex_unlock(&trace_types_lock);
1978 2526
1979 seq_release(inode, file); 2527 seq_release(inode, file);
@@ -2098,13 +2646,7 @@ static struct file_operations show_traces_fops = {
2098/* 2646/*
2099 * Only trace on a CPU if the bitmask is set: 2647 * Only trace on a CPU if the bitmask is set:
2100 */ 2648 */
2101static cpumask_t tracing_cpumask = CPU_MASK_ALL; 2649static cpumask_var_t tracing_cpumask;
2102
2103/*
2104 * When tracing/tracing_cpu_mask is modified then this holds
2105 * the new bitmask we are about to install:
2106 */
2107static cpumask_t tracing_cpumask_new;
2108 2650
2109/* 2651/*
2110 * The tracer itself will not take this lock, but still we want 2652 * The tracer itself will not take this lock, but still we want
@@ -2145,39 +2687,45 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2145 size_t count, loff_t *ppos) 2687 size_t count, loff_t *ppos)
2146{ 2688{
2147 int err, cpu; 2689 int err, cpu;
2690 cpumask_var_t tracing_cpumask_new;
2691
2692 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2693 return -ENOMEM;
2148 2694
2149 mutex_lock(&tracing_cpumask_update_lock); 2695 mutex_lock(&tracing_cpumask_update_lock);
2150 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new); 2696 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2151 if (err) 2697 if (err)
2152 goto err_unlock; 2698 goto err_unlock;
2153 2699
2154 raw_local_irq_disable(); 2700 local_irq_disable();
2155 __raw_spin_lock(&ftrace_max_lock); 2701 __raw_spin_lock(&ftrace_max_lock);
2156 for_each_tracing_cpu(cpu) { 2702 for_each_tracing_cpu(cpu) {
2157 /* 2703 /*
2158 * Increase/decrease the disabled counter if we are 2704 * Increase/decrease the disabled counter if we are
2159 * about to flip a bit in the cpumask: 2705 * about to flip a bit in the cpumask:
2160 */ 2706 */
2161 if (cpu_isset(cpu, tracing_cpumask) && 2707 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2162 !cpu_isset(cpu, tracing_cpumask_new)) { 2708 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2163 atomic_inc(&global_trace.data[cpu]->disabled); 2709 atomic_inc(&global_trace.data[cpu]->disabled);
2164 } 2710 }
2165 if (!cpu_isset(cpu, tracing_cpumask) && 2711 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2166 cpu_isset(cpu, tracing_cpumask_new)) { 2712 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2167 atomic_dec(&global_trace.data[cpu]->disabled); 2713 atomic_dec(&global_trace.data[cpu]->disabled);
2168 } 2714 }
2169 } 2715 }
2170 __raw_spin_unlock(&ftrace_max_lock); 2716 __raw_spin_unlock(&ftrace_max_lock);
2171 raw_local_irq_enable(); 2717 local_irq_enable();
2172 2718
2173 tracing_cpumask = tracing_cpumask_new; 2719 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2174 2720
2175 mutex_unlock(&tracing_cpumask_update_lock); 2721 mutex_unlock(&tracing_cpumask_update_lock);
2722 free_cpumask_var(tracing_cpumask_new);
2176 2723
2177 return count; 2724 return count;
2178 2725
2179err_unlock: 2726err_unlock:
2180 mutex_unlock(&tracing_cpumask_update_lock); 2727 mutex_unlock(&tracing_cpumask_update_lock);
2728 free_cpumask_var(tracing_cpumask);
2181 2729
2182 return err; 2730 return err;
2183} 2731}
@@ -2189,13 +2737,16 @@ static struct file_operations tracing_cpumask_fops = {
2189}; 2737};
2190 2738
2191static ssize_t 2739static ssize_t
2192tracing_iter_ctrl_read(struct file *filp, char __user *ubuf, 2740tracing_trace_options_read(struct file *filp, char __user *ubuf,
2193 size_t cnt, loff_t *ppos) 2741 size_t cnt, loff_t *ppos)
2194{ 2742{
2743 int i;
2195 char *buf; 2744 char *buf;
2196 int r = 0; 2745 int r = 0;
2197 int len = 0; 2746 int len = 0;
2198 int i; 2747 u32 tracer_flags = current_trace->flags->val;
2748 struct tracer_opt *trace_opts = current_trace->flags->opts;
2749
2199 2750
2200 /* calulate max size */ 2751 /* calulate max size */
2201 for (i = 0; trace_options[i]; i++) { 2752 for (i = 0; trace_options[i]; i++) {
@@ -2203,6 +2754,15 @@ tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2203 len += 3; /* "no" and space */ 2754 len += 3; /* "no" and space */
2204 } 2755 }
2205 2756
2757 /*
2758 * Increase the size with names of options specific
2759 * of the current tracer.
2760 */
2761 for (i = 0; trace_opts[i].name; i++) {
2762 len += strlen(trace_opts[i].name);
2763 len += 3; /* "no" and space */
2764 }
2765
2206 /* +2 for \n and \0 */ 2766 /* +2 for \n and \0 */
2207 buf = kmalloc(len + 2, GFP_KERNEL); 2767 buf = kmalloc(len + 2, GFP_KERNEL);
2208 if (!buf) 2768 if (!buf)
@@ -2215,6 +2775,15 @@ tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2215 r += sprintf(buf + r, "no%s ", trace_options[i]); 2775 r += sprintf(buf + r, "no%s ", trace_options[i]);
2216 } 2776 }
2217 2777
2778 for (i = 0; trace_opts[i].name; i++) {
2779 if (tracer_flags & trace_opts[i].bit)
2780 r += sprintf(buf + r, "%s ",
2781 trace_opts[i].name);
2782 else
2783 r += sprintf(buf + r, "no%s ",
2784 trace_opts[i].name);
2785 }
2786
2218 r += sprintf(buf + r, "\n"); 2787 r += sprintf(buf + r, "\n");
2219 WARN_ON(r >= len + 2); 2788 WARN_ON(r >= len + 2);
2220 2789
@@ -2225,13 +2794,48 @@ tracing_iter_ctrl_read(struct file *filp, char __user *ubuf,
2225 return r; 2794 return r;
2226} 2795}
2227 2796
2797/* Try to assign a tracer specific option */
2798static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2799{
2800 struct tracer_flags *trace_flags = trace->flags;
2801 struct tracer_opt *opts = NULL;
2802 int ret = 0, i = 0;
2803 int len;
2804
2805 for (i = 0; trace_flags->opts[i].name; i++) {
2806 opts = &trace_flags->opts[i];
2807 len = strlen(opts->name);
2808
2809 if (strncmp(cmp, opts->name, len) == 0) {
2810 ret = trace->set_flag(trace_flags->val,
2811 opts->bit, !neg);
2812 break;
2813 }
2814 }
2815 /* Not found */
2816 if (!trace_flags->opts[i].name)
2817 return -EINVAL;
2818
2819 /* Refused to handle */
2820 if (ret)
2821 return ret;
2822
2823 if (neg)
2824 trace_flags->val &= ~opts->bit;
2825 else
2826 trace_flags->val |= opts->bit;
2827
2828 return 0;
2829}
2830
2228static ssize_t 2831static ssize_t
2229tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf, 2832tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2230 size_t cnt, loff_t *ppos) 2833 size_t cnt, loff_t *ppos)
2231{ 2834{
2232 char buf[64]; 2835 char buf[64];
2233 char *cmp = buf; 2836 char *cmp = buf;
2234 int neg = 0; 2837 int neg = 0;
2838 int ret;
2235 int i; 2839 int i;
2236 2840
2237 if (cnt >= sizeof(buf)) 2841 if (cnt >= sizeof(buf))
@@ -2258,11 +2862,13 @@ tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2258 break; 2862 break;
2259 } 2863 }
2260 } 2864 }
2261 /* 2865
2262 * If no option could be set, return an error: 2866 /* If no option could be set, test the specific tracer options */
2263 */ 2867 if (!trace_options[i]) {
2264 if (!trace_options[i]) 2868 ret = set_tracer_option(current_trace, cmp, neg);
2265 return -EINVAL; 2869 if (ret)
2870 return ret;
2871 }
2266 2872
2267 filp->f_pos += cnt; 2873 filp->f_pos += cnt;
2268 2874
@@ -2271,8 +2877,8 @@ tracing_iter_ctrl_write(struct file *filp, const char __user *ubuf,
2271 2877
2272static struct file_operations tracing_iter_fops = { 2878static struct file_operations tracing_iter_fops = {
2273 .open = tracing_open_generic, 2879 .open = tracing_open_generic,
2274 .read = tracing_iter_ctrl_read, 2880 .read = tracing_trace_options_read,
2275 .write = tracing_iter_ctrl_write, 2881 .write = tracing_trace_options_write,
2276}; 2882};
2277 2883
2278static const char readme_msg[] = 2884static const char readme_msg[] =
@@ -2286,9 +2892,9 @@ static const char readme_msg[] =
2286 "# echo sched_switch > /debug/tracing/current_tracer\n" 2892 "# echo sched_switch > /debug/tracing/current_tracer\n"
2287 "# cat /debug/tracing/current_tracer\n" 2893 "# cat /debug/tracing/current_tracer\n"
2288 "sched_switch\n" 2894 "sched_switch\n"
2289 "# cat /debug/tracing/iter_ctrl\n" 2895 "# cat /debug/tracing/trace_options\n"
2290 "noprint-parent nosym-offset nosym-addr noverbose\n" 2896 "noprint-parent nosym-offset nosym-addr noverbose\n"
2291 "# echo print-parent > /debug/tracing/iter_ctrl\n" 2897 "# echo print-parent > /debug/tracing/trace_options\n"
2292 "# echo 1 > /debug/tracing/tracing_enabled\n" 2898 "# echo 1 > /debug/tracing/tracing_enabled\n"
2293 "# cat /debug/tracing/trace > /tmp/trace.txt\n" 2899 "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2294 "echo 0 > /debug/tracing/tracing_enabled\n" 2900 "echo 0 > /debug/tracing/tracing_enabled\n"
@@ -2311,11 +2917,10 @@ static ssize_t
2311tracing_ctrl_read(struct file *filp, char __user *ubuf, 2917tracing_ctrl_read(struct file *filp, char __user *ubuf,
2312 size_t cnt, loff_t *ppos) 2918 size_t cnt, loff_t *ppos)
2313{ 2919{
2314 struct trace_array *tr = filp->private_data;
2315 char buf[64]; 2920 char buf[64];
2316 int r; 2921 int r;
2317 2922
2318 r = sprintf(buf, "%ld\n", tr->ctrl); 2923 r = sprintf(buf, "%u\n", tracer_enabled);
2319 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2924 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2320} 2925}
2321 2926
@@ -2343,16 +2948,18 @@ tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2343 val = !!val; 2948 val = !!val;
2344 2949
2345 mutex_lock(&trace_types_lock); 2950 mutex_lock(&trace_types_lock);
2346 if (tr->ctrl ^ val) { 2951 if (tracer_enabled ^ val) {
2347 if (val) 2952 if (val) {
2348 tracer_enabled = 1; 2953 tracer_enabled = 1;
2349 else 2954 if (current_trace->start)
2955 current_trace->start(tr);
2956 tracing_start();
2957 } else {
2350 tracer_enabled = 0; 2958 tracer_enabled = 0;
2351 2959 tracing_stop();
2352 tr->ctrl = val; 2960 if (current_trace->stop)
2353 2961 current_trace->stop(tr);
2354 if (current_trace && current_trace->ctrl_update) 2962 }
2355 current_trace->ctrl_update(tr);
2356 } 2963 }
2357 mutex_unlock(&trace_types_lock); 2964 mutex_unlock(&trace_types_lock);
2358 2965
@@ -2378,29 +2985,11 @@ tracing_set_trace_read(struct file *filp, char __user *ubuf,
2378 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 2985 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2379} 2986}
2380 2987
2381static ssize_t 2988static int tracing_set_tracer(char *buf)
2382tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2383 size_t cnt, loff_t *ppos)
2384{ 2989{
2385 struct trace_array *tr = &global_trace; 2990 struct trace_array *tr = &global_trace;
2386 struct tracer *t; 2991 struct tracer *t;
2387 char buf[max_tracer_type_len+1]; 2992 int ret = 0;
2388 int i;
2389 size_t ret;
2390
2391 ret = cnt;
2392
2393 if (cnt > max_tracer_type_len)
2394 cnt = max_tracer_type_len;
2395
2396 if (copy_from_user(&buf, ubuf, cnt))
2397 return -EFAULT;
2398
2399 buf[cnt] = 0;
2400
2401 /* strip ending whitespace. */
2402 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2403 buf[i] = 0;
2404 2993
2405 mutex_lock(&trace_types_lock); 2994 mutex_lock(&trace_types_lock);
2406 for (t = trace_types; t; t = t->next) { 2995 for (t = trace_types; t; t = t->next) {
@@ -2414,18 +3003,52 @@ tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2414 if (t == current_trace) 3003 if (t == current_trace)
2415 goto out; 3004 goto out;
2416 3005
3006 trace_branch_disable();
2417 if (current_trace && current_trace->reset) 3007 if (current_trace && current_trace->reset)
2418 current_trace->reset(tr); 3008 current_trace->reset(tr);
2419 3009
2420 current_trace = t; 3010 current_trace = t;
2421 if (t->init) 3011 if (t->init) {
2422 t->init(tr); 3012 ret = t->init(tr);
3013 if (ret)
3014 goto out;
3015 }
2423 3016
3017 trace_branch_enable(tr);
2424 out: 3018 out:
2425 mutex_unlock(&trace_types_lock); 3019 mutex_unlock(&trace_types_lock);
2426 3020
2427 if (ret > 0) 3021 return ret;
2428 filp->f_pos += ret; 3022}
3023
3024static ssize_t
3025tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3026 size_t cnt, loff_t *ppos)
3027{
3028 char buf[max_tracer_type_len+1];
3029 int i;
3030 size_t ret;
3031 int err;
3032
3033 ret = cnt;
3034
3035 if (cnt > max_tracer_type_len)
3036 cnt = max_tracer_type_len;
3037
3038 if (copy_from_user(&buf, ubuf, cnt))
3039 return -EFAULT;
3040
3041 buf[cnt] = 0;
3042
3043 /* strip ending whitespace. */
3044 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3045 buf[i] = 0;
3046
3047 err = tracing_set_tracer(buf);
3048 if (err)
3049 return err;
3050
3051 filp->f_pos += ret;
2429 3052
2430 return ret; 3053 return ret;
2431} 3054}
@@ -2491,7 +3114,16 @@ static int tracing_open_pipe(struct inode *inode, struct file *filp)
2491 if (!iter) 3114 if (!iter)
2492 return -ENOMEM; 3115 return -ENOMEM;
2493 3116
3117 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3118 kfree(iter);
3119 return -ENOMEM;
3120 }
3121
2494 mutex_lock(&trace_types_lock); 3122 mutex_lock(&trace_types_lock);
3123
3124 /* trace pipe does not show start of buffer */
3125 cpumask_setall(iter->started);
3126
2495 iter->tr = &global_trace; 3127 iter->tr = &global_trace;
2496 iter->trace = current_trace; 3128 iter->trace = current_trace;
2497 filp->private_data = iter; 3129 filp->private_data = iter;
@@ -2507,6 +3139,7 @@ static int tracing_release_pipe(struct inode *inode, struct file *file)
2507{ 3139{
2508 struct trace_iterator *iter = file->private_data; 3140 struct trace_iterator *iter = file->private_data;
2509 3141
3142 free_cpumask_var(iter->started);
2510 kfree(iter); 3143 kfree(iter);
2511 atomic_dec(&tracing_reader); 3144 atomic_dec(&tracing_reader);
2512 3145
@@ -2667,7 +3300,7 @@ tracing_entries_read(struct file *filp, char __user *ubuf,
2667 char buf[64]; 3300 char buf[64];
2668 int r; 3301 int r;
2669 3302
2670 r = sprintf(buf, "%lu\n", tr->entries); 3303 r = sprintf(buf, "%lu\n", tr->entries >> 10);
2671 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3304 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2672} 3305}
2673 3306
@@ -2678,7 +3311,6 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
2678 unsigned long val; 3311 unsigned long val;
2679 char buf[64]; 3312 char buf[64];
2680 int ret, cpu; 3313 int ret, cpu;
2681 struct trace_array *tr = filp->private_data;
2682 3314
2683 if (cnt >= sizeof(buf)) 3315 if (cnt >= sizeof(buf))
2684 return -EINVAL; 3316 return -EINVAL;
@@ -2698,12 +3330,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
2698 3330
2699 mutex_lock(&trace_types_lock); 3331 mutex_lock(&trace_types_lock);
2700 3332
2701 if (tr->ctrl) { 3333 tracing_stop();
2702 cnt = -EBUSY;
2703 pr_info("ftrace: please disable tracing"
2704 " before modifying buffer size\n");
2705 goto out;
2706 }
2707 3334
2708 /* disable all cpu buffers */ 3335 /* disable all cpu buffers */
2709 for_each_tracing_cpu(cpu) { 3336 for_each_tracing_cpu(cpu) {
@@ -2713,6 +3340,9 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
2713 atomic_inc(&max_tr.data[cpu]->disabled); 3340 atomic_inc(&max_tr.data[cpu]->disabled);
2714 } 3341 }
2715 3342
3343 /* value is in KB */
3344 val <<= 10;
3345
2716 if (val != global_trace.entries) { 3346 if (val != global_trace.entries) {
2717 ret = ring_buffer_resize(global_trace.buffer, val); 3347 ret = ring_buffer_resize(global_trace.buffer, val);
2718 if (ret < 0) { 3348 if (ret < 0) {
@@ -2751,6 +3381,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
2751 atomic_dec(&max_tr.data[cpu]->disabled); 3381 atomic_dec(&max_tr.data[cpu]->disabled);
2752 } 3382 }
2753 3383
3384 tracing_start();
2754 max_tr.entries = global_trace.entries; 3385 max_tr.entries = global_trace.entries;
2755 mutex_unlock(&trace_types_lock); 3386 mutex_unlock(&trace_types_lock);
2756 3387
@@ -2762,7 +3393,7 @@ static int mark_printk(const char *fmt, ...)
2762 int ret; 3393 int ret;
2763 va_list args; 3394 va_list args;
2764 va_start(args, fmt); 3395 va_start(args, fmt);
2765 ret = trace_vprintk(0, fmt, args); 3396 ret = trace_vprintk(0, -1, fmt, args);
2766 va_end(args); 3397 va_end(args);
2767 return ret; 3398 return ret;
2768} 3399}
@@ -2773,9 +3404,8 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
2773{ 3404{
2774 char *buf; 3405 char *buf;
2775 char *end; 3406 char *end;
2776 struct trace_array *tr = &global_trace;
2777 3407
2778 if (!tr->ctrl || tracing_disabled) 3408 if (tracing_disabled)
2779 return -EINVAL; 3409 return -EINVAL;
2780 3410
2781 if (cnt > TRACE_BUF_SIZE) 3411 if (cnt > TRACE_BUF_SIZE)
@@ -2841,22 +3471,38 @@ static struct file_operations tracing_mark_fops = {
2841 3471
2842#ifdef CONFIG_DYNAMIC_FTRACE 3472#ifdef CONFIG_DYNAMIC_FTRACE
2843 3473
3474int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3475{
3476 return 0;
3477}
3478
2844static ssize_t 3479static ssize_t
2845tracing_read_long(struct file *filp, char __user *ubuf, 3480tracing_read_dyn_info(struct file *filp, char __user *ubuf,
2846 size_t cnt, loff_t *ppos) 3481 size_t cnt, loff_t *ppos)
2847{ 3482{
3483 static char ftrace_dyn_info_buffer[1024];
3484 static DEFINE_MUTEX(dyn_info_mutex);
2848 unsigned long *p = filp->private_data; 3485 unsigned long *p = filp->private_data;
2849 char buf[64]; 3486 char *buf = ftrace_dyn_info_buffer;
3487 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
2850 int r; 3488 int r;
2851 3489
2852 r = sprintf(buf, "%ld\n", *p); 3490 mutex_lock(&dyn_info_mutex);
3491 r = sprintf(buf, "%ld ", *p);
2853 3492
2854 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3493 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3494 buf[r++] = '\n';
3495
3496 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3497
3498 mutex_unlock(&dyn_info_mutex);
3499
3500 return r;
2855} 3501}
2856 3502
2857static struct file_operations tracing_read_long_fops = { 3503static struct file_operations tracing_dyn_info_fops = {
2858 .open = tracing_open_generic, 3504 .open = tracing_open_generic,
2859 .read = tracing_read_long, 3505 .read = tracing_read_dyn_info,
2860}; 3506};
2861#endif 3507#endif
2862 3508
@@ -2897,10 +3543,10 @@ static __init int tracer_init_debugfs(void)
2897 if (!entry) 3543 if (!entry)
2898 pr_warning("Could not create debugfs 'tracing_enabled' entry\n"); 3544 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
2899 3545
2900 entry = debugfs_create_file("iter_ctrl", 0644, d_tracer, 3546 entry = debugfs_create_file("trace_options", 0644, d_tracer,
2901 NULL, &tracing_iter_fops); 3547 NULL, &tracing_iter_fops);
2902 if (!entry) 3548 if (!entry)
2903 pr_warning("Could not create debugfs 'iter_ctrl' entry\n"); 3549 pr_warning("Could not create debugfs 'trace_options' entry\n");
2904 3550
2905 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer, 3551 entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
2906 NULL, &tracing_cpumask_fops); 3552 NULL, &tracing_cpumask_fops);
@@ -2950,11 +3596,11 @@ static __init int tracer_init_debugfs(void)
2950 pr_warning("Could not create debugfs " 3596 pr_warning("Could not create debugfs "
2951 "'trace_pipe' entry\n"); 3597 "'trace_pipe' entry\n");
2952 3598
2953 entry = debugfs_create_file("trace_entries", 0644, d_tracer, 3599 entry = debugfs_create_file("buffer_size_kb", 0644, d_tracer,
2954 &global_trace, &tracing_entries_fops); 3600 &global_trace, &tracing_entries_fops);
2955 if (!entry) 3601 if (!entry)
2956 pr_warning("Could not create debugfs " 3602 pr_warning("Could not create debugfs "
2957 "'trace_entries' entry\n"); 3603 "'buffer_size_kb' entry\n");
2958 3604
2959 entry = debugfs_create_file("trace_marker", 0220, d_tracer, 3605 entry = debugfs_create_file("trace_marker", 0220, d_tracer,
2960 NULL, &tracing_mark_fops); 3606 NULL, &tracing_mark_fops);
@@ -2965,7 +3611,7 @@ static __init int tracer_init_debugfs(void)
2965#ifdef CONFIG_DYNAMIC_FTRACE 3611#ifdef CONFIG_DYNAMIC_FTRACE
2966 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer, 3612 entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
2967 &ftrace_update_tot_cnt, 3613 &ftrace_update_tot_cnt,
2968 &tracing_read_long_fops); 3614 &tracing_dyn_info_fops);
2969 if (!entry) 3615 if (!entry)
2970 pr_warning("Could not create debugfs " 3616 pr_warning("Could not create debugfs "
2971 "'dyn_ftrace_total_info' entry\n"); 3617 "'dyn_ftrace_total_info' entry\n");
@@ -2976,7 +3622,7 @@ static __init int tracer_init_debugfs(void)
2976 return 0; 3622 return 0;
2977} 3623}
2978 3624
2979int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 3625int trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args)
2980{ 3626{
2981 static DEFINE_SPINLOCK(trace_buf_lock); 3627 static DEFINE_SPINLOCK(trace_buf_lock);
2982 static char trace_buf[TRACE_BUF_SIZE]; 3628 static char trace_buf[TRACE_BUF_SIZE];
@@ -2984,11 +3630,11 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2984 struct ring_buffer_event *event; 3630 struct ring_buffer_event *event;
2985 struct trace_array *tr = &global_trace; 3631 struct trace_array *tr = &global_trace;
2986 struct trace_array_cpu *data; 3632 struct trace_array_cpu *data;
2987 struct print_entry *entry;
2988 unsigned long flags, irq_flags;
2989 int cpu, len = 0, size, pc; 3633 int cpu, len = 0, size, pc;
3634 struct print_entry *entry;
3635 unsigned long irq_flags;
2990 3636
2991 if (!tr->ctrl || tracing_disabled) 3637 if (tracing_disabled || tracing_selftest_running)
2992 return 0; 3638 return 0;
2993 3639
2994 pc = preempt_count(); 3640 pc = preempt_count();
@@ -2999,7 +3645,8 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2999 if (unlikely(atomic_read(&data->disabled))) 3645 if (unlikely(atomic_read(&data->disabled)))
3000 goto out; 3646 goto out;
3001 3647
3002 spin_lock_irqsave(&trace_buf_lock, flags); 3648 pause_graph_tracing();
3649 spin_lock_irqsave(&trace_buf_lock, irq_flags);
3003 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args); 3650 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3004 3651
3005 len = min(len, TRACE_BUF_SIZE-1); 3652 len = min(len, TRACE_BUF_SIZE-1);
@@ -3010,17 +3657,18 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3010 if (!event) 3657 if (!event)
3011 goto out_unlock; 3658 goto out_unlock;
3012 entry = ring_buffer_event_data(event); 3659 entry = ring_buffer_event_data(event);
3013 tracing_generic_entry_update(&entry->ent, flags, pc); 3660 tracing_generic_entry_update(&entry->ent, irq_flags, pc);
3014 entry->ent.type = TRACE_PRINT; 3661 entry->ent.type = TRACE_PRINT;
3015 entry->ip = ip; 3662 entry->ip = ip;
3663 entry->depth = depth;
3016 3664
3017 memcpy(&entry->buf, trace_buf, len); 3665 memcpy(&entry->buf, trace_buf, len);
3018 entry->buf[len] = 0; 3666 entry->buf[len] = 0;
3019 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 3667 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
3020 3668
3021 out_unlock: 3669 out_unlock:
3022 spin_unlock_irqrestore(&trace_buf_lock, flags); 3670 spin_unlock_irqrestore(&trace_buf_lock, irq_flags);
3023 3671 unpause_graph_tracing();
3024 out: 3672 out:
3025 preempt_enable_notrace(); 3673 preempt_enable_notrace();
3026 3674
@@ -3037,7 +3685,7 @@ int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3037 return 0; 3685 return 0;
3038 3686
3039 va_start(ap, fmt); 3687 va_start(ap, fmt);
3040 ret = trace_vprintk(ip, fmt, ap); 3688 ret = trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
3041 va_end(ap); 3689 va_end(ap);
3042 return ret; 3690 return ret;
3043} 3691}
@@ -3046,7 +3694,8 @@ EXPORT_SYMBOL_GPL(__ftrace_printk);
3046static int trace_panic_handler(struct notifier_block *this, 3694static int trace_panic_handler(struct notifier_block *this,
3047 unsigned long event, void *unused) 3695 unsigned long event, void *unused)
3048{ 3696{
3049 ftrace_dump(); 3697 if (ftrace_dump_on_oops)
3698 ftrace_dump();
3050 return NOTIFY_OK; 3699 return NOTIFY_OK;
3051} 3700}
3052 3701
@@ -3062,7 +3711,8 @@ static int trace_die_handler(struct notifier_block *self,
3062{ 3711{
3063 switch (val) { 3712 switch (val) {
3064 case DIE_OOPS: 3713 case DIE_OOPS:
3065 ftrace_dump(); 3714 if (ftrace_dump_on_oops)
3715 ftrace_dump();
3066 break; 3716 break;
3067 default: 3717 default:
3068 break; 3718 break;
@@ -3103,13 +3753,11 @@ trace_printk_seq(struct trace_seq *s)
3103 trace_seq_reset(s); 3753 trace_seq_reset(s);
3104} 3754}
3105 3755
3106
3107void ftrace_dump(void) 3756void ftrace_dump(void)
3108{ 3757{
3109 static DEFINE_SPINLOCK(ftrace_dump_lock); 3758 static DEFINE_SPINLOCK(ftrace_dump_lock);
3110 /* use static because iter can be a bit big for the stack */ 3759 /* use static because iter can be a bit big for the stack */
3111 static struct trace_iterator iter; 3760 static struct trace_iterator iter;
3112 static cpumask_t mask;
3113 static int dump_ran; 3761 static int dump_ran;
3114 unsigned long flags; 3762 unsigned long flags;
3115 int cnt = 0, cpu; 3763 int cnt = 0, cpu;
@@ -3128,6 +3776,9 @@ void ftrace_dump(void)
3128 atomic_inc(&global_trace.data[cpu]->disabled); 3776 atomic_inc(&global_trace.data[cpu]->disabled);
3129 } 3777 }
3130 3778
3779 /* don't look at user memory in panic mode */
3780 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
3781
3131 printk(KERN_TRACE "Dumping ftrace buffer:\n"); 3782 printk(KERN_TRACE "Dumping ftrace buffer:\n");
3132 3783
3133 iter.tr = &global_trace; 3784 iter.tr = &global_trace;
@@ -3140,8 +3791,6 @@ void ftrace_dump(void)
3140 * and then release the locks again. 3791 * and then release the locks again.
3141 */ 3792 */
3142 3793
3143 cpus_clear(mask);
3144
3145 while (!trace_empty(&iter)) { 3794 while (!trace_empty(&iter)) {
3146 3795
3147 if (!cnt) 3796 if (!cnt)
@@ -3177,19 +3826,28 @@ __init static int tracer_alloc_buffers(void)
3177{ 3826{
3178 struct trace_array_cpu *data; 3827 struct trace_array_cpu *data;
3179 int i; 3828 int i;
3829 int ret = -ENOMEM;
3180 3830
3181 /* TODO: make the number of buffers hot pluggable with CPUS */ 3831 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
3182 tracing_buffer_mask = cpu_possible_map; 3832 goto out;
3833
3834 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
3835 goto out_free_buffer_mask;
3836
3837 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
3838 cpumask_copy(tracing_cpumask, cpu_all_mask);
3183 3839
3840 /* TODO: make the number of buffers hot pluggable with CPUS */
3184 global_trace.buffer = ring_buffer_alloc(trace_buf_size, 3841 global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3185 TRACE_BUFFER_FLAGS); 3842 TRACE_BUFFER_FLAGS);
3186 if (!global_trace.buffer) { 3843 if (!global_trace.buffer) {
3187 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n"); 3844 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3188 WARN_ON(1); 3845 WARN_ON(1);
3189 return 0; 3846 goto out_free_cpumask;
3190 } 3847 }
3191 global_trace.entries = ring_buffer_size(global_trace.buffer); 3848 global_trace.entries = ring_buffer_size(global_trace.buffer);
3192 3849
3850
3193#ifdef CONFIG_TRACER_MAX_TRACE 3851#ifdef CONFIG_TRACER_MAX_TRACE
3194 max_tr.buffer = ring_buffer_alloc(trace_buf_size, 3852 max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3195 TRACE_BUFFER_FLAGS); 3853 TRACE_BUFFER_FLAGS);
@@ -3197,7 +3855,7 @@ __init static int tracer_alloc_buffers(void)
3197 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n"); 3855 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3198 WARN_ON(1); 3856 WARN_ON(1);
3199 ring_buffer_free(global_trace.buffer); 3857 ring_buffer_free(global_trace.buffer);
3200 return 0; 3858 goto out_free_cpumask;
3201 } 3859 }
3202 max_tr.entries = ring_buffer_size(max_tr.buffer); 3860 max_tr.entries = ring_buffer_size(max_tr.buffer);
3203 WARN_ON(max_tr.entries != global_trace.entries); 3861 WARN_ON(max_tr.entries != global_trace.entries);
@@ -3221,15 +3879,20 @@ __init static int tracer_alloc_buffers(void)
3221#endif 3879#endif
3222 3880
3223 /* All seems OK, enable tracing */ 3881 /* All seems OK, enable tracing */
3224 global_trace.ctrl = tracer_enabled;
3225 tracing_disabled = 0; 3882 tracing_disabled = 0;
3226 3883
3227 atomic_notifier_chain_register(&panic_notifier_list, 3884 atomic_notifier_chain_register(&panic_notifier_list,
3228 &trace_panic_notifier); 3885 &trace_panic_notifier);
3229 3886
3230 register_die_notifier(&trace_die_notifier); 3887 register_die_notifier(&trace_die_notifier);
3888 ret = 0;
3231 3889
3232 return 0; 3890out_free_cpumask:
3891 free_cpumask_var(tracing_cpumask);
3892out_free_buffer_mask:
3893 free_cpumask_var(tracing_buffer_mask);
3894out:
3895 return ret;
3233} 3896}
3234early_initcall(tracer_alloc_buffers); 3897early_initcall(tracer_alloc_buffers);
3235fs_initcall(tracer_init_debugfs); 3898fs_initcall(tracer_init_debugfs);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 8465ad052707..4d3d381bfd95 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -8,6 +8,7 @@
8#include <linux/ring_buffer.h> 8#include <linux/ring_buffer.h>
9#include <linux/mmiotrace.h> 9#include <linux/mmiotrace.h>
10#include <linux/ftrace.h> 10#include <linux/ftrace.h>
11#include <trace/boot.h>
11 12
12enum trace_type { 13enum trace_type {
13 __TRACE_FIRST_TYPE = 0, 14 __TRACE_FIRST_TYPE = 0,
@@ -21,7 +22,14 @@ enum trace_type {
21 TRACE_SPECIAL, 22 TRACE_SPECIAL,
22 TRACE_MMIO_RW, 23 TRACE_MMIO_RW,
23 TRACE_MMIO_MAP, 24 TRACE_MMIO_MAP,
24 TRACE_BOOT, 25 TRACE_BRANCH,
26 TRACE_BOOT_CALL,
27 TRACE_BOOT_RET,
28 TRACE_GRAPH_RET,
29 TRACE_GRAPH_ENT,
30 TRACE_USER_STACK,
31 TRACE_HW_BRANCHES,
32 TRACE_POWER,
25 33
26 __TRACE_LAST_TYPE 34 __TRACE_LAST_TYPE
27}; 35};
@@ -38,6 +46,7 @@ struct trace_entry {
38 unsigned char flags; 46 unsigned char flags;
39 unsigned char preempt_count; 47 unsigned char preempt_count;
40 int pid; 48 int pid;
49 int tgid;
41}; 50};
42 51
43/* 52/*
@@ -48,6 +57,18 @@ struct ftrace_entry {
48 unsigned long ip; 57 unsigned long ip;
49 unsigned long parent_ip; 58 unsigned long parent_ip;
50}; 59};
60
61/* Function call entry */
62struct ftrace_graph_ent_entry {
63 struct trace_entry ent;
64 struct ftrace_graph_ent graph_ent;
65};
66
67/* Function return entry */
68struct ftrace_graph_ret_entry {
69 struct trace_entry ent;
70 struct ftrace_graph_ret ret;
71};
51extern struct tracer boot_tracer; 72extern struct tracer boot_tracer;
52 73
53/* 74/*
@@ -85,12 +106,18 @@ struct stack_entry {
85 unsigned long caller[FTRACE_STACK_ENTRIES]; 106 unsigned long caller[FTRACE_STACK_ENTRIES];
86}; 107};
87 108
109struct userstack_entry {
110 struct trace_entry ent;
111 unsigned long caller[FTRACE_STACK_ENTRIES];
112};
113
88/* 114/*
89 * ftrace_printk entry: 115 * ftrace_printk entry:
90 */ 116 */
91struct print_entry { 117struct print_entry {
92 struct trace_entry ent; 118 struct trace_entry ent;
93 unsigned long ip; 119 unsigned long ip;
120 int depth;
94 char buf[]; 121 char buf[];
95}; 122};
96 123
@@ -112,9 +139,35 @@ struct trace_mmiotrace_map {
112 struct mmiotrace_map map; 139 struct mmiotrace_map map;
113}; 140};
114 141
115struct trace_boot { 142struct trace_boot_call {
116 struct trace_entry ent; 143 struct trace_entry ent;
117 struct boot_trace initcall; 144 struct boot_trace_call boot_call;
145};
146
147struct trace_boot_ret {
148 struct trace_entry ent;
149 struct boot_trace_ret boot_ret;
150};
151
152#define TRACE_FUNC_SIZE 30
153#define TRACE_FILE_SIZE 20
154struct trace_branch {
155 struct trace_entry ent;
156 unsigned line;
157 char func[TRACE_FUNC_SIZE+1];
158 char file[TRACE_FILE_SIZE+1];
159 char correct;
160};
161
162struct hw_branch_entry {
163 struct trace_entry ent;
164 u64 from;
165 u64 to;
166};
167
168struct trace_power {
169 struct trace_entry ent;
170 struct power_trace state_data;
118}; 171};
119 172
120/* 173/*
@@ -172,7 +225,6 @@ struct trace_iterator;
172struct trace_array { 225struct trace_array {
173 struct ring_buffer *buffer; 226 struct ring_buffer *buffer;
174 unsigned long entries; 227 unsigned long entries;
175 long ctrl;
176 int cpu; 228 int cpu;
177 cycle_t time_start; 229 cycle_t time_start;
178 struct task_struct *waiter; 230 struct task_struct *waiter;
@@ -212,13 +264,22 @@ extern void __ftrace_bad_type(void);
212 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \ 264 IF_ASSIGN(var, ent, struct ctx_switch_entry, 0); \
213 IF_ASSIGN(var, ent, struct trace_field_cont, TRACE_CONT); \ 265 IF_ASSIGN(var, ent, struct trace_field_cont, TRACE_CONT); \
214 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \ 266 IF_ASSIGN(var, ent, struct stack_entry, TRACE_STACK); \
267 IF_ASSIGN(var, ent, struct userstack_entry, TRACE_USER_STACK);\
215 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \ 268 IF_ASSIGN(var, ent, struct print_entry, TRACE_PRINT); \
216 IF_ASSIGN(var, ent, struct special_entry, 0); \ 269 IF_ASSIGN(var, ent, struct special_entry, 0); \
217 IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \ 270 IF_ASSIGN(var, ent, struct trace_mmiotrace_rw, \
218 TRACE_MMIO_RW); \ 271 TRACE_MMIO_RW); \
219 IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \ 272 IF_ASSIGN(var, ent, struct trace_mmiotrace_map, \
220 TRACE_MMIO_MAP); \ 273 TRACE_MMIO_MAP); \
221 IF_ASSIGN(var, ent, struct trace_boot, TRACE_BOOT); \ 274 IF_ASSIGN(var, ent, struct trace_boot_call, TRACE_BOOT_CALL);\
275 IF_ASSIGN(var, ent, struct trace_boot_ret, TRACE_BOOT_RET);\
276 IF_ASSIGN(var, ent, struct trace_branch, TRACE_BRANCH); \
277 IF_ASSIGN(var, ent, struct ftrace_graph_ent_entry, \
278 TRACE_GRAPH_ENT); \
279 IF_ASSIGN(var, ent, struct ftrace_graph_ret_entry, \
280 TRACE_GRAPH_RET); \
281 IF_ASSIGN(var, ent, struct hw_branch_entry, TRACE_HW_BRANCHES);\
282 IF_ASSIGN(var, ent, struct trace_power, TRACE_POWER); \
222 __ftrace_bad_type(); \ 283 __ftrace_bad_type(); \
223 } while (0) 284 } while (0)
224 285
@@ -229,29 +290,56 @@ enum print_line_t {
229 TRACE_TYPE_UNHANDLED = 2 /* Relay to other output functions */ 290 TRACE_TYPE_UNHANDLED = 2 /* Relay to other output functions */
230}; 291};
231 292
293
294/*
295 * An option specific to a tracer. This is a boolean value.
296 * The bit is the bit index that sets its value on the
297 * flags value in struct tracer_flags.
298 */
299struct tracer_opt {
300 const char *name; /* Will appear on the trace_options file */
301 u32 bit; /* Mask assigned in val field in tracer_flags */
302};
303
304/*
305 * The set of specific options for a tracer. Your tracer
306 * have to set the initial value of the flags val.
307 */
308struct tracer_flags {
309 u32 val;
310 struct tracer_opt *opts;
311};
312
313/* Makes more easy to define a tracer opt */
314#define TRACER_OPT(s, b) .name = #s, .bit = b
315
232/* 316/*
233 * A specific tracer, represented by methods that operate on a trace array: 317 * A specific tracer, represented by methods that operate on a trace array:
234 */ 318 */
235struct tracer { 319struct tracer {
236 const char *name; 320 const char *name;
237 void (*init)(struct trace_array *tr); 321 /* Your tracer should raise a warning if init fails */
322 int (*init)(struct trace_array *tr);
238 void (*reset)(struct trace_array *tr); 323 void (*reset)(struct trace_array *tr);
324 void (*start)(struct trace_array *tr);
325 void (*stop)(struct trace_array *tr);
239 void (*open)(struct trace_iterator *iter); 326 void (*open)(struct trace_iterator *iter);
240 void (*pipe_open)(struct trace_iterator *iter); 327 void (*pipe_open)(struct trace_iterator *iter);
241 void (*close)(struct trace_iterator *iter); 328 void (*close)(struct trace_iterator *iter);
242 void (*start)(struct trace_iterator *iter);
243 void (*stop)(struct trace_iterator *iter);
244 ssize_t (*read)(struct trace_iterator *iter, 329 ssize_t (*read)(struct trace_iterator *iter,
245 struct file *filp, char __user *ubuf, 330 struct file *filp, char __user *ubuf,
246 size_t cnt, loff_t *ppos); 331 size_t cnt, loff_t *ppos);
247 void (*ctrl_update)(struct trace_array *tr);
248#ifdef CONFIG_FTRACE_STARTUP_TEST 332#ifdef CONFIG_FTRACE_STARTUP_TEST
249 int (*selftest)(struct tracer *trace, 333 int (*selftest)(struct tracer *trace,
250 struct trace_array *tr); 334 struct trace_array *tr);
251#endif 335#endif
336 void (*print_header)(struct seq_file *m);
252 enum print_line_t (*print_line)(struct trace_iterator *iter); 337 enum print_line_t (*print_line)(struct trace_iterator *iter);
338 /* If you handled the flag setting, return 0 */
339 int (*set_flag)(u32 old_flags, u32 bit, int set);
253 struct tracer *next; 340 struct tracer *next;
254 int print_max; 341 int print_max;
342 struct tracer_flags *flags;
255}; 343};
256 344
257struct trace_seq { 345struct trace_seq {
@@ -279,10 +367,14 @@ struct trace_iterator {
279 unsigned long iter_flags; 367 unsigned long iter_flags;
280 loff_t pos; 368 loff_t pos;
281 long idx; 369 long idx;
370
371 cpumask_var_t started;
282}; 372};
283 373
374int tracing_is_enabled(void);
284void trace_wake_up(void); 375void trace_wake_up(void);
285void tracing_reset(struct trace_array *tr, int cpu); 376void tracing_reset(struct trace_array *tr, int cpu);
377void tracing_reset_online_cpus(struct trace_array *tr);
286int tracing_open_generic(struct inode *inode, struct file *filp); 378int tracing_open_generic(struct inode *inode, struct file *filp);
287struct dentry *tracing_init_dentry(void); 379struct dentry *tracing_init_dentry(void);
288void init_tracer_sysprof_debugfs(struct dentry *d_tracer); 380void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
@@ -321,8 +413,15 @@ void trace_function(struct trace_array *tr,
321 unsigned long parent_ip, 413 unsigned long parent_ip,
322 unsigned long flags, int pc); 414 unsigned long flags, int pc);
323 415
416void trace_graph_return(struct ftrace_graph_ret *trace);
417int trace_graph_entry(struct ftrace_graph_ent *trace);
418void trace_hw_branch(struct trace_array *tr, u64 from, u64 to);
419
324void tracing_start_cmdline_record(void); 420void tracing_start_cmdline_record(void);
325void tracing_stop_cmdline_record(void); 421void tracing_stop_cmdline_record(void);
422void tracing_sched_switch_assign_trace(struct trace_array *tr);
423void tracing_stop_sched_switch_record(void);
424void tracing_start_sched_switch_record(void);
326int register_tracer(struct tracer *type); 425int register_tracer(struct tracer *type);
327void unregister_tracer(struct tracer *type); 426void unregister_tracer(struct tracer *type);
328 427
@@ -358,6 +457,7 @@ struct tracer_switch_ops {
358 struct tracer_switch_ops *next; 457 struct tracer_switch_ops *next;
359}; 458};
360 459
460char *trace_find_cmdline(int pid);
361#endif /* CONFIG_CONTEXT_SWITCH_TRACER */ 461#endif /* CONFIG_CONTEXT_SWITCH_TRACER */
362 462
363#ifdef CONFIG_DYNAMIC_FTRACE 463#ifdef CONFIG_DYNAMIC_FTRACE
@@ -383,19 +483,79 @@ extern int trace_selftest_startup_sched_switch(struct tracer *trace,
383 struct trace_array *tr); 483 struct trace_array *tr);
384extern int trace_selftest_startup_sysprof(struct tracer *trace, 484extern int trace_selftest_startup_sysprof(struct tracer *trace,
385 struct trace_array *tr); 485 struct trace_array *tr);
486extern int trace_selftest_startup_branch(struct tracer *trace,
487 struct trace_array *tr);
386#endif /* CONFIG_FTRACE_STARTUP_TEST */ 488#endif /* CONFIG_FTRACE_STARTUP_TEST */
387 489
388extern void *head_page(struct trace_array_cpu *data); 490extern void *head_page(struct trace_array_cpu *data);
389extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...); 491extern int trace_seq_printf(struct trace_seq *s, const char *fmt, ...);
390extern void trace_seq_print_cont(struct trace_seq *s, 492extern void trace_seq_print_cont(struct trace_seq *s,
391 struct trace_iterator *iter); 493 struct trace_iterator *iter);
494
495extern int
496seq_print_ip_sym(struct trace_seq *s, unsigned long ip,
497 unsigned long sym_flags);
392extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, 498extern ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf,
393 size_t cnt); 499 size_t cnt);
394extern long ns2usecs(cycle_t nsec); 500extern long ns2usecs(cycle_t nsec);
395extern int trace_vprintk(unsigned long ip, const char *fmt, va_list args); 501extern int
502trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args);
396 503
397extern unsigned long trace_flags; 504extern unsigned long trace_flags;
398 505
506/* Standard output formatting function used for function return traces */
507#ifdef CONFIG_FUNCTION_GRAPH_TRACER
508extern enum print_line_t print_graph_function(struct trace_iterator *iter);
509
510#ifdef CONFIG_DYNAMIC_FTRACE
511/* TODO: make this variable */
512#define FTRACE_GRAPH_MAX_FUNCS 32
513extern int ftrace_graph_count;
514extern unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS];
515
516static inline int ftrace_graph_addr(unsigned long addr)
517{
518 int i;
519
520 if (!ftrace_graph_count || test_tsk_trace_graph(current))
521 return 1;
522
523 for (i = 0; i < ftrace_graph_count; i++) {
524 if (addr == ftrace_graph_funcs[i])
525 return 1;
526 }
527
528 return 0;
529}
530#else
531static inline int ftrace_trace_addr(unsigned long addr)
532{
533 return 1;
534}
535static inline int ftrace_graph_addr(unsigned long addr)
536{
537 return 1;
538}
539#endif /* CONFIG_DYNAMIC_FTRACE */
540
541#else /* CONFIG_FUNCTION_GRAPH_TRACER */
542static inline enum print_line_t
543print_graph_function(struct trace_iterator *iter)
544{
545 return TRACE_TYPE_UNHANDLED;
546}
547#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
548
549extern struct pid *ftrace_pid_trace;
550
551static inline int ftrace_trace_task(struct task_struct *task)
552{
553 if (!ftrace_pid_trace)
554 return 1;
555
556 return test_tsk_trace_trace(task);
557}
558
399/* 559/*
400 * trace_iterator_flags is an enumeration that defines bit 560 * trace_iterator_flags is an enumeration that defines bit
401 * positions into trace_flags that controls the output. 561 * positions into trace_flags that controls the output.
@@ -415,8 +575,93 @@ enum trace_iterator_flags {
415 TRACE_ITER_STACKTRACE = 0x100, 575 TRACE_ITER_STACKTRACE = 0x100,
416 TRACE_ITER_SCHED_TREE = 0x200, 576 TRACE_ITER_SCHED_TREE = 0x200,
417 TRACE_ITER_PRINTK = 0x400, 577 TRACE_ITER_PRINTK = 0x400,
578 TRACE_ITER_PREEMPTONLY = 0x800,
579 TRACE_ITER_BRANCH = 0x1000,
580 TRACE_ITER_ANNOTATE = 0x2000,
581 TRACE_ITER_USERSTACKTRACE = 0x4000,
582 TRACE_ITER_SYM_USEROBJ = 0x8000,
583 TRACE_ITER_PRINTK_MSGONLY = 0x10000
418}; 584};
419 585
586/*
587 * TRACE_ITER_SYM_MASK masks the options in trace_flags that
588 * control the output of kernel symbols.
589 */
590#define TRACE_ITER_SYM_MASK \
591 (TRACE_ITER_PRINT_PARENT|TRACE_ITER_SYM_OFFSET|TRACE_ITER_SYM_ADDR)
592
420extern struct tracer nop_trace; 593extern struct tracer nop_trace;
421 594
595/**
596 * ftrace_preempt_disable - disable preemption scheduler safe
597 *
598 * When tracing can happen inside the scheduler, there exists
599 * cases that the tracing might happen before the need_resched
600 * flag is checked. If this happens and the tracer calls
601 * preempt_enable (after a disable), a schedule might take place
602 * causing an infinite recursion.
603 *
604 * To prevent this, we read the need_recshed flag before
605 * disabling preemption. When we want to enable preemption we
606 * check the flag, if it is set, then we call preempt_enable_no_resched.
607 * Otherwise, we call preempt_enable.
608 *
609 * The rational for doing the above is that if need resched is set
610 * and we have yet to reschedule, we are either in an atomic location
611 * (where we do not need to check for scheduling) or we are inside
612 * the scheduler and do not want to resched.
613 */
614static inline int ftrace_preempt_disable(void)
615{
616 int resched;
617
618 resched = need_resched();
619 preempt_disable_notrace();
620
621 return resched;
622}
623
624/**
625 * ftrace_preempt_enable - enable preemption scheduler safe
626 * @resched: the return value from ftrace_preempt_disable
627 *
628 * This is a scheduler safe way to enable preemption and not miss
629 * any preemption checks. The disabled saved the state of preemption.
630 * If resched is set, then we were either inside an atomic or
631 * are inside the scheduler (we would have already scheduled
632 * otherwise). In this case, we do not want to call normal
633 * preempt_enable, but preempt_enable_no_resched instead.
634 */
635static inline void ftrace_preempt_enable(int resched)
636{
637 if (resched)
638 preempt_enable_no_resched_notrace();
639 else
640 preempt_enable_notrace();
641}
642
643#ifdef CONFIG_BRANCH_TRACER
644extern int enable_branch_tracing(struct trace_array *tr);
645extern void disable_branch_tracing(void);
646static inline int trace_branch_enable(struct trace_array *tr)
647{
648 if (trace_flags & TRACE_ITER_BRANCH)
649 return enable_branch_tracing(tr);
650 return 0;
651}
652static inline void trace_branch_disable(void)
653{
654 /* due to races, always disable */
655 disable_branch_tracing();
656}
657#else
658static inline int trace_branch_enable(struct trace_array *tr)
659{
660 return 0;
661}
662static inline void trace_branch_disable(void)
663{
664}
665#endif /* CONFIG_BRANCH_TRACER */
666
422#endif /* _LINUX_KERNEL_TRACE_H */ 667#endif /* _LINUX_KERNEL_TRACE_H */
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index d0a5e50eeff2..366c8c333e13 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -13,101 +13,161 @@
13#include "trace.h" 13#include "trace.h"
14 14
15static struct trace_array *boot_trace; 15static struct trace_array *boot_trace;
16static int trace_boot_enabled; 16static bool pre_initcalls_finished;
17 17
18 18/* Tells the boot tracer that the pre_smp_initcalls are finished.
19/* Should be started after do_pre_smp_initcalls() in init/main.c */ 19 * So we are ready .
20 * It doesn't enable sched events tracing however.
21 * You have to call enable_boot_trace to do so.
22 */
20void start_boot_trace(void) 23void start_boot_trace(void)
21{ 24{
22 trace_boot_enabled = 1; 25 pre_initcalls_finished = true;
23} 26}
24 27
25void stop_boot_trace(void) 28void enable_boot_trace(void)
26{ 29{
27 trace_boot_enabled = 0; 30 if (pre_initcalls_finished)
31 tracing_start_sched_switch_record();
28} 32}
29 33
30void reset_boot_trace(struct trace_array *tr) 34void disable_boot_trace(void)
31{ 35{
32 stop_boot_trace(); 36 if (pre_initcalls_finished)
37 tracing_stop_sched_switch_record();
33} 38}
34 39
35static void boot_trace_init(struct trace_array *tr) 40static int boot_trace_init(struct trace_array *tr)
36{ 41{
37 int cpu; 42 int cpu;
38 boot_trace = tr; 43 boot_trace = tr;
39 44
40 trace_boot_enabled = 0; 45 for_each_cpu(cpu, cpu_possible_mask)
41
42 for_each_cpu_mask(cpu, cpu_possible_map)
43 tracing_reset(tr, cpu); 46 tracing_reset(tr, cpu);
47
48 tracing_sched_switch_assign_trace(tr);
49 return 0;
44} 50}
45 51
46static void boot_trace_ctrl_update(struct trace_array *tr) 52static enum print_line_t
53initcall_call_print_line(struct trace_iterator *iter)
47{ 54{
48 if (tr->ctrl) 55 struct trace_entry *entry = iter->ent;
49 start_boot_trace(); 56 struct trace_seq *s = &iter->seq;
57 struct trace_boot_call *field;
58 struct boot_trace_call *call;
59 u64 ts;
60 unsigned long nsec_rem;
61 int ret;
62
63 trace_assign_type(field, entry);
64 call = &field->boot_call;
65 ts = iter->ts;
66 nsec_rem = do_div(ts, 1000000000);
67
68 ret = trace_seq_printf(s, "[%5ld.%09ld] calling %s @ %i\n",
69 (unsigned long)ts, nsec_rem, call->func, call->caller);
70
71 if (!ret)
72 return TRACE_TYPE_PARTIAL_LINE;
50 else 73 else
51 stop_boot_trace(); 74 return TRACE_TYPE_HANDLED;
52} 75}
53 76
54static enum print_line_t initcall_print_line(struct trace_iterator *iter) 77static enum print_line_t
78initcall_ret_print_line(struct trace_iterator *iter)
55{ 79{
56 int ret;
57 struct trace_entry *entry = iter->ent; 80 struct trace_entry *entry = iter->ent;
58 struct trace_boot *field = (struct trace_boot *)entry;
59 struct boot_trace *it = &field->initcall;
60 struct trace_seq *s = &iter->seq; 81 struct trace_seq *s = &iter->seq;
61 struct timespec calltime = ktime_to_timespec(it->calltime); 82 struct trace_boot_ret *field;
62 struct timespec rettime = ktime_to_timespec(it->rettime); 83 struct boot_trace_ret *init_ret;
63 84 u64 ts;
64 if (entry->type == TRACE_BOOT) { 85 unsigned long nsec_rem;
65 ret = trace_seq_printf(s, "[%5ld.%09ld] calling %s @ %i\n", 86 int ret;
66 calltime.tv_sec, 87
67 calltime.tv_nsec, 88 trace_assign_type(field, entry);
68 it->func, it->caller); 89 init_ret = &field->boot_ret;
69 if (!ret) 90 ts = iter->ts;
70 return TRACE_TYPE_PARTIAL_LINE; 91 nsec_rem = do_div(ts, 1000000000);
71 92
72 ret = trace_seq_printf(s, "[%5ld.%09ld] initcall %s " 93 ret = trace_seq_printf(s, "[%5ld.%09ld] initcall %s "
73 "returned %d after %lld msecs\n", 94 "returned %d after %llu msecs\n",
74 rettime.tv_sec, 95 (unsigned long) ts,
75 rettime.tv_nsec, 96 nsec_rem,
76 it->func, it->result, it->duration); 97 init_ret->func, init_ret->result, init_ret->duration);
77 98
78 if (!ret) 99 if (!ret)
79 return TRACE_TYPE_PARTIAL_LINE; 100 return TRACE_TYPE_PARTIAL_LINE;
101 else
80 return TRACE_TYPE_HANDLED; 102 return TRACE_TYPE_HANDLED;
103}
104
105static enum print_line_t initcall_print_line(struct trace_iterator *iter)
106{
107 struct trace_entry *entry = iter->ent;
108
109 switch (entry->type) {
110 case TRACE_BOOT_CALL:
111 return initcall_call_print_line(iter);
112 case TRACE_BOOT_RET:
113 return initcall_ret_print_line(iter);
114 default:
115 return TRACE_TYPE_UNHANDLED;
81 } 116 }
82 return TRACE_TYPE_UNHANDLED;
83} 117}
84 118
85struct tracer boot_tracer __read_mostly = 119struct tracer boot_tracer __read_mostly =
86{ 120{
87 .name = "initcall", 121 .name = "initcall",
88 .init = boot_trace_init, 122 .init = boot_trace_init,
89 .reset = reset_boot_trace, 123 .reset = tracing_reset_online_cpus,
90 .ctrl_update = boot_trace_ctrl_update,
91 .print_line = initcall_print_line, 124 .print_line = initcall_print_line,
92}; 125};
93 126
94void trace_boot(struct boot_trace *it, initcall_t fn) 127void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
95{ 128{
96 struct ring_buffer_event *event; 129 struct ring_buffer_event *event;
97 struct trace_boot *entry; 130 struct trace_boot_call *entry;
98 struct trace_array_cpu *data;
99 unsigned long irq_flags; 131 unsigned long irq_flags;
100 struct trace_array *tr = boot_trace; 132 struct trace_array *tr = boot_trace;
101 133
102 if (!trace_boot_enabled) 134 if (!pre_initcalls_finished)
103 return; 135 return;
104 136
105 /* Get its name now since this function could 137 /* Get its name now since this function could
106 * disappear because it is in the .init section. 138 * disappear because it is in the .init section.
107 */ 139 */
108 sprint_symbol(it->func, (unsigned long)fn); 140 sprint_symbol(bt->func, (unsigned long)fn);
141 preempt_disable();
142
143 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
144 &irq_flags);
145 if (!event)
146 goto out;
147 entry = ring_buffer_event_data(event);
148 tracing_generic_entry_update(&entry->ent, 0, 0);
149 entry->ent.type = TRACE_BOOT_CALL;
150 entry->boot_call = *bt;
151 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
152
153 trace_wake_up();
154
155 out:
156 preempt_enable();
157}
158
159void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
160{
161 struct ring_buffer_event *event;
162 struct trace_boot_ret *entry;
163 unsigned long irq_flags;
164 struct trace_array *tr = boot_trace;
165
166 if (!pre_initcalls_finished)
167 return;
168
169 sprint_symbol(bt->func, (unsigned long)fn);
109 preempt_disable(); 170 preempt_disable();
110 data = tr->data[smp_processor_id()];
111 171
112 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), 172 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
113 &irq_flags); 173 &irq_flags);
@@ -115,8 +175,8 @@ void trace_boot(struct boot_trace *it, initcall_t fn)
115 goto out; 175 goto out;
116 entry = ring_buffer_event_data(event); 176 entry = ring_buffer_event_data(event);
117 tracing_generic_entry_update(&entry->ent, 0, 0); 177 tracing_generic_entry_update(&entry->ent, 0, 0);
118 entry->ent.type = TRACE_BOOT; 178 entry->ent.type = TRACE_BOOT_RET;
119 entry->initcall = *it; 179 entry->boot_ret = *bt;
120 ring_buffer_unlock_commit(tr->buffer, event, irq_flags); 180 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
121 181
122 trace_wake_up(); 182 trace_wake_up();
diff --git a/kernel/trace/trace_branch.c b/kernel/trace/trace_branch.c
new file mode 100644
index 000000000000..6c00feb3bac7
--- /dev/null
+++ b/kernel/trace/trace_branch.c
@@ -0,0 +1,342 @@
1/*
2 * unlikely profiler
3 *
4 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
5 */
6#include <linux/kallsyms.h>
7#include <linux/seq_file.h>
8#include <linux/spinlock.h>
9#include <linux/irqflags.h>
10#include <linux/debugfs.h>
11#include <linux/uaccess.h>
12#include <linux/module.h>
13#include <linux/ftrace.h>
14#include <linux/hash.h>
15#include <linux/fs.h>
16#include <asm/local.h>
17#include "trace.h"
18
19#ifdef CONFIG_BRANCH_TRACER
20
21static int branch_tracing_enabled __read_mostly;
22static DEFINE_MUTEX(branch_tracing_mutex);
23static struct trace_array *branch_tracer;
24
25static void
26probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
27{
28 struct trace_array *tr = branch_tracer;
29 struct ring_buffer_event *event;
30 struct trace_branch *entry;
31 unsigned long flags, irq_flags;
32 int cpu, pc;
33 const char *p;
34
35 /*
36 * I would love to save just the ftrace_likely_data pointer, but
37 * this code can also be used by modules. Ugly things can happen
38 * if the module is unloaded, and then we go and read the
39 * pointer. This is slower, but much safer.
40 */
41
42 if (unlikely(!tr))
43 return;
44
45 local_irq_save(flags);
46 cpu = raw_smp_processor_id();
47 if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
48 goto out;
49
50 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
51 &irq_flags);
52 if (!event)
53 goto out;
54
55 pc = preempt_count();
56 entry = ring_buffer_event_data(event);
57 tracing_generic_entry_update(&entry->ent, flags, pc);
58 entry->ent.type = TRACE_BRANCH;
59
60 /* Strip off the path, only save the file */
61 p = f->file + strlen(f->file);
62 while (p >= f->file && *p != '/')
63 p--;
64 p++;
65
66 strncpy(entry->func, f->func, TRACE_FUNC_SIZE);
67 strncpy(entry->file, p, TRACE_FILE_SIZE);
68 entry->func[TRACE_FUNC_SIZE] = 0;
69 entry->file[TRACE_FILE_SIZE] = 0;
70 entry->line = f->line;
71 entry->correct = val == expect;
72
73 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
74
75 out:
76 atomic_dec(&tr->data[cpu]->disabled);
77 local_irq_restore(flags);
78}
79
80static inline
81void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
82{
83 if (!branch_tracing_enabled)
84 return;
85
86 probe_likely_condition(f, val, expect);
87}
88
89int enable_branch_tracing(struct trace_array *tr)
90{
91 int ret = 0;
92
93 mutex_lock(&branch_tracing_mutex);
94 branch_tracer = tr;
95 /*
96 * Must be seen before enabling. The reader is a condition
97 * where we do not need a matching rmb()
98 */
99 smp_wmb();
100 branch_tracing_enabled++;
101 mutex_unlock(&branch_tracing_mutex);
102
103 return ret;
104}
105
106void disable_branch_tracing(void)
107{
108 mutex_lock(&branch_tracing_mutex);
109
110 if (!branch_tracing_enabled)
111 goto out_unlock;
112
113 branch_tracing_enabled--;
114
115 out_unlock:
116 mutex_unlock(&branch_tracing_mutex);
117}
118
119static void start_branch_trace(struct trace_array *tr)
120{
121 enable_branch_tracing(tr);
122}
123
124static void stop_branch_trace(struct trace_array *tr)
125{
126 disable_branch_tracing();
127}
128
129static int branch_trace_init(struct trace_array *tr)
130{
131 int cpu;
132
133 for_each_online_cpu(cpu)
134 tracing_reset(tr, cpu);
135
136 start_branch_trace(tr);
137 return 0;
138}
139
140static void branch_trace_reset(struct trace_array *tr)
141{
142 stop_branch_trace(tr);
143}
144
145struct tracer branch_trace __read_mostly =
146{
147 .name = "branch",
148 .init = branch_trace_init,
149 .reset = branch_trace_reset,
150#ifdef CONFIG_FTRACE_SELFTEST
151 .selftest = trace_selftest_startup_branch,
152#endif
153};
154
155__init static int init_branch_trace(void)
156{
157 return register_tracer(&branch_trace);
158}
159
160device_initcall(init_branch_trace);
161#else
162static inline
163void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
164{
165}
166#endif /* CONFIG_BRANCH_TRACER */
167
168void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
169{
170 /*
171 * I would love to have a trace point here instead, but the
172 * trace point code is so inundated with unlikely and likely
173 * conditions that the recursive nightmare that exists is too
174 * much to try to get working. At least for now.
175 */
176 trace_likely_condition(f, val, expect);
177
178 /* FIXME: Make this atomic! */
179 if (val == expect)
180 f->correct++;
181 else
182 f->incorrect++;
183}
184EXPORT_SYMBOL(ftrace_likely_update);
185
186struct ftrace_pointer {
187 void *start;
188 void *stop;
189 int hit;
190};
191
192static void *
193t_next(struct seq_file *m, void *v, loff_t *pos)
194{
195 const struct ftrace_pointer *f = m->private;
196 struct ftrace_branch_data *p = v;
197
198 (*pos)++;
199
200 if (v == (void *)1)
201 return f->start;
202
203 ++p;
204
205 if ((void *)p >= (void *)f->stop)
206 return NULL;
207
208 return p;
209}
210
211static void *t_start(struct seq_file *m, loff_t *pos)
212{
213 void *t = (void *)1;
214 loff_t l = 0;
215
216 for (; t && l < *pos; t = t_next(m, t, &l))
217 ;
218
219 return t;
220}
221
222static void t_stop(struct seq_file *m, void *p)
223{
224}
225
226static int t_show(struct seq_file *m, void *v)
227{
228 const struct ftrace_pointer *fp = m->private;
229 struct ftrace_branch_data *p = v;
230 const char *f;
231 long percent;
232
233 if (v == (void *)1) {
234 if (fp->hit)
235 seq_printf(m, " miss hit %% ");
236 else
237 seq_printf(m, " correct incorrect %% ");
238 seq_printf(m, " Function "
239 " File Line\n"
240 " ------- --------- - "
241 " -------- "
242 " ---- ----\n");
243 return 0;
244 }
245
246 /* Only print the file, not the path */
247 f = p->file + strlen(p->file);
248 while (f >= p->file && *f != '/')
249 f--;
250 f++;
251
252 /*
253 * The miss is overlayed on correct, and hit on incorrect.
254 */
255 if (p->correct) {
256 percent = p->incorrect * 100;
257 percent /= p->correct + p->incorrect;
258 } else
259 percent = p->incorrect ? 100 : -1;
260
261 seq_printf(m, "%8lu %8lu ", p->correct, p->incorrect);
262 if (percent < 0)
263 seq_printf(m, " X ");
264 else
265 seq_printf(m, "%3ld ", percent);
266 seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
267 return 0;
268}
269
270static struct seq_operations tracing_likely_seq_ops = {
271 .start = t_start,
272 .next = t_next,
273 .stop = t_stop,
274 .show = t_show,
275};
276
277static int tracing_branch_open(struct inode *inode, struct file *file)
278{
279 int ret;
280
281 ret = seq_open(file, &tracing_likely_seq_ops);
282 if (!ret) {
283 struct seq_file *m = file->private_data;
284 m->private = (void *)inode->i_private;
285 }
286
287 return ret;
288}
289
290static const struct file_operations tracing_branch_fops = {
291 .open = tracing_branch_open,
292 .read = seq_read,
293 .llseek = seq_lseek,
294};
295
296#ifdef CONFIG_PROFILE_ALL_BRANCHES
297extern unsigned long __start_branch_profile[];
298extern unsigned long __stop_branch_profile[];
299
300static const struct ftrace_pointer ftrace_branch_pos = {
301 .start = __start_branch_profile,
302 .stop = __stop_branch_profile,
303 .hit = 1,
304};
305
306#endif /* CONFIG_PROFILE_ALL_BRANCHES */
307
308extern unsigned long __start_annotated_branch_profile[];
309extern unsigned long __stop_annotated_branch_profile[];
310
311static const struct ftrace_pointer ftrace_annotated_branch_pos = {
312 .start = __start_annotated_branch_profile,
313 .stop = __stop_annotated_branch_profile,
314};
315
316static __init int ftrace_branch_init(void)
317{
318 struct dentry *d_tracer;
319 struct dentry *entry;
320
321 d_tracer = tracing_init_dentry();
322
323 entry = debugfs_create_file("profile_annotated_branch", 0444, d_tracer,
324 (void *)&ftrace_annotated_branch_pos,
325 &tracing_branch_fops);
326 if (!entry)
327 pr_warning("Could not create debugfs "
328 "'profile_annotatet_branch' entry\n");
329
330#ifdef CONFIG_PROFILE_ALL_BRANCHES
331 entry = debugfs_create_file("profile_branch", 0444, d_tracer,
332 (void *)&ftrace_branch_pos,
333 &tracing_branch_fops);
334 if (!entry)
335 pr_warning("Could not create debugfs"
336 " 'profile_branch' entry\n");
337#endif
338
339 return 0;
340}
341
342device_initcall(ftrace_branch_init);
diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c
index 0f85a64003d3..9236d7e25a16 100644
--- a/kernel/trace/trace_functions.c
+++ b/kernel/trace/trace_functions.c
@@ -16,20 +16,10 @@
16 16
17#include "trace.h" 17#include "trace.h"
18 18
19static void function_reset(struct trace_array *tr)
20{
21 int cpu;
22
23 tr->time_start = ftrace_now(tr->cpu);
24
25 for_each_online_cpu(cpu)
26 tracing_reset(tr, cpu);
27}
28
29static void start_function_trace(struct trace_array *tr) 19static void start_function_trace(struct trace_array *tr)
30{ 20{
31 tr->cpu = get_cpu(); 21 tr->cpu = get_cpu();
32 function_reset(tr); 22 tracing_reset_online_cpus(tr);
33 put_cpu(); 23 put_cpu();
34 24
35 tracing_start_cmdline_record(); 25 tracing_start_cmdline_record();
@@ -42,24 +32,20 @@ static void stop_function_trace(struct trace_array *tr)
42 tracing_stop_cmdline_record(); 32 tracing_stop_cmdline_record();
43} 33}
44 34
45static void function_trace_init(struct trace_array *tr) 35static int function_trace_init(struct trace_array *tr)
46{ 36{
47 if (tr->ctrl) 37 start_function_trace(tr);
48 start_function_trace(tr); 38 return 0;
49} 39}
50 40
51static void function_trace_reset(struct trace_array *tr) 41static void function_trace_reset(struct trace_array *tr)
52{ 42{
53 if (tr->ctrl) 43 stop_function_trace(tr);
54 stop_function_trace(tr);
55} 44}
56 45
57static void function_trace_ctrl_update(struct trace_array *tr) 46static void function_trace_start(struct trace_array *tr)
58{ 47{
59 if (tr->ctrl) 48 tracing_reset_online_cpus(tr);
60 start_function_trace(tr);
61 else
62 stop_function_trace(tr);
63} 49}
64 50
65static struct tracer function_trace __read_mostly = 51static struct tracer function_trace __read_mostly =
@@ -67,7 +53,7 @@ static struct tracer function_trace __read_mostly =
67 .name = "function", 53 .name = "function",
68 .init = function_trace_init, 54 .init = function_trace_init,
69 .reset = function_trace_reset, 55 .reset = function_trace_reset,
70 .ctrl_update = function_trace_ctrl_update, 56 .start = function_trace_start,
71#ifdef CONFIG_FTRACE_SELFTEST 57#ifdef CONFIG_FTRACE_SELFTEST
72 .selftest = trace_selftest_startup_function, 58 .selftest = trace_selftest_startup_function,
73#endif 59#endif
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
new file mode 100644
index 000000000000..930c08e5b38e
--- /dev/null
+++ b/kernel/trace/trace_functions_graph.c
@@ -0,0 +1,669 @@
1/*
2 *
3 * Function graph tracer.
4 * Copyright (c) 2008 Frederic Weisbecker <fweisbec@gmail.com>
5 * Mostly borrowed from function tracer which
6 * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
7 *
8 */
9#include <linux/debugfs.h>
10#include <linux/uaccess.h>
11#include <linux/ftrace.h>
12#include <linux/fs.h>
13
14#include "trace.h"
15
16#define TRACE_GRAPH_INDENT 2
17
18/* Flag options */
19#define TRACE_GRAPH_PRINT_OVERRUN 0x1
20#define TRACE_GRAPH_PRINT_CPU 0x2
21#define TRACE_GRAPH_PRINT_OVERHEAD 0x4
22#define TRACE_GRAPH_PRINT_PROC 0x8
23
24static struct tracer_opt trace_opts[] = {
25 /* Display overruns ? */
26 { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
27 /* Display CPU ? */
28 { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
29 /* Display Overhead ? */
30 { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
31 /* Display proc name/pid */
32 { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
33 { } /* Empty entry */
34};
35
36static struct tracer_flags tracer_flags = {
37 /* Don't display overruns and proc by default */
38 .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD,
39 .opts = trace_opts
40};
41
42/* pid on the last trace processed */
43static pid_t last_pid[NR_CPUS] = { [0 ... NR_CPUS-1] = -1 };
44
45static int graph_trace_init(struct trace_array *tr)
46{
47 int cpu, ret;
48
49 for_each_online_cpu(cpu)
50 tracing_reset(tr, cpu);
51
52 ret = register_ftrace_graph(&trace_graph_return,
53 &trace_graph_entry);
54 if (ret)
55 return ret;
56 tracing_start_cmdline_record();
57
58 return 0;
59}
60
61static void graph_trace_reset(struct trace_array *tr)
62{
63 tracing_stop_cmdline_record();
64 unregister_ftrace_graph();
65}
66
67static inline int log10_cpu(int nb)
68{
69 if (nb / 100)
70 return 3;
71 if (nb / 10)
72 return 2;
73 return 1;
74}
75
76static enum print_line_t
77print_graph_cpu(struct trace_seq *s, int cpu)
78{
79 int i;
80 int ret;
81 int log10_this = log10_cpu(cpu);
82 int log10_all = log10_cpu(cpumask_weight(cpu_online_mask));
83
84
85 /*
86 * Start with a space character - to make it stand out
87 * to the right a bit when trace output is pasted into
88 * email:
89 */
90 ret = trace_seq_printf(s, " ");
91
92 /*
93 * Tricky - we space the CPU field according to the max
94 * number of online CPUs. On a 2-cpu system it would take
95 * a maximum of 1 digit - on a 128 cpu system it would
96 * take up to 3 digits:
97 */
98 for (i = 0; i < log10_all - log10_this; i++) {
99 ret = trace_seq_printf(s, " ");
100 if (!ret)
101 return TRACE_TYPE_PARTIAL_LINE;
102 }
103 ret = trace_seq_printf(s, "%d) ", cpu);
104 if (!ret)
105 return TRACE_TYPE_PARTIAL_LINE;
106
107 return TRACE_TYPE_HANDLED;
108}
109
110#define TRACE_GRAPH_PROCINFO_LENGTH 14
111
112static enum print_line_t
113print_graph_proc(struct trace_seq *s, pid_t pid)
114{
115 int i;
116 int ret;
117 int len;
118 char comm[8];
119 int spaces = 0;
120 /* sign + log10(MAX_INT) + '\0' */
121 char pid_str[11];
122
123 strncpy(comm, trace_find_cmdline(pid), 7);
124 comm[7] = '\0';
125 sprintf(pid_str, "%d", pid);
126
127 /* 1 stands for the "-" character */
128 len = strlen(comm) + strlen(pid_str) + 1;
129
130 if (len < TRACE_GRAPH_PROCINFO_LENGTH)
131 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len;
132
133 /* First spaces to align center */
134 for (i = 0; i < spaces / 2; i++) {
135 ret = trace_seq_printf(s, " ");
136 if (!ret)
137 return TRACE_TYPE_PARTIAL_LINE;
138 }
139
140 ret = trace_seq_printf(s, "%s-%s", comm, pid_str);
141 if (!ret)
142 return TRACE_TYPE_PARTIAL_LINE;
143
144 /* Last spaces to align center */
145 for (i = 0; i < spaces - (spaces / 2); i++) {
146 ret = trace_seq_printf(s, " ");
147 if (!ret)
148 return TRACE_TYPE_PARTIAL_LINE;
149 }
150 return TRACE_TYPE_HANDLED;
151}
152
153
154/* If the pid changed since the last trace, output this event */
155static enum print_line_t
156verif_pid(struct trace_seq *s, pid_t pid, int cpu)
157{
158 pid_t prev_pid;
159 int ret;
160
161 if (last_pid[cpu] != -1 && last_pid[cpu] == pid)
162 return TRACE_TYPE_HANDLED;
163
164 prev_pid = last_pid[cpu];
165 last_pid[cpu] = pid;
166
167/*
168 * Context-switch trace line:
169
170 ------------------------------------------
171 | 1) migration/0--1 => sshd-1755
172 ------------------------------------------
173
174 */
175 ret = trace_seq_printf(s,
176 " ------------------------------------------\n");
177 if (!ret)
178 TRACE_TYPE_PARTIAL_LINE;
179
180 ret = print_graph_cpu(s, cpu);
181 if (ret == TRACE_TYPE_PARTIAL_LINE)
182 TRACE_TYPE_PARTIAL_LINE;
183
184 ret = print_graph_proc(s, prev_pid);
185 if (ret == TRACE_TYPE_PARTIAL_LINE)
186 TRACE_TYPE_PARTIAL_LINE;
187
188 ret = trace_seq_printf(s, " => ");
189 if (!ret)
190 TRACE_TYPE_PARTIAL_LINE;
191
192 ret = print_graph_proc(s, pid);
193 if (ret == TRACE_TYPE_PARTIAL_LINE)
194 TRACE_TYPE_PARTIAL_LINE;
195
196 ret = trace_seq_printf(s,
197 "\n ------------------------------------------\n\n");
198 if (!ret)
199 TRACE_TYPE_PARTIAL_LINE;
200
201 return ret;
202}
203
204static bool
205trace_branch_is_leaf(struct trace_iterator *iter,
206 struct ftrace_graph_ent_entry *curr)
207{
208 struct ring_buffer_iter *ring_iter;
209 struct ring_buffer_event *event;
210 struct ftrace_graph_ret_entry *next;
211
212 ring_iter = iter->buffer_iter[iter->cpu];
213
214 if (!ring_iter)
215 return false;
216
217 event = ring_buffer_iter_peek(ring_iter, NULL);
218
219 if (!event)
220 return false;
221
222 next = ring_buffer_event_data(event);
223
224 if (next->ent.type != TRACE_GRAPH_RET)
225 return false;
226
227 if (curr->ent.pid != next->ent.pid ||
228 curr->graph_ent.func != next->ret.func)
229 return false;
230
231 return true;
232}
233
234static enum print_line_t
235print_graph_irq(struct trace_seq *s, unsigned long addr,
236 enum trace_type type, int cpu, pid_t pid)
237{
238 int ret;
239
240 if (addr < (unsigned long)__irqentry_text_start ||
241 addr >= (unsigned long)__irqentry_text_end)
242 return TRACE_TYPE_UNHANDLED;
243
244 if (type == TRACE_GRAPH_ENT) {
245 ret = trace_seq_printf(s, "==========> | ");
246 } else {
247 /* Cpu */
248 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
249 ret = print_graph_cpu(s, cpu);
250 if (ret == TRACE_TYPE_PARTIAL_LINE)
251 return TRACE_TYPE_PARTIAL_LINE;
252 }
253 /* Proc */
254 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
255 ret = print_graph_proc(s, pid);
256 if (ret == TRACE_TYPE_PARTIAL_LINE)
257 return TRACE_TYPE_PARTIAL_LINE;
258
259 ret = trace_seq_printf(s, " | ");
260 if (!ret)
261 return TRACE_TYPE_PARTIAL_LINE;
262 }
263
264 /* No overhead */
265 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
266 ret = trace_seq_printf(s, " ");
267 if (!ret)
268 return TRACE_TYPE_PARTIAL_LINE;
269 }
270
271 ret = trace_seq_printf(s, "<========== |\n");
272 }
273 if (!ret)
274 return TRACE_TYPE_PARTIAL_LINE;
275 return TRACE_TYPE_HANDLED;
276}
277
278static enum print_line_t
279print_graph_duration(unsigned long long duration, struct trace_seq *s)
280{
281 unsigned long nsecs_rem = do_div(duration, 1000);
282 /* log10(ULONG_MAX) + '\0' */
283 char msecs_str[21];
284 char nsecs_str[5];
285 int ret, len;
286 int i;
287
288 sprintf(msecs_str, "%lu", (unsigned long) duration);
289
290 /* Print msecs */
291 ret = trace_seq_printf(s, msecs_str);
292 if (!ret)
293 return TRACE_TYPE_PARTIAL_LINE;
294
295 len = strlen(msecs_str);
296
297 /* Print nsecs (we don't want to exceed 7 numbers) */
298 if (len < 7) {
299 snprintf(nsecs_str, 8 - len, "%03lu", nsecs_rem);
300 ret = trace_seq_printf(s, ".%s", nsecs_str);
301 if (!ret)
302 return TRACE_TYPE_PARTIAL_LINE;
303 len += strlen(nsecs_str);
304 }
305
306 ret = trace_seq_printf(s, " us ");
307 if (!ret)
308 return TRACE_TYPE_PARTIAL_LINE;
309
310 /* Print remaining spaces to fit the row's width */
311 for (i = len; i < 7; i++) {
312 ret = trace_seq_printf(s, " ");
313 if (!ret)
314 return TRACE_TYPE_PARTIAL_LINE;
315 }
316
317 ret = trace_seq_printf(s, "| ");
318 if (!ret)
319 return TRACE_TYPE_PARTIAL_LINE;
320 return TRACE_TYPE_HANDLED;
321
322}
323
324/* Signal a overhead of time execution to the output */
325static int
326print_graph_overhead(unsigned long long duration, struct trace_seq *s)
327{
328 /* Duration exceeded 100 msecs */
329 if (duration > 100000ULL)
330 return trace_seq_printf(s, "! ");
331
332 /* Duration exceeded 10 msecs */
333 if (duration > 10000ULL)
334 return trace_seq_printf(s, "+ ");
335
336 return trace_seq_printf(s, " ");
337}
338
339/* Case of a leaf function on its call entry */
340static enum print_line_t
341print_graph_entry_leaf(struct trace_iterator *iter,
342 struct ftrace_graph_ent_entry *entry, struct trace_seq *s)
343{
344 struct ftrace_graph_ret_entry *ret_entry;
345 struct ftrace_graph_ret *graph_ret;
346 struct ring_buffer_event *event;
347 struct ftrace_graph_ent *call;
348 unsigned long long duration;
349 int ret;
350 int i;
351
352 event = ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
353 ret_entry = ring_buffer_event_data(event);
354 graph_ret = &ret_entry->ret;
355 call = &entry->graph_ent;
356 duration = graph_ret->rettime - graph_ret->calltime;
357
358 /* Overhead */
359 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
360 ret = print_graph_overhead(duration, s);
361 if (!ret)
362 return TRACE_TYPE_PARTIAL_LINE;
363 }
364
365 /* Duration */
366 ret = print_graph_duration(duration, s);
367 if (ret == TRACE_TYPE_PARTIAL_LINE)
368 return TRACE_TYPE_PARTIAL_LINE;
369
370 /* Function */
371 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
372 ret = trace_seq_printf(s, " ");
373 if (!ret)
374 return TRACE_TYPE_PARTIAL_LINE;
375 }
376
377 ret = seq_print_ip_sym(s, call->func, 0);
378 if (!ret)
379 return TRACE_TYPE_PARTIAL_LINE;
380
381 ret = trace_seq_printf(s, "();\n");
382 if (!ret)
383 return TRACE_TYPE_PARTIAL_LINE;
384
385 return TRACE_TYPE_HANDLED;
386}
387
388static enum print_line_t
389print_graph_entry_nested(struct ftrace_graph_ent_entry *entry,
390 struct trace_seq *s, pid_t pid, int cpu)
391{
392 int i;
393 int ret;
394 struct ftrace_graph_ent *call = &entry->graph_ent;
395
396 /* No overhead */
397 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
398 ret = trace_seq_printf(s, " ");
399 if (!ret)
400 return TRACE_TYPE_PARTIAL_LINE;
401 }
402
403 /* Interrupt */
404 ret = print_graph_irq(s, call->func, TRACE_GRAPH_ENT, cpu, pid);
405 if (ret == TRACE_TYPE_UNHANDLED) {
406 /* No time */
407 ret = trace_seq_printf(s, " | ");
408 if (!ret)
409 return TRACE_TYPE_PARTIAL_LINE;
410 } else {
411 if (ret == TRACE_TYPE_PARTIAL_LINE)
412 return TRACE_TYPE_PARTIAL_LINE;
413 }
414
415
416 /* Function */
417 for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
418 ret = trace_seq_printf(s, " ");
419 if (!ret)
420 return TRACE_TYPE_PARTIAL_LINE;
421 }
422
423 ret = seq_print_ip_sym(s, call->func, 0);
424 if (!ret)
425 return TRACE_TYPE_PARTIAL_LINE;
426
427 ret = trace_seq_printf(s, "() {\n");
428 if (!ret)
429 return TRACE_TYPE_PARTIAL_LINE;
430
431 return TRACE_TYPE_HANDLED;
432}
433
434static enum print_line_t
435print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
436 struct trace_iterator *iter, int cpu)
437{
438 int ret;
439 struct trace_entry *ent = iter->ent;
440
441 /* Pid */
442 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE)
443 return TRACE_TYPE_PARTIAL_LINE;
444
445 /* Cpu */
446 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
447 ret = print_graph_cpu(s, cpu);
448 if (ret == TRACE_TYPE_PARTIAL_LINE)
449 return TRACE_TYPE_PARTIAL_LINE;
450 }
451
452 /* Proc */
453 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
454 ret = print_graph_proc(s, ent->pid);
455 if (ret == TRACE_TYPE_PARTIAL_LINE)
456 return TRACE_TYPE_PARTIAL_LINE;
457
458 ret = trace_seq_printf(s, " | ");
459 if (!ret)
460 return TRACE_TYPE_PARTIAL_LINE;
461 }
462
463 if (trace_branch_is_leaf(iter, field))
464 return print_graph_entry_leaf(iter, field, s);
465 else
466 return print_graph_entry_nested(field, s, iter->ent->pid, cpu);
467
468}
469
470static enum print_line_t
471print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
472 struct trace_entry *ent, int cpu)
473{
474 int i;
475 int ret;
476 unsigned long long duration = trace->rettime - trace->calltime;
477
478 /* Pid */
479 if (verif_pid(s, ent->pid, cpu) == TRACE_TYPE_PARTIAL_LINE)
480 return TRACE_TYPE_PARTIAL_LINE;
481
482 /* Cpu */
483 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
484 ret = print_graph_cpu(s, cpu);
485 if (ret == TRACE_TYPE_PARTIAL_LINE)
486 return TRACE_TYPE_PARTIAL_LINE;
487 }
488
489 /* Proc */
490 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
491 ret = print_graph_proc(s, ent->pid);
492 if (ret == TRACE_TYPE_PARTIAL_LINE)
493 return TRACE_TYPE_PARTIAL_LINE;
494
495 ret = trace_seq_printf(s, " | ");
496 if (!ret)
497 return TRACE_TYPE_PARTIAL_LINE;
498 }
499
500 /* Overhead */
501 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
502 ret = print_graph_overhead(duration, s);
503 if (!ret)
504 return TRACE_TYPE_PARTIAL_LINE;
505 }
506
507 /* Duration */
508 ret = print_graph_duration(duration, s);
509 if (ret == TRACE_TYPE_PARTIAL_LINE)
510 return TRACE_TYPE_PARTIAL_LINE;
511
512 /* Closing brace */
513 for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
514 ret = trace_seq_printf(s, " ");
515 if (!ret)
516 return TRACE_TYPE_PARTIAL_LINE;
517 }
518
519 ret = trace_seq_printf(s, "}\n");
520 if (!ret)
521 return TRACE_TYPE_PARTIAL_LINE;
522
523 /* Overrun */
524 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
525 ret = trace_seq_printf(s, " (Overruns: %lu)\n",
526 trace->overrun);
527 if (!ret)
528 return TRACE_TYPE_PARTIAL_LINE;
529 }
530
531 ret = print_graph_irq(s, trace->func, TRACE_GRAPH_RET, cpu, ent->pid);
532 if (ret == TRACE_TYPE_PARTIAL_LINE)
533 return TRACE_TYPE_PARTIAL_LINE;
534
535 return TRACE_TYPE_HANDLED;
536}
537
538static enum print_line_t
539print_graph_comment(struct print_entry *trace, struct trace_seq *s,
540 struct trace_entry *ent, struct trace_iterator *iter)
541{
542 int i;
543 int ret;
544
545 /* Pid */
546 if (verif_pid(s, ent->pid, iter->cpu) == TRACE_TYPE_PARTIAL_LINE)
547 return TRACE_TYPE_PARTIAL_LINE;
548
549 /* Cpu */
550 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
551 ret = print_graph_cpu(s, iter->cpu);
552 if (ret == TRACE_TYPE_PARTIAL_LINE)
553 return TRACE_TYPE_PARTIAL_LINE;
554 }
555
556 /* Proc */
557 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
558 ret = print_graph_proc(s, ent->pid);
559 if (ret == TRACE_TYPE_PARTIAL_LINE)
560 return TRACE_TYPE_PARTIAL_LINE;
561
562 ret = trace_seq_printf(s, " | ");
563 if (!ret)
564 return TRACE_TYPE_PARTIAL_LINE;
565 }
566
567 /* No overhead */
568 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
569 ret = trace_seq_printf(s, " ");
570 if (!ret)
571 return TRACE_TYPE_PARTIAL_LINE;
572 }
573
574 /* No time */
575 ret = trace_seq_printf(s, " | ");
576 if (!ret)
577 return TRACE_TYPE_PARTIAL_LINE;
578
579 /* Indentation */
580 if (trace->depth > 0)
581 for (i = 0; i < (trace->depth + 1) * TRACE_GRAPH_INDENT; i++) {
582 ret = trace_seq_printf(s, " ");
583 if (!ret)
584 return TRACE_TYPE_PARTIAL_LINE;
585 }
586
587 /* The comment */
588 ret = trace_seq_printf(s, "/* %s", trace->buf);
589 if (!ret)
590 return TRACE_TYPE_PARTIAL_LINE;
591
592 if (ent->flags & TRACE_FLAG_CONT)
593 trace_seq_print_cont(s, iter);
594
595 ret = trace_seq_printf(s, " */\n");
596 if (!ret)
597 return TRACE_TYPE_PARTIAL_LINE;
598
599 return TRACE_TYPE_HANDLED;
600}
601
602
603enum print_line_t
604print_graph_function(struct trace_iterator *iter)
605{
606 struct trace_seq *s = &iter->seq;
607 struct trace_entry *entry = iter->ent;
608
609 switch (entry->type) {
610 case TRACE_GRAPH_ENT: {
611 struct ftrace_graph_ent_entry *field;
612 trace_assign_type(field, entry);
613 return print_graph_entry(field, s, iter,
614 iter->cpu);
615 }
616 case TRACE_GRAPH_RET: {
617 struct ftrace_graph_ret_entry *field;
618 trace_assign_type(field, entry);
619 return print_graph_return(&field->ret, s, entry, iter->cpu);
620 }
621 case TRACE_PRINT: {
622 struct print_entry *field;
623 trace_assign_type(field, entry);
624 return print_graph_comment(field, s, entry, iter);
625 }
626 default:
627 return TRACE_TYPE_UNHANDLED;
628 }
629}
630
631static void print_graph_headers(struct seq_file *s)
632{
633 /* 1st line */
634 seq_printf(s, "# ");
635 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
636 seq_printf(s, "CPU ");
637 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
638 seq_printf(s, "TASK/PID ");
639 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD)
640 seq_printf(s, "OVERHEAD/");
641 seq_printf(s, "DURATION FUNCTION CALLS\n");
642
643 /* 2nd line */
644 seq_printf(s, "# ");
645 if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
646 seq_printf(s, "| ");
647 if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
648 seq_printf(s, "| | ");
649 if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
650 seq_printf(s, "| ");
651 seq_printf(s, "| | | | |\n");
652 } else
653 seq_printf(s, " | | | | |\n");
654}
655static struct tracer graph_trace __read_mostly = {
656 .name = "function_graph",
657 .init = graph_trace_init,
658 .reset = graph_trace_reset,
659 .print_line = print_graph_function,
660 .print_header = print_graph_headers,
661 .flags = &tracer_flags,
662};
663
664static __init int init_graph_trace(void)
665{
666 return register_tracer(&graph_trace);
667}
668
669device_initcall(init_graph_trace);
diff --git a/kernel/trace/trace_hw_branches.c b/kernel/trace/trace_hw_branches.c
new file mode 100644
index 000000000000..649df22d435f
--- /dev/null
+++ b/kernel/trace/trace_hw_branches.c
@@ -0,0 +1,195 @@
1/*
2 * h/w branch tracer for x86 based on bts
3 *
4 * Copyright (C) 2008 Markus Metzger <markus.t.metzger@gmail.com>
5 *
6 */
7
8#include <linux/module.h>
9#include <linux/fs.h>
10#include <linux/debugfs.h>
11#include <linux/ftrace.h>
12#include <linux/kallsyms.h>
13
14#include <asm/ds.h>
15
16#include "trace.h"
17
18
19#define SIZEOF_BTS (1 << 13)
20
21static DEFINE_PER_CPU(struct bts_tracer *, tracer);
22static DEFINE_PER_CPU(unsigned char[SIZEOF_BTS], buffer);
23
24#define this_tracer per_cpu(tracer, smp_processor_id())
25#define this_buffer per_cpu(buffer, smp_processor_id())
26
27
28static void bts_trace_start_cpu(void *arg)
29{
30 if (this_tracer)
31 ds_release_bts(this_tracer);
32
33 this_tracer =
34 ds_request_bts(/* task = */ NULL, this_buffer, SIZEOF_BTS,
35 /* ovfl = */ NULL, /* th = */ (size_t)-1,
36 BTS_KERNEL);
37 if (IS_ERR(this_tracer)) {
38 this_tracer = NULL;
39 return;
40 }
41}
42
43static void bts_trace_start(struct trace_array *tr)
44{
45 int cpu;
46
47 tracing_reset_online_cpus(tr);
48
49 for_each_cpu(cpu, cpu_possible_mask)
50 smp_call_function_single(cpu, bts_trace_start_cpu, NULL, 1);
51}
52
53static void bts_trace_stop_cpu(void *arg)
54{
55 if (this_tracer) {
56 ds_release_bts(this_tracer);
57 this_tracer = NULL;
58 }
59}
60
61static void bts_trace_stop(struct trace_array *tr)
62{
63 int cpu;
64
65 for_each_cpu(cpu, cpu_possible_mask)
66 smp_call_function_single(cpu, bts_trace_stop_cpu, NULL, 1);
67}
68
69static int bts_trace_init(struct trace_array *tr)
70{
71 tracing_reset_online_cpus(tr);
72 bts_trace_start(tr);
73
74 return 0;
75}
76
77static void bts_trace_print_header(struct seq_file *m)
78{
79 seq_puts(m,
80 "# CPU# FROM TO FUNCTION\n");
81 seq_puts(m,
82 "# | | | |\n");
83}
84
85static enum print_line_t bts_trace_print_line(struct trace_iterator *iter)
86{
87 struct trace_entry *entry = iter->ent;
88 struct trace_seq *seq = &iter->seq;
89 struct hw_branch_entry *it;
90
91 trace_assign_type(it, entry);
92
93 if (entry->type == TRACE_HW_BRANCHES) {
94 if (trace_seq_printf(seq, "%4d ", entry->cpu) &&
95 trace_seq_printf(seq, "0x%016llx -> 0x%016llx ",
96 it->from, it->to) &&
97 (!it->from ||
98 seq_print_ip_sym(seq, it->from, /* sym_flags = */ 0)) &&
99 trace_seq_printf(seq, "\n"))
100 return TRACE_TYPE_HANDLED;
101 return TRACE_TYPE_PARTIAL_LINE;;
102 }
103 return TRACE_TYPE_UNHANDLED;
104}
105
106void trace_hw_branch(struct trace_array *tr, u64 from, u64 to)
107{
108 struct ring_buffer_event *event;
109 struct hw_branch_entry *entry;
110 unsigned long irq;
111
112 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry), &irq);
113 if (!event)
114 return;
115 entry = ring_buffer_event_data(event);
116 tracing_generic_entry_update(&entry->ent, 0, from);
117 entry->ent.type = TRACE_HW_BRANCHES;
118 entry->ent.cpu = smp_processor_id();
119 entry->from = from;
120 entry->to = to;
121 ring_buffer_unlock_commit(tr->buffer, event, irq);
122}
123
124static void trace_bts_at(struct trace_array *tr,
125 const struct bts_trace *trace, void *at)
126{
127 struct bts_struct bts;
128 int err = 0;
129
130 WARN_ON_ONCE(!trace->read);
131 if (!trace->read)
132 return;
133
134 err = trace->read(this_tracer, at, &bts);
135 if (err < 0)
136 return;
137
138 switch (bts.qualifier) {
139 case BTS_BRANCH:
140 trace_hw_branch(tr, bts.variant.lbr.from, bts.variant.lbr.to);
141 break;
142 }
143}
144
145static void trace_bts_cpu(void *arg)
146{
147 struct trace_array *tr = (struct trace_array *) arg;
148 const struct bts_trace *trace;
149 unsigned char *at;
150
151 if (!this_tracer)
152 return;
153
154 ds_suspend_bts(this_tracer);
155 trace = ds_read_bts(this_tracer);
156 if (!trace)
157 goto out;
158
159 for (at = trace->ds.top; (void *)at < trace->ds.end;
160 at += trace->ds.size)
161 trace_bts_at(tr, trace, at);
162
163 for (at = trace->ds.begin; (void *)at < trace->ds.top;
164 at += trace->ds.size)
165 trace_bts_at(tr, trace, at);
166
167out:
168 ds_resume_bts(this_tracer);
169}
170
171static void trace_bts_prepare(struct trace_iterator *iter)
172{
173 int cpu;
174
175 for_each_cpu(cpu, cpu_possible_mask)
176 smp_call_function_single(cpu, trace_bts_cpu, iter->tr, 1);
177}
178
179struct tracer bts_tracer __read_mostly =
180{
181 .name = "hw-branch-tracer",
182 .init = bts_trace_init,
183 .reset = bts_trace_stop,
184 .print_header = bts_trace_print_header,
185 .print_line = bts_trace_print_line,
186 .start = bts_trace_start,
187 .stop = bts_trace_stop,
188 .open = trace_bts_prepare
189};
190
191__init static int init_bts_trace(void)
192{
193 return register_tracer(&bts_tracer);
194}
195device_initcall(init_bts_trace);
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index 9c74071c10e0..7c2e326bbc8b 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -353,15 +353,28 @@ void trace_preempt_off(unsigned long a0, unsigned long a1)
353} 353}
354#endif /* CONFIG_PREEMPT_TRACER */ 354#endif /* CONFIG_PREEMPT_TRACER */
355 355
356/*
357 * save_tracer_enabled is used to save the state of the tracer_enabled
358 * variable when we disable it when we open a trace output file.
359 */
360static int save_tracer_enabled;
361
356static void start_irqsoff_tracer(struct trace_array *tr) 362static void start_irqsoff_tracer(struct trace_array *tr)
357{ 363{
358 register_ftrace_function(&trace_ops); 364 register_ftrace_function(&trace_ops);
359 tracer_enabled = 1; 365 if (tracing_is_enabled()) {
366 tracer_enabled = 1;
367 save_tracer_enabled = 1;
368 } else {
369 tracer_enabled = 0;
370 save_tracer_enabled = 0;
371 }
360} 372}
361 373
362static void stop_irqsoff_tracer(struct trace_array *tr) 374static void stop_irqsoff_tracer(struct trace_array *tr)
363{ 375{
364 tracer_enabled = 0; 376 tracer_enabled = 0;
377 save_tracer_enabled = 0;
365 unregister_ftrace_function(&trace_ops); 378 unregister_ftrace_function(&trace_ops);
366} 379}
367 380
@@ -370,53 +383,55 @@ static void __irqsoff_tracer_init(struct trace_array *tr)
370 irqsoff_trace = tr; 383 irqsoff_trace = tr;
371 /* make sure that the tracer is visible */ 384 /* make sure that the tracer is visible */
372 smp_wmb(); 385 smp_wmb();
373 386 start_irqsoff_tracer(tr);
374 if (tr->ctrl)
375 start_irqsoff_tracer(tr);
376} 387}
377 388
378static void irqsoff_tracer_reset(struct trace_array *tr) 389static void irqsoff_tracer_reset(struct trace_array *tr)
379{ 390{
380 if (tr->ctrl) 391 stop_irqsoff_tracer(tr);
381 stop_irqsoff_tracer(tr);
382} 392}
383 393
384static void irqsoff_tracer_ctrl_update(struct trace_array *tr) 394static void irqsoff_tracer_start(struct trace_array *tr)
385{ 395{
386 if (tr->ctrl) 396 tracer_enabled = 1;
387 start_irqsoff_tracer(tr); 397 save_tracer_enabled = 1;
388 else 398}
389 stop_irqsoff_tracer(tr); 399
400static void irqsoff_tracer_stop(struct trace_array *tr)
401{
402 tracer_enabled = 0;
403 save_tracer_enabled = 0;
390} 404}
391 405
392static void irqsoff_tracer_open(struct trace_iterator *iter) 406static void irqsoff_tracer_open(struct trace_iterator *iter)
393{ 407{
394 /* stop the trace while dumping */ 408 /* stop the trace while dumping */
395 if (iter->tr->ctrl) 409 tracer_enabled = 0;
396 stop_irqsoff_tracer(iter->tr);
397} 410}
398 411
399static void irqsoff_tracer_close(struct trace_iterator *iter) 412static void irqsoff_tracer_close(struct trace_iterator *iter)
400{ 413{
401 if (iter->tr->ctrl) 414 /* restart tracing */
402 start_irqsoff_tracer(iter->tr); 415 tracer_enabled = save_tracer_enabled;
403} 416}
404 417
405#ifdef CONFIG_IRQSOFF_TRACER 418#ifdef CONFIG_IRQSOFF_TRACER
406static void irqsoff_tracer_init(struct trace_array *tr) 419static int irqsoff_tracer_init(struct trace_array *tr)
407{ 420{
408 trace_type = TRACER_IRQS_OFF; 421 trace_type = TRACER_IRQS_OFF;
409 422
410 __irqsoff_tracer_init(tr); 423 __irqsoff_tracer_init(tr);
424 return 0;
411} 425}
412static struct tracer irqsoff_tracer __read_mostly = 426static struct tracer irqsoff_tracer __read_mostly =
413{ 427{
414 .name = "irqsoff", 428 .name = "irqsoff",
415 .init = irqsoff_tracer_init, 429 .init = irqsoff_tracer_init,
416 .reset = irqsoff_tracer_reset, 430 .reset = irqsoff_tracer_reset,
431 .start = irqsoff_tracer_start,
432 .stop = irqsoff_tracer_stop,
417 .open = irqsoff_tracer_open, 433 .open = irqsoff_tracer_open,
418 .close = irqsoff_tracer_close, 434 .close = irqsoff_tracer_close,
419 .ctrl_update = irqsoff_tracer_ctrl_update,
420 .print_max = 1, 435 .print_max = 1,
421#ifdef CONFIG_FTRACE_SELFTEST 436#ifdef CONFIG_FTRACE_SELFTEST
422 .selftest = trace_selftest_startup_irqsoff, 437 .selftest = trace_selftest_startup_irqsoff,
@@ -428,11 +443,12 @@ static struct tracer irqsoff_tracer __read_mostly =
428#endif 443#endif
429 444
430#ifdef CONFIG_PREEMPT_TRACER 445#ifdef CONFIG_PREEMPT_TRACER
431static void preemptoff_tracer_init(struct trace_array *tr) 446static int preemptoff_tracer_init(struct trace_array *tr)
432{ 447{
433 trace_type = TRACER_PREEMPT_OFF; 448 trace_type = TRACER_PREEMPT_OFF;
434 449
435 __irqsoff_tracer_init(tr); 450 __irqsoff_tracer_init(tr);
451 return 0;
436} 452}
437 453
438static struct tracer preemptoff_tracer __read_mostly = 454static struct tracer preemptoff_tracer __read_mostly =
@@ -440,9 +456,10 @@ static struct tracer preemptoff_tracer __read_mostly =
440 .name = "preemptoff", 456 .name = "preemptoff",
441 .init = preemptoff_tracer_init, 457 .init = preemptoff_tracer_init,
442 .reset = irqsoff_tracer_reset, 458 .reset = irqsoff_tracer_reset,
459 .start = irqsoff_tracer_start,
460 .stop = irqsoff_tracer_stop,
443 .open = irqsoff_tracer_open, 461 .open = irqsoff_tracer_open,
444 .close = irqsoff_tracer_close, 462 .close = irqsoff_tracer_close,
445 .ctrl_update = irqsoff_tracer_ctrl_update,
446 .print_max = 1, 463 .print_max = 1,
447#ifdef CONFIG_FTRACE_SELFTEST 464#ifdef CONFIG_FTRACE_SELFTEST
448 .selftest = trace_selftest_startup_preemptoff, 465 .selftest = trace_selftest_startup_preemptoff,
@@ -456,11 +473,12 @@ static struct tracer preemptoff_tracer __read_mostly =
456#if defined(CONFIG_IRQSOFF_TRACER) && \ 473#if defined(CONFIG_IRQSOFF_TRACER) && \
457 defined(CONFIG_PREEMPT_TRACER) 474 defined(CONFIG_PREEMPT_TRACER)
458 475
459static void preemptirqsoff_tracer_init(struct trace_array *tr) 476static int preemptirqsoff_tracer_init(struct trace_array *tr)
460{ 477{
461 trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF; 478 trace_type = TRACER_IRQS_OFF | TRACER_PREEMPT_OFF;
462 479
463 __irqsoff_tracer_init(tr); 480 __irqsoff_tracer_init(tr);
481 return 0;
464} 482}
465 483
466static struct tracer preemptirqsoff_tracer __read_mostly = 484static struct tracer preemptirqsoff_tracer __read_mostly =
@@ -468,9 +486,10 @@ static struct tracer preemptirqsoff_tracer __read_mostly =
468 .name = "preemptirqsoff", 486 .name = "preemptirqsoff",
469 .init = preemptirqsoff_tracer_init, 487 .init = preemptirqsoff_tracer_init,
470 .reset = irqsoff_tracer_reset, 488 .reset = irqsoff_tracer_reset,
489 .start = irqsoff_tracer_start,
490 .stop = irqsoff_tracer_stop,
471 .open = irqsoff_tracer_open, 491 .open = irqsoff_tracer_open,
472 .close = irqsoff_tracer_close, 492 .close = irqsoff_tracer_close,
473 .ctrl_update = irqsoff_tracer_ctrl_update,
474 .print_max = 1, 493 .print_max = 1,
475#ifdef CONFIG_FTRACE_SELFTEST 494#ifdef CONFIG_FTRACE_SELFTEST
476 .selftest = trace_selftest_startup_preemptirqsoff, 495 .selftest = trace_selftest_startup_preemptirqsoff,
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index f28484618ff0..fffcb069f1dc 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -18,46 +18,39 @@ struct header_iter {
18 18
19static struct trace_array *mmio_trace_array; 19static struct trace_array *mmio_trace_array;
20static bool overrun_detected; 20static bool overrun_detected;
21static unsigned long prev_overruns;
21 22
22static void mmio_reset_data(struct trace_array *tr) 23static void mmio_reset_data(struct trace_array *tr)
23{ 24{
24 int cpu;
25
26 overrun_detected = false; 25 overrun_detected = false;
27 tr->time_start = ftrace_now(tr->cpu); 26 prev_overruns = 0;
28 27
29 for_each_online_cpu(cpu) 28 tracing_reset_online_cpus(tr);
30 tracing_reset(tr, cpu);
31} 29}
32 30
33static void mmio_trace_init(struct trace_array *tr) 31static int mmio_trace_init(struct trace_array *tr)
34{ 32{
35 pr_debug("in %s\n", __func__); 33 pr_debug("in %s\n", __func__);
36 mmio_trace_array = tr; 34 mmio_trace_array = tr;
37 if (tr->ctrl) { 35
38 mmio_reset_data(tr); 36 mmio_reset_data(tr);
39 enable_mmiotrace(); 37 enable_mmiotrace();
40 } 38 return 0;
41} 39}
42 40
43static void mmio_trace_reset(struct trace_array *tr) 41static void mmio_trace_reset(struct trace_array *tr)
44{ 42{
45 pr_debug("in %s\n", __func__); 43 pr_debug("in %s\n", __func__);
46 if (tr->ctrl) 44
47 disable_mmiotrace(); 45 disable_mmiotrace();
48 mmio_reset_data(tr); 46 mmio_reset_data(tr);
49 mmio_trace_array = NULL; 47 mmio_trace_array = NULL;
50} 48}
51 49
52static void mmio_trace_ctrl_update(struct trace_array *tr) 50static void mmio_trace_start(struct trace_array *tr)
53{ 51{
54 pr_debug("in %s\n", __func__); 52 pr_debug("in %s\n", __func__);
55 if (tr->ctrl) { 53 mmio_reset_data(tr);
56 mmio_reset_data(tr);
57 enable_mmiotrace();
58 } else {
59 disable_mmiotrace();
60 }
61} 54}
62 55
63static int mmio_print_pcidev(struct trace_seq *s, const struct pci_dev *dev) 56static int mmio_print_pcidev(struct trace_seq *s, const struct pci_dev *dev)
@@ -128,16 +121,12 @@ static void mmio_close(struct trace_iterator *iter)
128 121
129static unsigned long count_overruns(struct trace_iterator *iter) 122static unsigned long count_overruns(struct trace_iterator *iter)
130{ 123{
131 int cpu;
132 unsigned long cnt = 0; 124 unsigned long cnt = 0;
133/* FIXME: */ 125 unsigned long over = ring_buffer_overruns(iter->tr->buffer);
134#if 0 126
135 for_each_online_cpu(cpu) { 127 if (over > prev_overruns)
136 cnt += iter->overrun[cpu]; 128 cnt = over - prev_overruns;
137 iter->overrun[cpu] = 0; 129 prev_overruns = over;
138 }
139#endif
140 (void)cpu;
141 return cnt; 130 return cnt;
142} 131}
143 132
@@ -298,10 +287,10 @@ static struct tracer mmio_tracer __read_mostly =
298 .name = "mmiotrace", 287 .name = "mmiotrace",
299 .init = mmio_trace_init, 288 .init = mmio_trace_init,
300 .reset = mmio_trace_reset, 289 .reset = mmio_trace_reset,
290 .start = mmio_trace_start,
301 .pipe_open = mmio_pipe_open, 291 .pipe_open = mmio_pipe_open,
302 .close = mmio_close, 292 .close = mmio_close,
303 .read = mmio_read, 293 .read = mmio_read,
304 .ctrl_update = mmio_trace_ctrl_update,
305 .print_line = mmio_print_line, 294 .print_line = mmio_print_line,
306}; 295};
307 296
@@ -373,5 +362,5 @@ void mmio_trace_mapping(struct mmiotrace_map *map)
373 362
374int mmio_trace_printk(const char *fmt, va_list args) 363int mmio_trace_printk(const char *fmt, va_list args)
375{ 364{
376 return trace_vprintk(0, fmt, args); 365 return trace_vprintk(0, -1, fmt, args);
377} 366}
diff --git a/kernel/trace/trace_nop.c b/kernel/trace/trace_nop.c
index 4592b4862515..b9767acd30ac 100644
--- a/kernel/trace/trace_nop.c
+++ b/kernel/trace/trace_nop.c
@@ -12,6 +12,27 @@
12 12
13#include "trace.h" 13#include "trace.h"
14 14
15/* Our two options */
16enum {
17 TRACE_NOP_OPT_ACCEPT = 0x1,
18 TRACE_NOP_OPT_REFUSE = 0x2
19};
20
21/* Options for the tracer (see trace_options file) */
22static struct tracer_opt nop_opts[] = {
23 /* Option that will be accepted by set_flag callback */
24 { TRACER_OPT(test_nop_accept, TRACE_NOP_OPT_ACCEPT) },
25 /* Option that will be refused by set_flag callback */
26 { TRACER_OPT(test_nop_refuse, TRACE_NOP_OPT_REFUSE) },
27 { } /* Always set a last empty entry */
28};
29
30static struct tracer_flags nop_flags = {
31 /* You can check your flags value here when you want. */
32 .val = 0, /* By default: all flags disabled */
33 .opts = nop_opts
34};
35
15static struct trace_array *ctx_trace; 36static struct trace_array *ctx_trace;
16 37
17static void start_nop_trace(struct trace_array *tr) 38static void start_nop_trace(struct trace_array *tr)
@@ -24,7 +45,7 @@ static void stop_nop_trace(struct trace_array *tr)
24 /* Nothing to do! */ 45 /* Nothing to do! */
25} 46}
26 47
27static void nop_trace_init(struct trace_array *tr) 48static int nop_trace_init(struct trace_array *tr)
28{ 49{
29 int cpu; 50 int cpu;
30 ctx_trace = tr; 51 ctx_trace = tr;
@@ -32,33 +53,53 @@ static void nop_trace_init(struct trace_array *tr)
32 for_each_online_cpu(cpu) 53 for_each_online_cpu(cpu)
33 tracing_reset(tr, cpu); 54 tracing_reset(tr, cpu);
34 55
35 if (tr->ctrl) 56 start_nop_trace(tr);
36 start_nop_trace(tr); 57 return 0;
37} 58}
38 59
39static void nop_trace_reset(struct trace_array *tr) 60static void nop_trace_reset(struct trace_array *tr)
40{ 61{
41 if (tr->ctrl) 62 stop_nop_trace(tr);
42 stop_nop_trace(tr);
43} 63}
44 64
45static void nop_trace_ctrl_update(struct trace_array *tr) 65/* It only serves as a signal handler and a callback to
66 * accept or refuse tthe setting of a flag.
67 * If you don't implement it, then the flag setting will be
68 * automatically accepted.
69 */
70static int nop_set_flag(u32 old_flags, u32 bit, int set)
46{ 71{
47 /* When starting a new trace, reset the buffers */ 72 /*
48 if (tr->ctrl) 73 * Note that you don't need to update nop_flags.val yourself.
49 start_nop_trace(tr); 74 * The tracing Api will do it automatically if you return 0
50 else 75 */
51 stop_nop_trace(tr); 76 if (bit == TRACE_NOP_OPT_ACCEPT) {
77 printk(KERN_DEBUG "nop_test_accept flag set to %d: we accept."
78 " Now cat trace_options to see the result\n",
79 set);
80 return 0;
81 }
82
83 if (bit == TRACE_NOP_OPT_REFUSE) {
84 printk(KERN_DEBUG "nop_test_refuse flag set to %d: we refuse."
85 "Now cat trace_options to see the result\n",
86 set);
87 return -EINVAL;
88 }
89
90 return 0;
52} 91}
53 92
93
54struct tracer nop_trace __read_mostly = 94struct tracer nop_trace __read_mostly =
55{ 95{
56 .name = "nop", 96 .name = "nop",
57 .init = nop_trace_init, 97 .init = nop_trace_init,
58 .reset = nop_trace_reset, 98 .reset = nop_trace_reset,
59 .ctrl_update = nop_trace_ctrl_update,
60#ifdef CONFIG_FTRACE_SELFTEST 99#ifdef CONFIG_FTRACE_SELFTEST
61 .selftest = trace_selftest_startup_nop, 100 .selftest = trace_selftest_startup_nop,
62#endif 101#endif
102 .flags = &nop_flags,
103 .set_flag = nop_set_flag
63}; 104};
64 105
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
new file mode 100644
index 000000000000..7bda248daf55
--- /dev/null
+++ b/kernel/trace/trace_power.c
@@ -0,0 +1,179 @@
1/*
2 * ring buffer based C-state tracer
3 *
4 * Arjan van de Ven <arjan@linux.intel.com>
5 * Copyright (C) 2008 Intel Corporation
6 *
7 * Much is borrowed from trace_boot.c which is
8 * Copyright (C) 2008 Frederic Weisbecker <fweisbec@gmail.com>
9 *
10 */
11
12#include <linux/init.h>
13#include <linux/debugfs.h>
14#include <linux/ftrace.h>
15#include <linux/kallsyms.h>
16#include <linux/module.h>
17
18#include "trace.h"
19
20static struct trace_array *power_trace;
21static int __read_mostly trace_power_enabled;
22
23
24static void start_power_trace(struct trace_array *tr)
25{
26 trace_power_enabled = 1;
27}
28
29static void stop_power_trace(struct trace_array *tr)
30{
31 trace_power_enabled = 0;
32}
33
34
35static int power_trace_init(struct trace_array *tr)
36{
37 int cpu;
38 power_trace = tr;
39
40 trace_power_enabled = 1;
41
42 for_each_cpu(cpu, cpu_possible_mask)
43 tracing_reset(tr, cpu);
44 return 0;
45}
46
47static enum print_line_t power_print_line(struct trace_iterator *iter)
48{
49 int ret = 0;
50 struct trace_entry *entry = iter->ent;
51 struct trace_power *field ;
52 struct power_trace *it;
53 struct trace_seq *s = &iter->seq;
54 struct timespec stamp;
55 struct timespec duration;
56
57 trace_assign_type(field, entry);
58 it = &field->state_data;
59 stamp = ktime_to_timespec(it->stamp);
60 duration = ktime_to_timespec(ktime_sub(it->end, it->stamp));
61
62 if (entry->type == TRACE_POWER) {
63 if (it->type == POWER_CSTATE)
64 ret = trace_seq_printf(s, "[%5ld.%09ld] CSTATE: Going to C%i on cpu %i for %ld.%09ld\n",
65 stamp.tv_sec,
66 stamp.tv_nsec,
67 it->state, iter->cpu,
68 duration.tv_sec,
69 duration.tv_nsec);
70 if (it->type == POWER_PSTATE)
71 ret = trace_seq_printf(s, "[%5ld.%09ld] PSTATE: Going to P%i on cpu %i\n",
72 stamp.tv_sec,
73 stamp.tv_nsec,
74 it->state, iter->cpu);
75 if (!ret)
76 return TRACE_TYPE_PARTIAL_LINE;
77 return TRACE_TYPE_HANDLED;
78 }
79 return TRACE_TYPE_UNHANDLED;
80}
81
82static struct tracer power_tracer __read_mostly =
83{
84 .name = "power",
85 .init = power_trace_init,
86 .start = start_power_trace,
87 .stop = stop_power_trace,
88 .reset = stop_power_trace,
89 .print_line = power_print_line,
90};
91
92static int init_power_trace(void)
93{
94 return register_tracer(&power_tracer);
95}
96device_initcall(init_power_trace);
97
98void trace_power_start(struct power_trace *it, unsigned int type,
99 unsigned int level)
100{
101 if (!trace_power_enabled)
102 return;
103
104 memset(it, 0, sizeof(struct power_trace));
105 it->state = level;
106 it->type = type;
107 it->stamp = ktime_get();
108}
109EXPORT_SYMBOL_GPL(trace_power_start);
110
111
112void trace_power_end(struct power_trace *it)
113{
114 struct ring_buffer_event *event;
115 struct trace_power *entry;
116 struct trace_array_cpu *data;
117 unsigned long irq_flags;
118 struct trace_array *tr = power_trace;
119
120 if (!trace_power_enabled)
121 return;
122
123 preempt_disable();
124 it->end = ktime_get();
125 data = tr->data[smp_processor_id()];
126
127 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
128 &irq_flags);
129 if (!event)
130 goto out;
131 entry = ring_buffer_event_data(event);
132 tracing_generic_entry_update(&entry->ent, 0, 0);
133 entry->ent.type = TRACE_POWER;
134 entry->state_data = *it;
135 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
136
137 trace_wake_up();
138
139 out:
140 preempt_enable();
141}
142EXPORT_SYMBOL_GPL(trace_power_end);
143
144void trace_power_mark(struct power_trace *it, unsigned int type,
145 unsigned int level)
146{
147 struct ring_buffer_event *event;
148 struct trace_power *entry;
149 struct trace_array_cpu *data;
150 unsigned long irq_flags;
151 struct trace_array *tr = power_trace;
152
153 if (!trace_power_enabled)
154 return;
155
156 memset(it, 0, sizeof(struct power_trace));
157 it->state = level;
158 it->type = type;
159 it->stamp = ktime_get();
160 preempt_disable();
161 it->end = it->stamp;
162 data = tr->data[smp_processor_id()];
163
164 event = ring_buffer_lock_reserve(tr->buffer, sizeof(*entry),
165 &irq_flags);
166 if (!event)
167 goto out;
168 entry = ring_buffer_event_data(event);
169 tracing_generic_entry_update(&entry->ent, 0, 0);
170 entry->ent.type = TRACE_POWER;
171 entry->state_data = *it;
172 ring_buffer_unlock_commit(tr->buffer, event, irq_flags);
173
174 trace_wake_up();
175
176 out:
177 preempt_enable();
178}
179EXPORT_SYMBOL_GPL(trace_power_mark);
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index b8f56beb1a62..df175cb4564f 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -16,7 +16,8 @@
16 16
17static struct trace_array *ctx_trace; 17static struct trace_array *ctx_trace;
18static int __read_mostly tracer_enabled; 18static int __read_mostly tracer_enabled;
19static atomic_t sched_ref; 19static int sched_ref;
20static DEFINE_MUTEX(sched_register_mutex);
20 21
21static void 22static void
22probe_sched_switch(struct rq *__rq, struct task_struct *prev, 23probe_sched_switch(struct rq *__rq, struct task_struct *prev,
@@ -27,7 +28,7 @@ probe_sched_switch(struct rq *__rq, struct task_struct *prev,
27 int cpu; 28 int cpu;
28 int pc; 29 int pc;
29 30
30 if (!atomic_read(&sched_ref)) 31 if (!sched_ref)
31 return; 32 return;
32 33
33 tracing_record_cmdline(prev); 34 tracing_record_cmdline(prev);
@@ -48,7 +49,7 @@ probe_sched_switch(struct rq *__rq, struct task_struct *prev,
48} 49}
49 50
50static void 51static void
51probe_sched_wakeup(struct rq *__rq, struct task_struct *wakee) 52probe_sched_wakeup(struct rq *__rq, struct task_struct *wakee, int success)
52{ 53{
53 struct trace_array_cpu *data; 54 struct trace_array_cpu *data;
54 unsigned long flags; 55 unsigned long flags;
@@ -71,16 +72,6 @@ probe_sched_wakeup(struct rq *__rq, struct task_struct *wakee)
71 local_irq_restore(flags); 72 local_irq_restore(flags);
72} 73}
73 74
74static void sched_switch_reset(struct trace_array *tr)
75{
76 int cpu;
77
78 tr->time_start = ftrace_now(tr->cpu);
79
80 for_each_online_cpu(cpu)
81 tracing_reset(tr, cpu);
82}
83
84static int tracing_sched_register(void) 75static int tracing_sched_register(void)
85{ 76{
86 int ret; 77 int ret;
@@ -123,20 +114,18 @@ static void tracing_sched_unregister(void)
123 114
124static void tracing_start_sched_switch(void) 115static void tracing_start_sched_switch(void)
125{ 116{
126 long ref; 117 mutex_lock(&sched_register_mutex);
127 118 if (!(sched_ref++))
128 ref = atomic_inc_return(&sched_ref);
129 if (ref == 1)
130 tracing_sched_register(); 119 tracing_sched_register();
120 mutex_unlock(&sched_register_mutex);
131} 121}
132 122
133static void tracing_stop_sched_switch(void) 123static void tracing_stop_sched_switch(void)
134{ 124{
135 long ref; 125 mutex_lock(&sched_register_mutex);
136 126 if (!(--sched_ref))
137 ref = atomic_dec_and_test(&sched_ref);
138 if (ref)
139 tracing_sched_unregister(); 127 tracing_sched_unregister();
128 mutex_unlock(&sched_register_mutex);
140} 129}
141 130
142void tracing_start_cmdline_record(void) 131void tracing_start_cmdline_record(void)
@@ -149,40 +138,86 @@ void tracing_stop_cmdline_record(void)
149 tracing_stop_sched_switch(); 138 tracing_stop_sched_switch();
150} 139}
151 140
141/**
142 * tracing_start_sched_switch_record - start tracing context switches
143 *
144 * Turns on context switch tracing for a tracer.
145 */
146void tracing_start_sched_switch_record(void)
147{
148 if (unlikely(!ctx_trace)) {
149 WARN_ON(1);
150 return;
151 }
152
153 tracing_start_sched_switch();
154
155 mutex_lock(&sched_register_mutex);
156 tracer_enabled++;
157 mutex_unlock(&sched_register_mutex);
158}
159
160/**
161 * tracing_stop_sched_switch_record - start tracing context switches
162 *
163 * Turns off context switch tracing for a tracer.
164 */
165void tracing_stop_sched_switch_record(void)
166{
167 mutex_lock(&sched_register_mutex);
168 tracer_enabled--;
169 WARN_ON(tracer_enabled < 0);
170 mutex_unlock(&sched_register_mutex);
171
172 tracing_stop_sched_switch();
173}
174
175/**
176 * tracing_sched_switch_assign_trace - assign a trace array for ctx switch
177 * @tr: trace array pointer to assign
178 *
179 * Some tracers might want to record the context switches in their
180 * trace. This function lets those tracers assign the trace array
181 * to use.
182 */
183void tracing_sched_switch_assign_trace(struct trace_array *tr)
184{
185 ctx_trace = tr;
186}
187
152static void start_sched_trace(struct trace_array *tr) 188static void start_sched_trace(struct trace_array *tr)
153{ 189{
154 sched_switch_reset(tr); 190 tracing_reset_online_cpus(tr);
155 tracing_start_cmdline_record(); 191 tracing_start_sched_switch_record();
156 tracer_enabled = 1;
157} 192}
158 193
159static void stop_sched_trace(struct trace_array *tr) 194static void stop_sched_trace(struct trace_array *tr)
160{ 195{
161 tracer_enabled = 0; 196 tracing_stop_sched_switch_record();
162 tracing_stop_cmdline_record();
163} 197}
164 198
165static void sched_switch_trace_init(struct trace_array *tr) 199static int sched_switch_trace_init(struct trace_array *tr)
166{ 200{
167 ctx_trace = tr; 201 ctx_trace = tr;
168 202 start_sched_trace(tr);
169 if (tr->ctrl) 203 return 0;
170 start_sched_trace(tr);
171} 204}
172 205
173static void sched_switch_trace_reset(struct trace_array *tr) 206static void sched_switch_trace_reset(struct trace_array *tr)
174{ 207{
175 if (tr->ctrl) 208 if (sched_ref)
176 stop_sched_trace(tr); 209 stop_sched_trace(tr);
177} 210}
178 211
179static void sched_switch_trace_ctrl_update(struct trace_array *tr) 212static void sched_switch_trace_start(struct trace_array *tr)
180{ 213{
181 /* When starting a new trace, reset the buffers */ 214 tracing_reset_online_cpus(tr);
182 if (tr->ctrl) 215 tracing_start_sched_switch();
183 start_sched_trace(tr); 216}
184 else 217
185 stop_sched_trace(tr); 218static void sched_switch_trace_stop(struct trace_array *tr)
219{
220 tracing_stop_sched_switch();
186} 221}
187 222
188static struct tracer sched_switch_trace __read_mostly = 223static struct tracer sched_switch_trace __read_mostly =
@@ -190,7 +225,8 @@ static struct tracer sched_switch_trace __read_mostly =
190 .name = "sched_switch", 225 .name = "sched_switch",
191 .init = sched_switch_trace_init, 226 .init = sched_switch_trace_init,
192 .reset = sched_switch_trace_reset, 227 .reset = sched_switch_trace_reset,
193 .ctrl_update = sched_switch_trace_ctrl_update, 228 .start = sched_switch_trace_start,
229 .stop = sched_switch_trace_stop,
194#ifdef CONFIG_FTRACE_SELFTEST 230#ifdef CONFIG_FTRACE_SELFTEST
195 .selftest = trace_selftest_startup_sched_switch, 231 .selftest = trace_selftest_startup_sched_switch,
196#endif 232#endif
@@ -198,14 +234,7 @@ static struct tracer sched_switch_trace __read_mostly =
198 234
199__init static int init_sched_switch_trace(void) 235__init static int init_sched_switch_trace(void)
200{ 236{
201 int ret = 0;
202
203 if (atomic_read(&sched_ref))
204 ret = tracing_sched_register();
205 if (ret) {
206 pr_info("error registering scheduler trace\n");
207 return ret;
208 }
209 return register_tracer(&sched_switch_trace); 237 return register_tracer(&sched_switch_trace);
210} 238}
211device_initcall(init_sched_switch_trace); 239device_initcall(init_sched_switch_trace);
240
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index 3ae93f16b565..43586b689e31 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -50,8 +50,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
50 return; 50 return;
51 51
52 pc = preempt_count(); 52 pc = preempt_count();
53 resched = need_resched(); 53 resched = ftrace_preempt_disable();
54 preempt_disable_notrace();
55 54
56 cpu = raw_smp_processor_id(); 55 cpu = raw_smp_processor_id();
57 data = tr->data[cpu]; 56 data = tr->data[cpu];
@@ -81,15 +80,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
81 out: 80 out:
82 atomic_dec(&data->disabled); 81 atomic_dec(&data->disabled);
83 82
84 /* 83 ftrace_preempt_enable(resched);
85 * To prevent recursion from the scheduler, if the
86 * resched flag was set before we entered, then
87 * don't reschedule.
88 */
89 if (resched)
90 preempt_enable_no_resched_notrace();
91 else
92 preempt_enable_notrace();
93} 84}
94 85
95static struct ftrace_ops trace_ops __read_mostly = 86static struct ftrace_ops trace_ops __read_mostly =
@@ -220,7 +211,7 @@ static void wakeup_reset(struct trace_array *tr)
220} 211}
221 212
222static void 213static void
223probe_wakeup(struct rq *rq, struct task_struct *p) 214probe_wakeup(struct rq *rq, struct task_struct *p, int success)
224{ 215{
225 int cpu = smp_processor_id(); 216 int cpu = smp_processor_id();
226 unsigned long flags; 217 unsigned long flags;
@@ -271,6 +262,12 @@ out:
271 atomic_dec(&wakeup_trace->data[cpu]->disabled); 262 atomic_dec(&wakeup_trace->data[cpu]->disabled);
272} 263}
273 264
265/*
266 * save_tracer_enabled is used to save the state of the tracer_enabled
267 * variable when we disable it when we open a trace output file.
268 */
269static int save_tracer_enabled;
270
274static void start_wakeup_tracer(struct trace_array *tr) 271static void start_wakeup_tracer(struct trace_array *tr)
275{ 272{
276 int ret; 273 int ret;
@@ -309,7 +306,13 @@ static void start_wakeup_tracer(struct trace_array *tr)
309 306
310 register_ftrace_function(&trace_ops); 307 register_ftrace_function(&trace_ops);
311 308
312 tracer_enabled = 1; 309 if (tracing_is_enabled()) {
310 tracer_enabled = 1;
311 save_tracer_enabled = 1;
312 } else {
313 tracer_enabled = 0;
314 save_tracer_enabled = 0;
315 }
313 316
314 return; 317 return;
315fail_deprobe_wake_new: 318fail_deprobe_wake_new:
@@ -321,49 +324,53 @@ fail_deprobe:
321static void stop_wakeup_tracer(struct trace_array *tr) 324static void stop_wakeup_tracer(struct trace_array *tr)
322{ 325{
323 tracer_enabled = 0; 326 tracer_enabled = 0;
327 save_tracer_enabled = 0;
324 unregister_ftrace_function(&trace_ops); 328 unregister_ftrace_function(&trace_ops);
325 unregister_trace_sched_switch(probe_wakeup_sched_switch); 329 unregister_trace_sched_switch(probe_wakeup_sched_switch);
326 unregister_trace_sched_wakeup_new(probe_wakeup); 330 unregister_trace_sched_wakeup_new(probe_wakeup);
327 unregister_trace_sched_wakeup(probe_wakeup); 331 unregister_trace_sched_wakeup(probe_wakeup);
328} 332}
329 333
330static void wakeup_tracer_init(struct trace_array *tr) 334static int wakeup_tracer_init(struct trace_array *tr)
331{ 335{
332 wakeup_trace = tr; 336 wakeup_trace = tr;
333 337 start_wakeup_tracer(tr);
334 if (tr->ctrl) 338 return 0;
335 start_wakeup_tracer(tr);
336} 339}
337 340
338static void wakeup_tracer_reset(struct trace_array *tr) 341static void wakeup_tracer_reset(struct trace_array *tr)
339{ 342{
340 if (tr->ctrl) { 343 stop_wakeup_tracer(tr);
341 stop_wakeup_tracer(tr); 344 /* make sure we put back any tasks we are tracing */
342 /* make sure we put back any tasks we are tracing */ 345 wakeup_reset(tr);
343 wakeup_reset(tr); 346}
344 } 347
348static void wakeup_tracer_start(struct trace_array *tr)
349{
350 wakeup_reset(tr);
351 tracer_enabled = 1;
352 save_tracer_enabled = 1;
345} 353}
346 354
347static void wakeup_tracer_ctrl_update(struct trace_array *tr) 355static void wakeup_tracer_stop(struct trace_array *tr)
348{ 356{
349 if (tr->ctrl) 357 tracer_enabled = 0;
350 start_wakeup_tracer(tr); 358 save_tracer_enabled = 0;
351 else
352 stop_wakeup_tracer(tr);
353} 359}
354 360
355static void wakeup_tracer_open(struct trace_iterator *iter) 361static void wakeup_tracer_open(struct trace_iterator *iter)
356{ 362{
357 /* stop the trace while dumping */ 363 /* stop the trace while dumping */
358 if (iter->tr->ctrl) 364 tracer_enabled = 0;
359 stop_wakeup_tracer(iter->tr);
360} 365}
361 366
362static void wakeup_tracer_close(struct trace_iterator *iter) 367static void wakeup_tracer_close(struct trace_iterator *iter)
363{ 368{
364 /* forget about any processes we were recording */ 369 /* forget about any processes we were recording */
365 if (iter->tr->ctrl) 370 if (save_tracer_enabled) {
366 start_wakeup_tracer(iter->tr); 371 wakeup_reset(iter->tr);
372 tracer_enabled = 1;
373 }
367} 374}
368 375
369static struct tracer wakeup_tracer __read_mostly = 376static struct tracer wakeup_tracer __read_mostly =
@@ -371,9 +378,10 @@ static struct tracer wakeup_tracer __read_mostly =
371 .name = "wakeup", 378 .name = "wakeup",
372 .init = wakeup_tracer_init, 379 .init = wakeup_tracer_init,
373 .reset = wakeup_tracer_reset, 380 .reset = wakeup_tracer_reset,
381 .start = wakeup_tracer_start,
382 .stop = wakeup_tracer_stop,
374 .open = wakeup_tracer_open, 383 .open = wakeup_tracer_open,
375 .close = wakeup_tracer_close, 384 .close = wakeup_tracer_close,
376 .ctrl_update = wakeup_tracer_ctrl_update,
377 .print_max = 1, 385 .print_max = 1,
378#ifdef CONFIG_FTRACE_SELFTEST 386#ifdef CONFIG_FTRACE_SELFTEST
379 .selftest = trace_selftest_startup_wakeup, 387 .selftest = trace_selftest_startup_wakeup,
diff --git a/kernel/trace/trace_selftest.c b/kernel/trace/trace_selftest.c
index 90bc752a7580..88c8eb70f54a 100644
--- a/kernel/trace/trace_selftest.c
+++ b/kernel/trace/trace_selftest.c
@@ -13,6 +13,7 @@ static inline int trace_valid_entry(struct trace_entry *entry)
13 case TRACE_STACK: 13 case TRACE_STACK:
14 case TRACE_PRINT: 14 case TRACE_PRINT:
15 case TRACE_SPECIAL: 15 case TRACE_SPECIAL:
16 case TRACE_BRANCH:
16 return 1; 17 return 1;
17 } 18 }
18 return 0; 19 return 0;
@@ -51,7 +52,7 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count)
51 int cpu, ret = 0; 52 int cpu, ret = 0;
52 53
53 /* Don't allow flipping of max traces now */ 54 /* Don't allow flipping of max traces now */
54 raw_local_irq_save(flags); 55 local_irq_save(flags);
55 __raw_spin_lock(&ftrace_max_lock); 56 __raw_spin_lock(&ftrace_max_lock);
56 57
57 cnt = ring_buffer_entries(tr->buffer); 58 cnt = ring_buffer_entries(tr->buffer);
@@ -62,7 +63,7 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count)
62 break; 63 break;
63 } 64 }
64 __raw_spin_unlock(&ftrace_max_lock); 65 __raw_spin_unlock(&ftrace_max_lock);
65 raw_local_irq_restore(flags); 66 local_irq_restore(flags);
66 67
67 if (count) 68 if (count)
68 *count = cnt; 69 *count = cnt;
@@ -70,6 +71,11 @@ static int trace_test_buffer(struct trace_array *tr, unsigned long *count)
70 return ret; 71 return ret;
71} 72}
72 73
74static inline void warn_failed_init_tracer(struct tracer *trace, int init_ret)
75{
76 printk(KERN_WARNING "Failed to init %s tracer, init returned %d\n",
77 trace->name, init_ret);
78}
73#ifdef CONFIG_FUNCTION_TRACER 79#ifdef CONFIG_FUNCTION_TRACER
74 80
75#ifdef CONFIG_DYNAMIC_FTRACE 81#ifdef CONFIG_DYNAMIC_FTRACE
@@ -110,8 +116,11 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
110 ftrace_set_filter(func_name, strlen(func_name), 1); 116 ftrace_set_filter(func_name, strlen(func_name), 1);
111 117
112 /* enable tracing */ 118 /* enable tracing */
113 tr->ctrl = 1; 119 ret = trace->init(tr);
114 trace->init(tr); 120 if (ret) {
121 warn_failed_init_tracer(trace, ret);
122 goto out;
123 }
115 124
116 /* Sleep for a 1/10 of a second */ 125 /* Sleep for a 1/10 of a second */
117 msleep(100); 126 msleep(100);
@@ -134,13 +143,13 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
134 msleep(100); 143 msleep(100);
135 144
136 /* stop the tracing. */ 145 /* stop the tracing. */
137 tr->ctrl = 0; 146 tracing_stop();
138 trace->ctrl_update(tr);
139 ftrace_enabled = 0; 147 ftrace_enabled = 0;
140 148
141 /* check the trace buffer */ 149 /* check the trace buffer */
142 ret = trace_test_buffer(tr, &count); 150 ret = trace_test_buffer(tr, &count);
143 trace->reset(tr); 151 trace->reset(tr);
152 tracing_start();
144 153
145 /* we should only have one item */ 154 /* we should only have one item */
146 if (!ret && count != 1) { 155 if (!ret && count != 1) {
@@ -148,6 +157,7 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
148 ret = -1; 157 ret = -1;
149 goto out; 158 goto out;
150 } 159 }
160
151 out: 161 out:
152 ftrace_enabled = save_ftrace_enabled; 162 ftrace_enabled = save_ftrace_enabled;
153 tracer_enabled = save_tracer_enabled; 163 tracer_enabled = save_tracer_enabled;
@@ -180,18 +190,22 @@ trace_selftest_startup_function(struct tracer *trace, struct trace_array *tr)
180 ftrace_enabled = 1; 190 ftrace_enabled = 1;
181 tracer_enabled = 1; 191 tracer_enabled = 1;
182 192
183 tr->ctrl = 1; 193 ret = trace->init(tr);
184 trace->init(tr); 194 if (ret) {
195 warn_failed_init_tracer(trace, ret);
196 goto out;
197 }
198
185 /* Sleep for a 1/10 of a second */ 199 /* Sleep for a 1/10 of a second */
186 msleep(100); 200 msleep(100);
187 /* stop the tracing. */ 201 /* stop the tracing. */
188 tr->ctrl = 0; 202 tracing_stop();
189 trace->ctrl_update(tr);
190 ftrace_enabled = 0; 203 ftrace_enabled = 0;
191 204
192 /* check the trace buffer */ 205 /* check the trace buffer */
193 ret = trace_test_buffer(tr, &count); 206 ret = trace_test_buffer(tr, &count);
194 trace->reset(tr); 207 trace->reset(tr);
208 tracing_start();
195 209
196 if (!ret && !count) { 210 if (!ret && !count) {
197 printk(KERN_CONT ".. no entries found .."); 211 printk(KERN_CONT ".. no entries found ..");
@@ -223,8 +237,12 @@ trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
223 int ret; 237 int ret;
224 238
225 /* start the tracing */ 239 /* start the tracing */
226 tr->ctrl = 1; 240 ret = trace->init(tr);
227 trace->init(tr); 241 if (ret) {
242 warn_failed_init_tracer(trace, ret);
243 return ret;
244 }
245
228 /* reset the max latency */ 246 /* reset the max latency */
229 tracing_max_latency = 0; 247 tracing_max_latency = 0;
230 /* disable interrupts for a bit */ 248 /* disable interrupts for a bit */
@@ -232,13 +250,13 @@ trace_selftest_startup_irqsoff(struct tracer *trace, struct trace_array *tr)
232 udelay(100); 250 udelay(100);
233 local_irq_enable(); 251 local_irq_enable();
234 /* stop the tracing. */ 252 /* stop the tracing. */
235 tr->ctrl = 0; 253 tracing_stop();
236 trace->ctrl_update(tr);
237 /* check both trace buffers */ 254 /* check both trace buffers */
238 ret = trace_test_buffer(tr, NULL); 255 ret = trace_test_buffer(tr, NULL);
239 if (!ret) 256 if (!ret)
240 ret = trace_test_buffer(&max_tr, &count); 257 ret = trace_test_buffer(&max_tr, &count);
241 trace->reset(tr); 258 trace->reset(tr);
259 tracing_start();
242 260
243 if (!ret && !count) { 261 if (!ret && !count) {
244 printk(KERN_CONT ".. no entries found .."); 262 printk(KERN_CONT ".. no entries found ..");
@@ -259,9 +277,26 @@ trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr)
259 unsigned long count; 277 unsigned long count;
260 int ret; 278 int ret;
261 279
280 /*
281 * Now that the big kernel lock is no longer preemptable,
282 * and this is called with the BKL held, it will always
283 * fail. If preemption is already disabled, simply
284 * pass the test. When the BKL is removed, or becomes
285 * preemptible again, we will once again test this,
286 * so keep it in.
287 */
288 if (preempt_count()) {
289 printk(KERN_CONT "can not test ... force ");
290 return 0;
291 }
292
262 /* start the tracing */ 293 /* start the tracing */
263 tr->ctrl = 1; 294 ret = trace->init(tr);
264 trace->init(tr); 295 if (ret) {
296 warn_failed_init_tracer(trace, ret);
297 return ret;
298 }
299
265 /* reset the max latency */ 300 /* reset the max latency */
266 tracing_max_latency = 0; 301 tracing_max_latency = 0;
267 /* disable preemption for a bit */ 302 /* disable preemption for a bit */
@@ -269,13 +304,13 @@ trace_selftest_startup_preemptoff(struct tracer *trace, struct trace_array *tr)
269 udelay(100); 304 udelay(100);
270 preempt_enable(); 305 preempt_enable();
271 /* stop the tracing. */ 306 /* stop the tracing. */
272 tr->ctrl = 0; 307 tracing_stop();
273 trace->ctrl_update(tr);
274 /* check both trace buffers */ 308 /* check both trace buffers */
275 ret = trace_test_buffer(tr, NULL); 309 ret = trace_test_buffer(tr, NULL);
276 if (!ret) 310 if (!ret)
277 ret = trace_test_buffer(&max_tr, &count); 311 ret = trace_test_buffer(&max_tr, &count);
278 trace->reset(tr); 312 trace->reset(tr);
313 tracing_start();
279 314
280 if (!ret && !count) { 315 if (!ret && !count) {
281 printk(KERN_CONT ".. no entries found .."); 316 printk(KERN_CONT ".. no entries found ..");
@@ -296,9 +331,25 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *
296 unsigned long count; 331 unsigned long count;
297 int ret; 332 int ret;
298 333
334 /*
335 * Now that the big kernel lock is no longer preemptable,
336 * and this is called with the BKL held, it will always
337 * fail. If preemption is already disabled, simply
338 * pass the test. When the BKL is removed, or becomes
339 * preemptible again, we will once again test this,
340 * so keep it in.
341 */
342 if (preempt_count()) {
343 printk(KERN_CONT "can not test ... force ");
344 return 0;
345 }
346
299 /* start the tracing */ 347 /* start the tracing */
300 tr->ctrl = 1; 348 ret = trace->init(tr);
301 trace->init(tr); 349 if (ret) {
350 warn_failed_init_tracer(trace, ret);
351 goto out;
352 }
302 353
303 /* reset the max latency */ 354 /* reset the max latency */
304 tracing_max_latency = 0; 355 tracing_max_latency = 0;
@@ -312,27 +363,30 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *
312 local_irq_enable(); 363 local_irq_enable();
313 364
314 /* stop the tracing. */ 365 /* stop the tracing. */
315 tr->ctrl = 0; 366 tracing_stop();
316 trace->ctrl_update(tr);
317 /* check both trace buffers */ 367 /* check both trace buffers */
318 ret = trace_test_buffer(tr, NULL); 368 ret = trace_test_buffer(tr, NULL);
319 if (ret) 369 if (ret) {
370 tracing_start();
320 goto out; 371 goto out;
372 }
321 373
322 ret = trace_test_buffer(&max_tr, &count); 374 ret = trace_test_buffer(&max_tr, &count);
323 if (ret) 375 if (ret) {
376 tracing_start();
324 goto out; 377 goto out;
378 }
325 379
326 if (!ret && !count) { 380 if (!ret && !count) {
327 printk(KERN_CONT ".. no entries found .."); 381 printk(KERN_CONT ".. no entries found ..");
328 ret = -1; 382 ret = -1;
383 tracing_start();
329 goto out; 384 goto out;
330 } 385 }
331 386
332 /* do the test by disabling interrupts first this time */ 387 /* do the test by disabling interrupts first this time */
333 tracing_max_latency = 0; 388 tracing_max_latency = 0;
334 tr->ctrl = 1; 389 tracing_start();
335 trace->ctrl_update(tr);
336 preempt_disable(); 390 preempt_disable();
337 local_irq_disable(); 391 local_irq_disable();
338 udelay(100); 392 udelay(100);
@@ -341,8 +395,7 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *
341 local_irq_enable(); 395 local_irq_enable();
342 396
343 /* stop the tracing. */ 397 /* stop the tracing. */
344 tr->ctrl = 0; 398 tracing_stop();
345 trace->ctrl_update(tr);
346 /* check both trace buffers */ 399 /* check both trace buffers */
347 ret = trace_test_buffer(tr, NULL); 400 ret = trace_test_buffer(tr, NULL);
348 if (ret) 401 if (ret)
@@ -358,6 +411,7 @@ trace_selftest_startup_preemptirqsoff(struct tracer *trace, struct trace_array *
358 411
359 out: 412 out:
360 trace->reset(tr); 413 trace->reset(tr);
414 tracing_start();
361 tracing_max_latency = save_max; 415 tracing_max_latency = save_max;
362 416
363 return ret; 417 return ret;
@@ -423,8 +477,12 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
423 wait_for_completion(&isrt); 477 wait_for_completion(&isrt);
424 478
425 /* start the tracing */ 479 /* start the tracing */
426 tr->ctrl = 1; 480 ret = trace->init(tr);
427 trace->init(tr); 481 if (ret) {
482 warn_failed_init_tracer(trace, ret);
483 return ret;
484 }
485
428 /* reset the max latency */ 486 /* reset the max latency */
429 tracing_max_latency = 0; 487 tracing_max_latency = 0;
430 488
@@ -448,8 +506,7 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
448 msleep(100); 506 msleep(100);
449 507
450 /* stop the tracing. */ 508 /* stop the tracing. */
451 tr->ctrl = 0; 509 tracing_stop();
452 trace->ctrl_update(tr);
453 /* check both trace buffers */ 510 /* check both trace buffers */
454 ret = trace_test_buffer(tr, NULL); 511 ret = trace_test_buffer(tr, NULL);
455 if (!ret) 512 if (!ret)
@@ -457,6 +514,7 @@ trace_selftest_startup_wakeup(struct tracer *trace, struct trace_array *tr)
457 514
458 515
459 trace->reset(tr); 516 trace->reset(tr);
517 tracing_start();
460 518
461 tracing_max_latency = save_max; 519 tracing_max_latency = save_max;
462 520
@@ -480,16 +538,20 @@ trace_selftest_startup_sched_switch(struct tracer *trace, struct trace_array *tr
480 int ret; 538 int ret;
481 539
482 /* start the tracing */ 540 /* start the tracing */
483 tr->ctrl = 1; 541 ret = trace->init(tr);
484 trace->init(tr); 542 if (ret) {
543 warn_failed_init_tracer(trace, ret);
544 return ret;
545 }
546
485 /* Sleep for a 1/10 of a second */ 547 /* Sleep for a 1/10 of a second */
486 msleep(100); 548 msleep(100);
487 /* stop the tracing. */ 549 /* stop the tracing. */
488 tr->ctrl = 0; 550 tracing_stop();
489 trace->ctrl_update(tr);
490 /* check the trace buffer */ 551 /* check the trace buffer */
491 ret = trace_test_buffer(tr, &count); 552 ret = trace_test_buffer(tr, &count);
492 trace->reset(tr); 553 trace->reset(tr);
554 tracing_start();
493 555
494 if (!ret && !count) { 556 if (!ret && !count) {
495 printk(KERN_CONT ".. no entries found .."); 557 printk(KERN_CONT ".. no entries found ..");
@@ -508,17 +570,48 @@ trace_selftest_startup_sysprof(struct tracer *trace, struct trace_array *tr)
508 int ret; 570 int ret;
509 571
510 /* start the tracing */ 572 /* start the tracing */
511 tr->ctrl = 1; 573 ret = trace->init(tr);
512 trace->init(tr); 574 if (ret) {
575 warn_failed_init_tracer(trace, ret);
576 return 0;
577 }
578
513 /* Sleep for a 1/10 of a second */ 579 /* Sleep for a 1/10 of a second */
514 msleep(100); 580 msleep(100);
515 /* stop the tracing. */ 581 /* stop the tracing. */
516 tr->ctrl = 0; 582 tracing_stop();
517 trace->ctrl_update(tr);
518 /* check the trace buffer */ 583 /* check the trace buffer */
519 ret = trace_test_buffer(tr, &count); 584 ret = trace_test_buffer(tr, &count);
520 trace->reset(tr); 585 trace->reset(tr);
586 tracing_start();
521 587
522 return ret; 588 return ret;
523} 589}
524#endif /* CONFIG_SYSPROF_TRACER */ 590#endif /* CONFIG_SYSPROF_TRACER */
591
592#ifdef CONFIG_BRANCH_TRACER
593int
594trace_selftest_startup_branch(struct tracer *trace, struct trace_array *tr)
595{
596 unsigned long count;
597 int ret;
598
599 /* start the tracing */
600 ret = trace->init(tr);
601 if (ret) {
602 warn_failed_init_tracer(trace, ret);
603 return ret;
604 }
605
606 /* Sleep for a 1/10 of a second */
607 msleep(100);
608 /* stop the tracing. */
609 tracing_stop();
610 /* check the trace buffer */
611 ret = trace_test_buffer(tr, &count);
612 trace->reset(tr);
613 tracing_start();
614
615 return ret;
616}
617#endif /* CONFIG_BRANCH_TRACER */
diff --git a/kernel/trace/trace_stack.c b/kernel/trace/trace_stack.c
index be682b62fe58..d0871bc0aca5 100644
--- a/kernel/trace/trace_stack.c
+++ b/kernel/trace/trace_stack.c
@@ -10,6 +10,7 @@
10#include <linux/debugfs.h> 10#include <linux/debugfs.h>
11#include <linux/ftrace.h> 11#include <linux/ftrace.h>
12#include <linux/module.h> 12#include <linux/module.h>
13#include <linux/sysctl.h>
13#include <linux/init.h> 14#include <linux/init.h>
14#include <linux/fs.h> 15#include <linux/fs.h>
15#include "trace.h" 16#include "trace.h"
@@ -31,6 +32,10 @@ static raw_spinlock_t max_stack_lock =
31 32
32static int stack_trace_disabled __read_mostly; 33static int stack_trace_disabled __read_mostly;
33static DEFINE_PER_CPU(int, trace_active); 34static DEFINE_PER_CPU(int, trace_active);
35static DEFINE_MUTEX(stack_sysctl_mutex);
36
37int stack_tracer_enabled;
38static int last_stack_tracer_enabled;
34 39
35static inline void check_stack(void) 40static inline void check_stack(void)
36{ 41{
@@ -48,7 +53,7 @@ static inline void check_stack(void)
48 if (!object_is_on_stack(&this_size)) 53 if (!object_is_on_stack(&this_size))
49 return; 54 return;
50 55
51 raw_local_irq_save(flags); 56 local_irq_save(flags);
52 __raw_spin_lock(&max_stack_lock); 57 __raw_spin_lock(&max_stack_lock);
53 58
54 /* a race could have already updated it */ 59 /* a race could have already updated it */
@@ -78,6 +83,7 @@ static inline void check_stack(void)
78 * on a new max, so it is far from a fast path. 83 * on a new max, so it is far from a fast path.
79 */ 84 */
80 while (i < max_stack_trace.nr_entries) { 85 while (i < max_stack_trace.nr_entries) {
86 int found = 0;
81 87
82 stack_dump_index[i] = this_size; 88 stack_dump_index[i] = this_size;
83 p = start; 89 p = start;
@@ -86,17 +92,19 @@ static inline void check_stack(void)
86 if (*p == stack_dump_trace[i]) { 92 if (*p == stack_dump_trace[i]) {
87 this_size = stack_dump_index[i++] = 93 this_size = stack_dump_index[i++] =
88 (top - p) * sizeof(unsigned long); 94 (top - p) * sizeof(unsigned long);
95 found = 1;
89 /* Start the search from here */ 96 /* Start the search from here */
90 start = p + 1; 97 start = p + 1;
91 } 98 }
92 } 99 }
93 100
94 i++; 101 if (!found)
102 i++;
95 } 103 }
96 104
97 out: 105 out:
98 __raw_spin_unlock(&max_stack_lock); 106 __raw_spin_unlock(&max_stack_lock);
99 raw_local_irq_restore(flags); 107 local_irq_restore(flags);
100} 108}
101 109
102static void 110static void
@@ -107,8 +115,7 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip)
107 if (unlikely(!ftrace_enabled || stack_trace_disabled)) 115 if (unlikely(!ftrace_enabled || stack_trace_disabled))
108 return; 116 return;
109 117
110 resched = need_resched(); 118 resched = ftrace_preempt_disable();
111 preempt_disable_notrace();
112 119
113 cpu = raw_smp_processor_id(); 120 cpu = raw_smp_processor_id();
114 /* no atomic needed, we only modify this variable by this cpu */ 121 /* no atomic needed, we only modify this variable by this cpu */
@@ -120,10 +127,7 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip)
120 out: 127 out:
121 per_cpu(trace_active, cpu)--; 128 per_cpu(trace_active, cpu)--;
122 /* prevent recursion in schedule */ 129 /* prevent recursion in schedule */
123 if (resched) 130 ftrace_preempt_enable(resched);
124 preempt_enable_no_resched_notrace();
125 else
126 preempt_enable_notrace();
127} 131}
128 132
129static struct ftrace_ops trace_ops __read_mostly = 133static struct ftrace_ops trace_ops __read_mostly =
@@ -166,16 +170,16 @@ stack_max_size_write(struct file *filp, const char __user *ubuf,
166 if (ret < 0) 170 if (ret < 0)
167 return ret; 171 return ret;
168 172
169 raw_local_irq_save(flags); 173 local_irq_save(flags);
170 __raw_spin_lock(&max_stack_lock); 174 __raw_spin_lock(&max_stack_lock);
171 *ptr = val; 175 *ptr = val;
172 __raw_spin_unlock(&max_stack_lock); 176 __raw_spin_unlock(&max_stack_lock);
173 raw_local_irq_restore(flags); 177 local_irq_restore(flags);
174 178
175 return count; 179 return count;
176} 180}
177 181
178static struct file_operations stack_max_size_fops = { 182static const struct file_operations stack_max_size_fops = {
179 .open = tracing_open_generic, 183 .open = tracing_open_generic,
180 .read = stack_max_size_read, 184 .read = stack_max_size_read,
181 .write = stack_max_size_write, 185 .write = stack_max_size_write,
@@ -184,11 +188,16 @@ static struct file_operations stack_max_size_fops = {
184static void * 188static void *
185t_next(struct seq_file *m, void *v, loff_t *pos) 189t_next(struct seq_file *m, void *v, loff_t *pos)
186{ 190{
187 long i = (long)m->private; 191 long i;
188 192
189 (*pos)++; 193 (*pos)++;
190 194
191 i++; 195 if (v == SEQ_START_TOKEN)
196 i = 0;
197 else {
198 i = *(long *)v;
199 i++;
200 }
192 201
193 if (i >= max_stack_trace.nr_entries || 202 if (i >= max_stack_trace.nr_entries ||
194 stack_dump_trace[i] == ULONG_MAX) 203 stack_dump_trace[i] == ULONG_MAX)
@@ -201,12 +210,15 @@ t_next(struct seq_file *m, void *v, loff_t *pos)
201 210
202static void *t_start(struct seq_file *m, loff_t *pos) 211static void *t_start(struct seq_file *m, loff_t *pos)
203{ 212{
204 void *t = &m->private; 213 void *t = SEQ_START_TOKEN;
205 loff_t l = 0; 214 loff_t l = 0;
206 215
207 local_irq_disable(); 216 local_irq_disable();
208 __raw_spin_lock(&max_stack_lock); 217 __raw_spin_lock(&max_stack_lock);
209 218
219 if (*pos == 0)
220 return SEQ_START_TOKEN;
221
210 for (; t && l < *pos; t = t_next(m, t, &l)) 222 for (; t && l < *pos; t = t_next(m, t, &l))
211 ; 223 ;
212 224
@@ -235,10 +247,10 @@ static int trace_lookup_stack(struct seq_file *m, long i)
235 247
236static int t_show(struct seq_file *m, void *v) 248static int t_show(struct seq_file *m, void *v)
237{ 249{
238 long i = *(long *)v; 250 long i;
239 int size; 251 int size;
240 252
241 if (i < 0) { 253 if (v == SEQ_START_TOKEN) {
242 seq_printf(m, " Depth Size Location" 254 seq_printf(m, " Depth Size Location"
243 " (%d entries)\n" 255 " (%d entries)\n"
244 " ----- ---- --------\n", 256 " ----- ---- --------\n",
@@ -246,6 +258,8 @@ static int t_show(struct seq_file *m, void *v)
246 return 0; 258 return 0;
247 } 259 }
248 260
261 i = *(long *)v;
262
249 if (i >= max_stack_trace.nr_entries || 263 if (i >= max_stack_trace.nr_entries ||
250 stack_dump_trace[i] == ULONG_MAX) 264 stack_dump_trace[i] == ULONG_MAX)
251 return 0; 265 return 0;
@@ -263,7 +277,7 @@ static int t_show(struct seq_file *m, void *v)
263 return 0; 277 return 0;
264} 278}
265 279
266static struct seq_operations stack_trace_seq_ops = { 280static const struct seq_operations stack_trace_seq_ops = {
267 .start = t_start, 281 .start = t_start,
268 .next = t_next, 282 .next = t_next,
269 .stop = t_stop, 283 .stop = t_stop,
@@ -275,20 +289,51 @@ static int stack_trace_open(struct inode *inode, struct file *file)
275 int ret; 289 int ret;
276 290
277 ret = seq_open(file, &stack_trace_seq_ops); 291 ret = seq_open(file, &stack_trace_seq_ops);
278 if (!ret) {
279 struct seq_file *m = file->private_data;
280 m->private = (void *)-1;
281 }
282 292
283 return ret; 293 return ret;
284} 294}
285 295
286static struct file_operations stack_trace_fops = { 296static const struct file_operations stack_trace_fops = {
287 .open = stack_trace_open, 297 .open = stack_trace_open,
288 .read = seq_read, 298 .read = seq_read,
289 .llseek = seq_lseek, 299 .llseek = seq_lseek,
290}; 300};
291 301
302int
303stack_trace_sysctl(struct ctl_table *table, int write,
304 struct file *file, void __user *buffer, size_t *lenp,
305 loff_t *ppos)
306{
307 int ret;
308
309 mutex_lock(&stack_sysctl_mutex);
310
311 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
312
313 if (ret || !write ||
314 (last_stack_tracer_enabled == stack_tracer_enabled))
315 goto out;
316
317 last_stack_tracer_enabled = stack_tracer_enabled;
318
319 if (stack_tracer_enabled)
320 register_ftrace_function(&trace_ops);
321 else
322 unregister_ftrace_function(&trace_ops);
323
324 out:
325 mutex_unlock(&stack_sysctl_mutex);
326 return ret;
327}
328
329static __init int enable_stacktrace(char *str)
330{
331 stack_tracer_enabled = 1;
332 last_stack_tracer_enabled = 1;
333 return 1;
334}
335__setup("stacktrace", enable_stacktrace);
336
292static __init int stack_trace_init(void) 337static __init int stack_trace_init(void)
293{ 338{
294 struct dentry *d_tracer; 339 struct dentry *d_tracer;
@@ -306,7 +351,8 @@ static __init int stack_trace_init(void)
306 if (!entry) 351 if (!entry)
307 pr_warning("Could not create debugfs 'stack_trace' entry\n"); 352 pr_warning("Could not create debugfs 'stack_trace' entry\n");
308 353
309 register_ftrace_function(&trace_ops); 354 if (stack_tracer_enabled)
355 register_ftrace_function(&trace_ops);
310 356
311 return 0; 357 return 0;
312} 358}
diff --git a/kernel/trace/trace_sysprof.c b/kernel/trace/trace_sysprof.c
index 9587d3bcba55..eaca5ad803ff 100644
--- a/kernel/trace/trace_sysprof.c
+++ b/kernel/trace/trace_sysprof.c
@@ -196,27 +196,19 @@ static enum hrtimer_restart stack_trace_timer_fn(struct hrtimer *hrtimer)
196 return HRTIMER_RESTART; 196 return HRTIMER_RESTART;
197} 197}
198 198
199static void start_stack_timer(int cpu) 199static void start_stack_timer(void *unused)
200{ 200{
201 struct hrtimer *hrtimer = &per_cpu(stack_trace_hrtimer, cpu); 201 struct hrtimer *hrtimer = &__get_cpu_var(stack_trace_hrtimer);
202 202
203 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 203 hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
204 hrtimer->function = stack_trace_timer_fn; 204 hrtimer->function = stack_trace_timer_fn;
205 hrtimer->cb_mode = HRTIMER_CB_IRQSAFE_PERCPU;
206 205
207 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL); 206 hrtimer_start(hrtimer, ns_to_ktime(sample_period), HRTIMER_MODE_REL);
208} 207}
209 208
210static void start_stack_timers(void) 209static void start_stack_timers(void)
211{ 210{
212 cpumask_t saved_mask = current->cpus_allowed; 211 on_each_cpu(start_stack_timer, NULL, 1);
213 int cpu;
214
215 for_each_online_cpu(cpu) {
216 set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu));
217 start_stack_timer(cpu);
218 }
219 set_cpus_allowed_ptr(current, &saved_mask);
220} 212}
221 213
222static void stop_stack_timer(int cpu) 214static void stop_stack_timer(int cpu)
@@ -234,20 +226,10 @@ static void stop_stack_timers(void)
234 stop_stack_timer(cpu); 226 stop_stack_timer(cpu);
235} 227}
236 228
237static void stack_reset(struct trace_array *tr)
238{
239 int cpu;
240
241 tr->time_start = ftrace_now(tr->cpu);
242
243 for_each_online_cpu(cpu)
244 tracing_reset(tr, cpu);
245}
246
247static void start_stack_trace(struct trace_array *tr) 229static void start_stack_trace(struct trace_array *tr)
248{ 230{
249 mutex_lock(&sample_timer_lock); 231 mutex_lock(&sample_timer_lock);
250 stack_reset(tr); 232 tracing_reset_online_cpus(tr);
251 start_stack_timers(); 233 start_stack_timers();
252 tracer_enabled = 1; 234 tracer_enabled = 1;
253 mutex_unlock(&sample_timer_lock); 235 mutex_unlock(&sample_timer_lock);
@@ -261,27 +243,17 @@ static void stop_stack_trace(struct trace_array *tr)
261 mutex_unlock(&sample_timer_lock); 243 mutex_unlock(&sample_timer_lock);
262} 244}
263 245
264static void stack_trace_init(struct trace_array *tr) 246static int stack_trace_init(struct trace_array *tr)
265{ 247{
266 sysprof_trace = tr; 248 sysprof_trace = tr;
267 249
268 if (tr->ctrl) 250 start_stack_trace(tr);
269 start_stack_trace(tr); 251 return 0;
270} 252}
271 253
272static void stack_trace_reset(struct trace_array *tr) 254static void stack_trace_reset(struct trace_array *tr)
273{ 255{
274 if (tr->ctrl) 256 stop_stack_trace(tr);
275 stop_stack_trace(tr);
276}
277
278static void stack_trace_ctrl_update(struct trace_array *tr)
279{
280 /* When starting a new trace, reset the buffers */
281 if (tr->ctrl)
282 start_stack_trace(tr);
283 else
284 stop_stack_trace(tr);
285} 257}
286 258
287static struct tracer stack_trace __read_mostly = 259static struct tracer stack_trace __read_mostly =
@@ -289,7 +261,6 @@ static struct tracer stack_trace __read_mostly =
289 .name = "sysprof", 261 .name = "sysprof",
290 .init = stack_trace_init, 262 .init = stack_trace_init,
291 .reset = stack_trace_reset, 263 .reset = stack_trace_reset,
292 .ctrl_update = stack_trace_ctrl_update,
293#ifdef CONFIG_FTRACE_SELFTEST 264#ifdef CONFIG_FTRACE_SELFTEST
294 .selftest = trace_selftest_startup_sysprof, 265 .selftest = trace_selftest_startup_sysprof,
295#endif 266#endif
diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index af8c85664882..79602740bbb5 100644
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -43,6 +43,7 @@ static DEFINE_MUTEX(tracepoints_mutex);
43 */ 43 */
44#define TRACEPOINT_HASH_BITS 6 44#define TRACEPOINT_HASH_BITS 6
45#define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS) 45#define TRACEPOINT_TABLE_SIZE (1 << TRACEPOINT_HASH_BITS)
46static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE];
46 47
47/* 48/*
48 * Note about RCU : 49 * Note about RCU :
@@ -54,40 +55,43 @@ struct tracepoint_entry {
54 struct hlist_node hlist; 55 struct hlist_node hlist;
55 void **funcs; 56 void **funcs;
56 int refcount; /* Number of times armed. 0 if disarmed. */ 57 int refcount; /* Number of times armed. 0 if disarmed. */
57 struct rcu_head rcu;
58 void *oldptr;
59 unsigned char rcu_pending:1;
60 char name[0]; 58 char name[0];
61}; 59};
62 60
63static struct hlist_head tracepoint_table[TRACEPOINT_TABLE_SIZE]; 61struct tp_probes {
62 union {
63 struct rcu_head rcu;
64 struct list_head list;
65 } u;
66 void *probes[0];
67};
64 68
65static void free_old_closure(struct rcu_head *head) 69static inline void *allocate_probes(int count)
66{ 70{
67 struct tracepoint_entry *entry = container_of(head, 71 struct tp_probes *p = kmalloc(count * sizeof(void *)
68 struct tracepoint_entry, rcu); 72 + sizeof(struct tp_probes), GFP_KERNEL);
69 kfree(entry->oldptr); 73 return p == NULL ? NULL : p->probes;
70 /* Make sure we free the data before setting the pending flag to 0 */
71 smp_wmb();
72 entry->rcu_pending = 0;
73} 74}
74 75
75static void tracepoint_entry_free_old(struct tracepoint_entry *entry, void *old) 76static void rcu_free_old_probes(struct rcu_head *head)
76{ 77{
77 if (!old) 78 kfree(container_of(head, struct tp_probes, u.rcu));
78 return; 79}
79 entry->oldptr = old; 80
80 entry->rcu_pending = 1; 81static inline void release_probes(void *old)
81 /* write rcu_pending before calling the RCU callback */ 82{
82 smp_wmb(); 83 if (old) {
83 call_rcu_sched(&entry->rcu, free_old_closure); 84 struct tp_probes *tp_probes = container_of(old,
85 struct tp_probes, probes[0]);
86 call_rcu_sched(&tp_probes->u.rcu, rcu_free_old_probes);
87 }
84} 88}
85 89
86static void debug_print_probes(struct tracepoint_entry *entry) 90static void debug_print_probes(struct tracepoint_entry *entry)
87{ 91{
88 int i; 92 int i;
89 93
90 if (!tracepoint_debug) 94 if (!tracepoint_debug || !entry->funcs)
91 return; 95 return;
92 96
93 for (i = 0; entry->funcs[i]; i++) 97 for (i = 0; entry->funcs[i]; i++)
@@ -111,12 +115,13 @@ tracepoint_entry_add_probe(struct tracepoint_entry *entry, void *probe)
111 return ERR_PTR(-EEXIST); 115 return ERR_PTR(-EEXIST);
112 } 116 }
113 /* + 2 : one for new probe, one for NULL func */ 117 /* + 2 : one for new probe, one for NULL func */
114 new = kzalloc((nr_probes + 2) * sizeof(void *), GFP_KERNEL); 118 new = allocate_probes(nr_probes + 2);
115 if (new == NULL) 119 if (new == NULL)
116 return ERR_PTR(-ENOMEM); 120 return ERR_PTR(-ENOMEM);
117 if (old) 121 if (old)
118 memcpy(new, old, nr_probes * sizeof(void *)); 122 memcpy(new, old, nr_probes * sizeof(void *));
119 new[nr_probes] = probe; 123 new[nr_probes] = probe;
124 new[nr_probes + 1] = NULL;
120 entry->refcount = nr_probes + 1; 125 entry->refcount = nr_probes + 1;
121 entry->funcs = new; 126 entry->funcs = new;
122 debug_print_probes(entry); 127 debug_print_probes(entry);
@@ -132,7 +137,7 @@ tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe)
132 old = entry->funcs; 137 old = entry->funcs;
133 138
134 if (!old) 139 if (!old)
135 return NULL; 140 return ERR_PTR(-ENOENT);
136 141
137 debug_print_probes(entry); 142 debug_print_probes(entry);
138 /* (N -> M), (N > 1, M >= 0) probes */ 143 /* (N -> M), (N > 1, M >= 0) probes */
@@ -151,13 +156,13 @@ tracepoint_entry_remove_probe(struct tracepoint_entry *entry, void *probe)
151 int j = 0; 156 int j = 0;
152 /* N -> M, (N > 1, M > 0) */ 157 /* N -> M, (N > 1, M > 0) */
153 /* + 1 for NULL */ 158 /* + 1 for NULL */
154 new = kzalloc((nr_probes - nr_del + 1) 159 new = allocate_probes(nr_probes - nr_del + 1);
155 * sizeof(void *), GFP_KERNEL);
156 if (new == NULL) 160 if (new == NULL)
157 return ERR_PTR(-ENOMEM); 161 return ERR_PTR(-ENOMEM);
158 for (i = 0; old[i]; i++) 162 for (i = 0; old[i]; i++)
159 if ((probe && old[i] != probe)) 163 if ((probe && old[i] != probe))
160 new[j++] = old[i]; 164 new[j++] = old[i];
165 new[nr_probes - nr_del] = NULL;
161 entry->refcount = nr_probes - nr_del; 166 entry->refcount = nr_probes - nr_del;
162 entry->funcs = new; 167 entry->funcs = new;
163 } 168 }
@@ -215,7 +220,6 @@ static struct tracepoint_entry *add_tracepoint(const char *name)
215 memcpy(&e->name[0], name, name_len); 220 memcpy(&e->name[0], name, name_len);
216 e->funcs = NULL; 221 e->funcs = NULL;
217 e->refcount = 0; 222 e->refcount = 0;
218 e->rcu_pending = 0;
219 hlist_add_head(&e->hlist, head); 223 hlist_add_head(&e->hlist, head);
220 return e; 224 return e;
221} 225}
@@ -224,32 +228,10 @@ static struct tracepoint_entry *add_tracepoint(const char *name)
224 * Remove the tracepoint from the tracepoint hash table. Must be called with 228 * Remove the tracepoint from the tracepoint hash table. Must be called with
225 * mutex_lock held. 229 * mutex_lock held.
226 */ 230 */
227static int remove_tracepoint(const char *name) 231static inline void remove_tracepoint(struct tracepoint_entry *e)
228{ 232{
229 struct hlist_head *head;
230 struct hlist_node *node;
231 struct tracepoint_entry *e;
232 int found = 0;
233 size_t len = strlen(name) + 1;
234 u32 hash = jhash(name, len-1, 0);
235
236 head = &tracepoint_table[hash & (TRACEPOINT_TABLE_SIZE - 1)];
237 hlist_for_each_entry(e, node, head, hlist) {
238 if (!strcmp(name, e->name)) {
239 found = 1;
240 break;
241 }
242 }
243 if (!found)
244 return -ENOENT;
245 if (e->refcount)
246 return -EBUSY;
247 hlist_del(&e->hlist); 233 hlist_del(&e->hlist);
248 /* Make sure the call_rcu_sched has been executed */
249 if (e->rcu_pending)
250 rcu_barrier_sched();
251 kfree(e); 234 kfree(e);
252 return 0;
253} 235}
254 236
255/* 237/*
@@ -280,6 +262,7 @@ static void set_tracepoint(struct tracepoint_entry **entry,
280static void disable_tracepoint(struct tracepoint *elem) 262static void disable_tracepoint(struct tracepoint *elem)
281{ 263{
282 elem->state = 0; 264 elem->state = 0;
265 rcu_assign_pointer(elem->funcs, NULL);
283} 266}
284 267
285/** 268/**
@@ -320,6 +303,23 @@ static void tracepoint_update_probes(void)
320 module_update_tracepoints(); 303 module_update_tracepoints();
321} 304}
322 305
306static void *tracepoint_add_probe(const char *name, void *probe)
307{
308 struct tracepoint_entry *entry;
309 void *old;
310
311 entry = get_tracepoint(name);
312 if (!entry) {
313 entry = add_tracepoint(name);
314 if (IS_ERR(entry))
315 return entry;
316 }
317 old = tracepoint_entry_add_probe(entry, probe);
318 if (IS_ERR(old) && !entry->refcount)
319 remove_tracepoint(entry);
320 return old;
321}
322
323/** 323/**
324 * tracepoint_probe_register - Connect a probe to a tracepoint 324 * tracepoint_probe_register - Connect a probe to a tracepoint
325 * @name: tracepoint name 325 * @name: tracepoint name
@@ -330,44 +330,36 @@ static void tracepoint_update_probes(void)
330 */ 330 */
331int tracepoint_probe_register(const char *name, void *probe) 331int tracepoint_probe_register(const char *name, void *probe)
332{ 332{
333 struct tracepoint_entry *entry;
334 int ret = 0;
335 void *old; 333 void *old;
336 334
337 mutex_lock(&tracepoints_mutex); 335 mutex_lock(&tracepoints_mutex);
338 entry = get_tracepoint(name); 336 old = tracepoint_add_probe(name, probe);
339 if (!entry) {
340 entry = add_tracepoint(name);
341 if (IS_ERR(entry)) {
342 ret = PTR_ERR(entry);
343 goto end;
344 }
345 }
346 /*
347 * If we detect that a call_rcu_sched is pending for this tracepoint,
348 * make sure it's executed now.
349 */
350 if (entry->rcu_pending)
351 rcu_barrier_sched();
352 old = tracepoint_entry_add_probe(entry, probe);
353 if (IS_ERR(old)) {
354 ret = PTR_ERR(old);
355 goto end;
356 }
357 mutex_unlock(&tracepoints_mutex); 337 mutex_unlock(&tracepoints_mutex);
338 if (IS_ERR(old))
339 return PTR_ERR(old);
340
358 tracepoint_update_probes(); /* may update entry */ 341 tracepoint_update_probes(); /* may update entry */
359 mutex_lock(&tracepoints_mutex); 342 release_probes(old);
360 entry = get_tracepoint(name); 343 return 0;
361 WARN_ON(!entry);
362 if (entry->rcu_pending)
363 rcu_barrier_sched();
364 tracepoint_entry_free_old(entry, old);
365end:
366 mutex_unlock(&tracepoints_mutex);
367 return ret;
368} 344}
369EXPORT_SYMBOL_GPL(tracepoint_probe_register); 345EXPORT_SYMBOL_GPL(tracepoint_probe_register);
370 346
347static void *tracepoint_remove_probe(const char *name, void *probe)
348{
349 struct tracepoint_entry *entry;
350 void *old;
351
352 entry = get_tracepoint(name);
353 if (!entry)
354 return ERR_PTR(-ENOENT);
355 old = tracepoint_entry_remove_probe(entry, probe);
356 if (IS_ERR(old))
357 return old;
358 if (!entry->refcount)
359 remove_tracepoint(entry);
360 return old;
361}
362
371/** 363/**
372 * tracepoint_probe_unregister - Disconnect a probe from a tracepoint 364 * tracepoint_probe_unregister - Disconnect a probe from a tracepoint
373 * @name: tracepoint name 365 * @name: tracepoint name
@@ -380,38 +372,104 @@ EXPORT_SYMBOL_GPL(tracepoint_probe_register);
380 */ 372 */
381int tracepoint_probe_unregister(const char *name, void *probe) 373int tracepoint_probe_unregister(const char *name, void *probe)
382{ 374{
383 struct tracepoint_entry *entry;
384 void *old; 375 void *old;
385 int ret = -ENOENT;
386 376
387 mutex_lock(&tracepoints_mutex); 377 mutex_lock(&tracepoints_mutex);
388 entry = get_tracepoint(name); 378 old = tracepoint_remove_probe(name, probe);
389 if (!entry)
390 goto end;
391 if (entry->rcu_pending)
392 rcu_barrier_sched();
393 old = tracepoint_entry_remove_probe(entry, probe);
394 if (!old) {
395 printk(KERN_WARNING "Warning: Trying to unregister a probe"
396 "that doesn't exist\n");
397 goto end;
398 }
399 mutex_unlock(&tracepoints_mutex); 379 mutex_unlock(&tracepoints_mutex);
380 if (IS_ERR(old))
381 return PTR_ERR(old);
382
400 tracepoint_update_probes(); /* may update entry */ 383 tracepoint_update_probes(); /* may update entry */
384 release_probes(old);
385 return 0;
386}
387EXPORT_SYMBOL_GPL(tracepoint_probe_unregister);
388
389static LIST_HEAD(old_probes);
390static int need_update;
391
392static void tracepoint_add_old_probes(void *old)
393{
394 need_update = 1;
395 if (old) {
396 struct tp_probes *tp_probes = container_of(old,
397 struct tp_probes, probes[0]);
398 list_add(&tp_probes->u.list, &old_probes);
399 }
400}
401
402/**
403 * tracepoint_probe_register_noupdate - register a probe but not connect
404 * @name: tracepoint name
405 * @probe: probe handler
406 *
407 * caller must call tracepoint_probe_update_all()
408 */
409int tracepoint_probe_register_noupdate(const char *name, void *probe)
410{
411 void *old;
412
401 mutex_lock(&tracepoints_mutex); 413 mutex_lock(&tracepoints_mutex);
402 entry = get_tracepoint(name); 414 old = tracepoint_add_probe(name, probe);
403 if (!entry) 415 if (IS_ERR(old)) {
404 goto end; 416 mutex_unlock(&tracepoints_mutex);
405 if (entry->rcu_pending) 417 return PTR_ERR(old);
406 rcu_barrier_sched(); 418 }
407 tracepoint_entry_free_old(entry, old); 419 tracepoint_add_old_probes(old);
408 remove_tracepoint(name); /* Ignore busy error message */
409 ret = 0;
410end:
411 mutex_unlock(&tracepoints_mutex); 420 mutex_unlock(&tracepoints_mutex);
412 return ret; 421 return 0;
413} 422}
414EXPORT_SYMBOL_GPL(tracepoint_probe_unregister); 423EXPORT_SYMBOL_GPL(tracepoint_probe_register_noupdate);
424
425/**
426 * tracepoint_probe_unregister_noupdate - remove a probe but not disconnect
427 * @name: tracepoint name
428 * @probe: probe function pointer
429 *
430 * caller must call tracepoint_probe_update_all()
431 */
432int tracepoint_probe_unregister_noupdate(const char *name, void *probe)
433{
434 void *old;
435
436 mutex_lock(&tracepoints_mutex);
437 old = tracepoint_remove_probe(name, probe);
438 if (IS_ERR(old)) {
439 mutex_unlock(&tracepoints_mutex);
440 return PTR_ERR(old);
441 }
442 tracepoint_add_old_probes(old);
443 mutex_unlock(&tracepoints_mutex);
444 return 0;
445}
446EXPORT_SYMBOL_GPL(tracepoint_probe_unregister_noupdate);
447
448/**
449 * tracepoint_probe_update_all - update tracepoints
450 */
451void tracepoint_probe_update_all(void)
452{
453 LIST_HEAD(release_probes);
454 struct tp_probes *pos, *next;
455
456 mutex_lock(&tracepoints_mutex);
457 if (!need_update) {
458 mutex_unlock(&tracepoints_mutex);
459 return;
460 }
461 if (!list_empty(&old_probes))
462 list_replace_init(&old_probes, &release_probes);
463 need_update = 0;
464 mutex_unlock(&tracepoints_mutex);
465
466 tracepoint_update_probes();
467 list_for_each_entry_safe(pos, next, &release_probes, u.list) {
468 list_del(&pos->u.list);
469 call_rcu_sched(&pos->u.rcu, rcu_free_old_probes);
470 }
471}
472EXPORT_SYMBOL_GPL(tracepoint_probe_update_all);
415 473
416/** 474/**
417 * tracepoint_get_iter_range - Get a next tracepoint iterator given a range. 475 * tracepoint_get_iter_range - Get a next tracepoint iterator given a range.
@@ -483,3 +541,36 @@ void tracepoint_iter_reset(struct tracepoint_iter *iter)
483 iter->tracepoint = NULL; 541 iter->tracepoint = NULL;
484} 542}
485EXPORT_SYMBOL_GPL(tracepoint_iter_reset); 543EXPORT_SYMBOL_GPL(tracepoint_iter_reset);
544
545#ifdef CONFIG_MODULES
546
547int tracepoint_module_notify(struct notifier_block *self,
548 unsigned long val, void *data)
549{
550 struct module *mod = data;
551
552 switch (val) {
553 case MODULE_STATE_COMING:
554 tracepoint_update_probe_range(mod->tracepoints,
555 mod->tracepoints + mod->num_tracepoints);
556 break;
557 case MODULE_STATE_GOING:
558 tracepoint_update_probe_range(mod->tracepoints,
559 mod->tracepoints + mod->num_tracepoints);
560 break;
561 }
562 return 0;
563}
564
565struct notifier_block tracepoint_module_nb = {
566 .notifier_call = tracepoint_module_notify,
567 .priority = 0,
568};
569
570static int init_tracepoints(void)
571{
572 return register_module_notifier(&tracepoint_module_nb);
573}
574__initcall(init_tracepoints);
575
576#endif /* CONFIG_MODULES */
diff --git a/kernel/tsacct.c b/kernel/tsacct.c
index 8ebcd8532dfb..2dc06ab35716 100644
--- a/kernel/tsacct.c
+++ b/kernel/tsacct.c
@@ -27,6 +27,7 @@
27 */ 27 */
28void bacct_add_tsk(struct taskstats *stats, struct task_struct *tsk) 28void bacct_add_tsk(struct taskstats *stats, struct task_struct *tsk)
29{ 29{
30 const struct cred *tcred;
30 struct timespec uptime, ts; 31 struct timespec uptime, ts;
31 u64 ac_etime; 32 u64 ac_etime;
32 33
@@ -53,10 +54,11 @@ void bacct_add_tsk(struct taskstats *stats, struct task_struct *tsk)
53 stats->ac_flag |= AXSIG; 54 stats->ac_flag |= AXSIG;
54 stats->ac_nice = task_nice(tsk); 55 stats->ac_nice = task_nice(tsk);
55 stats->ac_sched = tsk->policy; 56 stats->ac_sched = tsk->policy;
56 stats->ac_uid = tsk->uid;
57 stats->ac_gid = tsk->gid;
58 stats->ac_pid = tsk->pid; 57 stats->ac_pid = tsk->pid;
59 rcu_read_lock(); 58 rcu_read_lock();
59 tcred = __task_cred(tsk);
60 stats->ac_uid = tcred->uid;
61 stats->ac_gid = tcred->gid;
60 stats->ac_ppid = pid_alive(tsk) ? 62 stats->ac_ppid = pid_alive(tsk) ?
61 rcu_dereference(tsk->real_parent)->tgid : 0; 63 rcu_dereference(tsk->real_parent)->tgid : 0;
62 rcu_read_unlock(); 64 rcu_read_unlock();
diff --git a/kernel/uid16.c b/kernel/uid16.c
index 3e41c1673e2f..2460c3199b5a 100644
--- a/kernel/uid16.c
+++ b/kernel/uid16.c
@@ -84,11 +84,12 @@ asmlinkage long sys_setresuid16(old_uid_t ruid, old_uid_t euid, old_uid_t suid)
84 84
85asmlinkage long sys_getresuid16(old_uid_t __user *ruid, old_uid_t __user *euid, old_uid_t __user *suid) 85asmlinkage long sys_getresuid16(old_uid_t __user *ruid, old_uid_t __user *euid, old_uid_t __user *suid)
86{ 86{
87 const struct cred *cred = current_cred();
87 int retval; 88 int retval;
88 89
89 if (!(retval = put_user(high2lowuid(current->uid), ruid)) && 90 if (!(retval = put_user(high2lowuid(cred->uid), ruid)) &&
90 !(retval = put_user(high2lowuid(current->euid), euid))) 91 !(retval = put_user(high2lowuid(cred->euid), euid)))
91 retval = put_user(high2lowuid(current->suid), suid); 92 retval = put_user(high2lowuid(cred->suid), suid);
92 93
93 return retval; 94 return retval;
94} 95}
@@ -104,11 +105,12 @@ asmlinkage long sys_setresgid16(old_gid_t rgid, old_gid_t egid, old_gid_t sgid)
104 105
105asmlinkage long sys_getresgid16(old_gid_t __user *rgid, old_gid_t __user *egid, old_gid_t __user *sgid) 106asmlinkage long sys_getresgid16(old_gid_t __user *rgid, old_gid_t __user *egid, old_gid_t __user *sgid)
106{ 107{
108 const struct cred *cred = current_cred();
107 int retval; 109 int retval;
108 110
109 if (!(retval = put_user(high2lowgid(current->gid), rgid)) && 111 if (!(retval = put_user(high2lowgid(cred->gid), rgid)) &&
110 !(retval = put_user(high2lowgid(current->egid), egid))) 112 !(retval = put_user(high2lowgid(cred->egid), egid)))
111 retval = put_user(high2lowgid(current->sgid), sgid); 113 retval = put_user(high2lowgid(cred->sgid), sgid);
112 114
113 return retval; 115 return retval;
114} 116}
@@ -161,25 +163,24 @@ static int groups16_from_user(struct group_info *group_info,
161 163
162asmlinkage long sys_getgroups16(int gidsetsize, old_gid_t __user *grouplist) 164asmlinkage long sys_getgroups16(int gidsetsize, old_gid_t __user *grouplist)
163{ 165{
164 int i = 0; 166 const struct cred *cred = current_cred();
167 int i;
165 168
166 if (gidsetsize < 0) 169 if (gidsetsize < 0)
167 return -EINVAL; 170 return -EINVAL;
168 171
169 get_group_info(current->group_info); 172 i = cred->group_info->ngroups;
170 i = current->group_info->ngroups;
171 if (gidsetsize) { 173 if (gidsetsize) {
172 if (i > gidsetsize) { 174 if (i > gidsetsize) {
173 i = -EINVAL; 175 i = -EINVAL;
174 goto out; 176 goto out;
175 } 177 }
176 if (groups16_to_user(grouplist, current->group_info)) { 178 if (groups16_to_user(grouplist, cred->group_info)) {
177 i = -EFAULT; 179 i = -EFAULT;
178 goto out; 180 goto out;
179 } 181 }
180 } 182 }
181out: 183out:
182 put_group_info(current->group_info);
183 return i; 184 return i;
184} 185}
185 186
@@ -210,20 +211,20 @@ asmlinkage long sys_setgroups16(int gidsetsize, old_gid_t __user *grouplist)
210 211
211asmlinkage long sys_getuid16(void) 212asmlinkage long sys_getuid16(void)
212{ 213{
213 return high2lowuid(current->uid); 214 return high2lowuid(current_uid());
214} 215}
215 216
216asmlinkage long sys_geteuid16(void) 217asmlinkage long sys_geteuid16(void)
217{ 218{
218 return high2lowuid(current->euid); 219 return high2lowuid(current_euid());
219} 220}
220 221
221asmlinkage long sys_getgid16(void) 222asmlinkage long sys_getgid16(void)
222{ 223{
223 return high2lowgid(current->gid); 224 return high2lowgid(current_gid());
224} 225}
225 226
226asmlinkage long sys_getegid16(void) 227asmlinkage long sys_getegid16(void)
227{ 228{
228 return high2lowgid(current->egid); 229 return high2lowgid(current_egid());
229} 230}
diff --git a/kernel/user.c b/kernel/user.c
index 39d6159fae43..477b6660f447 100644
--- a/kernel/user.c
+++ b/kernel/user.c
@@ -16,12 +16,13 @@
16#include <linux/interrupt.h> 16#include <linux/interrupt.h>
17#include <linux/module.h> 17#include <linux/module.h>
18#include <linux/user_namespace.h> 18#include <linux/user_namespace.h>
19#include "cred-internals.h"
19 20
20struct user_namespace init_user_ns = { 21struct user_namespace init_user_ns = {
21 .kref = { 22 .kref = {
22 .refcount = ATOMIC_INIT(2), 23 .refcount = ATOMIC_INIT(1),
23 }, 24 },
24 .root_user = &root_user, 25 .creator = &root_user,
25}; 26};
26EXPORT_SYMBOL_GPL(init_user_ns); 27EXPORT_SYMBOL_GPL(init_user_ns);
27 28
@@ -47,12 +48,14 @@ static struct kmem_cache *uid_cachep;
47 */ 48 */
48static DEFINE_SPINLOCK(uidhash_lock); 49static DEFINE_SPINLOCK(uidhash_lock);
49 50
51/* root_user.__count is 2, 1 for init task cred, 1 for init_user_ns->creator */
50struct user_struct root_user = { 52struct user_struct root_user = {
51 .__count = ATOMIC_INIT(1), 53 .__count = ATOMIC_INIT(2),
52 .processes = ATOMIC_INIT(1), 54 .processes = ATOMIC_INIT(1),
53 .files = ATOMIC_INIT(0), 55 .files = ATOMIC_INIT(0),
54 .sigpending = ATOMIC_INIT(0), 56 .sigpending = ATOMIC_INIT(0),
55 .locked_shm = 0, 57 .locked_shm = 0,
58 .user_ns = &init_user_ns,
56#ifdef CONFIG_USER_SCHED 59#ifdef CONFIG_USER_SCHED
57 .tg = &init_task_group, 60 .tg = &init_task_group,
58#endif 61#endif
@@ -101,19 +104,15 @@ static int sched_create_user(struct user_struct *up)
101 if (IS_ERR(up->tg)) 104 if (IS_ERR(up->tg))
102 rc = -ENOMEM; 105 rc = -ENOMEM;
103 106
104 return rc; 107 set_tg_uid(up);
105}
106 108
107static void sched_switch_user(struct task_struct *p) 109 return rc;
108{
109 sched_move_task(p);
110} 110}
111 111
112#else /* CONFIG_USER_SCHED */ 112#else /* CONFIG_USER_SCHED */
113 113
114static void sched_destroy_user(struct user_struct *up) { } 114static void sched_destroy_user(struct user_struct *up) { }
115static int sched_create_user(struct user_struct *up) { return 0; } 115static int sched_create_user(struct user_struct *up) { return 0; }
116static void sched_switch_user(struct task_struct *p) { }
117 116
118#endif /* CONFIG_USER_SCHED */ 117#endif /* CONFIG_USER_SCHED */
119 118
@@ -242,13 +241,21 @@ static struct kobj_type uids_ktype = {
242 .release = uids_release, 241 .release = uids_release,
243}; 242};
244 243
245/* create /sys/kernel/uids/<uid>/cpu_share file for this user */ 244/*
245 * Create /sys/kernel/uids/<uid>/cpu_share file for this user
246 * We do not create this file for users in a user namespace (until
247 * sysfs tagging is implemented).
248 *
249 * See Documentation/scheduler/sched-design-CFS.txt for ramifications.
250 */
246static int uids_user_create(struct user_struct *up) 251static int uids_user_create(struct user_struct *up)
247{ 252{
248 struct kobject *kobj = &up->kobj; 253 struct kobject *kobj = &up->kobj;
249 int error; 254 int error;
250 255
251 memset(kobj, 0, sizeof(struct kobject)); 256 memset(kobj, 0, sizeof(struct kobject));
257 if (up->user_ns != &init_user_ns)
258 return 0;
252 kobj->kset = uids_kset; 259 kobj->kset = uids_kset;
253 error = kobject_init_and_add(kobj, &uids_ktype, NULL, "%d", up->uid); 260 error = kobject_init_and_add(kobj, &uids_ktype, NULL, "%d", up->uid);
254 if (error) { 261 if (error) {
@@ -284,6 +291,8 @@ static void remove_user_sysfs_dir(struct work_struct *w)
284 unsigned long flags; 291 unsigned long flags;
285 int remove_user = 0; 292 int remove_user = 0;
286 293
294 if (up->user_ns != &init_user_ns)
295 return;
287 /* Make uid_hash_remove() + sysfs_remove_file() + kobject_del() 296 /* Make uid_hash_remove() + sysfs_remove_file() + kobject_del()
288 * atomic. 297 * atomic.
289 */ 298 */
@@ -319,12 +328,13 @@ done:
319 * IRQ state (as stored in flags) is restored and uidhash_lock released 328 * IRQ state (as stored in flags) is restored and uidhash_lock released
320 * upon function exit. 329 * upon function exit.
321 */ 330 */
322static inline void free_user(struct user_struct *up, unsigned long flags) 331static void free_user(struct user_struct *up, unsigned long flags)
323{ 332{
324 /* restore back the count */ 333 /* restore back the count */
325 atomic_inc(&up->__count); 334 atomic_inc(&up->__count);
326 spin_unlock_irqrestore(&uidhash_lock, flags); 335 spin_unlock_irqrestore(&uidhash_lock, flags);
327 336
337 put_user_ns(up->user_ns);
328 INIT_WORK(&up->work, remove_user_sysfs_dir); 338 INIT_WORK(&up->work, remove_user_sysfs_dir);
329 schedule_work(&up->work); 339 schedule_work(&up->work);
330} 340}
@@ -340,13 +350,14 @@ static inline void uids_mutex_unlock(void) { }
340 * IRQ state (as stored in flags) is restored and uidhash_lock released 350 * IRQ state (as stored in flags) is restored and uidhash_lock released
341 * upon function exit. 351 * upon function exit.
342 */ 352 */
343static inline void free_user(struct user_struct *up, unsigned long flags) 353static void free_user(struct user_struct *up, unsigned long flags)
344{ 354{
345 uid_hash_remove(up); 355 uid_hash_remove(up);
346 spin_unlock_irqrestore(&uidhash_lock, flags); 356 spin_unlock_irqrestore(&uidhash_lock, flags);
347 sched_destroy_user(up); 357 sched_destroy_user(up);
348 key_put(up->uid_keyring); 358 key_put(up->uid_keyring);
349 key_put(up->session_keyring); 359 key_put(up->session_keyring);
360 put_user_ns(up->user_ns);
350 kmem_cache_free(uid_cachep, up); 361 kmem_cache_free(uid_cachep, up);
351} 362}
352 363
@@ -362,7 +373,7 @@ struct user_struct *find_user(uid_t uid)
362{ 373{
363 struct user_struct *ret; 374 struct user_struct *ret;
364 unsigned long flags; 375 unsigned long flags;
365 struct user_namespace *ns = current->nsproxy->user_ns; 376 struct user_namespace *ns = current_user_ns();
366 377
367 spin_lock_irqsave(&uidhash_lock, flags); 378 spin_lock_irqsave(&uidhash_lock, flags);
368 ret = uid_hash_find(uid, uidhashentry(ns, uid)); 379 ret = uid_hash_find(uid, uidhashentry(ns, uid));
@@ -409,6 +420,8 @@ struct user_struct *alloc_uid(struct user_namespace *ns, uid_t uid)
409 if (sched_create_user(new) < 0) 420 if (sched_create_user(new) < 0)
410 goto out_free_user; 421 goto out_free_user;
411 422
423 new->user_ns = get_user_ns(ns);
424
412 if (uids_user_create(new)) 425 if (uids_user_create(new))
413 goto out_destoy_sched; 426 goto out_destoy_sched;
414 427
@@ -432,7 +445,6 @@ struct user_struct *alloc_uid(struct user_namespace *ns, uid_t uid)
432 up = new; 445 up = new;
433 } 446 }
434 spin_unlock_irq(&uidhash_lock); 447 spin_unlock_irq(&uidhash_lock);
435
436 } 448 }
437 449
438 uids_mutex_unlock(); 450 uids_mutex_unlock();
@@ -441,6 +453,7 @@ struct user_struct *alloc_uid(struct user_namespace *ns, uid_t uid)
441 453
442out_destoy_sched: 454out_destoy_sched:
443 sched_destroy_user(new); 455 sched_destroy_user(new);
456 put_user_ns(new->user_ns);
444out_free_user: 457out_free_user:
445 kmem_cache_free(uid_cachep, new); 458 kmem_cache_free(uid_cachep, new);
446out_unlock: 459out_unlock:
@@ -448,63 +461,6 @@ out_unlock:
448 return NULL; 461 return NULL;
449} 462}
450 463
451void switch_uid(struct user_struct *new_user)
452{
453 struct user_struct *old_user;
454
455 /* What if a process setreuid()'s and this brings the
456 * new uid over his NPROC rlimit? We can check this now
457 * cheaply with the new uid cache, so if it matters
458 * we should be checking for it. -DaveM
459 */
460 old_user = current->user;
461 atomic_inc(&new_user->processes);
462 atomic_dec(&old_user->processes);
463 switch_uid_keyring(new_user);
464 current->user = new_user;
465 sched_switch_user(current);
466
467 /*
468 * We need to synchronize with __sigqueue_alloc()
469 * doing a get_uid(p->user).. If that saw the old
470 * user value, we need to wait until it has exited
471 * its critical region before we can free the old
472 * structure.
473 */
474 smp_mb();
475 spin_unlock_wait(&current->sighand->siglock);
476
477 free_uid(old_user);
478 suid_keys(current);
479}
480
481#ifdef CONFIG_USER_NS
482void release_uids(struct user_namespace *ns)
483{
484 int i;
485 unsigned long flags;
486 struct hlist_head *head;
487 struct hlist_node *nd;
488
489 spin_lock_irqsave(&uidhash_lock, flags);
490 /*
491 * collapse the chains so that the user_struct-s will
492 * be still alive, but not in hashes. subsequent free_uid()
493 * will free them.
494 */
495 for (i = 0; i < UIDHASH_SZ; i++) {
496 head = ns->uidhash_table + i;
497 while (!hlist_empty(head)) {
498 nd = head->first;
499 hlist_del_init(nd);
500 }
501 }
502 spin_unlock_irqrestore(&uidhash_lock, flags);
503
504 free_uid(ns->root_user);
505}
506#endif
507
508static int __init uid_cache_init(void) 464static int __init uid_cache_init(void)
509{ 465{
510 int n; 466 int n;
diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
index 532858fa5b88..79084311ee57 100644
--- a/kernel/user_namespace.c
+++ b/kernel/user_namespace.c
@@ -9,60 +9,55 @@
9#include <linux/nsproxy.h> 9#include <linux/nsproxy.h>
10#include <linux/slab.h> 10#include <linux/slab.h>
11#include <linux/user_namespace.h> 11#include <linux/user_namespace.h>
12#include <linux/cred.h>
12 13
13/* 14/*
14 * Clone a new ns copying an original user ns, setting refcount to 1 15 * Create a new user namespace, deriving the creator from the user in the
15 * @old_ns: namespace to clone 16 * passed credentials, and replacing that user with the new root user for the
16 * Return NULL on error (failure to kmalloc), new ns otherwise 17 * new namespace.
18 *
19 * This is called by copy_creds(), which will finish setting the target task's
20 * credentials.
17 */ 21 */
18static struct user_namespace *clone_user_ns(struct user_namespace *old_ns) 22int create_user_ns(struct cred *new)
19{ 23{
20 struct user_namespace *ns; 24 struct user_namespace *ns;
21 struct user_struct *new_user; 25 struct user_struct *root_user;
22 int n; 26 int n;
23 27
24 ns = kmalloc(sizeof(struct user_namespace), GFP_KERNEL); 28 ns = kmalloc(sizeof(struct user_namespace), GFP_KERNEL);
25 if (!ns) 29 if (!ns)
26 return ERR_PTR(-ENOMEM); 30 return -ENOMEM;
27 31
28 kref_init(&ns->kref); 32 kref_init(&ns->kref);
29 33
30 for (n = 0; n < UIDHASH_SZ; ++n) 34 for (n = 0; n < UIDHASH_SZ; ++n)
31 INIT_HLIST_HEAD(ns->uidhash_table + n); 35 INIT_HLIST_HEAD(ns->uidhash_table + n);
32 36
33 /* Insert new root user. */ 37 /* Alloc new root user. */
34 ns->root_user = alloc_uid(ns, 0); 38 root_user = alloc_uid(ns, 0);
35 if (!ns->root_user) { 39 if (!root_user) {
36 kfree(ns); 40 kfree(ns);
37 return ERR_PTR(-ENOMEM); 41 return -ENOMEM;
38 } 42 }
39 43
40 /* Reset current->user with a new one */ 44 /* set the new root user in the credentials under preparation */
41 new_user = alloc_uid(ns, current->uid); 45 ns->creator = new->user;
42 if (!new_user) { 46 new->user = root_user;
43 free_uid(ns->root_user); 47 new->uid = new->euid = new->suid = new->fsuid = 0;
44 kfree(ns); 48 new->gid = new->egid = new->sgid = new->fsgid = 0;
45 return ERR_PTR(-ENOMEM); 49 put_group_info(new->group_info);
46 } 50 new->group_info = get_group_info(&init_groups);
47 51#ifdef CONFIG_KEYS
48 switch_uid(new_user); 52 key_put(new->request_key_auth);
49 return ns; 53 new->request_key_auth = NULL;
50} 54#endif
51 55 /* tgcred will be cleared in our caller bc CLONE_THREAD won't be set */
52struct user_namespace * copy_user_ns(int flags, struct user_namespace *old_ns)
53{
54 struct user_namespace *new_ns;
55
56 BUG_ON(!old_ns);
57 get_user_ns(old_ns);
58
59 if (!(flags & CLONE_NEWUSER))
60 return old_ns;
61 56
62 new_ns = clone_user_ns(old_ns); 57 /* alloc_uid() incremented the userns refcount. Just set it to 1 */
58 kref_set(&ns->kref, 1);
63 59
64 put_user_ns(old_ns); 60 return 0;
65 return new_ns;
66} 61}
67 62
68void free_user_ns(struct kref *kref) 63void free_user_ns(struct kref *kref)
@@ -70,7 +65,7 @@ void free_user_ns(struct kref *kref)
70 struct user_namespace *ns; 65 struct user_namespace *ns;
71 66
72 ns = container_of(kref, struct user_namespace, kref); 67 ns = container_of(kref, struct user_namespace, kref);
73 release_uids(ns); 68 free_uid(ns->creator);
74 kfree(ns); 69 kfree(ns);
75} 70}
76EXPORT_SYMBOL(free_user_ns); 71EXPORT_SYMBOL(free_user_ns);
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index d4dc69ddebd7..2f445833ae37 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -73,7 +73,7 @@ static DEFINE_SPINLOCK(workqueue_lock);
73static LIST_HEAD(workqueues); 73static LIST_HEAD(workqueues);
74 74
75static int singlethread_cpu __read_mostly; 75static int singlethread_cpu __read_mostly;
76static cpumask_t cpu_singlethread_map __read_mostly; 76static const struct cpumask *cpu_singlethread_map __read_mostly;
77/* 77/*
78 * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD 78 * _cpu_down() first removes CPU from cpu_online_map, then CPU_DEAD
79 * flushes cwq->worklist. This means that flush_workqueue/wait_on_work 79 * flushes cwq->worklist. This means that flush_workqueue/wait_on_work
@@ -81,24 +81,24 @@ static cpumask_t cpu_singlethread_map __read_mostly;
81 * use cpu_possible_map, the cpumask below is more a documentation 81 * use cpu_possible_map, the cpumask below is more a documentation
82 * than optimization. 82 * than optimization.
83 */ 83 */
84static cpumask_t cpu_populated_map __read_mostly; 84static cpumask_var_t cpu_populated_map __read_mostly;
85 85
86/* If it's single threaded, it isn't in the list of workqueues. */ 86/* If it's single threaded, it isn't in the list of workqueues. */
87static inline int is_single_threaded(struct workqueue_struct *wq) 87static inline int is_wq_single_threaded(struct workqueue_struct *wq)
88{ 88{
89 return wq->singlethread; 89 return wq->singlethread;
90} 90}
91 91
92static const cpumask_t *wq_cpu_map(struct workqueue_struct *wq) 92static const struct cpumask *wq_cpu_map(struct workqueue_struct *wq)
93{ 93{
94 return is_single_threaded(wq) 94 return is_wq_single_threaded(wq)
95 ? &cpu_singlethread_map : &cpu_populated_map; 95 ? cpu_singlethread_map : cpu_populated_map;
96} 96}
97 97
98static 98static
99struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu) 99struct cpu_workqueue_struct *wq_per_cpu(struct workqueue_struct *wq, int cpu)
100{ 100{
101 if (unlikely(is_single_threaded(wq))) 101 if (unlikely(is_wq_single_threaded(wq)))
102 cpu = singlethread_cpu; 102 cpu = singlethread_cpu;
103 return per_cpu_ptr(wq->cpu_wq, cpu); 103 return per_cpu_ptr(wq->cpu_wq, cpu);
104} 104}
@@ -410,7 +410,7 @@ static int flush_cpu_workqueue(struct cpu_workqueue_struct *cwq)
410 */ 410 */
411void flush_workqueue(struct workqueue_struct *wq) 411void flush_workqueue(struct workqueue_struct *wq)
412{ 412{
413 const cpumask_t *cpu_map = wq_cpu_map(wq); 413 const struct cpumask *cpu_map = wq_cpu_map(wq);
414 int cpu; 414 int cpu;
415 415
416 might_sleep(); 416 might_sleep();
@@ -532,7 +532,7 @@ static void wait_on_work(struct work_struct *work)
532{ 532{
533 struct cpu_workqueue_struct *cwq; 533 struct cpu_workqueue_struct *cwq;
534 struct workqueue_struct *wq; 534 struct workqueue_struct *wq;
535 const cpumask_t *cpu_map; 535 const struct cpumask *cpu_map;
536 int cpu; 536 int cpu;
537 537
538 might_sleep(); 538 might_sleep();
@@ -769,7 +769,7 @@ static int create_workqueue_thread(struct cpu_workqueue_struct *cwq, int cpu)
769{ 769{
770 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 770 struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 };
771 struct workqueue_struct *wq = cwq->wq; 771 struct workqueue_struct *wq = cwq->wq;
772 const char *fmt = is_single_threaded(wq) ? "%s" : "%s/%d"; 772 const char *fmt = is_wq_single_threaded(wq) ? "%s" : "%s/%d";
773 struct task_struct *p; 773 struct task_struct *p;
774 774
775 p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu); 775 p = kthread_create(worker_thread, cwq, fmt, wq->name, cpu);
@@ -903,7 +903,7 @@ static void cleanup_workqueue_thread(struct cpu_workqueue_struct *cwq)
903 */ 903 */
904void destroy_workqueue(struct workqueue_struct *wq) 904void destroy_workqueue(struct workqueue_struct *wq)
905{ 905{
906 const cpumask_t *cpu_map = wq_cpu_map(wq); 906 const struct cpumask *cpu_map = wq_cpu_map(wq);
907 int cpu; 907 int cpu;
908 908
909 cpu_maps_update_begin(); 909 cpu_maps_update_begin();
@@ -933,7 +933,7 @@ static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,
933 933
934 switch (action) { 934 switch (action) {
935 case CPU_UP_PREPARE: 935 case CPU_UP_PREPARE:
936 cpu_set(cpu, cpu_populated_map); 936 cpumask_set_cpu(cpu, cpu_populated_map);
937 } 937 }
938undo: 938undo:
939 list_for_each_entry(wq, &workqueues, list) { 939 list_for_each_entry(wq, &workqueues, list) {
@@ -964,7 +964,7 @@ undo:
964 switch (action) { 964 switch (action) {
965 case CPU_UP_CANCELED: 965 case CPU_UP_CANCELED:
966 case CPU_POST_DEAD: 966 case CPU_POST_DEAD:
967 cpu_clear(cpu, cpu_populated_map); 967 cpumask_clear_cpu(cpu, cpu_populated_map);
968 } 968 }
969 969
970 return ret; 970 return ret;
@@ -1017,9 +1017,11 @@ EXPORT_SYMBOL_GPL(work_on_cpu);
1017 1017
1018void __init init_workqueues(void) 1018void __init init_workqueues(void)
1019{ 1019{
1020 cpu_populated_map = cpu_online_map; 1020 alloc_cpumask_var(&cpu_populated_map, GFP_KERNEL);
1021 singlethread_cpu = first_cpu(cpu_possible_map); 1021
1022 cpu_singlethread_map = cpumask_of_cpu(singlethread_cpu); 1022 cpumask_copy(cpu_populated_map, cpu_online_mask);
1023 singlethread_cpu = cpumask_first(cpu_possible_mask);
1024 cpu_singlethread_map = cpumask_of(singlethread_cpu);
1023 hotcpu_notifier(workqueue_cpu_callback, 0); 1025 hotcpu_notifier(workqueue_cpu_callback, 0);
1024 keventd_wq = create_workqueue("events"); 1026 keventd_wq = create_workqueue("events");
1025 BUG_ON(!keventd_wq); 1027 BUG_ON(!keventd_wq);