aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/fork.c47
-rw-r--r--kernel/futex.c28
-rw-r--r--kernel/futex_compat.c6
-rw-r--r--kernel/irq/manage.c27
-rw-r--r--kernel/irq/numa_migrate.c4
-rw-r--r--kernel/lockdep_proc.c3
-rw-r--r--kernel/module.c10
-rw-r--r--kernel/panic.c1
-rw-r--r--kernel/perf_counter.c680
-rw-r--r--kernel/posix-cpu-timers.c7
-rw-r--r--kernel/posix-timers.c7
-rw-r--r--kernel/rtmutex.c4
-rw-r--r--kernel/sched_cpupri.c15
-rw-r--r--kernel/sched_fair.c32
-rw-r--r--kernel/signal.c25
-rw-r--r--kernel/smp.c2
-rw-r--r--kernel/sysctl.c7
-rw-r--r--kernel/time/clockevents.c16
-rw-r--r--kernel/time/tick-broadcast.c7
-rw-r--r--kernel/time/timer_list.c2
-rw-r--r--kernel/trace/blktrace.c12
-rw-r--r--kernel/trace/ftrace.c21
-rw-r--r--kernel/trace/ring_buffer.c15
-rw-r--r--kernel/trace/trace.c25
-rw-r--r--kernel/trace/trace.h4
-rw-r--r--kernel/trace/trace_event_profile.c2
-rw-r--r--kernel/trace/trace_events.c4
-rw-r--r--kernel/trace/trace_events_filter.c20
-rw-r--r--kernel/trace/trace_functions_graph.c11
-rw-r--r--kernel/trace/trace_printk.c2
-rw-r--r--kernel/wait.c5
31 files changed, 681 insertions, 370 deletions
diff --git a/kernel/fork.c b/kernel/fork.c
index 29b532e718f7..e6c04d462ab2 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -426,7 +426,6 @@ static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
426 init_rwsem(&mm->mmap_sem); 426 init_rwsem(&mm->mmap_sem);
427 INIT_LIST_HEAD(&mm->mmlist); 427 INIT_LIST_HEAD(&mm->mmlist);
428 mm->flags = (current->mm) ? current->mm->flags : default_dump_filter; 428 mm->flags = (current->mm) ? current->mm->flags : default_dump_filter;
429 mm->oom_adj = (current->mm) ? current->mm->oom_adj : 0;
430 mm->core_state = NULL; 429 mm->core_state = NULL;
431 mm->nr_ptes = 0; 430 mm->nr_ptes = 0;
432 set_mm_counter(mm, file_rss, 0); 431 set_mm_counter(mm, file_rss, 0);
@@ -568,18 +567,18 @@ void mm_release(struct task_struct *tsk, struct mm_struct *mm)
568 * the value intact in a core dump, and to save the unnecessary 567 * the value intact in a core dump, and to save the unnecessary
569 * trouble otherwise. Userland only wants this done for a sys_exit. 568 * trouble otherwise. Userland only wants this done for a sys_exit.
570 */ 569 */
571 if (tsk->clear_child_tid 570 if (tsk->clear_child_tid) {
572 && !(tsk->flags & PF_SIGNALED) 571 if (!(tsk->flags & PF_SIGNALED) &&
573 && atomic_read(&mm->mm_users) > 1) { 572 atomic_read(&mm->mm_users) > 1) {
574 u32 __user * tidptr = tsk->clear_child_tid; 573 /*
574 * We don't check the error code - if userspace has
575 * not set up a proper pointer then tough luck.
576 */
577 put_user(0, tsk->clear_child_tid);
578 sys_futex(tsk->clear_child_tid, FUTEX_WAKE,
579 1, NULL, NULL, 0);
580 }
575 tsk->clear_child_tid = NULL; 581 tsk->clear_child_tid = NULL;
576
577 /*
578 * We don't check the error code - if userspace has
579 * not set up a proper pointer then tough luck.
580 */
581 put_user(0, tidptr);
582 sys_futex(tidptr, FUTEX_WAKE, 1, NULL, NULL, 0);
583 } 582 }
584} 583}
585 584
@@ -816,11 +815,8 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
816{ 815{
817 struct signal_struct *sig; 816 struct signal_struct *sig;
818 817
819 if (clone_flags & CLONE_THREAD) { 818 if (clone_flags & CLONE_THREAD)
820 atomic_inc(&current->signal->count);
821 atomic_inc(&current->signal->live);
822 return 0; 819 return 0;
823 }
824 820
825 sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL); 821 sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL);
826 tsk->signal = sig; 822 tsk->signal = sig;
@@ -878,16 +874,6 @@ void __cleanup_signal(struct signal_struct *sig)
878 kmem_cache_free(signal_cachep, sig); 874 kmem_cache_free(signal_cachep, sig);
879} 875}
880 876
881static void cleanup_signal(struct task_struct *tsk)
882{
883 struct signal_struct *sig = tsk->signal;
884
885 atomic_dec(&sig->live);
886
887 if (atomic_dec_and_test(&sig->count))
888 __cleanup_signal(sig);
889}
890
891static void copy_flags(unsigned long clone_flags, struct task_struct *p) 877static void copy_flags(unsigned long clone_flags, struct task_struct *p)
892{ 878{
893 unsigned long new_flags = p->flags; 879 unsigned long new_flags = p->flags;
@@ -1240,6 +1226,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1240 } 1226 }
1241 1227
1242 if (clone_flags & CLONE_THREAD) { 1228 if (clone_flags & CLONE_THREAD) {
1229 atomic_inc(&current->signal->count);
1230 atomic_inc(&current->signal->live);
1243 p->group_leader = current->group_leader; 1231 p->group_leader = current->group_leader;
1244 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group); 1232 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
1245 } 1233 }
@@ -1269,6 +1257,7 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1269 write_unlock_irq(&tasklist_lock); 1257 write_unlock_irq(&tasklist_lock);
1270 proc_fork_connector(p); 1258 proc_fork_connector(p);
1271 cgroup_post_fork(p); 1259 cgroup_post_fork(p);
1260 perf_counter_fork(p);
1272 return p; 1261 return p;
1273 1262
1274bad_fork_free_pid: 1263bad_fork_free_pid:
@@ -1282,7 +1271,8 @@ bad_fork_cleanup_mm:
1282 if (p->mm) 1271 if (p->mm)
1283 mmput(p->mm); 1272 mmput(p->mm);
1284bad_fork_cleanup_signal: 1273bad_fork_cleanup_signal:
1285 cleanup_signal(p); 1274 if (!(clone_flags & CLONE_THREAD))
1275 __cleanup_signal(p->signal);
1286bad_fork_cleanup_sighand: 1276bad_fork_cleanup_sighand:
1287 __cleanup_sighand(p->sighand); 1277 __cleanup_sighand(p->sighand);
1288bad_fork_cleanup_fs: 1278bad_fork_cleanup_fs:
@@ -1410,9 +1400,6 @@ long do_fork(unsigned long clone_flags,
1410 init_completion(&vfork); 1400 init_completion(&vfork);
1411 } 1401 }
1412 1402
1413 if (!(clone_flags & CLONE_THREAD))
1414 perf_counter_fork(p);
1415
1416 audit_finish_fork(p); 1403 audit_finish_fork(p);
1417 tracehook_report_clone(regs, clone_flags, nr, p); 1404 tracehook_report_clone(regs, clone_flags, nr, p);
1418 1405
diff --git a/kernel/futex.c b/kernel/futex.c
index 0672ff88f159..e18cfbdc7190 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1010,15 +1010,19 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
1010 * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue 1010 * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue
1011 * q: the futex_q 1011 * q: the futex_q
1012 * key: the key of the requeue target futex 1012 * key: the key of the requeue target futex
1013 * hb: the hash_bucket of the requeue target futex
1013 * 1014 *
1014 * During futex_requeue, with requeue_pi=1, it is possible to acquire the 1015 * During futex_requeue, with requeue_pi=1, it is possible to acquire the
1015 * target futex if it is uncontended or via a lock steal. Set the futex_q key 1016 * target futex if it is uncontended or via a lock steal. Set the futex_q key
1016 * to the requeue target futex so the waiter can detect the wakeup on the right 1017 * to the requeue target futex so the waiter can detect the wakeup on the right
1017 * futex, but remove it from the hb and NULL the rt_waiter so it can detect 1018 * futex, but remove it from the hb and NULL the rt_waiter so it can detect
1018 * atomic lock acquisition. Must be called with the q->lock_ptr held. 1019 * atomic lock acquisition. Set the q->lock_ptr to the requeue target hb->lock
1020 * to protect access to the pi_state to fixup the owner later. Must be called
1021 * with both q->lock_ptr and hb->lock held.
1019 */ 1022 */
1020static inline 1023static inline
1021void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key) 1024void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
1025 struct futex_hash_bucket *hb)
1022{ 1026{
1023 drop_futex_key_refs(&q->key); 1027 drop_futex_key_refs(&q->key);
1024 get_futex_key_refs(key); 1028 get_futex_key_refs(key);
@@ -1030,6 +1034,11 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key)
1030 WARN_ON(!q->rt_waiter); 1034 WARN_ON(!q->rt_waiter);
1031 q->rt_waiter = NULL; 1035 q->rt_waiter = NULL;
1032 1036
1037 q->lock_ptr = &hb->lock;
1038#ifdef CONFIG_DEBUG_PI_LIST
1039 q->list.plist.lock = &hb->lock;
1040#endif
1041
1033 wake_up_state(q->task, TASK_NORMAL); 1042 wake_up_state(q->task, TASK_NORMAL);
1034} 1043}
1035 1044
@@ -1088,7 +1097,7 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
1088 ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task, 1097 ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
1089 set_waiters); 1098 set_waiters);
1090 if (ret == 1) 1099 if (ret == 1)
1091 requeue_pi_wake_futex(top_waiter, key2); 1100 requeue_pi_wake_futex(top_waiter, key2, hb2);
1092 1101
1093 return ret; 1102 return ret;
1094} 1103}
@@ -1247,8 +1256,15 @@ retry_private:
1247 if (!match_futex(&this->key, &key1)) 1256 if (!match_futex(&this->key, &key1))
1248 continue; 1257 continue;
1249 1258
1250 WARN_ON(!requeue_pi && this->rt_waiter); 1259 /*
1251 WARN_ON(requeue_pi && !this->rt_waiter); 1260 * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always
1261 * be paired with each other and no other futex ops.
1262 */
1263 if ((requeue_pi && !this->rt_waiter) ||
1264 (!requeue_pi && this->rt_waiter)) {
1265 ret = -EINVAL;
1266 break;
1267 }
1252 1268
1253 /* 1269 /*
1254 * Wake nr_wake waiters. For requeue_pi, if we acquired the 1270 * Wake nr_wake waiters. For requeue_pi, if we acquired the
@@ -1273,7 +1289,7 @@ retry_private:
1273 this->task, 1); 1289 this->task, 1);
1274 if (ret == 1) { 1290 if (ret == 1) {
1275 /* We got the lock. */ 1291 /* We got the lock. */
1276 requeue_pi_wake_futex(this, &key2); 1292 requeue_pi_wake_futex(this, &key2, hb2);
1277 continue; 1293 continue;
1278 } else if (ret) { 1294 } else if (ret) {
1279 /* -EDEADLK */ 1295 /* -EDEADLK */
diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c
index d607a5b9ee29..235716556bf1 100644
--- a/kernel/futex_compat.c
+++ b/kernel/futex_compat.c
@@ -180,7 +180,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
180 int cmd = op & FUTEX_CMD_MASK; 180 int cmd = op & FUTEX_CMD_MASK;
181 181
182 if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || 182 if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
183 cmd == FUTEX_WAIT_BITSET)) { 183 cmd == FUTEX_WAIT_BITSET ||
184 cmd == FUTEX_WAIT_REQUEUE_PI)) {
184 if (get_compat_timespec(&ts, utime)) 185 if (get_compat_timespec(&ts, utime))
185 return -EFAULT; 186 return -EFAULT;
186 if (!timespec_valid(&ts)) 187 if (!timespec_valid(&ts))
@@ -191,7 +192,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
191 t = ktime_add_safe(ktime_get(), t); 192 t = ktime_add_safe(ktime_get(), t);
192 tp = &t; 193 tp = &t;
193 } 194 }
194 if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE) 195 if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
196 cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
195 val2 = (int) (unsigned long) utime; 197 val2 = (int) (unsigned long) utime;
196 198
197 return do_futex(uaddr, op, val, tp, uaddr2, val2, val3); 199 return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 61c679db4687..0ec9ed831737 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -607,7 +607,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
607 */ 607 */
608 get_task_struct(t); 608 get_task_struct(t);
609 new->thread = t; 609 new->thread = t;
610 wake_up_process(t);
611 } 610 }
612 611
613 /* 612 /*
@@ -690,6 +689,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
690 (int)(new->flags & IRQF_TRIGGER_MASK)); 689 (int)(new->flags & IRQF_TRIGGER_MASK));
691 } 690 }
692 691
692 new->irq = irq;
693 *old_ptr = new; 693 *old_ptr = new;
694 694
695 /* Reset broken irq detection when installing new handler */ 695 /* Reset broken irq detection when installing new handler */
@@ -707,7 +707,13 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
707 707
708 spin_unlock_irqrestore(&desc->lock, flags); 708 spin_unlock_irqrestore(&desc->lock, flags);
709 709
710 new->irq = irq; 710 /*
711 * Strictly no need to wake it up, but hung_task complains
712 * when no hard interrupt wakes the thread up.
713 */
714 if (new->thread)
715 wake_up_process(new->thread);
716
711 register_irq_proc(irq, desc); 717 register_irq_proc(irq, desc);
712 new->dir = NULL; 718 new->dir = NULL;
713 register_handler_proc(irq, new); 719 register_handler_proc(irq, new);
@@ -761,7 +767,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
761{ 767{
762 struct irq_desc *desc = irq_to_desc(irq); 768 struct irq_desc *desc = irq_to_desc(irq);
763 struct irqaction *action, **action_ptr; 769 struct irqaction *action, **action_ptr;
764 struct task_struct *irqthread;
765 unsigned long flags; 770 unsigned long flags;
766 771
767 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq); 772 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
@@ -809,9 +814,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
809 desc->chip->disable(irq); 814 desc->chip->disable(irq);
810 } 815 }
811 816
812 irqthread = action->thread;
813 action->thread = NULL;
814
815 spin_unlock_irqrestore(&desc->lock, flags); 817 spin_unlock_irqrestore(&desc->lock, flags);
816 818
817 unregister_handler_proc(irq, action); 819 unregister_handler_proc(irq, action);
@@ -819,12 +821,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
819 /* Make sure it's not being used on another CPU: */ 821 /* Make sure it's not being used on another CPU: */
820 synchronize_irq(irq); 822 synchronize_irq(irq);
821 823
822 if (irqthread) {
823 if (!test_bit(IRQTF_DIED, &action->thread_flags))
824 kthread_stop(irqthread);
825 put_task_struct(irqthread);
826 }
827
828#ifdef CONFIG_DEBUG_SHIRQ 824#ifdef CONFIG_DEBUG_SHIRQ
829 /* 825 /*
830 * It's a shared IRQ -- the driver ought to be prepared for an IRQ 826 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
@@ -840,6 +836,13 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
840 local_irq_restore(flags); 836 local_irq_restore(flags);
841 } 837 }
842#endif 838#endif
839
840 if (action->thread) {
841 if (!test_bit(IRQTF_DIED, &action->thread_flags))
842 kthread_stop(action->thread);
843 put_task_struct(action->thread);
844 }
845
843 return action; 846 return action;
844} 847}
845 848
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c
index 2f69bee57bf2..3fd30197da2e 100644
--- a/kernel/irq/numa_migrate.c
+++ b/kernel/irq/numa_migrate.c
@@ -107,8 +107,8 @@ out_unlock:
107 107
108struct irq_desc *move_irq_desc(struct irq_desc *desc, int node) 108struct irq_desc *move_irq_desc(struct irq_desc *desc, int node)
109{ 109{
110 /* those all static, do move them */ 110 /* those static or target node is -1, do not move them */
111 if (desc->irq < NR_IRQS_LEGACY) 111 if (desc->irq < NR_IRQS_LEGACY || node == -1)
112 return desc; 112 return desc;
113 113
114 if (desc->node != node) 114 if (desc->node != node)
diff --git a/kernel/lockdep_proc.c b/kernel/lockdep_proc.c
index d7135aa2d2c4..e94caa666dba 100644
--- a/kernel/lockdep_proc.c
+++ b/kernel/lockdep_proc.c
@@ -758,7 +758,8 @@ static int __init lockdep_proc_init(void)
758 &proc_lockdep_stats_operations); 758 &proc_lockdep_stats_operations);
759 759
760#ifdef CONFIG_LOCK_STAT 760#ifdef CONFIG_LOCK_STAT
761 proc_create("lock_stat", S_IRUSR, NULL, &proc_lock_stat_operations); 761 proc_create("lock_stat", S_IRUSR | S_IWUSR, NULL,
762 &proc_lock_stat_operations);
762#endif 763#endif
763 764
764 return 0; 765 return 0;
diff --git a/kernel/module.c b/kernel/module.c
index fd1411403558..2d537186191f 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -909,16 +909,18 @@ void __symbol_put(const char *symbol)
909} 909}
910EXPORT_SYMBOL(__symbol_put); 910EXPORT_SYMBOL(__symbol_put);
911 911
912/* Note this assumes addr is a function, which it currently always is. */
912void symbol_put_addr(void *addr) 913void symbol_put_addr(void *addr)
913{ 914{
914 struct module *modaddr; 915 struct module *modaddr;
916 unsigned long a = (unsigned long)dereference_function_descriptor(addr);
915 917
916 if (core_kernel_text((unsigned long)addr)) 918 if (core_kernel_text(a))
917 return; 919 return;
918 920
919 /* module_text_address is safe here: we're supposed to have reference 921 /* module_text_address is safe here: we're supposed to have reference
920 * to module from symbol_get, so it can't go away. */ 922 * to module from symbol_get, so it can't go away. */
921 modaddr = __module_text_address((unsigned long)addr); 923 modaddr = __module_text_address(a);
922 BUG_ON(!modaddr); 924 BUG_ON(!modaddr);
923 module_put(modaddr); 925 module_put(modaddr);
924} 926}
@@ -1272,6 +1274,10 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
1272 struct module_notes_attrs *notes_attrs; 1274 struct module_notes_attrs *notes_attrs;
1273 struct bin_attribute *nattr; 1275 struct bin_attribute *nattr;
1274 1276
1277 /* failed to create section attributes, so can't create notes */
1278 if (!mod->sect_attrs)
1279 return;
1280
1275 /* Count notes sections and allocate structures. */ 1281 /* Count notes sections and allocate structures. */
1276 notes = 0; 1282 notes = 0;
1277 for (i = 0; i < nsect; i++) 1283 for (i = 0; i < nsect; i++)
diff --git a/kernel/panic.c b/kernel/panic.c
index 984b3ecbd72c..512ab73b0ca3 100644
--- a/kernel/panic.c
+++ b/kernel/panic.c
@@ -301,6 +301,7 @@ int oops_may_print(void)
301 */ 301 */
302void oops_enter(void) 302void oops_enter(void)
303{ 303{
304 tracing_off();
304 /* can't trust the integrity of the kernel anymore: */ 305 /* can't trust the integrity of the kernel anymore: */
305 debug_locks_off(); 306 debug_locks_off();
306 do_oops_enter_exit(); 307 do_oops_enter_exit();
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index 950931041954..d7cbc579fc80 100644
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -42,6 +42,7 @@ static int perf_overcommit __read_mostly = 1;
42static atomic_t nr_counters __read_mostly; 42static atomic_t nr_counters __read_mostly;
43static atomic_t nr_mmap_counters __read_mostly; 43static atomic_t nr_mmap_counters __read_mostly;
44static atomic_t nr_comm_counters __read_mostly; 44static atomic_t nr_comm_counters __read_mostly;
45static atomic_t nr_task_counters __read_mostly;
45 46
46/* 47/*
47 * perf counter paranoia level: 48 * perf counter paranoia level:
@@ -49,7 +50,7 @@ static atomic_t nr_comm_counters __read_mostly;
49 * 1 - disallow cpu counters to unpriv 50 * 1 - disallow cpu counters to unpriv
50 * 2 - disallow kernel profiling to unpriv 51 * 2 - disallow kernel profiling to unpriv
51 */ 52 */
52int sysctl_perf_counter_paranoid __read_mostly; 53int sysctl_perf_counter_paranoid __read_mostly = 1;
53 54
54static inline bool perf_paranoid_cpu(void) 55static inline bool perf_paranoid_cpu(void)
55{ 56{
@@ -87,6 +88,7 @@ void __weak hw_perf_disable(void) { barrier(); }
87void __weak hw_perf_enable(void) { barrier(); } 88void __weak hw_perf_enable(void) { barrier(); }
88 89
89void __weak hw_perf_counter_setup(int cpu) { barrier(); } 90void __weak hw_perf_counter_setup(int cpu) { barrier(); }
91void __weak hw_perf_counter_setup_online(int cpu) { barrier(); }
90 92
91int __weak 93int __weak
92hw_perf_group_sched_in(struct perf_counter *group_leader, 94hw_perf_group_sched_in(struct perf_counter *group_leader,
@@ -305,6 +307,10 @@ counter_sched_out(struct perf_counter *counter,
305 return; 307 return;
306 308
307 counter->state = PERF_COUNTER_STATE_INACTIVE; 309 counter->state = PERF_COUNTER_STATE_INACTIVE;
310 if (counter->pending_disable) {
311 counter->pending_disable = 0;
312 counter->state = PERF_COUNTER_STATE_OFF;
313 }
308 counter->tstamp_stopped = ctx->time; 314 counter->tstamp_stopped = ctx->time;
309 counter->pmu->disable(counter); 315 counter->pmu->disable(counter);
310 counter->oncpu = -1; 316 counter->oncpu = -1;
@@ -1103,7 +1109,7 @@ static void perf_counter_sync_stat(struct perf_counter_context *ctx,
1103 __perf_counter_sync_stat(counter, next_counter); 1109 __perf_counter_sync_stat(counter, next_counter);
1104 1110
1105 counter = list_next_entry(counter, event_entry); 1111 counter = list_next_entry(counter, event_entry);
1106 next_counter = list_next_entry(counter, event_entry); 1112 next_counter = list_next_entry(next_counter, event_entry);
1107 } 1113 }
1108} 1114}
1109 1115
@@ -1497,10 +1503,21 @@ static void perf_counter_enable_on_exec(struct task_struct *task)
1497 */ 1503 */
1498static void __perf_counter_read(void *info) 1504static void __perf_counter_read(void *info)
1499{ 1505{
1506 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1500 struct perf_counter *counter = info; 1507 struct perf_counter *counter = info;
1501 struct perf_counter_context *ctx = counter->ctx; 1508 struct perf_counter_context *ctx = counter->ctx;
1502 unsigned long flags; 1509 unsigned long flags;
1503 1510
1511 /*
1512 * If this is a task context, we need to check whether it is
1513 * the current task context of this cpu. If not it has been
1514 * scheduled out before the smp call arrived. In that case
1515 * counter->count would have been updated to a recent sample
1516 * when the counter was scheduled out.
1517 */
1518 if (ctx->task && cpuctx->task_ctx != ctx)
1519 return;
1520
1504 local_irq_save(flags); 1521 local_irq_save(flags);
1505 if (ctx->is_active) 1522 if (ctx->is_active)
1506 update_context_time(ctx); 1523 update_context_time(ctx);
@@ -1654,6 +1671,8 @@ static void free_counter(struct perf_counter *counter)
1654 atomic_dec(&nr_mmap_counters); 1671 atomic_dec(&nr_mmap_counters);
1655 if (counter->attr.comm) 1672 if (counter->attr.comm)
1656 atomic_dec(&nr_comm_counters); 1673 atomic_dec(&nr_comm_counters);
1674 if (counter->attr.task)
1675 atomic_dec(&nr_task_counters);
1657 } 1676 }
1658 1677
1659 if (counter->destroy) 1678 if (counter->destroy)
@@ -1688,14 +1707,133 @@ static int perf_release(struct inode *inode, struct file *file)
1688 return 0; 1707 return 0;
1689} 1708}
1690 1709
1710static int perf_counter_read_size(struct perf_counter *counter)
1711{
1712 int entry = sizeof(u64); /* value */
1713 int size = 0;
1714 int nr = 1;
1715
1716 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1717 size += sizeof(u64);
1718
1719 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1720 size += sizeof(u64);
1721
1722 if (counter->attr.read_format & PERF_FORMAT_ID)
1723 entry += sizeof(u64);
1724
1725 if (counter->attr.read_format & PERF_FORMAT_GROUP) {
1726 nr += counter->group_leader->nr_siblings;
1727 size += sizeof(u64);
1728 }
1729
1730 size += entry * nr;
1731
1732 return size;
1733}
1734
1735static u64 perf_counter_read_value(struct perf_counter *counter)
1736{
1737 struct perf_counter *child;
1738 u64 total = 0;
1739
1740 total += perf_counter_read(counter);
1741 list_for_each_entry(child, &counter->child_list, child_list)
1742 total += perf_counter_read(child);
1743
1744 return total;
1745}
1746
1747static int perf_counter_read_entry(struct perf_counter *counter,
1748 u64 read_format, char __user *buf)
1749{
1750 int n = 0, count = 0;
1751 u64 values[2];
1752
1753 values[n++] = perf_counter_read_value(counter);
1754 if (read_format & PERF_FORMAT_ID)
1755 values[n++] = primary_counter_id(counter);
1756
1757 count = n * sizeof(u64);
1758
1759 if (copy_to_user(buf, values, count))
1760 return -EFAULT;
1761
1762 return count;
1763}
1764
1765static int perf_counter_read_group(struct perf_counter *counter,
1766 u64 read_format, char __user *buf)
1767{
1768 struct perf_counter *leader = counter->group_leader, *sub;
1769 int n = 0, size = 0, err = -EFAULT;
1770 u64 values[3];
1771
1772 values[n++] = 1 + leader->nr_siblings;
1773 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1774 values[n++] = leader->total_time_enabled +
1775 atomic64_read(&leader->child_total_time_enabled);
1776 }
1777 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1778 values[n++] = leader->total_time_running +
1779 atomic64_read(&leader->child_total_time_running);
1780 }
1781
1782 size = n * sizeof(u64);
1783
1784 if (copy_to_user(buf, values, size))
1785 return -EFAULT;
1786
1787 err = perf_counter_read_entry(leader, read_format, buf + size);
1788 if (err < 0)
1789 return err;
1790
1791 size += err;
1792
1793 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1794 err = perf_counter_read_entry(sub, read_format,
1795 buf + size);
1796 if (err < 0)
1797 return err;
1798
1799 size += err;
1800 }
1801
1802 return size;
1803}
1804
1805static int perf_counter_read_one(struct perf_counter *counter,
1806 u64 read_format, char __user *buf)
1807{
1808 u64 values[4];
1809 int n = 0;
1810
1811 values[n++] = perf_counter_read_value(counter);
1812 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1813 values[n++] = counter->total_time_enabled +
1814 atomic64_read(&counter->child_total_time_enabled);
1815 }
1816 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1817 values[n++] = counter->total_time_running +
1818 atomic64_read(&counter->child_total_time_running);
1819 }
1820 if (read_format & PERF_FORMAT_ID)
1821 values[n++] = primary_counter_id(counter);
1822
1823 if (copy_to_user(buf, values, n * sizeof(u64)))
1824 return -EFAULT;
1825
1826 return n * sizeof(u64);
1827}
1828
1691/* 1829/*
1692 * Read the performance counter - simple non blocking version for now 1830 * Read the performance counter - simple non blocking version for now
1693 */ 1831 */
1694static ssize_t 1832static ssize_t
1695perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count) 1833perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1696{ 1834{
1697 u64 values[4]; 1835 u64 read_format = counter->attr.read_format;
1698 int n; 1836 int ret;
1699 1837
1700 /* 1838 /*
1701 * Return end-of-file for a read on a counter that is in 1839 * Return end-of-file for a read on a counter that is in
@@ -1705,28 +1843,18 @@ perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1705 if (counter->state == PERF_COUNTER_STATE_ERROR) 1843 if (counter->state == PERF_COUNTER_STATE_ERROR)
1706 return 0; 1844 return 0;
1707 1845
1846 if (count < perf_counter_read_size(counter))
1847 return -ENOSPC;
1848
1708 WARN_ON_ONCE(counter->ctx->parent_ctx); 1849 WARN_ON_ONCE(counter->ctx->parent_ctx);
1709 mutex_lock(&counter->child_mutex); 1850 mutex_lock(&counter->child_mutex);
1710 values[0] = perf_counter_read(counter); 1851 if (read_format & PERF_FORMAT_GROUP)
1711 n = 1; 1852 ret = perf_counter_read_group(counter, read_format, buf);
1712 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1853 else
1713 values[n++] = counter->total_time_enabled + 1854 ret = perf_counter_read_one(counter, read_format, buf);
1714 atomic64_read(&counter->child_total_time_enabled);
1715 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1716 values[n++] = counter->total_time_running +
1717 atomic64_read(&counter->child_total_time_running);
1718 if (counter->attr.read_format & PERF_FORMAT_ID)
1719 values[n++] = primary_counter_id(counter);
1720 mutex_unlock(&counter->child_mutex); 1855 mutex_unlock(&counter->child_mutex);
1721 1856
1722 if (count < n * sizeof(u64)) 1857 return ret;
1723 return -EINVAL;
1724 count = n * sizeof(u64);
1725
1726 if (copy_to_user(buf, values, count))
1727 return -EFAULT;
1728
1729 return count;
1730} 1858}
1731 1859
1732static ssize_t 1860static ssize_t
@@ -1891,6 +2019,10 @@ int perf_counter_task_disable(void)
1891 return 0; 2019 return 0;
1892} 2020}
1893 2021
2022#ifndef PERF_COUNTER_INDEX_OFFSET
2023# define PERF_COUNTER_INDEX_OFFSET 0
2024#endif
2025
1894static int perf_counter_index(struct perf_counter *counter) 2026static int perf_counter_index(struct perf_counter *counter)
1895{ 2027{
1896 if (counter->state != PERF_COUNTER_STATE_ACTIVE) 2028 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
@@ -2230,7 +2362,7 @@ static void perf_pending_counter(struct perf_pending_entry *entry)
2230 2362
2231 if (counter->pending_disable) { 2363 if (counter->pending_disable) {
2232 counter->pending_disable = 0; 2364 counter->pending_disable = 0;
2233 perf_counter_disable(counter); 2365 __perf_counter_disable(counter);
2234 } 2366 }
2235 2367
2236 if (counter->pending_wakeup) { 2368 if (counter->pending_wakeup) {
@@ -2615,7 +2747,80 @@ static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
2615 return task_pid_nr_ns(p, counter->ns); 2747 return task_pid_nr_ns(p, counter->ns);
2616} 2748}
2617 2749
2618static void perf_counter_output(struct perf_counter *counter, int nmi, 2750static void perf_output_read_one(struct perf_output_handle *handle,
2751 struct perf_counter *counter)
2752{
2753 u64 read_format = counter->attr.read_format;
2754 u64 values[4];
2755 int n = 0;
2756
2757 values[n++] = atomic64_read(&counter->count);
2758 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2759 values[n++] = counter->total_time_enabled +
2760 atomic64_read(&counter->child_total_time_enabled);
2761 }
2762 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2763 values[n++] = counter->total_time_running +
2764 atomic64_read(&counter->child_total_time_running);
2765 }
2766 if (read_format & PERF_FORMAT_ID)
2767 values[n++] = primary_counter_id(counter);
2768
2769 perf_output_copy(handle, values, n * sizeof(u64));
2770}
2771
2772/*
2773 * XXX PERF_FORMAT_GROUP vs inherited counters seems difficult.
2774 */
2775static void perf_output_read_group(struct perf_output_handle *handle,
2776 struct perf_counter *counter)
2777{
2778 struct perf_counter *leader = counter->group_leader, *sub;
2779 u64 read_format = counter->attr.read_format;
2780 u64 values[5];
2781 int n = 0;
2782
2783 values[n++] = 1 + leader->nr_siblings;
2784
2785 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2786 values[n++] = leader->total_time_enabled;
2787
2788 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2789 values[n++] = leader->total_time_running;
2790
2791 if (leader != counter)
2792 leader->pmu->read(leader);
2793
2794 values[n++] = atomic64_read(&leader->count);
2795 if (read_format & PERF_FORMAT_ID)
2796 values[n++] = primary_counter_id(leader);
2797
2798 perf_output_copy(handle, values, n * sizeof(u64));
2799
2800 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2801 n = 0;
2802
2803 if (sub != counter)
2804 sub->pmu->read(sub);
2805
2806 values[n++] = atomic64_read(&sub->count);
2807 if (read_format & PERF_FORMAT_ID)
2808 values[n++] = primary_counter_id(sub);
2809
2810 perf_output_copy(handle, values, n * sizeof(u64));
2811 }
2812}
2813
2814static void perf_output_read(struct perf_output_handle *handle,
2815 struct perf_counter *counter)
2816{
2817 if (counter->attr.read_format & PERF_FORMAT_GROUP)
2818 perf_output_read_group(handle, counter);
2819 else
2820 perf_output_read_one(handle, counter);
2821}
2822
2823void perf_counter_output(struct perf_counter *counter, int nmi,
2619 struct perf_sample_data *data) 2824 struct perf_sample_data *data)
2620{ 2825{
2621 int ret; 2826 int ret;
@@ -2626,10 +2831,6 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2626 struct { 2831 struct {
2627 u32 pid, tid; 2832 u32 pid, tid;
2628 } tid_entry; 2833 } tid_entry;
2629 struct {
2630 u64 id;
2631 u64 counter;
2632 } group_entry;
2633 struct perf_callchain_entry *callchain = NULL; 2834 struct perf_callchain_entry *callchain = NULL;
2634 int callchain_size = 0; 2835 int callchain_size = 0;
2635 u64 time; 2836 u64 time;
@@ -2684,10 +2885,8 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2684 if (sample_type & PERF_SAMPLE_PERIOD) 2885 if (sample_type & PERF_SAMPLE_PERIOD)
2685 header.size += sizeof(u64); 2886 header.size += sizeof(u64);
2686 2887
2687 if (sample_type & PERF_SAMPLE_GROUP) { 2888 if (sample_type & PERF_SAMPLE_READ)
2688 header.size += sizeof(u64) + 2889 header.size += perf_counter_read_size(counter);
2689 counter->nr_siblings * sizeof(group_entry);
2690 }
2691 2890
2692 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 2891 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2693 callchain = perf_callchain(data->regs); 2892 callchain = perf_callchain(data->regs);
@@ -2699,6 +2898,18 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2699 header.size += sizeof(u64); 2898 header.size += sizeof(u64);
2700 } 2899 }
2701 2900
2901 if (sample_type & PERF_SAMPLE_RAW) {
2902 int size = sizeof(u32);
2903
2904 if (data->raw)
2905 size += data->raw->size;
2906 else
2907 size += sizeof(u32);
2908
2909 WARN_ON_ONCE(size & (sizeof(u64)-1));
2910 header.size += size;
2911 }
2912
2702 ret = perf_output_begin(&handle, counter, header.size, nmi, 1); 2913 ret = perf_output_begin(&handle, counter, header.size, nmi, 1);
2703 if (ret) 2914 if (ret)
2704 return; 2915 return;
@@ -2732,26 +2943,8 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2732 if (sample_type & PERF_SAMPLE_PERIOD) 2943 if (sample_type & PERF_SAMPLE_PERIOD)
2733 perf_output_put(&handle, data->period); 2944 perf_output_put(&handle, data->period);
2734 2945
2735 /* 2946 if (sample_type & PERF_SAMPLE_READ)
2736 * XXX PERF_SAMPLE_GROUP vs inherited counters seems difficult. 2947 perf_output_read(&handle, counter);
2737 */
2738 if (sample_type & PERF_SAMPLE_GROUP) {
2739 struct perf_counter *leader, *sub;
2740 u64 nr = counter->nr_siblings;
2741
2742 perf_output_put(&handle, nr);
2743
2744 leader = counter->group_leader;
2745 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2746 if (sub != counter)
2747 sub->pmu->read(sub);
2748
2749 group_entry.id = primary_counter_id(sub);
2750 group_entry.counter = atomic64_read(&sub->count);
2751
2752 perf_output_put(&handle, group_entry);
2753 }
2754 }
2755 2948
2756 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 2949 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2757 if (callchain) 2950 if (callchain)
@@ -2762,6 +2955,22 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2762 } 2955 }
2763 } 2956 }
2764 2957
2958 if (sample_type & PERF_SAMPLE_RAW) {
2959 if (data->raw) {
2960 perf_output_put(&handle, data->raw->size);
2961 perf_output_copy(&handle, data->raw->data, data->raw->size);
2962 } else {
2963 struct {
2964 u32 size;
2965 u32 data;
2966 } raw = {
2967 .size = sizeof(u32),
2968 .data = 0,
2969 };
2970 perf_output_put(&handle, raw);
2971 }
2972 }
2973
2765 perf_output_end(&handle); 2974 perf_output_end(&handle);
2766} 2975}
2767 2976
@@ -2774,8 +2983,6 @@ struct perf_read_event {
2774 2983
2775 u32 pid; 2984 u32 pid;
2776 u32 tid; 2985 u32 tid;
2777 u64 value;
2778 u64 format[3];
2779}; 2986};
2780 2987
2781static void 2988static void
@@ -2787,80 +2994,74 @@ perf_counter_read_event(struct perf_counter *counter,
2787 .header = { 2994 .header = {
2788 .type = PERF_EVENT_READ, 2995 .type = PERF_EVENT_READ,
2789 .misc = 0, 2996 .misc = 0,
2790 .size = sizeof(event) - sizeof(event.format), 2997 .size = sizeof(event) + perf_counter_read_size(counter),
2791 }, 2998 },
2792 .pid = perf_counter_pid(counter, task), 2999 .pid = perf_counter_pid(counter, task),
2793 .tid = perf_counter_tid(counter, task), 3000 .tid = perf_counter_tid(counter, task),
2794 .value = atomic64_read(&counter->count),
2795 }; 3001 };
2796 int ret, i = 0; 3002 int ret;
2797
2798 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2799 event.header.size += sizeof(u64);
2800 event.format[i++] = counter->total_time_enabled;
2801 }
2802
2803 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2804 event.header.size += sizeof(u64);
2805 event.format[i++] = counter->total_time_running;
2806 }
2807
2808 if (counter->attr.read_format & PERF_FORMAT_ID) {
2809 event.header.size += sizeof(u64);
2810 event.format[i++] = primary_counter_id(counter);
2811 }
2812 3003
2813 ret = perf_output_begin(&handle, counter, event.header.size, 0, 0); 3004 ret = perf_output_begin(&handle, counter, event.header.size, 0, 0);
2814 if (ret) 3005 if (ret)
2815 return; 3006 return;
2816 3007
2817 perf_output_copy(&handle, &event, event.header.size); 3008 perf_output_put(&handle, event);
3009 perf_output_read(&handle, counter);
3010
2818 perf_output_end(&handle); 3011 perf_output_end(&handle);
2819} 3012}
2820 3013
2821/* 3014/*
2822 * fork tracking 3015 * task tracking -- fork/exit
3016 *
3017 * enabled by: attr.comm | attr.mmap | attr.task
2823 */ 3018 */
2824 3019
2825struct perf_fork_event { 3020struct perf_task_event {
2826 struct task_struct *task; 3021 struct task_struct *task;
3022 struct perf_counter_context *task_ctx;
2827 3023
2828 struct { 3024 struct {
2829 struct perf_event_header header; 3025 struct perf_event_header header;
2830 3026
2831 u32 pid; 3027 u32 pid;
2832 u32 ppid; 3028 u32 ppid;
3029 u32 tid;
3030 u32 ptid;
2833 } event; 3031 } event;
2834}; 3032};
2835 3033
2836static void perf_counter_fork_output(struct perf_counter *counter, 3034static void perf_counter_task_output(struct perf_counter *counter,
2837 struct perf_fork_event *fork_event) 3035 struct perf_task_event *task_event)
2838{ 3036{
2839 struct perf_output_handle handle; 3037 struct perf_output_handle handle;
2840 int size = fork_event->event.header.size; 3038 int size = task_event->event.header.size;
2841 struct task_struct *task = fork_event->task; 3039 struct task_struct *task = task_event->task;
2842 int ret = perf_output_begin(&handle, counter, size, 0, 0); 3040 int ret = perf_output_begin(&handle, counter, size, 0, 0);
2843 3041
2844 if (ret) 3042 if (ret)
2845 return; 3043 return;
2846 3044
2847 fork_event->event.pid = perf_counter_pid(counter, task); 3045 task_event->event.pid = perf_counter_pid(counter, task);
2848 fork_event->event.ppid = perf_counter_pid(counter, task->real_parent); 3046 task_event->event.ppid = perf_counter_pid(counter, current);
2849 3047
2850 perf_output_put(&handle, fork_event->event); 3048 task_event->event.tid = perf_counter_tid(counter, task);
3049 task_event->event.ptid = perf_counter_tid(counter, current);
3050
3051 perf_output_put(&handle, task_event->event);
2851 perf_output_end(&handle); 3052 perf_output_end(&handle);
2852} 3053}
2853 3054
2854static int perf_counter_fork_match(struct perf_counter *counter) 3055static int perf_counter_task_match(struct perf_counter *counter)
2855{ 3056{
2856 if (counter->attr.comm || counter->attr.mmap) 3057 if (counter->attr.comm || counter->attr.mmap || counter->attr.task)
2857 return 1; 3058 return 1;
2858 3059
2859 return 0; 3060 return 0;
2860} 3061}
2861 3062
2862static void perf_counter_fork_ctx(struct perf_counter_context *ctx, 3063static void perf_counter_task_ctx(struct perf_counter_context *ctx,
2863 struct perf_fork_event *fork_event) 3064 struct perf_task_event *task_event)
2864{ 3065{
2865 struct perf_counter *counter; 3066 struct perf_counter *counter;
2866 3067
@@ -2869,54 +3070,62 @@ static void perf_counter_fork_ctx(struct perf_counter_context *ctx,
2869 3070
2870 rcu_read_lock(); 3071 rcu_read_lock();
2871 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) { 3072 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
2872 if (perf_counter_fork_match(counter)) 3073 if (perf_counter_task_match(counter))
2873 perf_counter_fork_output(counter, fork_event); 3074 perf_counter_task_output(counter, task_event);
2874 } 3075 }
2875 rcu_read_unlock(); 3076 rcu_read_unlock();
2876} 3077}
2877 3078
2878static void perf_counter_fork_event(struct perf_fork_event *fork_event) 3079static void perf_counter_task_event(struct perf_task_event *task_event)
2879{ 3080{
2880 struct perf_cpu_context *cpuctx; 3081 struct perf_cpu_context *cpuctx;
2881 struct perf_counter_context *ctx; 3082 struct perf_counter_context *ctx = task_event->task_ctx;
2882 3083
2883 cpuctx = &get_cpu_var(perf_cpu_context); 3084 cpuctx = &get_cpu_var(perf_cpu_context);
2884 perf_counter_fork_ctx(&cpuctx->ctx, fork_event); 3085 perf_counter_task_ctx(&cpuctx->ctx, task_event);
2885 put_cpu_var(perf_cpu_context); 3086 put_cpu_var(perf_cpu_context);
2886 3087
2887 rcu_read_lock(); 3088 rcu_read_lock();
2888 /* 3089 if (!ctx)
2889 * doesn't really matter which of the child contexts the 3090 ctx = rcu_dereference(task_event->task->perf_counter_ctxp);
2890 * events ends up in.
2891 */
2892 ctx = rcu_dereference(current->perf_counter_ctxp);
2893 if (ctx) 3091 if (ctx)
2894 perf_counter_fork_ctx(ctx, fork_event); 3092 perf_counter_task_ctx(ctx, task_event);
2895 rcu_read_unlock(); 3093 rcu_read_unlock();
2896} 3094}
2897 3095
2898void perf_counter_fork(struct task_struct *task) 3096static void perf_counter_task(struct task_struct *task,
3097 struct perf_counter_context *task_ctx,
3098 int new)
2899{ 3099{
2900 struct perf_fork_event fork_event; 3100 struct perf_task_event task_event;
2901 3101
2902 if (!atomic_read(&nr_comm_counters) && 3102 if (!atomic_read(&nr_comm_counters) &&
2903 !atomic_read(&nr_mmap_counters)) 3103 !atomic_read(&nr_mmap_counters) &&
3104 !atomic_read(&nr_task_counters))
2904 return; 3105 return;
2905 3106
2906 fork_event = (struct perf_fork_event){ 3107 task_event = (struct perf_task_event){
2907 .task = task, 3108 .task = task,
2908 .event = { 3109 .task_ctx = task_ctx,
3110 .event = {
2909 .header = { 3111 .header = {
2910 .type = PERF_EVENT_FORK, 3112 .type = new ? PERF_EVENT_FORK : PERF_EVENT_EXIT,
2911 .misc = 0, 3113 .misc = 0,
2912 .size = sizeof(fork_event.event), 3114 .size = sizeof(task_event.event),
2913 }, 3115 },
2914 /* .pid */ 3116 /* .pid */
2915 /* .ppid */ 3117 /* .ppid */
3118 /* .tid */
3119 /* .ptid */
2916 }, 3120 },
2917 }; 3121 };
2918 3122
2919 perf_counter_fork_event(&fork_event); 3123 perf_counter_task_event(&task_event);
3124}
3125
3126void perf_counter_fork(struct task_struct *task)
3127{
3128 perf_counter_task(task, NULL, 1);
2920} 3129}
2921 3130
2922/* 3131/*
@@ -3305,125 +3514,111 @@ int perf_counter_overflow(struct perf_counter *counter, int nmi,
3305 * Generic software counter infrastructure 3514 * Generic software counter infrastructure
3306 */ 3515 */
3307 3516
3308static void perf_swcounter_update(struct perf_counter *counter) 3517/*
3518 * We directly increment counter->count and keep a second value in
3519 * counter->hw.period_left to count intervals. This period counter
3520 * is kept in the range [-sample_period, 0] so that we can use the
3521 * sign as trigger.
3522 */
3523
3524static u64 perf_swcounter_set_period(struct perf_counter *counter)
3309{ 3525{
3310 struct hw_perf_counter *hwc = &counter->hw; 3526 struct hw_perf_counter *hwc = &counter->hw;
3311 u64 prev, now; 3527 u64 period = hwc->last_period;
3312 s64 delta; 3528 u64 nr, offset;
3529 s64 old, val;
3530
3531 hwc->last_period = hwc->sample_period;
3313 3532
3314again: 3533again:
3315 prev = atomic64_read(&hwc->prev_count); 3534 old = val = atomic64_read(&hwc->period_left);
3316 now = atomic64_read(&hwc->count); 3535 if (val < 0)
3317 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev) 3536 return 0;
3318 goto again;
3319 3537
3320 delta = now - prev; 3538 nr = div64_u64(period + val, period);
3539 offset = nr * period;
3540 val -= offset;
3541 if (atomic64_cmpxchg(&hwc->period_left, old, val) != old)
3542 goto again;
3321 3543
3322 atomic64_add(delta, &counter->count); 3544 return nr;
3323 atomic64_sub(delta, &hwc->period_left);
3324} 3545}
3325 3546
3326static void perf_swcounter_set_period(struct perf_counter *counter) 3547static void perf_swcounter_overflow(struct perf_counter *counter,
3548 int nmi, struct perf_sample_data *data)
3327{ 3549{
3328 struct hw_perf_counter *hwc = &counter->hw; 3550 struct hw_perf_counter *hwc = &counter->hw;
3329 s64 left = atomic64_read(&hwc->period_left); 3551 u64 overflow;
3330 s64 period = hwc->sample_period;
3331 3552
3332 if (unlikely(left <= -period)) { 3553 data->period = counter->hw.last_period;
3333 left = period; 3554 overflow = perf_swcounter_set_period(counter);
3334 atomic64_set(&hwc->period_left, left);
3335 hwc->last_period = period;
3336 }
3337 3555
3338 if (unlikely(left <= 0)) { 3556 if (hwc->interrupts == MAX_INTERRUPTS)
3339 left += period; 3557 return;
3340 atomic64_add(period, &hwc->period_left);
3341 hwc->last_period = period;
3342 }
3343 3558
3344 atomic64_set(&hwc->prev_count, -left); 3559 for (; overflow; overflow--) {
3345 atomic64_set(&hwc->count, -left); 3560 if (perf_counter_overflow(counter, nmi, data)) {
3561 /*
3562 * We inhibit the overflow from happening when
3563 * hwc->interrupts == MAX_INTERRUPTS.
3564 */
3565 break;
3566 }
3567 }
3346} 3568}
3347 3569
3348static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer) 3570static void perf_swcounter_unthrottle(struct perf_counter *counter)
3349{ 3571{
3350 enum hrtimer_restart ret = HRTIMER_RESTART;
3351 struct perf_sample_data data;
3352 struct perf_counter *counter;
3353 u64 period;
3354
3355 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
3356 counter->pmu->read(counter);
3357
3358 data.addr = 0;
3359 data.regs = get_irq_regs();
3360 /* 3572 /*
3361 * In case we exclude kernel IPs or are somehow not in interrupt 3573 * Nothing to do, we already reset hwc->interrupts.
3362 * context, provide the next best thing, the user IP.
3363 */ 3574 */
3364 if ((counter->attr.exclude_kernel || !data.regs) && 3575}
3365 !counter->attr.exclude_user)
3366 data.regs = task_pt_regs(current);
3367 3576
3368 if (data.regs) { 3577static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
3369 if (perf_counter_overflow(counter, 0, &data)) 3578 int nmi, struct perf_sample_data *data)
3370 ret = HRTIMER_NORESTART; 3579{
3371 } 3580 struct hw_perf_counter *hwc = &counter->hw;
3372 3581
3373 period = max_t(u64, 10000, counter->hw.sample_period); 3582 atomic64_add(nr, &counter->count);
3374 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
3375 3583
3376 return ret; 3584 if (!hwc->sample_period)
3377} 3585 return;
3378 3586
3379static void perf_swcounter_overflow(struct perf_counter *counter, 3587 if (!data->regs)
3380 int nmi, struct perf_sample_data *data) 3588 return;
3381{
3382 data->period = counter->hw.last_period;
3383 3589
3384 perf_swcounter_update(counter); 3590 if (!atomic64_add_negative(nr, &hwc->period_left))
3385 perf_swcounter_set_period(counter); 3591 perf_swcounter_overflow(counter, nmi, data);
3386 if (perf_counter_overflow(counter, nmi, data))
3387 /* soft-disable the counter */
3388 ;
3389} 3592}
3390 3593
3391static int perf_swcounter_is_counting(struct perf_counter *counter) 3594static int perf_swcounter_is_counting(struct perf_counter *counter)
3392{ 3595{
3393 struct perf_counter_context *ctx; 3596 /*
3394 unsigned long flags; 3597 * The counter is active, we're good!
3395 int count; 3598 */
3396
3397 if (counter->state == PERF_COUNTER_STATE_ACTIVE) 3599 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
3398 return 1; 3600 return 1;
3399 3601
3602 /*
3603 * The counter is off/error, not counting.
3604 */
3400 if (counter->state != PERF_COUNTER_STATE_INACTIVE) 3605 if (counter->state != PERF_COUNTER_STATE_INACTIVE)
3401 return 0; 3606 return 0;
3402 3607
3403 /* 3608 /*
3404 * If the counter is inactive, it could be just because 3609 * The counter is inactive, if the context is active
3405 * its task is scheduled out, or because it's in a group 3610 * we're part of a group that didn't make it on the 'pmu',
3406 * which could not go on the PMU. We want to count in 3611 * not counting.
3407 * the first case but not the second. If the context is
3408 * currently active then an inactive software counter must
3409 * be the second case. If it's not currently active then
3410 * we need to know whether the counter was active when the
3411 * context was last active, which we can determine by
3412 * comparing counter->tstamp_stopped with ctx->time.
3413 *
3414 * We are within an RCU read-side critical section,
3415 * which protects the existence of *ctx.
3416 */ 3612 */
3417 ctx = counter->ctx; 3613 if (counter->ctx->is_active)
3418 spin_lock_irqsave(&ctx->lock, flags); 3614 return 0;
3419 count = 1; 3615
3420 /* Re-check state now we have the lock */ 3616 /*
3421 if (counter->state < PERF_COUNTER_STATE_INACTIVE || 3617 * We're inactive and the context is too, this means the
3422 counter->ctx->is_active || 3618 * task is scheduled out, we're counting events that happen
3423 counter->tstamp_stopped < ctx->time) 3619 * to us, like migration events.
3424 count = 0; 3620 */
3425 spin_unlock_irqrestore(&ctx->lock, flags); 3621 return 1;
3426 return count;
3427} 3622}
3428 3623
3429static int perf_swcounter_match(struct perf_counter *counter, 3624static int perf_swcounter_match(struct perf_counter *counter,
@@ -3449,15 +3644,6 @@ static int perf_swcounter_match(struct perf_counter *counter,
3449 return 1; 3644 return 1;
3450} 3645}
3451 3646
3452static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
3453 int nmi, struct perf_sample_data *data)
3454{
3455 int neg = atomic64_add_negative(nr, &counter->hw.count);
3456
3457 if (counter->hw.sample_period && !neg && data->regs)
3458 perf_swcounter_overflow(counter, nmi, data);
3459}
3460
3461static void perf_swcounter_ctx_event(struct perf_counter_context *ctx, 3647static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
3462 enum perf_type_id type, 3648 enum perf_type_id type,
3463 u32 event, u64 nr, int nmi, 3649 u32 event, u64 nr, int nmi,
@@ -3536,27 +3722,66 @@ void __perf_swcounter_event(u32 event, u64 nr, int nmi,
3536 3722
3537static void perf_swcounter_read(struct perf_counter *counter) 3723static void perf_swcounter_read(struct perf_counter *counter)
3538{ 3724{
3539 perf_swcounter_update(counter);
3540} 3725}
3541 3726
3542static int perf_swcounter_enable(struct perf_counter *counter) 3727static int perf_swcounter_enable(struct perf_counter *counter)
3543{ 3728{
3544 perf_swcounter_set_period(counter); 3729 struct hw_perf_counter *hwc = &counter->hw;
3730
3731 if (hwc->sample_period) {
3732 hwc->last_period = hwc->sample_period;
3733 perf_swcounter_set_period(counter);
3734 }
3545 return 0; 3735 return 0;
3546} 3736}
3547 3737
3548static void perf_swcounter_disable(struct perf_counter *counter) 3738static void perf_swcounter_disable(struct perf_counter *counter)
3549{ 3739{
3550 perf_swcounter_update(counter);
3551} 3740}
3552 3741
3553static const struct pmu perf_ops_generic = { 3742static const struct pmu perf_ops_generic = {
3554 .enable = perf_swcounter_enable, 3743 .enable = perf_swcounter_enable,
3555 .disable = perf_swcounter_disable, 3744 .disable = perf_swcounter_disable,
3556 .read = perf_swcounter_read, 3745 .read = perf_swcounter_read,
3746 .unthrottle = perf_swcounter_unthrottle,
3557}; 3747};
3558 3748
3559/* 3749/*
3750 * hrtimer based swcounter callback
3751 */
3752
3753static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
3754{
3755 enum hrtimer_restart ret = HRTIMER_RESTART;
3756 struct perf_sample_data data;
3757 struct perf_counter *counter;
3758 u64 period;
3759
3760 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
3761 counter->pmu->read(counter);
3762
3763 data.addr = 0;
3764 data.regs = get_irq_regs();
3765 /*
3766 * In case we exclude kernel IPs or are somehow not in interrupt
3767 * context, provide the next best thing, the user IP.
3768 */
3769 if ((counter->attr.exclude_kernel || !data.regs) &&
3770 !counter->attr.exclude_user)
3771 data.regs = task_pt_regs(current);
3772
3773 if (data.regs) {
3774 if (perf_counter_overflow(counter, 0, &data))
3775 ret = HRTIMER_NORESTART;
3776 }
3777
3778 period = max_t(u64, 10000, counter->hw.sample_period);
3779 hrtimer_forward_now(hrtimer, ns_to_ktime(period));
3780
3781 return ret;
3782}
3783
3784/*
3560 * Software counter: cpu wall time clock 3785 * Software counter: cpu wall time clock
3561 */ 3786 */
3562 3787
@@ -3673,17 +3898,24 @@ static const struct pmu perf_ops_task_clock = {
3673}; 3898};
3674 3899
3675#ifdef CONFIG_EVENT_PROFILE 3900#ifdef CONFIG_EVENT_PROFILE
3676void perf_tpcounter_event(int event_id) 3901void perf_tpcounter_event(int event_id, u64 addr, u64 count, void *record,
3902 int entry_size)
3677{ 3903{
3904 struct perf_raw_record raw = {
3905 .size = entry_size,
3906 .data = record,
3907 };
3908
3678 struct perf_sample_data data = { 3909 struct perf_sample_data data = {
3679 .regs = get_irq_regs(), 3910 .regs = get_irq_regs(),
3680 .addr = 0, 3911 .addr = addr,
3912 .raw = &raw,
3681 }; 3913 };
3682 3914
3683 if (!data.regs) 3915 if (!data.regs)
3684 data.regs = task_pt_regs(current); 3916 data.regs = task_pt_regs(current);
3685 3917
3686 do_perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, &data); 3918 do_perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, count, 1, &data);
3687} 3919}
3688EXPORT_SYMBOL_GPL(perf_tpcounter_event); 3920EXPORT_SYMBOL_GPL(perf_tpcounter_event);
3689 3921
@@ -3697,6 +3929,14 @@ static void tp_perf_counter_destroy(struct perf_counter *counter)
3697 3929
3698static const struct pmu *tp_perf_counter_init(struct perf_counter *counter) 3930static const struct pmu *tp_perf_counter_init(struct perf_counter *counter)
3699{ 3931{
3932 /*
3933 * Raw tracepoint data is a severe data leak, only allow root to
3934 * have these.
3935 */
3936 if ((counter->attr.sample_type & PERF_SAMPLE_RAW) &&
3937 !capable(CAP_SYS_ADMIN))
3938 return ERR_PTR(-EPERM);
3939
3700 if (ftrace_profile_enable(counter->attr.config)) 3940 if (ftrace_profile_enable(counter->attr.config))
3701 return NULL; 3941 return NULL;
3702 3942
@@ -3826,13 +4066,14 @@ perf_counter_alloc(struct perf_counter_attr *attr,
3826 hwc->sample_period = attr->sample_period; 4066 hwc->sample_period = attr->sample_period;
3827 if (attr->freq && attr->sample_freq) 4067 if (attr->freq && attr->sample_freq)
3828 hwc->sample_period = 1; 4068 hwc->sample_period = 1;
4069 hwc->last_period = hwc->sample_period;
3829 4070
3830 atomic64_set(&hwc->period_left, hwc->sample_period); 4071 atomic64_set(&hwc->period_left, hwc->sample_period);
3831 4072
3832 /* 4073 /*
3833 * we currently do not support PERF_SAMPLE_GROUP on inherited counters 4074 * we currently do not support PERF_FORMAT_GROUP on inherited counters
3834 */ 4075 */
3835 if (attr->inherit && (attr->sample_type & PERF_SAMPLE_GROUP)) 4076 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
3836 goto done; 4077 goto done;
3837 4078
3838 switch (attr->type) { 4079 switch (attr->type) {
@@ -3875,6 +4116,8 @@ done:
3875 atomic_inc(&nr_mmap_counters); 4116 atomic_inc(&nr_mmap_counters);
3876 if (counter->attr.comm) 4117 if (counter->attr.comm)
3877 atomic_inc(&nr_comm_counters); 4118 atomic_inc(&nr_comm_counters);
4119 if (counter->attr.task)
4120 atomic_inc(&nr_task_counters);
3878 } 4121 }
3879 4122
3880 return counter; 4123 return counter;
@@ -4236,8 +4479,10 @@ void perf_counter_exit_task(struct task_struct *child)
4236 struct perf_counter_context *child_ctx; 4479 struct perf_counter_context *child_ctx;
4237 unsigned long flags; 4480 unsigned long flags;
4238 4481
4239 if (likely(!child->perf_counter_ctxp)) 4482 if (likely(!child->perf_counter_ctxp)) {
4483 perf_counter_task(child, NULL, 0);
4240 return; 4484 return;
4485 }
4241 4486
4242 local_irq_save(flags); 4487 local_irq_save(flags);
4243 /* 4488 /*
@@ -4262,8 +4507,14 @@ void perf_counter_exit_task(struct task_struct *child)
4262 * the counters from it. 4507 * the counters from it.
4263 */ 4508 */
4264 unclone_ctx(child_ctx); 4509 unclone_ctx(child_ctx);
4265 spin_unlock(&child_ctx->lock); 4510 spin_unlock_irqrestore(&child_ctx->lock, flags);
4266 local_irq_restore(flags); 4511
4512 /*
4513 * Report the task dead after unscheduling the counters so that we
4514 * won't get any samples after PERF_EVENT_EXIT. We can however still
4515 * get a few PERF_EVENT_READ events.
4516 */
4517 perf_counter_task(child, child_ctx, 0);
4267 4518
4268 /* 4519 /*
4269 * We can recurse on the same lock type through: 4520 * We can recurse on the same lock type through:
@@ -4484,6 +4735,11 @@ perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
4484 perf_counter_init_cpu(cpu); 4735 perf_counter_init_cpu(cpu);
4485 break; 4736 break;
4486 4737
4738 case CPU_ONLINE:
4739 case CPU_ONLINE_FROZEN:
4740 hw_perf_counter_setup_online(cpu);
4741 break;
4742
4487 case CPU_DOWN_PREPARE: 4743 case CPU_DOWN_PREPARE:
4488 case CPU_DOWN_PREPARE_FROZEN: 4744 case CPU_DOWN_PREPARE_FROZEN:
4489 perf_counter_exit_cpu(cpu); 4745 perf_counter_exit_cpu(cpu);
@@ -4508,6 +4764,8 @@ void __init perf_counter_init(void)
4508{ 4764{
4509 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE, 4765 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
4510 (void *)(long)smp_processor_id()); 4766 (void *)(long)smp_processor_id());
4767 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
4768 (void *)(long)smp_processor_id());
4511 register_cpu_notifier(&perf_cpu_nb); 4769 register_cpu_notifier(&perf_cpu_nb);
4512} 4770}
4513 4771
diff --git a/kernel/posix-cpu-timers.c b/kernel/posix-cpu-timers.c
index bece7c0b67b2..e33a21cb9407 100644
--- a/kernel/posix-cpu-timers.c
+++ b/kernel/posix-cpu-timers.c
@@ -521,11 +521,12 @@ void posix_cpu_timers_exit(struct task_struct *tsk)
521} 521}
522void posix_cpu_timers_exit_group(struct task_struct *tsk) 522void posix_cpu_timers_exit_group(struct task_struct *tsk)
523{ 523{
524 struct task_cputime cputime; 524 struct signal_struct *const sig = tsk->signal;
525 525
526 thread_group_cputimer(tsk, &cputime);
527 cleanup_timers(tsk->signal->cpu_timers, 526 cleanup_timers(tsk->signal->cpu_timers,
528 cputime.utime, cputime.stime, cputime.sum_exec_runtime); 527 cputime_add(tsk->utime, sig->utime),
528 cputime_add(tsk->stime, sig->stime),
529 tsk->se.sum_exec_runtime + sig->sum_sched_runtime);
529} 530}
530 531
531static void clear_dead_task(struct k_itimer *timer, union cpu_time_count now) 532static void clear_dead_task(struct k_itimer *timer, union cpu_time_count now)
diff --git a/kernel/posix-timers.c b/kernel/posix-timers.c
index 052ec4d195c7..d089d052c4a9 100644
--- a/kernel/posix-timers.c
+++ b/kernel/posix-timers.c
@@ -202,6 +202,12 @@ static int no_timer_create(struct k_itimer *new_timer)
202 return -EOPNOTSUPP; 202 return -EOPNOTSUPP;
203} 203}
204 204
205static int no_nsleep(const clockid_t which_clock, int flags,
206 struct timespec *tsave, struct timespec __user *rmtp)
207{
208 return -EOPNOTSUPP;
209}
210
205/* 211/*
206 * Return nonzero if we know a priori this clockid_t value is bogus. 212 * Return nonzero if we know a priori this clockid_t value is bogus.
207 */ 213 */
@@ -254,6 +260,7 @@ static __init int init_posix_timers(void)
254 .clock_get = posix_get_monotonic_raw, 260 .clock_get = posix_get_monotonic_raw,
255 .clock_set = do_posix_clock_nosettime, 261 .clock_set = do_posix_clock_nosettime,
256 .timer_create = no_timer_create, 262 .timer_create = no_timer_create,
263 .nsleep = no_nsleep,
257 }; 264 };
258 265
259 register_posix_clock(CLOCK_REALTIME, &clock_realtime); 266 register_posix_clock(CLOCK_REALTIME, &clock_realtime);
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index fcd107a78c5a..29bd4baf9e75 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -1039,16 +1039,14 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
1039 if (!rt_mutex_owner(lock) || try_to_steal_lock(lock, task)) { 1039 if (!rt_mutex_owner(lock) || try_to_steal_lock(lock, task)) {
1040 /* We got the lock for task. */ 1040 /* We got the lock for task. */
1041 debug_rt_mutex_lock(lock); 1041 debug_rt_mutex_lock(lock);
1042
1043 rt_mutex_set_owner(lock, task, 0); 1042 rt_mutex_set_owner(lock, task, 0);
1044 1043 spin_unlock(&lock->wait_lock);
1045 rt_mutex_deadlock_account_lock(lock, task); 1044 rt_mutex_deadlock_account_lock(lock, task);
1046 return 1; 1045 return 1;
1047 } 1046 }
1048 1047
1049 ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock); 1048 ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
1050 1049
1051
1052 if (ret && !waiter->task) { 1050 if (ret && !waiter->task) {
1053 /* 1051 /*
1054 * Reset the return value. We might have 1052 * Reset the return value. We might have
diff --git a/kernel/sched_cpupri.c b/kernel/sched_cpupri.c
index e6c251790dde..d014efbf947a 100644
--- a/kernel/sched_cpupri.c
+++ b/kernel/sched_cpupri.c
@@ -81,8 +81,21 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
81 if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids) 81 if (cpumask_any_and(&p->cpus_allowed, vec->mask) >= nr_cpu_ids)
82 continue; 82 continue;
83 83
84 if (lowest_mask) 84 if (lowest_mask) {
85 cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask); 85 cpumask_and(lowest_mask, &p->cpus_allowed, vec->mask);
86
87 /*
88 * We have to ensure that we have at least one bit
89 * still set in the array, since the map could have
90 * been concurrently emptied between the first and
91 * second reads of vec->mask. If we hit this
92 * condition, simply act as though we never hit this
93 * priority level and continue on.
94 */
95 if (cpumask_any(lowest_mask) >= nr_cpu_ids)
96 continue;
97 }
98
86 return 1; 99 return 1;
87 } 100 }
88 101
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 9ffb2b2ceba4..652e8bdef9aa 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -611,9 +611,13 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
611static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) 611static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
612{ 612{
613#ifdef CONFIG_SCHEDSTATS 613#ifdef CONFIG_SCHEDSTATS
614 struct task_struct *tsk = NULL;
615
616 if (entity_is_task(se))
617 tsk = task_of(se);
618
614 if (se->sleep_start) { 619 if (se->sleep_start) {
615 u64 delta = rq_of(cfs_rq)->clock - se->sleep_start; 620 u64 delta = rq_of(cfs_rq)->clock - se->sleep_start;
616 struct task_struct *tsk = task_of(se);
617 621
618 if ((s64)delta < 0) 622 if ((s64)delta < 0)
619 delta = 0; 623 delta = 0;
@@ -624,11 +628,11 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
624 se->sleep_start = 0; 628 se->sleep_start = 0;
625 se->sum_sleep_runtime += delta; 629 se->sum_sleep_runtime += delta;
626 630
627 account_scheduler_latency(tsk, delta >> 10, 1); 631 if (tsk)
632 account_scheduler_latency(tsk, delta >> 10, 1);
628 } 633 }
629 if (se->block_start) { 634 if (se->block_start) {
630 u64 delta = rq_of(cfs_rq)->clock - se->block_start; 635 u64 delta = rq_of(cfs_rq)->clock - se->block_start;
631 struct task_struct *tsk = task_of(se);
632 636
633 if ((s64)delta < 0) 637 if ((s64)delta < 0)
634 delta = 0; 638 delta = 0;
@@ -639,17 +643,19 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se)
639 se->block_start = 0; 643 se->block_start = 0;
640 se->sum_sleep_runtime += delta; 644 se->sum_sleep_runtime += delta;
641 645
642 /* 646 if (tsk) {
643 * Blocking time is in units of nanosecs, so shift by 20 to 647 /*
644 * get a milliseconds-range estimation of the amount of 648 * Blocking time is in units of nanosecs, so shift by
645 * time that the task spent sleeping: 649 * 20 to get a milliseconds-range estimation of the
646 */ 650 * amount of time that the task spent sleeping:
647 if (unlikely(prof_on == SLEEP_PROFILING)) { 651 */
648 652 if (unlikely(prof_on == SLEEP_PROFILING)) {
649 profile_hits(SLEEP_PROFILING, (void *)get_wchan(tsk), 653 profile_hits(SLEEP_PROFILING,
650 delta >> 20); 654 (void *)get_wchan(tsk),
655 delta >> 20);
656 }
657 account_scheduler_latency(tsk, delta >> 10, 0);
651 } 658 }
652 account_scheduler_latency(tsk, delta >> 10, 0);
653 } 659 }
654#endif 660#endif
655} 661}
diff --git a/kernel/signal.c b/kernel/signal.c
index ccf1ceedaebe..64c5deeaca5d 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -2454,11 +2454,9 @@ do_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, unsigned long s
2454 stack_t oss; 2454 stack_t oss;
2455 int error; 2455 int error;
2456 2456
2457 if (uoss) { 2457 oss.ss_sp = (void __user *) current->sas_ss_sp;
2458 oss.ss_sp = (void __user *) current->sas_ss_sp; 2458 oss.ss_size = current->sas_ss_size;
2459 oss.ss_size = current->sas_ss_size; 2459 oss.ss_flags = sas_ss_flags(sp);
2460 oss.ss_flags = sas_ss_flags(sp);
2461 }
2462 2460
2463 if (uss) { 2461 if (uss) {
2464 void __user *ss_sp; 2462 void __user *ss_sp;
@@ -2466,10 +2464,12 @@ do_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, unsigned long s
2466 int ss_flags; 2464 int ss_flags;
2467 2465
2468 error = -EFAULT; 2466 error = -EFAULT;
2469 if (!access_ok(VERIFY_READ, uss, sizeof(*uss)) 2467 if (!access_ok(VERIFY_READ, uss, sizeof(*uss)))
2470 || __get_user(ss_sp, &uss->ss_sp) 2468 goto out;
2471 || __get_user(ss_flags, &uss->ss_flags) 2469 error = __get_user(ss_sp, &uss->ss_sp) |
2472 || __get_user(ss_size, &uss->ss_size)) 2470 __get_user(ss_flags, &uss->ss_flags) |
2471 __get_user(ss_size, &uss->ss_size);
2472 if (error)
2473 goto out; 2473 goto out;
2474 2474
2475 error = -EPERM; 2475 error = -EPERM;
@@ -2501,13 +2501,16 @@ do_sigaltstack (const stack_t __user *uss, stack_t __user *uoss, unsigned long s
2501 current->sas_ss_size = ss_size; 2501 current->sas_ss_size = ss_size;
2502 } 2502 }
2503 2503
2504 error = 0;
2504 if (uoss) { 2505 if (uoss) {
2505 error = -EFAULT; 2506 error = -EFAULT;
2506 if (copy_to_user(uoss, &oss, sizeof(oss))) 2507 if (!access_ok(VERIFY_WRITE, uoss, sizeof(*uoss)))
2507 goto out; 2508 goto out;
2509 error = __put_user(oss.ss_sp, &uoss->ss_sp) |
2510 __put_user(oss.ss_size, &uoss->ss_size) |
2511 __put_user(oss.ss_flags, &uoss->ss_flags);
2508 } 2512 }
2509 2513
2510 error = 0;
2511out: 2514out:
2512 return error; 2515 return error;
2513} 2516}
diff --git a/kernel/smp.c b/kernel/smp.c
index ad63d8501207..94188b8ecc33 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -57,7 +57,7 @@ hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu)
57 return NOTIFY_BAD; 57 return NOTIFY_BAD;
58 break; 58 break;
59 59
60#ifdef CONFIG_CPU_HOTPLUG 60#ifdef CONFIG_HOTPLUG_CPU
61 case CPU_UP_CANCELED: 61 case CPU_UP_CANCELED:
62 case CPU_UP_CANCELED_FROZEN: 62 case CPU_UP_CANCELED_FROZEN:
63 63
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 98e02328c67d..58be76017fd0 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -49,6 +49,7 @@
49#include <linux/acpi.h> 49#include <linux/acpi.h>
50#include <linux/reboot.h> 50#include <linux/reboot.h>
51#include <linux/ftrace.h> 51#include <linux/ftrace.h>
52#include <linux/security.h>
52#include <linux/slow-work.h> 53#include <linux/slow-work.h>
53#include <linux/perf_counter.h> 54#include <linux/perf_counter.h>
54 55
@@ -1306,10 +1307,10 @@ static struct ctl_table vm_table[] = {
1306 { 1307 {
1307 .ctl_name = CTL_UNNUMBERED, 1308 .ctl_name = CTL_UNNUMBERED,
1308 .procname = "mmap_min_addr", 1309 .procname = "mmap_min_addr",
1309 .data = &mmap_min_addr, 1310 .data = &dac_mmap_min_addr,
1310 .maxlen = sizeof(unsigned long), 1311 .maxlen = sizeof(unsigned long),
1311 .mode = 0644, 1312 .mode = 0644,
1312 .proc_handler = &proc_doulongvec_minmax, 1313 .proc_handler = &mmap_min_addr_handler,
1313 }, 1314 },
1314#ifdef CONFIG_NUMA 1315#ifdef CONFIG_NUMA
1315 { 1316 {
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index a6dcd67b041d..620b58abdc32 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -137,11 +137,12 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
137 */ 137 */
138int clockevents_register_notifier(struct notifier_block *nb) 138int clockevents_register_notifier(struct notifier_block *nb)
139{ 139{
140 unsigned long flags;
140 int ret; 141 int ret;
141 142
142 spin_lock(&clockevents_lock); 143 spin_lock_irqsave(&clockevents_lock, flags);
143 ret = raw_notifier_chain_register(&clockevents_chain, nb); 144 ret = raw_notifier_chain_register(&clockevents_chain, nb);
144 spin_unlock(&clockevents_lock); 145 spin_unlock_irqrestore(&clockevents_lock, flags);
145 146
146 return ret; 147 return ret;
147} 148}
@@ -178,16 +179,18 @@ static void clockevents_notify_released(void)
178 */ 179 */
179void clockevents_register_device(struct clock_event_device *dev) 180void clockevents_register_device(struct clock_event_device *dev)
180{ 181{
182 unsigned long flags;
183
181 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); 184 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
182 BUG_ON(!dev->cpumask); 185 BUG_ON(!dev->cpumask);
183 186
184 spin_lock(&clockevents_lock); 187 spin_lock_irqsave(&clockevents_lock, flags);
185 188
186 list_add(&dev->list, &clockevent_devices); 189 list_add(&dev->list, &clockevent_devices);
187 clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev); 190 clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev);
188 clockevents_notify_released(); 191 clockevents_notify_released();
189 192
190 spin_unlock(&clockevents_lock); 193 spin_unlock_irqrestore(&clockevents_lock, flags);
191} 194}
192EXPORT_SYMBOL_GPL(clockevents_register_device); 195EXPORT_SYMBOL_GPL(clockevents_register_device);
193 196
@@ -235,8 +238,9 @@ void clockevents_exchange_device(struct clock_event_device *old,
235void clockevents_notify(unsigned long reason, void *arg) 238void clockevents_notify(unsigned long reason, void *arg)
236{ 239{
237 struct list_head *node, *tmp; 240 struct list_head *node, *tmp;
241 unsigned long flags;
238 242
239 spin_lock(&clockevents_lock); 243 spin_lock_irqsave(&clockevents_lock, flags);
240 clockevents_do_notify(reason, arg); 244 clockevents_do_notify(reason, arg);
241 245
242 switch (reason) { 246 switch (reason) {
@@ -251,7 +255,7 @@ void clockevents_notify(unsigned long reason, void *arg)
251 default: 255 default:
252 break; 256 break;
253 } 257 }
254 spin_unlock(&clockevents_lock); 258 spin_unlock_irqrestore(&clockevents_lock, flags);
255} 259}
256EXPORT_SYMBOL_GPL(clockevents_notify); 260EXPORT_SYMBOL_GPL(clockevents_notify);
257#endif 261#endif
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 877dbedc3118..c2ec25087a35 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -205,11 +205,11 @@ static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
205 * Powerstate information: The system enters/leaves a state, where 205 * Powerstate information: The system enters/leaves a state, where
206 * affected devices might stop 206 * affected devices might stop
207 */ 207 */
208static void tick_do_broadcast_on_off(void *why) 208static void tick_do_broadcast_on_off(unsigned long *reason)
209{ 209{
210 struct clock_event_device *bc, *dev; 210 struct clock_event_device *bc, *dev;
211 struct tick_device *td; 211 struct tick_device *td;
212 unsigned long flags, *reason = why; 212 unsigned long flags;
213 int cpu, bc_stopped; 213 int cpu, bc_stopped;
214 214
215 spin_lock_irqsave(&tick_broadcast_lock, flags); 215 spin_lock_irqsave(&tick_broadcast_lock, flags);
@@ -276,8 +276,7 @@ void tick_broadcast_on_off(unsigned long reason, int *oncpu)
276 printk(KERN_ERR "tick-broadcast: ignoring broadcast for " 276 printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
277 "offline CPU #%d\n", *oncpu); 277 "offline CPU #%d\n", *oncpu);
278 else 278 else
279 smp_call_function_single(*oncpu, tick_do_broadcast_on_off, 279 tick_do_broadcast_on_off(&reason);
280 &reason, 1);
281} 280}
282 281
283/* 282/*
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index a999b92a1277..fddd69d16e03 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -286,7 +286,7 @@ static int __init init_timer_list_procfs(void)
286{ 286{
287 struct proc_dir_entry *pe; 287 struct proc_dir_entry *pe;
288 288
289 pe = proc_create("timer_list", 0644, NULL, &timer_list_fops); 289 pe = proc_create("timer_list", 0444, NULL, &timer_list_fops);
290 if (!pe) 290 if (!pe)
291 return -ENOMEM; 291 return -ENOMEM;
292 return 0; 292 return 0;
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index 1090b0aed9ba..7a34cb563fec 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -267,8 +267,8 @@ static void blk_trace_free(struct blk_trace *bt)
267{ 267{
268 debugfs_remove(bt->msg_file); 268 debugfs_remove(bt->msg_file);
269 debugfs_remove(bt->dropped_file); 269 debugfs_remove(bt->dropped_file);
270 debugfs_remove(bt->dir);
271 relay_close(bt->rchan); 270 relay_close(bt->rchan);
271 debugfs_remove(bt->dir);
272 free_percpu(bt->sequence); 272 free_percpu(bt->sequence);
273 free_percpu(bt->msg_data); 273 free_percpu(bt->msg_data);
274 kfree(bt); 274 kfree(bt);
@@ -378,18 +378,8 @@ static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
378 378
379static int blk_remove_buf_file_callback(struct dentry *dentry) 379static int blk_remove_buf_file_callback(struct dentry *dentry)
380{ 380{
381 struct dentry *parent = dentry->d_parent;
382 debugfs_remove(dentry); 381 debugfs_remove(dentry);
383 382
384 /*
385 * this will fail for all but the last file, but that is ok. what we
386 * care about is the top level buts->name directory going away, when
387 * the last trace file is gone. Then we don't have to rmdir() that
388 * manually on trace stop, so it nicely solves the issue with
389 * force killing of running traces.
390 */
391
392 debugfs_remove(parent);
393 return 0; 383 return 0;
394} 384}
395 385
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 1f3ec2afa511..25edd5cc5935 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1662,7 +1662,7 @@ ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1662 1662
1663 mutex_lock(&ftrace_regex_lock); 1663 mutex_lock(&ftrace_regex_lock);
1664 if ((file->f_mode & FMODE_WRITE) && 1664 if ((file->f_mode & FMODE_WRITE) &&
1665 !(file->f_flags & O_APPEND)) 1665 (file->f_flags & O_TRUNC))
1666 ftrace_filter_reset(enable); 1666 ftrace_filter_reset(enable);
1667 1667
1668 if (file->f_mode & FMODE_READ) { 1668 if (file->f_mode & FMODE_READ) {
@@ -2278,7 +2278,11 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2278 read++; 2278 read++;
2279 cnt--; 2279 cnt--;
2280 2280
2281 if (!(iter->flags & ~FTRACE_ITER_CONT)) { 2281 /*
2282 * If the parser haven't finished with the last write,
2283 * continue reading the user input without skipping spaces.
2284 */
2285 if (!(iter->flags & FTRACE_ITER_CONT)) {
2282 /* skip white space */ 2286 /* skip white space */
2283 while (cnt && isspace(ch)) { 2287 while (cnt && isspace(ch)) {
2284 ret = get_user(ch, ubuf++); 2288 ret = get_user(ch, ubuf++);
@@ -2288,8 +2292,9 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2288 cnt--; 2292 cnt--;
2289 } 2293 }
2290 2294
2295 /* only spaces were written */
2291 if (isspace(ch)) { 2296 if (isspace(ch)) {
2292 file->f_pos += read; 2297 *ppos += read;
2293 ret = read; 2298 ret = read;
2294 goto out; 2299 goto out;
2295 } 2300 }
@@ -2319,12 +2324,12 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2319 if (ret) 2324 if (ret)
2320 goto out; 2325 goto out;
2321 iter->buffer_idx = 0; 2326 iter->buffer_idx = 0;
2322 } else 2327 } else {
2323 iter->flags |= FTRACE_ITER_CONT; 2328 iter->flags |= FTRACE_ITER_CONT;
2329 iter->buffer[iter->buffer_idx++] = ch;
2330 }
2324 2331
2325 2332 *ppos += read;
2326 file->f_pos += read;
2327
2328 ret = read; 2333 ret = read;
2329 out: 2334 out:
2330 mutex_unlock(&ftrace_regex_lock); 2335 mutex_unlock(&ftrace_regex_lock);
@@ -2577,7 +2582,7 @@ ftrace_graph_open(struct inode *inode, struct file *file)
2577 2582
2578 mutex_lock(&graph_lock); 2583 mutex_lock(&graph_lock);
2579 if ((file->f_mode & FMODE_WRITE) && 2584 if ((file->f_mode & FMODE_WRITE) &&
2580 !(file->f_flags & O_APPEND)) { 2585 (file->f_flags & O_TRUNC)) {
2581 ftrace_graph_count = 0; 2586 ftrace_graph_count = 0;
2582 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs)); 2587 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2583 } 2588 }
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index bf27bb7a63e2..a330513d96ce 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -735,6 +735,7 @@ ring_buffer_free(struct ring_buffer *buffer)
735 735
736 put_online_cpus(); 736 put_online_cpus();
737 737
738 kfree(buffer->buffers);
738 free_cpumask_var(buffer->cpumask); 739 free_cpumask_var(buffer->cpumask);
739 740
740 kfree(buffer); 741 kfree(buffer);
@@ -1785,7 +1786,7 @@ void ring_buffer_discard_commit(struct ring_buffer *buffer,
1785 */ 1786 */
1786 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); 1787 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
1787 1788
1788 if (!rb_try_to_discard(cpu_buffer, event)) 1789 if (rb_try_to_discard(cpu_buffer, event))
1789 goto out; 1790 goto out;
1790 1791
1791 /* 1792 /*
@@ -2383,7 +2384,6 @@ rb_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
2383 * the box. Return the padding, and we will release 2384 * the box. Return the padding, and we will release
2384 * the current locks, and try again. 2385 * the current locks, and try again.
2385 */ 2386 */
2386 rb_advance_reader(cpu_buffer);
2387 return event; 2387 return event;
2388 2388
2389 case RINGBUF_TYPE_TIME_EXTEND: 2389 case RINGBUF_TYPE_TIME_EXTEND:
@@ -2486,7 +2486,7 @@ static inline int rb_ok_to_lock(void)
2486 * buffer too. A one time deal is all you get from reading 2486 * buffer too. A one time deal is all you get from reading
2487 * the ring buffer from an NMI. 2487 * the ring buffer from an NMI.
2488 */ 2488 */
2489 if (likely(!in_nmi() && !oops_in_progress)) 2489 if (likely(!in_nmi()))
2490 return 1; 2490 return 1;
2491 2491
2492 tracing_off_permanent(); 2492 tracing_off_permanent();
@@ -2519,6 +2519,8 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
2519 if (dolock) 2519 if (dolock)
2520 spin_lock(&cpu_buffer->reader_lock); 2520 spin_lock(&cpu_buffer->reader_lock);
2521 event = rb_buffer_peek(buffer, cpu, ts); 2521 event = rb_buffer_peek(buffer, cpu, ts);
2522 if (event && event->type_len == RINGBUF_TYPE_PADDING)
2523 rb_advance_reader(cpu_buffer);
2522 if (dolock) 2524 if (dolock)
2523 spin_unlock(&cpu_buffer->reader_lock); 2525 spin_unlock(&cpu_buffer->reader_lock);
2524 local_irq_restore(flags); 2526 local_irq_restore(flags);
@@ -2590,12 +2592,9 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
2590 spin_lock(&cpu_buffer->reader_lock); 2592 spin_lock(&cpu_buffer->reader_lock);
2591 2593
2592 event = rb_buffer_peek(buffer, cpu, ts); 2594 event = rb_buffer_peek(buffer, cpu, ts);
2593 if (!event) 2595 if (event)
2594 goto out_unlock; 2596 rb_advance_reader(cpu_buffer);
2595
2596 rb_advance_reader(cpu_buffer);
2597 2597
2598 out_unlock:
2599 if (dolock) 2598 if (dolock)
2600 spin_unlock(&cpu_buffer->reader_lock); 2599 spin_unlock(&cpu_buffer->reader_lock);
2601 local_irq_restore(flags); 2600 local_irq_restore(flags);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 8bc8d8afea6a..8c358395d338 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -848,6 +848,7 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
848 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | 848 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
849 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0); 849 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
850} 850}
851EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
851 852
852struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr, 853struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
853 int type, 854 int type,
@@ -2031,7 +2032,7 @@ static int tracing_open(struct inode *inode, struct file *file)
2031 2032
2032 /* If this file was open for write, then erase contents */ 2033 /* If this file was open for write, then erase contents */
2033 if ((file->f_mode & FMODE_WRITE) && 2034 if ((file->f_mode & FMODE_WRITE) &&
2034 !(file->f_flags & O_APPEND)) { 2035 (file->f_flags & O_TRUNC)) {
2035 long cpu = (long) inode->i_private; 2036 long cpu = (long) inode->i_private;
2036 2037
2037 if (cpu == TRACE_PIPE_ALL_CPU) 2038 if (cpu == TRACE_PIPE_ALL_CPU)
@@ -3085,7 +3086,8 @@ tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3085 break; 3086 break;
3086 } 3087 }
3087 3088
3088 trace_consume(iter); 3089 if (ret != TRACE_TYPE_NO_CONSUME)
3090 trace_consume(iter);
3089 rem -= count; 3091 rem -= count;
3090 if (!find_next_entry_inc(iter)) { 3092 if (!find_next_entry_inc(iter)) {
3091 rem = 0; 3093 rem = 0;
@@ -3894,17 +3896,9 @@ trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3894 if (ret < 0) 3896 if (ret < 0)
3895 return ret; 3897 return ret;
3896 3898
3897 switch (val) { 3899 if (val != 0 && val != 1)
3898 case 0:
3899 trace_flags &= ~(1 << index);
3900 break;
3901 case 1:
3902 trace_flags |= 1 << index;
3903 break;
3904
3905 default:
3906 return -EINVAL; 3900 return -EINVAL;
3907 } 3901 set_tracer_flags(1 << index, val);
3908 3902
3909 *ppos += cnt; 3903 *ppos += cnt;
3910 3904
@@ -4233,8 +4227,11 @@ static void __ftrace_dump(bool disable_tracing)
4233 iter.pos = -1; 4227 iter.pos = -1;
4234 4228
4235 if (find_next_entry_inc(&iter) != NULL) { 4229 if (find_next_entry_inc(&iter) != NULL) {
4236 print_trace_line(&iter); 4230 int ret;
4237 trace_consume(&iter); 4231
4232 ret = print_trace_line(&iter);
4233 if (ret != TRACE_TYPE_NO_CONSUME)
4234 trace_consume(&iter);
4238 } 4235 }
4239 4236
4240 trace_printk_seq(&iter.seq); 4237 trace_printk_seq(&iter.seq);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index 3548ae5cc780..8b9f4f6e9559 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -438,10 +438,6 @@ struct trace_entry *tracing_get_trace_entry(struct trace_array *tr,
438struct trace_entry *trace_find_next_entry(struct trace_iterator *iter, 438struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
439 int *ent_cpu, u64 *ent_ts); 439 int *ent_cpu, u64 *ent_ts);
440 440
441void tracing_generic_entry_update(struct trace_entry *entry,
442 unsigned long flags,
443 int pc);
444
445void default_wait_pipe(struct trace_iterator *iter); 441void default_wait_pipe(struct trace_iterator *iter);
446void poll_wait_pipe(struct trace_iterator *iter); 442void poll_wait_pipe(struct trace_iterator *iter);
447 443
diff --git a/kernel/trace/trace_event_profile.c b/kernel/trace/trace_event_profile.c
index 5b5895afecfe..11ba5bb4ed0a 100644
--- a/kernel/trace/trace_event_profile.c
+++ b/kernel/trace/trace_event_profile.c
@@ -14,7 +14,7 @@ int ftrace_profile_enable(int event_id)
14 14
15 mutex_lock(&event_mutex); 15 mutex_lock(&event_mutex);
16 list_for_each_entry(event, &ftrace_events, list) { 16 list_for_each_entry(event, &ftrace_events, list) {
17 if (event->id == event_id) { 17 if (event->id == event_id && event->profile_enable) {
18 ret = event->profile_enable(event); 18 ret = event->profile_enable(event);
19 break; 19 break;
20 } 20 }
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index 53c8fd376a88..e75276a49cf5 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -376,7 +376,7 @@ ftrace_event_seq_open(struct inode *inode, struct file *file)
376 const struct seq_operations *seq_ops; 376 const struct seq_operations *seq_ops;
377 377
378 if ((file->f_mode & FMODE_WRITE) && 378 if ((file->f_mode & FMODE_WRITE) &&
379 !(file->f_flags & O_APPEND)) 379 (file->f_flags & O_TRUNC))
380 ftrace_clear_events(); 380 ftrace_clear_events();
381 381
382 seq_ops = inode->i_private; 382 seq_ops = inode->i_private;
@@ -940,7 +940,7 @@ event_create_dir(struct ftrace_event_call *call, struct dentry *d_events,
940 entry = trace_create_file("enable", 0644, call->dir, call, 940 entry = trace_create_file("enable", 0644, call->dir, call,
941 enable); 941 enable);
942 942
943 if (call->id) 943 if (call->id && call->profile_enable)
944 entry = trace_create_file("id", 0444, call->dir, call, 944 entry = trace_create_file("id", 0444, call->dir, call,
945 id); 945 id);
946 946
diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c
index 936c621bbf46..f32dc9d1ea7b 100644
--- a/kernel/trace/trace_events_filter.c
+++ b/kernel/trace/trace_events_filter.c
@@ -624,9 +624,6 @@ static int filter_add_subsystem_pred(struct filter_parse_state *ps,
624 return -ENOSPC; 624 return -ENOSPC;
625 } 625 }
626 626
627 filter->preds[filter->n_preds] = pred;
628 filter->n_preds++;
629
630 list_for_each_entry(call, &ftrace_events, list) { 627 list_for_each_entry(call, &ftrace_events, list) {
631 628
632 if (!call->define_fields) 629 if (!call->define_fields)
@@ -643,6 +640,9 @@ static int filter_add_subsystem_pred(struct filter_parse_state *ps,
643 } 640 }
644 replace_filter_string(call->filter, filter_string); 641 replace_filter_string(call->filter, filter_string);
645 } 642 }
643
644 filter->preds[filter->n_preds] = pred;
645 filter->n_preds++;
646out: 646out:
647 return err; 647 return err;
648} 648}
@@ -1029,12 +1029,17 @@ static int replace_preds(struct event_subsystem *system,
1029 1029
1030 if (elt->op == OP_AND || elt->op == OP_OR) { 1030 if (elt->op == OP_AND || elt->op == OP_OR) {
1031 pred = create_logical_pred(elt->op); 1031 pred = create_logical_pred(elt->op);
1032 if (!pred)
1033 return -ENOMEM;
1032 if (call) { 1034 if (call) {
1033 err = filter_add_pred(ps, call, pred); 1035 err = filter_add_pred(ps, call, pred);
1034 filter_free_pred(pred); 1036 filter_free_pred(pred);
1035 } else 1037 } else {
1036 err = filter_add_subsystem_pred(ps, system, 1038 err = filter_add_subsystem_pred(ps, system,
1037 pred, filter_string); 1039 pred, filter_string);
1040 if (err)
1041 filter_free_pred(pred);
1042 }
1038 if (err) 1043 if (err)
1039 return err; 1044 return err;
1040 1045
@@ -1048,12 +1053,17 @@ static int replace_preds(struct event_subsystem *system,
1048 } 1053 }
1049 1054
1050 pred = create_pred(elt->op, operand1, operand2); 1055 pred = create_pred(elt->op, operand1, operand2);
1056 if (!pred)
1057 return -ENOMEM;
1051 if (call) { 1058 if (call) {
1052 err = filter_add_pred(ps, call, pred); 1059 err = filter_add_pred(ps, call, pred);
1053 filter_free_pred(pred); 1060 filter_free_pred(pred);
1054 } else 1061 } else {
1055 err = filter_add_subsystem_pred(ps, system, pred, 1062 err = filter_add_subsystem_pred(ps, system, pred,
1056 filter_string); 1063 filter_string);
1064 if (err)
1065 filter_free_pred(pred);
1066 }
1057 if (err) 1067 if (err)
1058 return err; 1068 return err;
1059 1069
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index d2249abafb53..420ec3487579 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -843,9 +843,16 @@ print_graph_function(struct trace_iterator *iter)
843 843
844 switch (entry->type) { 844 switch (entry->type) {
845 case TRACE_GRAPH_ENT: { 845 case TRACE_GRAPH_ENT: {
846 struct ftrace_graph_ent_entry *field; 846 /*
847 * print_graph_entry() may consume the current event,
848 * thus @field may become invalid, so we need to save it.
849 * sizeof(struct ftrace_graph_ent_entry) is very small,
850 * it can be safely saved at the stack.
851 */
852 struct ftrace_graph_ent_entry *field, saved;
847 trace_assign_type(field, entry); 853 trace_assign_type(field, entry);
848 return print_graph_entry(field, s, iter); 854 saved = *field;
855 return print_graph_entry(&saved, s, iter);
849 } 856 }
850 case TRACE_GRAPH_RET: { 857 case TRACE_GRAPH_RET: {
851 struct ftrace_graph_ret_entry *field; 858 struct ftrace_graph_ret_entry *field;
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
index 7b6278110827..687699d365ae 100644
--- a/kernel/trace/trace_printk.c
+++ b/kernel/trace/trace_printk.c
@@ -176,7 +176,7 @@ static int t_show(struct seq_file *m, void *v)
176 const char *str = *fmt; 176 const char *str = *fmt;
177 int i; 177 int i;
178 178
179 seq_printf(m, "0x%lx : \"", (unsigned long)fmt); 179 seq_printf(m, "0x%lx : \"", *(unsigned long *)fmt);
180 180
181 /* 181 /*
182 * Tabs and new lines need to be converted. 182 * Tabs and new lines need to be converted.
diff --git a/kernel/wait.c b/kernel/wait.c
index ea7c3b4275cf..c4bd3d825f35 100644
--- a/kernel/wait.c
+++ b/kernel/wait.c
@@ -10,13 +10,14 @@
10#include <linux/wait.h> 10#include <linux/wait.h>
11#include <linux/hash.h> 11#include <linux/hash.h>
12 12
13void init_waitqueue_head(wait_queue_head_t *q) 13void __init_waitqueue_head(wait_queue_head_t *q, struct lock_class_key *key)
14{ 14{
15 spin_lock_init(&q->lock); 15 spin_lock_init(&q->lock);
16 lockdep_set_class(&q->lock, key);
16 INIT_LIST_HEAD(&q->task_list); 17 INIT_LIST_HEAD(&q->task_list);
17} 18}
18 19
19EXPORT_SYMBOL(init_waitqueue_head); 20EXPORT_SYMBOL(__init_waitqueue_head);
20 21
21void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) 22void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
22{ 23{
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-09 07:58:03 -0500