aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/fork.c21
-rw-r--r--kernel/futex.c28
-rw-r--r--kernel/futex_compat.c6
-rw-r--r--kernel/irq/manage.c27
-rw-r--r--kernel/module.c10
-rw-r--r--kernel/perf_counter.c356
-rw-r--r--kernel/sysctl.c7
-rw-r--r--kernel/time/clockevents.c16
-rw-r--r--kernel/time/tick-broadcast.c7
-rw-r--r--kernel/time/timer_list.c2
-rw-r--r--kernel/trace/Kconfig8
-rw-r--r--kernel/trace/blktrace.c24
-rw-r--r--kernel/trace/ftrace.c17
-rw-r--r--kernel/trace/kmemtrace.c4
-rw-r--r--kernel/trace/ring_buffer.c172
-rw-r--r--kernel/trace/trace.c309
-rw-r--r--kernel/trace/trace.h26
-rw-r--r--kernel/trace/trace_boot.c16
-rw-r--r--kernel/trace/trace_events.c6
-rw-r--r--kernel/trace/trace_functions_graph.c14
-rw-r--r--kernel/trace/trace_irqsoff.c3
-rw-r--r--kernel/trace/trace_mmiotrace.c10
-rw-r--r--kernel/trace/trace_power.c22
-rw-r--r--kernel/trace/trace_sched_switch.c18
-rw-r--r--kernel/trace/trace_sched_wakeup.c7
-rw-r--r--kernel/trace/trace_syscalls.c18
-rw-r--r--kernel/wait.c5
27 files changed, 757 insertions, 402 deletions
diff --git a/kernel/fork.c b/kernel/fork.c
index 021e1138556e..e6c04d462ab2 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -426,7 +426,6 @@ static struct mm_struct * mm_init(struct mm_struct * mm, struct task_struct *p)
426 init_rwsem(&mm->mmap_sem); 426 init_rwsem(&mm->mmap_sem);
427 INIT_LIST_HEAD(&mm->mmlist); 427 INIT_LIST_HEAD(&mm->mmlist);
428 mm->flags = (current->mm) ? current->mm->flags : default_dump_filter; 428 mm->flags = (current->mm) ? current->mm->flags : default_dump_filter;
429 mm->oom_adj = (current->mm) ? current->mm->oom_adj : 0;
430 mm->core_state = NULL; 429 mm->core_state = NULL;
431 mm->nr_ptes = 0; 430 mm->nr_ptes = 0;
432 set_mm_counter(mm, file_rss, 0); 431 set_mm_counter(mm, file_rss, 0);
@@ -816,11 +815,8 @@ static int copy_signal(unsigned long clone_flags, struct task_struct *tsk)
816{ 815{
817 struct signal_struct *sig; 816 struct signal_struct *sig;
818 817
819 if (clone_flags & CLONE_THREAD) { 818 if (clone_flags & CLONE_THREAD)
820 atomic_inc(&current->signal->count);
821 atomic_inc(&current->signal->live);
822 return 0; 819 return 0;
823 }
824 820
825 sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL); 821 sig = kmem_cache_alloc(signal_cachep, GFP_KERNEL);
826 tsk->signal = sig; 822 tsk->signal = sig;
@@ -878,16 +874,6 @@ void __cleanup_signal(struct signal_struct *sig)
878 kmem_cache_free(signal_cachep, sig); 874 kmem_cache_free(signal_cachep, sig);
879} 875}
880 876
881static void cleanup_signal(struct task_struct *tsk)
882{
883 struct signal_struct *sig = tsk->signal;
884
885 atomic_dec(&sig->live);
886
887 if (atomic_dec_and_test(&sig->count))
888 __cleanup_signal(sig);
889}
890
891static void copy_flags(unsigned long clone_flags, struct task_struct *p) 877static void copy_flags(unsigned long clone_flags, struct task_struct *p)
892{ 878{
893 unsigned long new_flags = p->flags; 879 unsigned long new_flags = p->flags;
@@ -1240,6 +1226,8 @@ static struct task_struct *copy_process(unsigned long clone_flags,
1240 } 1226 }
1241 1227
1242 if (clone_flags & CLONE_THREAD) { 1228 if (clone_flags & CLONE_THREAD) {
1229 atomic_inc(&current->signal->count);
1230 atomic_inc(&current->signal->live);
1243 p->group_leader = current->group_leader; 1231 p->group_leader = current->group_leader;
1244 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group); 1232 list_add_tail_rcu(&p->thread_group, &p->group_leader->thread_group);
1245 } 1233 }
@@ -1283,7 +1271,8 @@ bad_fork_cleanup_mm:
1283 if (p->mm) 1271 if (p->mm)
1284 mmput(p->mm); 1272 mmput(p->mm);
1285bad_fork_cleanup_signal: 1273bad_fork_cleanup_signal:
1286 cleanup_signal(p); 1274 if (!(clone_flags & CLONE_THREAD))
1275 __cleanup_signal(p->signal);
1287bad_fork_cleanup_sighand: 1276bad_fork_cleanup_sighand:
1288 __cleanup_sighand(p->sighand); 1277 __cleanup_sighand(p->sighand);
1289bad_fork_cleanup_fs: 1278bad_fork_cleanup_fs:
diff --git a/kernel/futex.c b/kernel/futex.c
index 0672ff88f159..e18cfbdc7190 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1010,15 +1010,19 @@ void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
1010 * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue 1010 * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue
1011 * q: the futex_q 1011 * q: the futex_q
1012 * key: the key of the requeue target futex 1012 * key: the key of the requeue target futex
1013 * hb: the hash_bucket of the requeue target futex
1013 * 1014 *
1014 * During futex_requeue, with requeue_pi=1, it is possible to acquire the 1015 * During futex_requeue, with requeue_pi=1, it is possible to acquire the
1015 * target futex if it is uncontended or via a lock steal. Set the futex_q key 1016 * target futex if it is uncontended or via a lock steal. Set the futex_q key
1016 * to the requeue target futex so the waiter can detect the wakeup on the right 1017 * to the requeue target futex so the waiter can detect the wakeup on the right
1017 * futex, but remove it from the hb and NULL the rt_waiter so it can detect 1018 * futex, but remove it from the hb and NULL the rt_waiter so it can detect
1018 * atomic lock acquisition. Must be called with the q->lock_ptr held. 1019 * atomic lock acquisition. Set the q->lock_ptr to the requeue target hb->lock
1020 * to protect access to the pi_state to fixup the owner later. Must be called
1021 * with both q->lock_ptr and hb->lock held.
1019 */ 1022 */
1020static inline 1023static inline
1021void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key) 1024void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
1025 struct futex_hash_bucket *hb)
1022{ 1026{
1023 drop_futex_key_refs(&q->key); 1027 drop_futex_key_refs(&q->key);
1024 get_futex_key_refs(key); 1028 get_futex_key_refs(key);
@@ -1030,6 +1034,11 @@ void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key)
1030 WARN_ON(!q->rt_waiter); 1034 WARN_ON(!q->rt_waiter);
1031 q->rt_waiter = NULL; 1035 q->rt_waiter = NULL;
1032 1036
1037 q->lock_ptr = &hb->lock;
1038#ifdef CONFIG_DEBUG_PI_LIST
1039 q->list.plist.lock = &hb->lock;
1040#endif
1041
1033 wake_up_state(q->task, TASK_NORMAL); 1042 wake_up_state(q->task, TASK_NORMAL);
1034} 1043}
1035 1044
@@ -1088,7 +1097,7 @@ static int futex_proxy_trylock_atomic(u32 __user *pifutex,
1088 ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task, 1097 ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
1089 set_waiters); 1098 set_waiters);
1090 if (ret == 1) 1099 if (ret == 1)
1091 requeue_pi_wake_futex(top_waiter, key2); 1100 requeue_pi_wake_futex(top_waiter, key2, hb2);
1092 1101
1093 return ret; 1102 return ret;
1094} 1103}
@@ -1247,8 +1256,15 @@ retry_private:
1247 if (!match_futex(&this->key, &key1)) 1256 if (!match_futex(&this->key, &key1))
1248 continue; 1257 continue;
1249 1258
1250 WARN_ON(!requeue_pi && this->rt_waiter); 1259 /*
1251 WARN_ON(requeue_pi && !this->rt_waiter); 1260 * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always
1261 * be paired with each other and no other futex ops.
1262 */
1263 if ((requeue_pi && !this->rt_waiter) ||
1264 (!requeue_pi && this->rt_waiter)) {
1265 ret = -EINVAL;
1266 break;
1267 }
1252 1268
1253 /* 1269 /*
1254 * Wake nr_wake waiters. For requeue_pi, if we acquired the 1270 * Wake nr_wake waiters. For requeue_pi, if we acquired the
@@ -1273,7 +1289,7 @@ retry_private:
1273 this->task, 1); 1289 this->task, 1);
1274 if (ret == 1) { 1290 if (ret == 1) {
1275 /* We got the lock. */ 1291 /* We got the lock. */
1276 requeue_pi_wake_futex(this, &key2); 1292 requeue_pi_wake_futex(this, &key2, hb2);
1277 continue; 1293 continue;
1278 } else if (ret) { 1294 } else if (ret) {
1279 /* -EDEADLK */ 1295 /* -EDEADLK */
diff --git a/kernel/futex_compat.c b/kernel/futex_compat.c
index d607a5b9ee29..235716556bf1 100644
--- a/kernel/futex_compat.c
+++ b/kernel/futex_compat.c
@@ -180,7 +180,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
180 int cmd = op & FUTEX_CMD_MASK; 180 int cmd = op & FUTEX_CMD_MASK;
181 181
182 if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI || 182 if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
183 cmd == FUTEX_WAIT_BITSET)) { 183 cmd == FUTEX_WAIT_BITSET ||
184 cmd == FUTEX_WAIT_REQUEUE_PI)) {
184 if (get_compat_timespec(&ts, utime)) 185 if (get_compat_timespec(&ts, utime))
185 return -EFAULT; 186 return -EFAULT;
186 if (!timespec_valid(&ts)) 187 if (!timespec_valid(&ts))
@@ -191,7 +192,8 @@ asmlinkage long compat_sys_futex(u32 __user *uaddr, int op, u32 val,
191 t = ktime_add_safe(ktime_get(), t); 192 t = ktime_add_safe(ktime_get(), t);
192 tp = &t; 193 tp = &t;
193 } 194 }
194 if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE) 195 if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
196 cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
195 val2 = (int) (unsigned long) utime; 197 val2 = (int) (unsigned long) utime;
196 198
197 return do_futex(uaddr, op, val, tp, uaddr2, val2, val3); 199 return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 61c679db4687..0ec9ed831737 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -607,7 +607,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
607 */ 607 */
608 get_task_struct(t); 608 get_task_struct(t);
609 new->thread = t; 609 new->thread = t;
610 wake_up_process(t);
611 } 610 }
612 611
613 /* 612 /*
@@ -690,6 +689,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
690 (int)(new->flags & IRQF_TRIGGER_MASK)); 689 (int)(new->flags & IRQF_TRIGGER_MASK));
691 } 690 }
692 691
692 new->irq = irq;
693 *old_ptr = new; 693 *old_ptr = new;
694 694
695 /* Reset broken irq detection when installing new handler */ 695 /* Reset broken irq detection when installing new handler */
@@ -707,7 +707,13 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
707 707
708 spin_unlock_irqrestore(&desc->lock, flags); 708 spin_unlock_irqrestore(&desc->lock, flags);
709 709
710 new->irq = irq; 710 /*
711 * Strictly no need to wake it up, but hung_task complains
712 * when no hard interrupt wakes the thread up.
713 */
714 if (new->thread)
715 wake_up_process(new->thread);
716
711 register_irq_proc(irq, desc); 717 register_irq_proc(irq, desc);
712 new->dir = NULL; 718 new->dir = NULL;
713 register_handler_proc(irq, new); 719 register_handler_proc(irq, new);
@@ -761,7 +767,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
761{ 767{
762 struct irq_desc *desc = irq_to_desc(irq); 768 struct irq_desc *desc = irq_to_desc(irq);
763 struct irqaction *action, **action_ptr; 769 struct irqaction *action, **action_ptr;
764 struct task_struct *irqthread;
765 unsigned long flags; 770 unsigned long flags;
766 771
767 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq); 772 WARN(in_interrupt(), "Trying to free IRQ %d from IRQ context!\n", irq);
@@ -809,9 +814,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
809 desc->chip->disable(irq); 814 desc->chip->disable(irq);
810 } 815 }
811 816
812 irqthread = action->thread;
813 action->thread = NULL;
814
815 spin_unlock_irqrestore(&desc->lock, flags); 817 spin_unlock_irqrestore(&desc->lock, flags);
816 818
817 unregister_handler_proc(irq, action); 819 unregister_handler_proc(irq, action);
@@ -819,12 +821,6 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
819 /* Make sure it's not being used on another CPU: */ 821 /* Make sure it's not being used on another CPU: */
820 synchronize_irq(irq); 822 synchronize_irq(irq);
821 823
822 if (irqthread) {
823 if (!test_bit(IRQTF_DIED, &action->thread_flags))
824 kthread_stop(irqthread);
825 put_task_struct(irqthread);
826 }
827
828#ifdef CONFIG_DEBUG_SHIRQ 824#ifdef CONFIG_DEBUG_SHIRQ
829 /* 825 /*
830 * It's a shared IRQ -- the driver ought to be prepared for an IRQ 826 * It's a shared IRQ -- the driver ought to be prepared for an IRQ
@@ -840,6 +836,13 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
840 local_irq_restore(flags); 836 local_irq_restore(flags);
841 } 837 }
842#endif 838#endif
839
840 if (action->thread) {
841 if (!test_bit(IRQTF_DIED, &action->thread_flags))
842 kthread_stop(action->thread);
843 put_task_struct(action->thread);
844 }
845
843 return action; 846 return action;
844} 847}
845 848
diff --git a/kernel/module.c b/kernel/module.c
index b1821438694e..46580edff0cb 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -914,16 +914,18 @@ void __symbol_put(const char *symbol)
914} 914}
915EXPORT_SYMBOL(__symbol_put); 915EXPORT_SYMBOL(__symbol_put);
916 916
917/* Note this assumes addr is a function, which it currently always is. */
917void symbol_put_addr(void *addr) 918void symbol_put_addr(void *addr)
918{ 919{
919 struct module *modaddr; 920 struct module *modaddr;
921 unsigned long a = (unsigned long)dereference_function_descriptor(addr);
920 922
921 if (core_kernel_text((unsigned long)addr)) 923 if (core_kernel_text(a))
922 return; 924 return;
923 925
924 /* module_text_address is safe here: we're supposed to have reference 926 /* module_text_address is safe here: we're supposed to have reference
925 * to module from symbol_get, so it can't go away. */ 927 * to module from symbol_get, so it can't go away. */
926 modaddr = __module_text_address((unsigned long)addr); 928 modaddr = __module_text_address(a);
927 BUG_ON(!modaddr); 929 BUG_ON(!modaddr);
928 module_put(modaddr); 930 module_put(modaddr);
929} 931}
@@ -1279,6 +1281,10 @@ static void add_notes_attrs(struct module *mod, unsigned int nsect,
1279 struct module_notes_attrs *notes_attrs; 1281 struct module_notes_attrs *notes_attrs;
1280 struct bin_attribute *nattr; 1282 struct bin_attribute *nattr;
1281 1283
1284 /* failed to create section attributes, so can't create notes */
1285 if (!mod->sect_attrs)
1286 return;
1287
1282 /* Count notes sections and allocate structures. */ 1288 /* Count notes sections and allocate structures. */
1283 notes = 0; 1289 notes = 0;
1284 for (i = 0; i < nsect; i++) 1290 for (i = 0; i < nsect; i++)
diff --git a/kernel/perf_counter.c b/kernel/perf_counter.c
index b0b20a07f394..d7cbc579fc80 100644
--- a/kernel/perf_counter.c
+++ b/kernel/perf_counter.c
@@ -50,7 +50,7 @@ static atomic_t nr_task_counters __read_mostly;
50 * 1 - disallow cpu counters to unpriv 50 * 1 - disallow cpu counters to unpriv
51 * 2 - disallow kernel profiling to unpriv 51 * 2 - disallow kernel profiling to unpriv
52 */ 52 */
53int sysctl_perf_counter_paranoid __read_mostly; 53int sysctl_perf_counter_paranoid __read_mostly = 1;
54 54
55static inline bool perf_paranoid_cpu(void) 55static inline bool perf_paranoid_cpu(void)
56{ 56{
@@ -88,6 +88,7 @@ void __weak hw_perf_disable(void) { barrier(); }
88void __weak hw_perf_enable(void) { barrier(); } 88void __weak hw_perf_enable(void) { barrier(); }
89 89
90void __weak hw_perf_counter_setup(int cpu) { barrier(); } 90void __weak hw_perf_counter_setup(int cpu) { barrier(); }
91void __weak hw_perf_counter_setup_online(int cpu) { barrier(); }
91 92
92int __weak 93int __weak
93hw_perf_group_sched_in(struct perf_counter *group_leader, 94hw_perf_group_sched_in(struct perf_counter *group_leader,
@@ -306,6 +307,10 @@ counter_sched_out(struct perf_counter *counter,
306 return; 307 return;
307 308
308 counter->state = PERF_COUNTER_STATE_INACTIVE; 309 counter->state = PERF_COUNTER_STATE_INACTIVE;
310 if (counter->pending_disable) {
311 counter->pending_disable = 0;
312 counter->state = PERF_COUNTER_STATE_OFF;
313 }
309 counter->tstamp_stopped = ctx->time; 314 counter->tstamp_stopped = ctx->time;
310 counter->pmu->disable(counter); 315 counter->pmu->disable(counter);
311 counter->oncpu = -1; 316 counter->oncpu = -1;
@@ -1498,10 +1503,21 @@ static void perf_counter_enable_on_exec(struct task_struct *task)
1498 */ 1503 */
1499static void __perf_counter_read(void *info) 1504static void __perf_counter_read(void *info)
1500{ 1505{
1506 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1501 struct perf_counter *counter = info; 1507 struct perf_counter *counter = info;
1502 struct perf_counter_context *ctx = counter->ctx; 1508 struct perf_counter_context *ctx = counter->ctx;
1503 unsigned long flags; 1509 unsigned long flags;
1504 1510
1511 /*
1512 * If this is a task context, we need to check whether it is
1513 * the current task context of this cpu. If not it has been
1514 * scheduled out before the smp call arrived. In that case
1515 * counter->count would have been updated to a recent sample
1516 * when the counter was scheduled out.
1517 */
1518 if (ctx->task && cpuctx->task_ctx != ctx)
1519 return;
1520
1505 local_irq_save(flags); 1521 local_irq_save(flags);
1506 if (ctx->is_active) 1522 if (ctx->is_active)
1507 update_context_time(ctx); 1523 update_context_time(ctx);
@@ -1691,7 +1707,32 @@ static int perf_release(struct inode *inode, struct file *file)
1691 return 0; 1707 return 0;
1692} 1708}
1693 1709
1694static u64 perf_counter_read_tree(struct perf_counter *counter) 1710static int perf_counter_read_size(struct perf_counter *counter)
1711{
1712 int entry = sizeof(u64); /* value */
1713 int size = 0;
1714 int nr = 1;
1715
1716 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1717 size += sizeof(u64);
1718
1719 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1720 size += sizeof(u64);
1721
1722 if (counter->attr.read_format & PERF_FORMAT_ID)
1723 entry += sizeof(u64);
1724
1725 if (counter->attr.read_format & PERF_FORMAT_GROUP) {
1726 nr += counter->group_leader->nr_siblings;
1727 size += sizeof(u64);
1728 }
1729
1730 size += entry * nr;
1731
1732 return size;
1733}
1734
1735static u64 perf_counter_read_value(struct perf_counter *counter)
1695{ 1736{
1696 struct perf_counter *child; 1737 struct perf_counter *child;
1697 u64 total = 0; 1738 u64 total = 0;
@@ -1703,14 +1744,96 @@ static u64 perf_counter_read_tree(struct perf_counter *counter)
1703 return total; 1744 return total;
1704} 1745}
1705 1746
1747static int perf_counter_read_entry(struct perf_counter *counter,
1748 u64 read_format, char __user *buf)
1749{
1750 int n = 0, count = 0;
1751 u64 values[2];
1752
1753 values[n++] = perf_counter_read_value(counter);
1754 if (read_format & PERF_FORMAT_ID)
1755 values[n++] = primary_counter_id(counter);
1756
1757 count = n * sizeof(u64);
1758
1759 if (copy_to_user(buf, values, count))
1760 return -EFAULT;
1761
1762 return count;
1763}
1764
1765static int perf_counter_read_group(struct perf_counter *counter,
1766 u64 read_format, char __user *buf)
1767{
1768 struct perf_counter *leader = counter->group_leader, *sub;
1769 int n = 0, size = 0, err = -EFAULT;
1770 u64 values[3];
1771
1772 values[n++] = 1 + leader->nr_siblings;
1773 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1774 values[n++] = leader->total_time_enabled +
1775 atomic64_read(&leader->child_total_time_enabled);
1776 }
1777 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1778 values[n++] = leader->total_time_running +
1779 atomic64_read(&leader->child_total_time_running);
1780 }
1781
1782 size = n * sizeof(u64);
1783
1784 if (copy_to_user(buf, values, size))
1785 return -EFAULT;
1786
1787 err = perf_counter_read_entry(leader, read_format, buf + size);
1788 if (err < 0)
1789 return err;
1790
1791 size += err;
1792
1793 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1794 err = perf_counter_read_entry(sub, read_format,
1795 buf + size);
1796 if (err < 0)
1797 return err;
1798
1799 size += err;
1800 }
1801
1802 return size;
1803}
1804
1805static int perf_counter_read_one(struct perf_counter *counter,
1806 u64 read_format, char __user *buf)
1807{
1808 u64 values[4];
1809 int n = 0;
1810
1811 values[n++] = perf_counter_read_value(counter);
1812 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
1813 values[n++] = counter->total_time_enabled +
1814 atomic64_read(&counter->child_total_time_enabled);
1815 }
1816 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
1817 values[n++] = counter->total_time_running +
1818 atomic64_read(&counter->child_total_time_running);
1819 }
1820 if (read_format & PERF_FORMAT_ID)
1821 values[n++] = primary_counter_id(counter);
1822
1823 if (copy_to_user(buf, values, n * sizeof(u64)))
1824 return -EFAULT;
1825
1826 return n * sizeof(u64);
1827}
1828
1706/* 1829/*
1707 * Read the performance counter - simple non blocking version for now 1830 * Read the performance counter - simple non blocking version for now
1708 */ 1831 */
1709static ssize_t 1832static ssize_t
1710perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count) 1833perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1711{ 1834{
1712 u64 values[4]; 1835 u64 read_format = counter->attr.read_format;
1713 int n; 1836 int ret;
1714 1837
1715 /* 1838 /*
1716 * Return end-of-file for a read on a counter that is in 1839 * Return end-of-file for a read on a counter that is in
@@ -1720,28 +1843,18 @@ perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1720 if (counter->state == PERF_COUNTER_STATE_ERROR) 1843 if (counter->state == PERF_COUNTER_STATE_ERROR)
1721 return 0; 1844 return 0;
1722 1845
1846 if (count < perf_counter_read_size(counter))
1847 return -ENOSPC;
1848
1723 WARN_ON_ONCE(counter->ctx->parent_ctx); 1849 WARN_ON_ONCE(counter->ctx->parent_ctx);
1724 mutex_lock(&counter->child_mutex); 1850 mutex_lock(&counter->child_mutex);
1725 values[0] = perf_counter_read_tree(counter); 1851 if (read_format & PERF_FORMAT_GROUP)
1726 n = 1; 1852 ret = perf_counter_read_group(counter, read_format, buf);
1727 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1853 else
1728 values[n++] = counter->total_time_enabled + 1854 ret = perf_counter_read_one(counter, read_format, buf);
1729 atomic64_read(&counter->child_total_time_enabled);
1730 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1731 values[n++] = counter->total_time_running +
1732 atomic64_read(&counter->child_total_time_running);
1733 if (counter->attr.read_format & PERF_FORMAT_ID)
1734 values[n++] = primary_counter_id(counter);
1735 mutex_unlock(&counter->child_mutex); 1855 mutex_unlock(&counter->child_mutex);
1736 1856
1737 if (count < n * sizeof(u64)) 1857 return ret;
1738 return -EINVAL;
1739 count = n * sizeof(u64);
1740
1741 if (copy_to_user(buf, values, count))
1742 return -EFAULT;
1743
1744 return count;
1745} 1858}
1746 1859
1747static ssize_t 1860static ssize_t
@@ -1906,6 +2019,10 @@ int perf_counter_task_disable(void)
1906 return 0; 2019 return 0;
1907} 2020}
1908 2021
2022#ifndef PERF_COUNTER_INDEX_OFFSET
2023# define PERF_COUNTER_INDEX_OFFSET 0
2024#endif
2025
1909static int perf_counter_index(struct perf_counter *counter) 2026static int perf_counter_index(struct perf_counter *counter)
1910{ 2027{
1911 if (counter->state != PERF_COUNTER_STATE_ACTIVE) 2028 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
@@ -2245,7 +2362,7 @@ static void perf_pending_counter(struct perf_pending_entry *entry)
2245 2362
2246 if (counter->pending_disable) { 2363 if (counter->pending_disable) {
2247 counter->pending_disable = 0; 2364 counter->pending_disable = 0;
2248 perf_counter_disable(counter); 2365 __perf_counter_disable(counter);
2249 } 2366 }
2250 2367
2251 if (counter->pending_wakeup) { 2368 if (counter->pending_wakeup) {
@@ -2630,7 +2747,80 @@ static u32 perf_counter_tid(struct perf_counter *counter, struct task_struct *p)
2630 return task_pid_nr_ns(p, counter->ns); 2747 return task_pid_nr_ns(p, counter->ns);
2631} 2748}
2632 2749
2633static void perf_counter_output(struct perf_counter *counter, int nmi, 2750static void perf_output_read_one(struct perf_output_handle *handle,
2751 struct perf_counter *counter)
2752{
2753 u64 read_format = counter->attr.read_format;
2754 u64 values[4];
2755 int n = 0;
2756
2757 values[n++] = atomic64_read(&counter->count);
2758 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2759 values[n++] = counter->total_time_enabled +
2760 atomic64_read(&counter->child_total_time_enabled);
2761 }
2762 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2763 values[n++] = counter->total_time_running +
2764 atomic64_read(&counter->child_total_time_running);
2765 }
2766 if (read_format & PERF_FORMAT_ID)
2767 values[n++] = primary_counter_id(counter);
2768
2769 perf_output_copy(handle, values, n * sizeof(u64));
2770}
2771
2772/*
2773 * XXX PERF_FORMAT_GROUP vs inherited counters seems difficult.
2774 */
2775static void perf_output_read_group(struct perf_output_handle *handle,
2776 struct perf_counter *counter)
2777{
2778 struct perf_counter *leader = counter->group_leader, *sub;
2779 u64 read_format = counter->attr.read_format;
2780 u64 values[5];
2781 int n = 0;
2782
2783 values[n++] = 1 + leader->nr_siblings;
2784
2785 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
2786 values[n++] = leader->total_time_enabled;
2787
2788 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
2789 values[n++] = leader->total_time_running;
2790
2791 if (leader != counter)
2792 leader->pmu->read(leader);
2793
2794 values[n++] = atomic64_read(&leader->count);
2795 if (read_format & PERF_FORMAT_ID)
2796 values[n++] = primary_counter_id(leader);
2797
2798 perf_output_copy(handle, values, n * sizeof(u64));
2799
2800 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2801 n = 0;
2802
2803 if (sub != counter)
2804 sub->pmu->read(sub);
2805
2806 values[n++] = atomic64_read(&sub->count);
2807 if (read_format & PERF_FORMAT_ID)
2808 values[n++] = primary_counter_id(sub);
2809
2810 perf_output_copy(handle, values, n * sizeof(u64));
2811 }
2812}
2813
2814static void perf_output_read(struct perf_output_handle *handle,
2815 struct perf_counter *counter)
2816{
2817 if (counter->attr.read_format & PERF_FORMAT_GROUP)
2818 perf_output_read_group(handle, counter);
2819 else
2820 perf_output_read_one(handle, counter);
2821}
2822
2823void perf_counter_output(struct perf_counter *counter, int nmi,
2634 struct perf_sample_data *data) 2824 struct perf_sample_data *data)
2635{ 2825{
2636 int ret; 2826 int ret;
@@ -2641,10 +2831,6 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2641 struct { 2831 struct {
2642 u32 pid, tid; 2832 u32 pid, tid;
2643 } tid_entry; 2833 } tid_entry;
2644 struct {
2645 u64 id;
2646 u64 counter;
2647 } group_entry;
2648 struct perf_callchain_entry *callchain = NULL; 2834 struct perf_callchain_entry *callchain = NULL;
2649 int callchain_size = 0; 2835 int callchain_size = 0;
2650 u64 time; 2836 u64 time;
@@ -2699,10 +2885,8 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2699 if (sample_type & PERF_SAMPLE_PERIOD) 2885 if (sample_type & PERF_SAMPLE_PERIOD)
2700 header.size += sizeof(u64); 2886 header.size += sizeof(u64);
2701 2887
2702 if (sample_type & PERF_SAMPLE_GROUP) { 2888 if (sample_type & PERF_SAMPLE_READ)
2703 header.size += sizeof(u64) + 2889 header.size += perf_counter_read_size(counter);
2704 counter->nr_siblings * sizeof(group_entry);
2705 }
2706 2890
2707 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 2891 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2708 callchain = perf_callchain(data->regs); 2892 callchain = perf_callchain(data->regs);
@@ -2759,26 +2943,8 @@ static void perf_counter_output(struct perf_counter *counter, int nmi,
2759 if (sample_type & PERF_SAMPLE_PERIOD) 2943 if (sample_type & PERF_SAMPLE_PERIOD)
2760 perf_output_put(&handle, data->period); 2944 perf_output_put(&handle, data->period);
2761 2945
2762 /* 2946 if (sample_type & PERF_SAMPLE_READ)
2763 * XXX PERF_SAMPLE_GROUP vs inherited counters seems difficult. 2947 perf_output_read(&handle, counter);
2764 */
2765 if (sample_type & PERF_SAMPLE_GROUP) {
2766 struct perf_counter *leader, *sub;
2767 u64 nr = counter->nr_siblings;
2768
2769 perf_output_put(&handle, nr);
2770
2771 leader = counter->group_leader;
2772 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
2773 if (sub != counter)
2774 sub->pmu->read(sub);
2775
2776 group_entry.id = primary_counter_id(sub);
2777 group_entry.counter = atomic64_read(&sub->count);
2778
2779 perf_output_put(&handle, group_entry);
2780 }
2781 }
2782 2948
2783 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 2949 if (sample_type & PERF_SAMPLE_CALLCHAIN) {
2784 if (callchain) 2950 if (callchain)
@@ -2817,8 +2983,6 @@ struct perf_read_event {
2817 2983
2818 u32 pid; 2984 u32 pid;
2819 u32 tid; 2985 u32 tid;
2820 u64 value;
2821 u64 format[3];
2822}; 2986};
2823 2987
2824static void 2988static void
@@ -2830,34 +2994,20 @@ perf_counter_read_event(struct perf_counter *counter,
2830 .header = { 2994 .header = {
2831 .type = PERF_EVENT_READ, 2995 .type = PERF_EVENT_READ,
2832 .misc = 0, 2996 .misc = 0,
2833 .size = sizeof(event) - sizeof(event.format), 2997 .size = sizeof(event) + perf_counter_read_size(counter),
2834 }, 2998 },
2835 .pid = perf_counter_pid(counter, task), 2999 .pid = perf_counter_pid(counter, task),
2836 .tid = perf_counter_tid(counter, task), 3000 .tid = perf_counter_tid(counter, task),
2837 .value = atomic64_read(&counter->count),
2838 }; 3001 };
2839 int ret, i = 0; 3002 int ret;
2840
2841 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
2842 event.header.size += sizeof(u64);
2843 event.format[i++] = counter->total_time_enabled;
2844 }
2845
2846 if (counter->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
2847 event.header.size += sizeof(u64);
2848 event.format[i++] = counter->total_time_running;
2849 }
2850
2851 if (counter->attr.read_format & PERF_FORMAT_ID) {
2852 event.header.size += sizeof(u64);
2853 event.format[i++] = primary_counter_id(counter);
2854 }
2855 3003
2856 ret = perf_output_begin(&handle, counter, event.header.size, 0, 0); 3004 ret = perf_output_begin(&handle, counter, event.header.size, 0, 0);
2857 if (ret) 3005 if (ret)
2858 return; 3006 return;
2859 3007
2860 perf_output_copy(&handle, &event, event.header.size); 3008 perf_output_put(&handle, event);
3009 perf_output_read(&handle, counter);
3010
2861 perf_output_end(&handle); 3011 perf_output_end(&handle);
2862} 3012}
2863 3013
@@ -2893,10 +3043,10 @@ static void perf_counter_task_output(struct perf_counter *counter,
2893 return; 3043 return;
2894 3044
2895 task_event->event.pid = perf_counter_pid(counter, task); 3045 task_event->event.pid = perf_counter_pid(counter, task);
2896 task_event->event.ppid = perf_counter_pid(counter, task->real_parent); 3046 task_event->event.ppid = perf_counter_pid(counter, current);
2897 3047
2898 task_event->event.tid = perf_counter_tid(counter, task); 3048 task_event->event.tid = perf_counter_tid(counter, task);
2899 task_event->event.ptid = perf_counter_tid(counter, task->real_parent); 3049 task_event->event.ptid = perf_counter_tid(counter, current);
2900 3050
2901 perf_output_put(&handle, task_event->event); 3051 perf_output_put(&handle, task_event->event);
2902 perf_output_end(&handle); 3052 perf_output_end(&handle);
@@ -3443,40 +3593,32 @@ static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
3443 3593
3444static int perf_swcounter_is_counting(struct perf_counter *counter) 3594static int perf_swcounter_is_counting(struct perf_counter *counter)
3445{ 3595{
3446 struct perf_counter_context *ctx; 3596 /*
3447 unsigned long flags; 3597 * The counter is active, we're good!
3448 int count; 3598 */
3449
3450 if (counter->state == PERF_COUNTER_STATE_ACTIVE) 3599 if (counter->state == PERF_COUNTER_STATE_ACTIVE)
3451 return 1; 3600 return 1;
3452 3601
3602 /*
3603 * The counter is off/error, not counting.
3604 */
3453 if (counter->state != PERF_COUNTER_STATE_INACTIVE) 3605 if (counter->state != PERF_COUNTER_STATE_INACTIVE)
3454 return 0; 3606 return 0;
3455 3607
3456 /* 3608 /*
3457 * If the counter is inactive, it could be just because 3609 * The counter is inactive, if the context is active
3458 * its task is scheduled out, or because it's in a group 3610 * we're part of a group that didn't make it on the 'pmu',
3459 * which could not go on the PMU. We want to count in 3611 * not counting.
3460 * the first case but not the second. If the context is
3461 * currently active then an inactive software counter must
3462 * be the second case. If it's not currently active then
3463 * we need to know whether the counter was active when the
3464 * context was last active, which we can determine by
3465 * comparing counter->tstamp_stopped with ctx->time.
3466 *
3467 * We are within an RCU read-side critical section,
3468 * which protects the existence of *ctx.
3469 */ 3612 */
3470 ctx = counter->ctx; 3613 if (counter->ctx->is_active)
3471 spin_lock_irqsave(&ctx->lock, flags); 3614 return 0;
3472 count = 1; 3615
3473 /* Re-check state now we have the lock */ 3616 /*
3474 if (counter->state < PERF_COUNTER_STATE_INACTIVE || 3617 * We're inactive and the context is too, this means the
3475 counter->ctx->is_active || 3618 * task is scheduled out, we're counting events that happen
3476 counter->tstamp_stopped < ctx->time) 3619 * to us, like migration events.
3477 count = 0; 3620 */
3478 spin_unlock_irqrestore(&ctx->lock, flags); 3621 return 1;
3479 return count;
3480} 3622}
3481 3623
3482static int perf_swcounter_match(struct perf_counter *counter, 3624static int perf_swcounter_match(struct perf_counter *counter,
@@ -3924,13 +4066,14 @@ perf_counter_alloc(struct perf_counter_attr *attr,
3924 hwc->sample_period = attr->sample_period; 4066 hwc->sample_period = attr->sample_period;
3925 if (attr->freq && attr->sample_freq) 4067 if (attr->freq && attr->sample_freq)
3926 hwc->sample_period = 1; 4068 hwc->sample_period = 1;
4069 hwc->last_period = hwc->sample_period;
3927 4070
3928 atomic64_set(&hwc->period_left, hwc->sample_period); 4071 atomic64_set(&hwc->period_left, hwc->sample_period);
3929 4072
3930 /* 4073 /*
3931 * we currently do not support PERF_SAMPLE_GROUP on inherited counters 4074 * we currently do not support PERF_FORMAT_GROUP on inherited counters
3932 */ 4075 */
3933 if (attr->inherit && (attr->sample_type & PERF_SAMPLE_GROUP)) 4076 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
3934 goto done; 4077 goto done;
3935 4078
3936 switch (attr->type) { 4079 switch (attr->type) {
@@ -4592,6 +4735,11 @@ perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
4592 perf_counter_init_cpu(cpu); 4735 perf_counter_init_cpu(cpu);
4593 break; 4736 break;
4594 4737
4738 case CPU_ONLINE:
4739 case CPU_ONLINE_FROZEN:
4740 hw_perf_counter_setup_online(cpu);
4741 break;
4742
4595 case CPU_DOWN_PREPARE: 4743 case CPU_DOWN_PREPARE:
4596 case CPU_DOWN_PREPARE_FROZEN: 4744 case CPU_DOWN_PREPARE_FROZEN:
4597 perf_counter_exit_cpu(cpu); 4745 perf_counter_exit_cpu(cpu);
@@ -4616,6 +4764,8 @@ void __init perf_counter_init(void)
4616{ 4764{
4617 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE, 4765 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
4618 (void *)(long)smp_processor_id()); 4766 (void *)(long)smp_processor_id());
4767 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_ONLINE,
4768 (void *)(long)smp_processor_id());
4619 register_cpu_notifier(&perf_cpu_nb); 4769 register_cpu_notifier(&perf_cpu_nb);
4620} 4770}
4621 4771
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 98e02328c67d..58be76017fd0 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -49,6 +49,7 @@
49#include <linux/acpi.h> 49#include <linux/acpi.h>
50#include <linux/reboot.h> 50#include <linux/reboot.h>
51#include <linux/ftrace.h> 51#include <linux/ftrace.h>
52#include <linux/security.h>
52#include <linux/slow-work.h> 53#include <linux/slow-work.h>
53#include <linux/perf_counter.h> 54#include <linux/perf_counter.h>
54 55
@@ -1306,10 +1307,10 @@ static struct ctl_table vm_table[] = {
1306 { 1307 {
1307 .ctl_name = CTL_UNNUMBERED, 1308 .ctl_name = CTL_UNNUMBERED,
1308 .procname = "mmap_min_addr", 1309 .procname = "mmap_min_addr",
1309 .data = &mmap_min_addr, 1310 .data = &dac_mmap_min_addr,
1310 .maxlen = sizeof(unsigned long), 1311 .maxlen = sizeof(unsigned long),
1311 .mode = 0644, 1312 .mode = 0644,
1312 .proc_handler = &proc_doulongvec_minmax, 1313 .proc_handler = &mmap_min_addr_handler,
1313 }, 1314 },
1314#ifdef CONFIG_NUMA 1315#ifdef CONFIG_NUMA
1315 { 1316 {
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c
index a6dcd67b041d..620b58abdc32 100644
--- a/kernel/time/clockevents.c
+++ b/kernel/time/clockevents.c
@@ -137,11 +137,12 @@ int clockevents_program_event(struct clock_event_device *dev, ktime_t expires,
137 */ 137 */
138int clockevents_register_notifier(struct notifier_block *nb) 138int clockevents_register_notifier(struct notifier_block *nb)
139{ 139{
140 unsigned long flags;
140 int ret; 141 int ret;
141 142
142 spin_lock(&clockevents_lock); 143 spin_lock_irqsave(&clockevents_lock, flags);
143 ret = raw_notifier_chain_register(&clockevents_chain, nb); 144 ret = raw_notifier_chain_register(&clockevents_chain, nb);
144 spin_unlock(&clockevents_lock); 145 spin_unlock_irqrestore(&clockevents_lock, flags);
145 146
146 return ret; 147 return ret;
147} 148}
@@ -178,16 +179,18 @@ static void clockevents_notify_released(void)
178 */ 179 */
179void clockevents_register_device(struct clock_event_device *dev) 180void clockevents_register_device(struct clock_event_device *dev)
180{ 181{
182 unsigned long flags;
183
181 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED); 184 BUG_ON(dev->mode != CLOCK_EVT_MODE_UNUSED);
182 BUG_ON(!dev->cpumask); 185 BUG_ON(!dev->cpumask);
183 186
184 spin_lock(&clockevents_lock); 187 spin_lock_irqsave(&clockevents_lock, flags);
185 188
186 list_add(&dev->list, &clockevent_devices); 189 list_add(&dev->list, &clockevent_devices);
187 clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev); 190 clockevents_do_notify(CLOCK_EVT_NOTIFY_ADD, dev);
188 clockevents_notify_released(); 191 clockevents_notify_released();
189 192
190 spin_unlock(&clockevents_lock); 193 spin_unlock_irqrestore(&clockevents_lock, flags);
191} 194}
192EXPORT_SYMBOL_GPL(clockevents_register_device); 195EXPORT_SYMBOL_GPL(clockevents_register_device);
193 196
@@ -235,8 +238,9 @@ void clockevents_exchange_device(struct clock_event_device *old,
235void clockevents_notify(unsigned long reason, void *arg) 238void clockevents_notify(unsigned long reason, void *arg)
236{ 239{
237 struct list_head *node, *tmp; 240 struct list_head *node, *tmp;
241 unsigned long flags;
238 242
239 spin_lock(&clockevents_lock); 243 spin_lock_irqsave(&clockevents_lock, flags);
240 clockevents_do_notify(reason, arg); 244 clockevents_do_notify(reason, arg);
241 245
242 switch (reason) { 246 switch (reason) {
@@ -251,7 +255,7 @@ void clockevents_notify(unsigned long reason, void *arg)
251 default: 255 default:
252 break; 256 break;
253 } 257 }
254 spin_unlock(&clockevents_lock); 258 spin_unlock_irqrestore(&clockevents_lock, flags);
255} 259}
256EXPORT_SYMBOL_GPL(clockevents_notify); 260EXPORT_SYMBOL_GPL(clockevents_notify);
257#endif 261#endif
diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c
index 877dbedc3118..c2ec25087a35 100644
--- a/kernel/time/tick-broadcast.c
+++ b/kernel/time/tick-broadcast.c
@@ -205,11 +205,11 @@ static void tick_handle_periodic_broadcast(struct clock_event_device *dev)
205 * Powerstate information: The system enters/leaves a state, where 205 * Powerstate information: The system enters/leaves a state, where
206 * affected devices might stop 206 * affected devices might stop
207 */ 207 */
208static void tick_do_broadcast_on_off(void *why) 208static void tick_do_broadcast_on_off(unsigned long *reason)
209{ 209{
210 struct clock_event_device *bc, *dev; 210 struct clock_event_device *bc, *dev;
211 struct tick_device *td; 211 struct tick_device *td;
212 unsigned long flags, *reason = why; 212 unsigned long flags;
213 int cpu, bc_stopped; 213 int cpu, bc_stopped;
214 214
215 spin_lock_irqsave(&tick_broadcast_lock, flags); 215 spin_lock_irqsave(&tick_broadcast_lock, flags);
@@ -276,8 +276,7 @@ void tick_broadcast_on_off(unsigned long reason, int *oncpu)
276 printk(KERN_ERR "tick-broadcast: ignoring broadcast for " 276 printk(KERN_ERR "tick-broadcast: ignoring broadcast for "
277 "offline CPU #%d\n", *oncpu); 277 "offline CPU #%d\n", *oncpu);
278 else 278 else
279 smp_call_function_single(*oncpu, tick_do_broadcast_on_off, 279 tick_do_broadcast_on_off(&reason);
280 &reason, 1);
281} 280}
282 281
283/* 282/*
diff --git a/kernel/time/timer_list.c b/kernel/time/timer_list.c
index a999b92a1277..fddd69d16e03 100644
--- a/kernel/time/timer_list.c
+++ b/kernel/time/timer_list.c
@@ -286,7 +286,7 @@ static int __init init_timer_list_procfs(void)
286{ 286{
287 struct proc_dir_entry *pe; 287 struct proc_dir_entry *pe;
288 288
289 pe = proc_create("timer_list", 0644, NULL, &timer_list_fops); 289 pe = proc_create("timer_list", 0444, NULL, &timer_list_fops);
290 if (!pe) 290 if (!pe)
291 return -ENOMEM; 291 return -ENOMEM;
292 return 0; 292 return 0;
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index 163fbfc2f39f..1ea0d1234f4a 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -62,6 +62,12 @@ config EVENT_TRACING
62config CONTEXT_SWITCH_TRACER 62config CONTEXT_SWITCH_TRACER
63 bool 63 bool
64 64
65config RING_BUFFER_ALLOW_SWAP
66 bool
67 help
68 Allow the use of ring_buffer_swap_cpu.
69 Adds a very slight overhead to tracing when enabled.
70
65# All tracer options should select GENERIC_TRACER. For those options that are 71# All tracer options should select GENERIC_TRACER. For those options that are
66# enabled by all tracers (context switch and event tracer) they select TRACING. 72# enabled by all tracers (context switch and event tracer) they select TRACING.
67# This allows those options to appear when no other tracer is selected. But the 73# This allows those options to appear when no other tracer is selected. But the
@@ -146,6 +152,7 @@ config IRQSOFF_TRACER
146 select TRACE_IRQFLAGS 152 select TRACE_IRQFLAGS
147 select GENERIC_TRACER 153 select GENERIC_TRACER
148 select TRACER_MAX_TRACE 154 select TRACER_MAX_TRACE
155 select RING_BUFFER_ALLOW_SWAP
149 help 156 help
150 This option measures the time spent in irqs-off critical 157 This option measures the time spent in irqs-off critical
151 sections, with microsecond accuracy. 158 sections, with microsecond accuracy.
@@ -167,6 +174,7 @@ config PREEMPT_TRACER
167 depends on PREEMPT 174 depends on PREEMPT
168 select GENERIC_TRACER 175 select GENERIC_TRACER
169 select TRACER_MAX_TRACE 176 select TRACER_MAX_TRACE
177 select RING_BUFFER_ALLOW_SWAP
170 help 178 help
171 This option measures the time spent in preemption off critical 179 This option measures the time spent in preemption off critical
172 sections, with microsecond accuracy. 180 sections, with microsecond accuracy.
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index 1090b0aed9ba..3eb159c277c8 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -65,13 +65,15 @@ static void trace_note(struct blk_trace *bt, pid_t pid, int action,
65{ 65{
66 struct blk_io_trace *t; 66 struct blk_io_trace *t;
67 struct ring_buffer_event *event = NULL; 67 struct ring_buffer_event *event = NULL;
68 struct ring_buffer *buffer = NULL;
68 int pc = 0; 69 int pc = 0;
69 int cpu = smp_processor_id(); 70 int cpu = smp_processor_id();
70 bool blk_tracer = blk_tracer_enabled; 71 bool blk_tracer = blk_tracer_enabled;
71 72
72 if (blk_tracer) { 73 if (blk_tracer) {
74 buffer = blk_tr->buffer;
73 pc = preempt_count(); 75 pc = preempt_count();
74 event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK, 76 event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
75 sizeof(*t) + len, 77 sizeof(*t) + len,
76 0, pc); 78 0, pc);
77 if (!event) 79 if (!event)
@@ -96,7 +98,7 @@ record_it:
96 memcpy((void *) t + sizeof(*t), data, len); 98 memcpy((void *) t + sizeof(*t), data, len);
97 99
98 if (blk_tracer) 100 if (blk_tracer)
99 trace_buffer_unlock_commit(blk_tr, event, 0, pc); 101 trace_buffer_unlock_commit(buffer, event, 0, pc);
100 } 102 }
101} 103}
102 104
@@ -179,6 +181,7 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
179{ 181{
180 struct task_struct *tsk = current; 182 struct task_struct *tsk = current;
181 struct ring_buffer_event *event = NULL; 183 struct ring_buffer_event *event = NULL;
184 struct ring_buffer *buffer = NULL;
182 struct blk_io_trace *t; 185 struct blk_io_trace *t;
183 unsigned long flags = 0; 186 unsigned long flags = 0;
184 unsigned long *sequence; 187 unsigned long *sequence;
@@ -204,8 +207,9 @@ static void __blk_add_trace(struct blk_trace *bt, sector_t sector, int bytes,
204 if (blk_tracer) { 207 if (blk_tracer) {
205 tracing_record_cmdline(current); 208 tracing_record_cmdline(current);
206 209
210 buffer = blk_tr->buffer;
207 pc = preempt_count(); 211 pc = preempt_count();
208 event = trace_buffer_lock_reserve(blk_tr, TRACE_BLK, 212 event = trace_buffer_lock_reserve(buffer, TRACE_BLK,
209 sizeof(*t) + pdu_len, 213 sizeof(*t) + pdu_len,
210 0, pc); 214 0, pc);
211 if (!event) 215 if (!event)
@@ -252,7 +256,7 @@ record_it:
252 memcpy((void *) t + sizeof(*t), pdu_data, pdu_len); 256 memcpy((void *) t + sizeof(*t), pdu_data, pdu_len);
253 257
254 if (blk_tracer) { 258 if (blk_tracer) {
255 trace_buffer_unlock_commit(blk_tr, event, 0, pc); 259 trace_buffer_unlock_commit(buffer, event, 0, pc);
256 return; 260 return;
257 } 261 }
258 } 262 }
@@ -267,8 +271,8 @@ static void blk_trace_free(struct blk_trace *bt)
267{ 271{
268 debugfs_remove(bt->msg_file); 272 debugfs_remove(bt->msg_file);
269 debugfs_remove(bt->dropped_file); 273 debugfs_remove(bt->dropped_file);
270 debugfs_remove(bt->dir);
271 relay_close(bt->rchan); 274 relay_close(bt->rchan);
275 debugfs_remove(bt->dir);
272 free_percpu(bt->sequence); 276 free_percpu(bt->sequence);
273 free_percpu(bt->msg_data); 277 free_percpu(bt->msg_data);
274 kfree(bt); 278 kfree(bt);
@@ -378,18 +382,8 @@ static int blk_subbuf_start_callback(struct rchan_buf *buf, void *subbuf,
378 382
379static int blk_remove_buf_file_callback(struct dentry *dentry) 383static int blk_remove_buf_file_callback(struct dentry *dentry)
380{ 384{
381 struct dentry *parent = dentry->d_parent;
382 debugfs_remove(dentry); 385 debugfs_remove(dentry);
383 386
384 /*
385 * this will fail for all but the last file, but that is ok. what we
386 * care about is the top level buts->name directory going away, when
387 * the last trace file is gone. Then we don't have to rmdir() that
388 * manually on trace stop, so it nicely solves the issue with
389 * force killing of running traces.
390 */
391
392 debugfs_remove(parent);
393 return 0; 387 return 0;
394} 388}
395 389
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 1993b7186cdb..8c804e24f96f 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2222,7 +2222,11 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2222 read++; 2222 read++;
2223 cnt--; 2223 cnt--;
2224 2224
2225 if (!(iter->flags & ~FTRACE_ITER_CONT)) { 2225 /*
2226 * If the parser haven't finished with the last write,
2227 * continue reading the user input without skipping spaces.
2228 */
2229 if (!(iter->flags & FTRACE_ITER_CONT)) {
2226 /* skip white space */ 2230 /* skip white space */
2227 while (cnt && isspace(ch)) { 2231 while (cnt && isspace(ch)) {
2228 ret = get_user(ch, ubuf++); 2232 ret = get_user(ch, ubuf++);
@@ -2232,8 +2236,9 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2232 cnt--; 2236 cnt--;
2233 } 2237 }
2234 2238
2239 /* only spaces were written */
2235 if (isspace(ch)) { 2240 if (isspace(ch)) {
2236 file->f_pos += read; 2241 *ppos += read;
2237 ret = read; 2242 ret = read;
2238 goto out; 2243 goto out;
2239 } 2244 }
@@ -2262,12 +2267,12 @@ ftrace_regex_write(struct file *file, const char __user *ubuf,
2262 if (ret) 2267 if (ret)
2263 goto out; 2268 goto out;
2264 iter->buffer_idx = 0; 2269 iter->buffer_idx = 0;
2265 } else 2270 } else {
2266 iter->flags |= FTRACE_ITER_CONT; 2271 iter->flags |= FTRACE_ITER_CONT;
2272 iter->buffer[iter->buffer_idx++] = ch;
2273 }
2267 2274
2268 2275 *ppos += read;
2269 file->f_pos += read;
2270
2271 ret = read; 2276 ret = read;
2272 out: 2277 out:
2273 mutex_unlock(&ftrace_regex_lock); 2278 mutex_unlock(&ftrace_regex_lock);
diff --git a/kernel/trace/kmemtrace.c b/kernel/trace/kmemtrace.c
index dda53ccf749b..81b1645c8549 100644
--- a/kernel/trace/kmemtrace.c
+++ b/kernel/trace/kmemtrace.c
@@ -183,11 +183,9 @@ static void kmemtrace_stop_probes(void)
183 183
184static int kmem_trace_init(struct trace_array *tr) 184static int kmem_trace_init(struct trace_array *tr)
185{ 185{
186 int cpu;
187 kmemtrace_array = tr; 186 kmemtrace_array = tr;
188 187
189 for_each_cpu(cpu, cpu_possible_mask) 188 tracing_reset_online_cpus(tr);
190 tracing_reset(tr, cpu);
191 189
192 kmemtrace_start_probes(); 190 kmemtrace_start_probes();
193 191
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index da2c59d8f486..454e74e718cf 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -218,17 +218,12 @@ enum {
218 218
219static inline int rb_null_event(struct ring_buffer_event *event) 219static inline int rb_null_event(struct ring_buffer_event *event)
220{ 220{
221 return event->type_len == RINGBUF_TYPE_PADDING 221 return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;
222 && event->time_delta == 0;
223}
224
225static inline int rb_discarded_event(struct ring_buffer_event *event)
226{
227 return event->type_len == RINGBUF_TYPE_PADDING && event->time_delta;
228} 222}
229 223
230static void rb_event_set_padding(struct ring_buffer_event *event) 224static void rb_event_set_padding(struct ring_buffer_event *event)
231{ 225{
226 /* padding has a NULL time_delta */
232 event->type_len = RINGBUF_TYPE_PADDING; 227 event->type_len = RINGBUF_TYPE_PADDING;
233 event->time_delta = 0; 228 event->time_delta = 0;
234} 229}
@@ -472,14 +467,19 @@ struct ring_buffer_iter {
472}; 467};
473 468
474/* buffer may be either ring_buffer or ring_buffer_per_cpu */ 469/* buffer may be either ring_buffer or ring_buffer_per_cpu */
475#define RB_WARN_ON(buffer, cond) \ 470#define RB_WARN_ON(b, cond) \
476 ({ \ 471 ({ \
477 int _____ret = unlikely(cond); \ 472 int _____ret = unlikely(cond); \
478 if (_____ret) { \ 473 if (_____ret) { \
479 atomic_inc(&buffer->record_disabled); \ 474 if (__same_type(*(b), struct ring_buffer_per_cpu)) { \
480 WARN_ON(1); \ 475 struct ring_buffer_per_cpu *__b = \
481 } \ 476 (void *)b; \
482 _____ret; \ 477 atomic_inc(&__b->buffer->record_disabled); \
478 } else \
479 atomic_inc(&b->record_disabled); \
480 WARN_ON(1); \
481 } \
482 _____ret; \
483 }) 483 })
484 484
485/* Up this if you want to test the TIME_EXTENTS and normalization */ 485/* Up this if you want to test the TIME_EXTENTS and normalization */
@@ -1778,9 +1778,6 @@ rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
1778 event->type_len = RINGBUF_TYPE_PADDING; 1778 event->type_len = RINGBUF_TYPE_PADDING;
1779 /* time delta must be non zero */ 1779 /* time delta must be non zero */
1780 event->time_delta = 1; 1780 event->time_delta = 1;
1781 /* Account for this as an entry */
1782 local_inc(&tail_page->entries);
1783 local_inc(&cpu_buffer->entries);
1784 1781
1785 /* Set write to end of buffer */ 1782 /* Set write to end of buffer */
1786 length = (tail + length) - BUF_PAGE_SIZE; 1783 length = (tail + length) - BUF_PAGE_SIZE;
@@ -2076,7 +2073,8 @@ static void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
2076} 2073}
2077 2074
2078static struct ring_buffer_event * 2075static struct ring_buffer_event *
2079rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer, 2076rb_reserve_next_event(struct ring_buffer *buffer,
2077 struct ring_buffer_per_cpu *cpu_buffer,
2080 unsigned long length) 2078 unsigned long length)
2081{ 2079{
2082 struct ring_buffer_event *event; 2080 struct ring_buffer_event *event;
@@ -2086,6 +2084,21 @@ rb_reserve_next_event(struct ring_buffer_per_cpu *cpu_buffer,
2086 2084
2087 rb_start_commit(cpu_buffer); 2085 rb_start_commit(cpu_buffer);
2088 2086
2087#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2088 /*
2089 * Due to the ability to swap a cpu buffer from a buffer
2090 * it is possible it was swapped before we committed.
2091 * (committing stops a swap). We check for it here and
2092 * if it happened, we have to fail the write.
2093 */
2094 barrier();
2095 if (unlikely(ACCESS_ONCE(cpu_buffer->buffer) != buffer)) {
2096 local_dec(&cpu_buffer->committing);
2097 local_dec(&cpu_buffer->commits);
2098 return NULL;
2099 }
2100#endif
2101
2089 length = rb_calculate_event_length(length); 2102 length = rb_calculate_event_length(length);
2090 again: 2103 again:
2091 /* 2104 /*
@@ -2246,7 +2259,7 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
2246 if (length > BUF_MAX_DATA_SIZE) 2259 if (length > BUF_MAX_DATA_SIZE)
2247 goto out; 2260 goto out;
2248 2261
2249 event = rb_reserve_next_event(cpu_buffer, length); 2262 event = rb_reserve_next_event(buffer, cpu_buffer, length);
2250 if (!event) 2263 if (!event)
2251 goto out; 2264 goto out;
2252 2265
@@ -2269,18 +2282,23 @@ ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
2269} 2282}
2270EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve); 2283EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
2271 2284
2272static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer, 2285static void
2286rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2273 struct ring_buffer_event *event) 2287 struct ring_buffer_event *event)
2274{ 2288{
2275 local_inc(&cpu_buffer->entries);
2276
2277 /* 2289 /*
2278 * The event first in the commit queue updates the 2290 * The event first in the commit queue updates the
2279 * time stamp. 2291 * time stamp.
2280 */ 2292 */
2281 if (rb_event_is_commit(cpu_buffer, event)) 2293 if (rb_event_is_commit(cpu_buffer, event))
2282 cpu_buffer->write_stamp += event->time_delta; 2294 cpu_buffer->write_stamp += event->time_delta;
2295}
2283 2296
2297static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
2298 struct ring_buffer_event *event)
2299{
2300 local_inc(&cpu_buffer->entries);
2301 rb_update_write_stamp(cpu_buffer, event);
2284 rb_end_commit(cpu_buffer); 2302 rb_end_commit(cpu_buffer);
2285} 2303}
2286 2304
@@ -2327,32 +2345,57 @@ static inline void rb_event_discard(struct ring_buffer_event *event)
2327 event->time_delta = 1; 2345 event->time_delta = 1;
2328} 2346}
2329 2347
2330/** 2348/*
2331 * ring_buffer_event_discard - discard any event in the ring buffer 2349 * Decrement the entries to the page that an event is on.
2332 * @event: the event to discard 2350 * The event does not even need to exist, only the pointer
2333 * 2351 * to the page it is on. This may only be called before the commit
2334 * Sometimes a event that is in the ring buffer needs to be ignored. 2352 * takes place.
2335 * This function lets the user discard an event in the ring buffer
2336 * and then that event will not be read later.
2337 *
2338 * Note, it is up to the user to be careful with this, and protect
2339 * against races. If the user discards an event that has been consumed
2340 * it is possible that it could corrupt the ring buffer.
2341 */ 2353 */
2342void ring_buffer_event_discard(struct ring_buffer_event *event) 2354static inline void
2355rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
2356 struct ring_buffer_event *event)
2343{ 2357{
2344 rb_event_discard(event); 2358 unsigned long addr = (unsigned long)event;
2359 struct buffer_page *bpage = cpu_buffer->commit_page;
2360 struct buffer_page *start;
2361
2362 addr &= PAGE_MASK;
2363
2364 /* Do the likely case first */
2365 if (likely(bpage->page == (void *)addr)) {
2366 local_dec(&bpage->entries);
2367 return;
2368 }
2369
2370 /*
2371 * Because the commit page may be on the reader page we
2372 * start with the next page and check the end loop there.
2373 */
2374 rb_inc_page(cpu_buffer, &bpage);
2375 start = bpage;
2376 do {
2377 if (bpage->page == (void *)addr) {
2378 local_dec(&bpage->entries);
2379 return;
2380 }
2381 rb_inc_page(cpu_buffer, &bpage);
2382 } while (bpage != start);
2383
2384 /* commit not part of this buffer?? */
2385 RB_WARN_ON(cpu_buffer, 1);
2345} 2386}
2346EXPORT_SYMBOL_GPL(ring_buffer_event_discard);
2347 2387
2348/** 2388/**
2349 * ring_buffer_commit_discard - discard an event that has not been committed 2389 * ring_buffer_commit_discard - discard an event that has not been committed
2350 * @buffer: the ring buffer 2390 * @buffer: the ring buffer
2351 * @event: non committed event to discard 2391 * @event: non committed event to discard
2352 * 2392 *
2353 * This is similar to ring_buffer_event_discard but must only be 2393 * Sometimes an event that is in the ring buffer needs to be ignored.
2354 * performed on an event that has not been committed yet. The difference 2394 * This function lets the user discard an event in the ring buffer
2355 * is that this will also try to free the event from the ring buffer 2395 * and then that event will not be read later.
2396 *
2397 * This function only works if it is called before the the item has been
2398 * committed. It will try to free the event from the ring buffer
2356 * if another event has not been added behind it. 2399 * if another event has not been added behind it.
2357 * 2400 *
2358 * If another event has been added behind it, it will set the event 2401 * If another event has been added behind it, it will set the event
@@ -2380,14 +2423,15 @@ void ring_buffer_discard_commit(struct ring_buffer *buffer,
2380 */ 2423 */
2381 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); 2424 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
2382 2425
2426 rb_decrement_entry(cpu_buffer, event);
2383 if (rb_try_to_discard(cpu_buffer, event)) 2427 if (rb_try_to_discard(cpu_buffer, event))
2384 goto out; 2428 goto out;
2385 2429
2386 /* 2430 /*
2387 * The commit is still visible by the reader, so we 2431 * The commit is still visible by the reader, so we
2388 * must increment entries. 2432 * must still update the timestamp.
2389 */ 2433 */
2390 local_inc(&cpu_buffer->entries); 2434 rb_update_write_stamp(cpu_buffer, event);
2391 out: 2435 out:
2392 rb_end_commit(cpu_buffer); 2436 rb_end_commit(cpu_buffer);
2393 2437
@@ -2448,7 +2492,7 @@ int ring_buffer_write(struct ring_buffer *buffer,
2448 if (length > BUF_MAX_DATA_SIZE) 2492 if (length > BUF_MAX_DATA_SIZE)
2449 goto out; 2493 goto out;
2450 2494
2451 event = rb_reserve_next_event(cpu_buffer, length); 2495 event = rb_reserve_next_event(buffer, cpu_buffer, length);
2452 if (!event) 2496 if (!event)
2453 goto out; 2497 goto out;
2454 2498
@@ -2899,8 +2943,7 @@ static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
2899 2943
2900 event = rb_reader_event(cpu_buffer); 2944 event = rb_reader_event(cpu_buffer);
2901 2945
2902 if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX 2946 if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
2903 || rb_discarded_event(event))
2904 cpu_buffer->read++; 2947 cpu_buffer->read++;
2905 2948
2906 rb_update_read_stamp(cpu_buffer, event); 2949 rb_update_read_stamp(cpu_buffer, event);
@@ -3132,10 +3175,8 @@ ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts)
3132 spin_unlock(&cpu_buffer->reader_lock); 3175 spin_unlock(&cpu_buffer->reader_lock);
3133 local_irq_restore(flags); 3176 local_irq_restore(flags);
3134 3177
3135 if (event && event->type_len == RINGBUF_TYPE_PADDING) { 3178 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3136 cpu_relax();
3137 goto again; 3179 goto again;
3138 }
3139 3180
3140 return event; 3181 return event;
3141} 3182}
@@ -3160,10 +3201,8 @@ ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
3160 event = rb_iter_peek(iter, ts); 3201 event = rb_iter_peek(iter, ts);
3161 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3202 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3162 3203
3163 if (event && event->type_len == RINGBUF_TYPE_PADDING) { 3204 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3164 cpu_relax();
3165 goto again; 3205 goto again;
3166 }
3167 3206
3168 return event; 3207 return event;
3169} 3208}
@@ -3209,10 +3248,8 @@ ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts)
3209 out: 3248 out:
3210 preempt_enable(); 3249 preempt_enable();
3211 3250
3212 if (event && event->type_len == RINGBUF_TYPE_PADDING) { 3251 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3213 cpu_relax();
3214 goto again; 3252 goto again;
3215 }
3216 3253
3217 return event; 3254 return event;
3218} 3255}
@@ -3292,21 +3329,19 @@ ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
3292 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 3329 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
3293 unsigned long flags; 3330 unsigned long flags;
3294 3331
3295 again:
3296 spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3332 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3333 again:
3297 event = rb_iter_peek(iter, ts); 3334 event = rb_iter_peek(iter, ts);
3298 if (!event) 3335 if (!event)
3299 goto out; 3336 goto out;
3300 3337
3338 if (event->type_len == RINGBUF_TYPE_PADDING)
3339 goto again;
3340
3301 rb_advance_iter(iter); 3341 rb_advance_iter(iter);
3302 out: 3342 out:
3303 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3343 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3304 3344
3305 if (event && event->type_len == RINGBUF_TYPE_PADDING) {
3306 cpu_relax();
3307 goto again;
3308 }
3309
3310 return event; 3345 return event;
3311} 3346}
3312EXPORT_SYMBOL_GPL(ring_buffer_read); 3347EXPORT_SYMBOL_GPL(ring_buffer_read);
@@ -3373,12 +3408,16 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
3373 3408
3374 spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3409 spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3375 3410
3411 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
3412 goto out;
3413
3376 __raw_spin_lock(&cpu_buffer->lock); 3414 __raw_spin_lock(&cpu_buffer->lock);
3377 3415
3378 rb_reset_cpu(cpu_buffer); 3416 rb_reset_cpu(cpu_buffer);
3379 3417
3380 __raw_spin_unlock(&cpu_buffer->lock); 3418 __raw_spin_unlock(&cpu_buffer->lock);
3381 3419
3420 out:
3382 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3421 spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3383 3422
3384 atomic_dec(&cpu_buffer->record_disabled); 3423 atomic_dec(&cpu_buffer->record_disabled);
@@ -3461,6 +3500,7 @@ int ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
3461} 3500}
3462EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu); 3501EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
3463 3502
3503#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3464/** 3504/**
3465 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers 3505 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
3466 * @buffer_a: One buffer to swap with 3506 * @buffer_a: One buffer to swap with
@@ -3515,20 +3555,28 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
3515 atomic_inc(&cpu_buffer_a->record_disabled); 3555 atomic_inc(&cpu_buffer_a->record_disabled);
3516 atomic_inc(&cpu_buffer_b->record_disabled); 3556 atomic_inc(&cpu_buffer_b->record_disabled);
3517 3557
3558 ret = -EBUSY;
3559 if (local_read(&cpu_buffer_a->committing))
3560 goto out_dec;
3561 if (local_read(&cpu_buffer_b->committing))
3562 goto out_dec;
3563
3518 buffer_a->buffers[cpu] = cpu_buffer_b; 3564 buffer_a->buffers[cpu] = cpu_buffer_b;
3519 buffer_b->buffers[cpu] = cpu_buffer_a; 3565 buffer_b->buffers[cpu] = cpu_buffer_a;
3520 3566
3521 cpu_buffer_b->buffer = buffer_a; 3567 cpu_buffer_b->buffer = buffer_a;
3522 cpu_buffer_a->buffer = buffer_b; 3568 cpu_buffer_a->buffer = buffer_b;
3523 3569
3570 ret = 0;
3571
3572out_dec:
3524 atomic_dec(&cpu_buffer_a->record_disabled); 3573 atomic_dec(&cpu_buffer_a->record_disabled);
3525 atomic_dec(&cpu_buffer_b->record_disabled); 3574 atomic_dec(&cpu_buffer_b->record_disabled);
3526
3527 ret = 0;
3528out: 3575out:
3529 return ret; 3576 return ret;
3530} 3577}
3531EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu); 3578EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
3579#endif /* CONFIG_RING_BUFFER_ALLOW_SWAP */
3532 3580
3533/** 3581/**
3534 * ring_buffer_alloc_read_page - allocate a page to read from buffer 3582 * ring_buffer_alloc_read_page - allocate a page to read from buffer
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 0f0881676dc9..5c75deeefe30 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -169,10 +169,11 @@ static struct trace_array global_trace;
169 169
170static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu); 170static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
171 171
172int filter_current_check_discard(struct ftrace_event_call *call, void *rec, 172int filter_current_check_discard(struct ring_buffer *buffer,
173 struct ftrace_event_call *call, void *rec,
173 struct ring_buffer_event *event) 174 struct ring_buffer_event *event)
174{ 175{
175 return filter_check_discard(call, rec, global_trace.buffer, event); 176 return filter_check_discard(call, rec, buffer, event);
176} 177}
177EXPORT_SYMBOL_GPL(filter_current_check_discard); 178EXPORT_SYMBOL_GPL(filter_current_check_discard);
178 179
@@ -263,6 +264,9 @@ unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
263 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME | 264 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
264 TRACE_ITER_GRAPH_TIME; 265 TRACE_ITER_GRAPH_TIME;
265 266
267static int trace_stop_count;
268static DEFINE_SPINLOCK(tracing_start_lock);
269
266/** 270/**
267 * trace_wake_up - wake up tasks waiting for trace input 271 * trace_wake_up - wake up tasks waiting for trace input
268 * 272 *
@@ -407,19 +411,22 @@ static void
407__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu) 411__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
408{ 412{
409 struct trace_array_cpu *data = tr->data[cpu]; 413 struct trace_array_cpu *data = tr->data[cpu];
414 struct trace_array_cpu *max_data = tr->data[cpu];
410 415
411 max_tr.cpu = cpu; 416 max_tr.cpu = cpu;
412 max_tr.time_start = data->preempt_timestamp; 417 max_tr.time_start = data->preempt_timestamp;
413 418
414 data = max_tr.data[cpu]; 419 max_data = max_tr.data[cpu];
415 data->saved_latency = tracing_max_latency; 420 max_data->saved_latency = tracing_max_latency;
421 max_data->critical_start = data->critical_start;
422 max_data->critical_end = data->critical_end;
416 423
417 memcpy(data->comm, tsk->comm, TASK_COMM_LEN); 424 memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
418 data->pid = tsk->pid; 425 max_data->pid = tsk->pid;
419 data->uid = task_uid(tsk); 426 max_data->uid = task_uid(tsk);
420 data->nice = tsk->static_prio - 20 - MAX_RT_PRIO; 427 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
421 data->policy = tsk->policy; 428 max_data->policy = tsk->policy;
422 data->rt_priority = tsk->rt_priority; 429 max_data->rt_priority = tsk->rt_priority;
423 430
424 /* record this tasks comm */ 431 /* record this tasks comm */
425 tracing_record_cmdline(tsk); 432 tracing_record_cmdline(tsk);
@@ -439,16 +446,15 @@ update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
439{ 446{
440 struct ring_buffer *buf = tr->buffer; 447 struct ring_buffer *buf = tr->buffer;
441 448
449 if (trace_stop_count)
450 return;
451
442 WARN_ON_ONCE(!irqs_disabled()); 452 WARN_ON_ONCE(!irqs_disabled());
443 __raw_spin_lock(&ftrace_max_lock); 453 __raw_spin_lock(&ftrace_max_lock);
444 454
445 tr->buffer = max_tr.buffer; 455 tr->buffer = max_tr.buffer;
446 max_tr.buffer = buf; 456 max_tr.buffer = buf;
447 457
448 ftrace_disable_cpu();
449 ring_buffer_reset(tr->buffer);
450 ftrace_enable_cpu();
451
452 __update_max_tr(tr, tsk, cpu); 458 __update_max_tr(tr, tsk, cpu);
453 __raw_spin_unlock(&ftrace_max_lock); 459 __raw_spin_unlock(&ftrace_max_lock);
454} 460}
@@ -466,17 +472,30 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
466{ 472{
467 int ret; 473 int ret;
468 474
475 if (trace_stop_count)
476 return;
477
469 WARN_ON_ONCE(!irqs_disabled()); 478 WARN_ON_ONCE(!irqs_disabled());
470 __raw_spin_lock(&ftrace_max_lock); 479 __raw_spin_lock(&ftrace_max_lock);
471 480
472 ftrace_disable_cpu(); 481 ftrace_disable_cpu();
473 482
474 ring_buffer_reset(max_tr.buffer);
475 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu); 483 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
476 484
485 if (ret == -EBUSY) {
486 /*
487 * We failed to swap the buffer due to a commit taking
488 * place on this CPU. We fail to record, but we reset
489 * the max trace buffer (no one writes directly to it)
490 * and flag that it failed.
491 */
492 trace_array_printk(&max_tr, _THIS_IP_,
493 "Failed to swap buffers due to commit in progress\n");
494 }
495
477 ftrace_enable_cpu(); 496 ftrace_enable_cpu();
478 497
479 WARN_ON_ONCE(ret && ret != -EAGAIN); 498 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
480 499
481 __update_max_tr(tr, tsk, cpu); 500 __update_max_tr(tr, tsk, cpu);
482 __raw_spin_unlock(&ftrace_max_lock); 501 __raw_spin_unlock(&ftrace_max_lock);
@@ -538,7 +557,6 @@ __acquires(kernel_lock)
538 if (type->selftest && !tracing_selftest_disabled) { 557 if (type->selftest && !tracing_selftest_disabled) {
539 struct tracer *saved_tracer = current_trace; 558 struct tracer *saved_tracer = current_trace;
540 struct trace_array *tr = &global_trace; 559 struct trace_array *tr = &global_trace;
541 int i;
542 560
543 /* 561 /*
544 * Run a selftest on this tracer. 562 * Run a selftest on this tracer.
@@ -547,8 +565,7 @@ __acquires(kernel_lock)
547 * internal tracing to verify that everything is in order. 565 * internal tracing to verify that everything is in order.
548 * If we fail, we do not register this tracer. 566 * If we fail, we do not register this tracer.
549 */ 567 */
550 for_each_tracing_cpu(i) 568 tracing_reset_online_cpus(tr);
551 tracing_reset(tr, i);
552 569
553 current_trace = type; 570 current_trace = type;
554 /* the test is responsible for initializing and enabling */ 571 /* the test is responsible for initializing and enabling */
@@ -561,8 +578,7 @@ __acquires(kernel_lock)
561 goto out; 578 goto out;
562 } 579 }
563 /* Only reset on passing, to avoid touching corrupted buffers */ 580 /* Only reset on passing, to avoid touching corrupted buffers */
564 for_each_tracing_cpu(i) 581 tracing_reset_online_cpus(tr);
565 tracing_reset(tr, i);
566 582
567 printk(KERN_CONT "PASSED\n"); 583 printk(KERN_CONT "PASSED\n");
568 } 584 }
@@ -637,21 +653,42 @@ void unregister_tracer(struct tracer *type)
637 mutex_unlock(&trace_types_lock); 653 mutex_unlock(&trace_types_lock);
638} 654}
639 655
640void tracing_reset(struct trace_array *tr, int cpu) 656static void __tracing_reset(struct trace_array *tr, int cpu)
641{ 657{
642 ftrace_disable_cpu(); 658 ftrace_disable_cpu();
643 ring_buffer_reset_cpu(tr->buffer, cpu); 659 ring_buffer_reset_cpu(tr->buffer, cpu);
644 ftrace_enable_cpu(); 660 ftrace_enable_cpu();
645} 661}
646 662
663void tracing_reset(struct trace_array *tr, int cpu)
664{
665 struct ring_buffer *buffer = tr->buffer;
666
667 ring_buffer_record_disable(buffer);
668
669 /* Make sure all commits have finished */
670 synchronize_sched();
671 __tracing_reset(tr, cpu);
672
673 ring_buffer_record_enable(buffer);
674}
675
647void tracing_reset_online_cpus(struct trace_array *tr) 676void tracing_reset_online_cpus(struct trace_array *tr)
648{ 677{
678 struct ring_buffer *buffer = tr->buffer;
649 int cpu; 679 int cpu;
650 680
681 ring_buffer_record_disable(buffer);
682
683 /* Make sure all commits have finished */
684 synchronize_sched();
685
651 tr->time_start = ftrace_now(tr->cpu); 686 tr->time_start = ftrace_now(tr->cpu);
652 687
653 for_each_online_cpu(cpu) 688 for_each_online_cpu(cpu)
654 tracing_reset(tr, cpu); 689 __tracing_reset(tr, cpu);
690
691 ring_buffer_record_enable(buffer);
655} 692}
656 693
657void tracing_reset_current(int cpu) 694void tracing_reset_current(int cpu)
@@ -682,9 +719,6 @@ static void trace_init_cmdlines(void)
682 cmdline_idx = 0; 719 cmdline_idx = 0;
683} 720}
684 721
685static int trace_stop_count;
686static DEFINE_SPINLOCK(tracing_start_lock);
687
688/** 722/**
689 * ftrace_off_permanent - disable all ftrace code permanently 723 * ftrace_off_permanent - disable all ftrace code permanently
690 * 724 *
@@ -865,14 +899,15 @@ tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
865} 899}
866EXPORT_SYMBOL_GPL(tracing_generic_entry_update); 900EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
867 901
868struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr, 902struct ring_buffer_event *
869 int type, 903trace_buffer_lock_reserve(struct ring_buffer *buffer,
870 unsigned long len, 904 int type,
871 unsigned long flags, int pc) 905 unsigned long len,
906 unsigned long flags, int pc)
872{ 907{
873 struct ring_buffer_event *event; 908 struct ring_buffer_event *event;
874 909
875 event = ring_buffer_lock_reserve(tr->buffer, len); 910 event = ring_buffer_lock_reserve(buffer, len);
876 if (event != NULL) { 911 if (event != NULL) {
877 struct trace_entry *ent = ring_buffer_event_data(event); 912 struct trace_entry *ent = ring_buffer_event_data(event);
878 913
@@ -883,53 +918,59 @@ struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
883 return event; 918 return event;
884} 919}
885 920
886static inline void __trace_buffer_unlock_commit(struct trace_array *tr, 921static inline void
887 struct ring_buffer_event *event, 922__trace_buffer_unlock_commit(struct ring_buffer *buffer,
888 unsigned long flags, int pc, 923 struct ring_buffer_event *event,
889 int wake) 924 unsigned long flags, int pc,
925 int wake)
890{ 926{
891 ring_buffer_unlock_commit(tr->buffer, event); 927 ring_buffer_unlock_commit(buffer, event);
892 928
893 ftrace_trace_stack(tr, flags, 6, pc); 929 ftrace_trace_stack(buffer, flags, 6, pc);
894 ftrace_trace_userstack(tr, flags, pc); 930 ftrace_trace_userstack(buffer, flags, pc);
895 931
896 if (wake) 932 if (wake)
897 trace_wake_up(); 933 trace_wake_up();
898} 934}
899 935
900void trace_buffer_unlock_commit(struct trace_array *tr, 936void trace_buffer_unlock_commit(struct ring_buffer *buffer,
901 struct ring_buffer_event *event, 937 struct ring_buffer_event *event,
902 unsigned long flags, int pc) 938 unsigned long flags, int pc)
903{ 939{
904 __trace_buffer_unlock_commit(tr, event, flags, pc, 1); 940 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
905} 941}
906 942
907struct ring_buffer_event * 943struct ring_buffer_event *
908trace_current_buffer_lock_reserve(int type, unsigned long len, 944trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
945 int type, unsigned long len,
909 unsigned long flags, int pc) 946 unsigned long flags, int pc)
910{ 947{
911 return trace_buffer_lock_reserve(&global_trace, 948 *current_rb = global_trace.buffer;
949 return trace_buffer_lock_reserve(*current_rb,
912 type, len, flags, pc); 950 type, len, flags, pc);
913} 951}
914EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve); 952EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
915 953
916void trace_current_buffer_unlock_commit(struct ring_buffer_event *event, 954void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
955 struct ring_buffer_event *event,
917 unsigned long flags, int pc) 956 unsigned long flags, int pc)
918{ 957{
919 __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 1); 958 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
920} 959}
921EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit); 960EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
922 961
923void trace_nowake_buffer_unlock_commit(struct ring_buffer_event *event, 962void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
924 unsigned long flags, int pc) 963 struct ring_buffer_event *event,
964 unsigned long flags, int pc)
925{ 965{
926 __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 0); 966 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
927} 967}
928EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit); 968EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
929 969
930void trace_current_buffer_discard_commit(struct ring_buffer_event *event) 970void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
971 struct ring_buffer_event *event)
931{ 972{
932 ring_buffer_discard_commit(global_trace.buffer, event); 973 ring_buffer_discard_commit(buffer, event);
933} 974}
934EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit); 975EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
935 976
@@ -939,6 +980,7 @@ trace_function(struct trace_array *tr,
939 int pc) 980 int pc)
940{ 981{
941 struct ftrace_event_call *call = &event_function; 982 struct ftrace_event_call *call = &event_function;
983 struct ring_buffer *buffer = tr->buffer;
942 struct ring_buffer_event *event; 984 struct ring_buffer_event *event;
943 struct ftrace_entry *entry; 985 struct ftrace_entry *entry;
944 986
@@ -946,7 +988,7 @@ trace_function(struct trace_array *tr,
946 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 988 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
947 return; 989 return;
948 990
949 event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry), 991 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
950 flags, pc); 992 flags, pc);
951 if (!event) 993 if (!event)
952 return; 994 return;
@@ -954,8 +996,8 @@ trace_function(struct trace_array *tr,
954 entry->ip = ip; 996 entry->ip = ip;
955 entry->parent_ip = parent_ip; 997 entry->parent_ip = parent_ip;
956 998
957 if (!filter_check_discard(call, entry, tr->buffer, event)) 999 if (!filter_check_discard(call, entry, buffer, event))
958 ring_buffer_unlock_commit(tr->buffer, event); 1000 ring_buffer_unlock_commit(buffer, event);
959} 1001}
960 1002
961void 1003void
@@ -968,7 +1010,7 @@ ftrace(struct trace_array *tr, struct trace_array_cpu *data,
968} 1010}
969 1011
970#ifdef CONFIG_STACKTRACE 1012#ifdef CONFIG_STACKTRACE
971static void __ftrace_trace_stack(struct trace_array *tr, 1013static void __ftrace_trace_stack(struct ring_buffer *buffer,
972 unsigned long flags, 1014 unsigned long flags,
973 int skip, int pc) 1015 int skip, int pc)
974{ 1016{
@@ -977,7 +1019,7 @@ static void __ftrace_trace_stack(struct trace_array *tr,
977 struct stack_entry *entry; 1019 struct stack_entry *entry;
978 struct stack_trace trace; 1020 struct stack_trace trace;
979 1021
980 event = trace_buffer_lock_reserve(tr, TRACE_STACK, 1022 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
981 sizeof(*entry), flags, pc); 1023 sizeof(*entry), flags, pc);
982 if (!event) 1024 if (!event)
983 return; 1025 return;
@@ -990,26 +1032,27 @@ static void __ftrace_trace_stack(struct trace_array *tr,
990 trace.entries = entry->caller; 1032 trace.entries = entry->caller;
991 1033
992 save_stack_trace(&trace); 1034 save_stack_trace(&trace);
993 if (!filter_check_discard(call, entry, tr->buffer, event)) 1035 if (!filter_check_discard(call, entry, buffer, event))
994 ring_buffer_unlock_commit(tr->buffer, event); 1036 ring_buffer_unlock_commit(buffer, event);
995} 1037}
996 1038
997void ftrace_trace_stack(struct trace_array *tr, unsigned long flags, int skip, 1039void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
998 int pc) 1040 int skip, int pc)
999{ 1041{
1000 if (!(trace_flags & TRACE_ITER_STACKTRACE)) 1042 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1001 return; 1043 return;
1002 1044
1003 __ftrace_trace_stack(tr, flags, skip, pc); 1045 __ftrace_trace_stack(buffer, flags, skip, pc);
1004} 1046}
1005 1047
1006void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 1048void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1007 int pc) 1049 int pc)
1008{ 1050{
1009 __ftrace_trace_stack(tr, flags, skip, pc); 1051 __ftrace_trace_stack(tr->buffer, flags, skip, pc);
1010} 1052}
1011 1053
1012void ftrace_trace_userstack(struct trace_array *tr, unsigned long flags, int pc) 1054void
1055ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1013{ 1056{
1014 struct ftrace_event_call *call = &event_user_stack; 1057 struct ftrace_event_call *call = &event_user_stack;
1015 struct ring_buffer_event *event; 1058 struct ring_buffer_event *event;
@@ -1019,7 +1062,7 @@ void ftrace_trace_userstack(struct trace_array *tr, unsigned long flags, int pc)
1019 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE)) 1062 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1020 return; 1063 return;
1021 1064
1022 event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK, 1065 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1023 sizeof(*entry), flags, pc); 1066 sizeof(*entry), flags, pc);
1024 if (!event) 1067 if (!event)
1025 return; 1068 return;
@@ -1033,8 +1076,8 @@ void ftrace_trace_userstack(struct trace_array *tr, unsigned long flags, int pc)
1033 trace.entries = entry->caller; 1076 trace.entries = entry->caller;
1034 1077
1035 save_stack_trace_user(&trace); 1078 save_stack_trace_user(&trace);
1036 if (!filter_check_discard(call, entry, tr->buffer, event)) 1079 if (!filter_check_discard(call, entry, buffer, event))
1037 ring_buffer_unlock_commit(tr->buffer, event); 1080 ring_buffer_unlock_commit(buffer, event);
1038} 1081}
1039 1082
1040#ifdef UNUSED 1083#ifdef UNUSED
@@ -1053,9 +1096,10 @@ ftrace_trace_special(void *__tr,
1053{ 1096{
1054 struct ring_buffer_event *event; 1097 struct ring_buffer_event *event;
1055 struct trace_array *tr = __tr; 1098 struct trace_array *tr = __tr;
1099 struct ring_buffer *buffer = tr->buffer;
1056 struct special_entry *entry; 1100 struct special_entry *entry;
1057 1101
1058 event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL, 1102 event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
1059 sizeof(*entry), 0, pc); 1103 sizeof(*entry), 0, pc);
1060 if (!event) 1104 if (!event)
1061 return; 1105 return;
@@ -1063,7 +1107,7 @@ ftrace_trace_special(void *__tr,
1063 entry->arg1 = arg1; 1107 entry->arg1 = arg1;
1064 entry->arg2 = arg2; 1108 entry->arg2 = arg2;
1065 entry->arg3 = arg3; 1109 entry->arg3 = arg3;
1066 trace_buffer_unlock_commit(tr, event, 0, pc); 1110 trace_buffer_unlock_commit(buffer, event, 0, pc);
1067} 1111}
1068 1112
1069void 1113void
@@ -1109,6 +1153,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1109 1153
1110 struct ftrace_event_call *call = &event_bprint; 1154 struct ftrace_event_call *call = &event_bprint;
1111 struct ring_buffer_event *event; 1155 struct ring_buffer_event *event;
1156 struct ring_buffer *buffer;
1112 struct trace_array *tr = &global_trace; 1157 struct trace_array *tr = &global_trace;
1113 struct trace_array_cpu *data; 1158 struct trace_array_cpu *data;
1114 struct bprint_entry *entry; 1159 struct bprint_entry *entry;
@@ -1141,7 +1186,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1141 goto out_unlock; 1186 goto out_unlock;
1142 1187
1143 size = sizeof(*entry) + sizeof(u32) * len; 1188 size = sizeof(*entry) + sizeof(u32) * len;
1144 event = trace_buffer_lock_reserve(tr, TRACE_BPRINT, size, flags, pc); 1189 buffer = tr->buffer;
1190 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1191 flags, pc);
1145 if (!event) 1192 if (!event)
1146 goto out_unlock; 1193 goto out_unlock;
1147 entry = ring_buffer_event_data(event); 1194 entry = ring_buffer_event_data(event);
@@ -1149,8 +1196,8 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1149 entry->fmt = fmt; 1196 entry->fmt = fmt;
1150 1197
1151 memcpy(entry->buf, trace_buf, sizeof(u32) * len); 1198 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1152 if (!filter_check_discard(call, entry, tr->buffer, event)) 1199 if (!filter_check_discard(call, entry, buffer, event))
1153 ring_buffer_unlock_commit(tr->buffer, event); 1200 ring_buffer_unlock_commit(buffer, event);
1154 1201
1155out_unlock: 1202out_unlock:
1156 __raw_spin_unlock(&trace_buf_lock); 1203 __raw_spin_unlock(&trace_buf_lock);
@@ -1165,14 +1212,30 @@ out:
1165} 1212}
1166EXPORT_SYMBOL_GPL(trace_vbprintk); 1213EXPORT_SYMBOL_GPL(trace_vbprintk);
1167 1214
1168int trace_vprintk(unsigned long ip, const char *fmt, va_list args) 1215int trace_array_printk(struct trace_array *tr,
1216 unsigned long ip, const char *fmt, ...)
1217{
1218 int ret;
1219 va_list ap;
1220
1221 if (!(trace_flags & TRACE_ITER_PRINTK))
1222 return 0;
1223
1224 va_start(ap, fmt);
1225 ret = trace_array_vprintk(tr, ip, fmt, ap);
1226 va_end(ap);
1227 return ret;
1228}
1229
1230int trace_array_vprintk(struct trace_array *tr,
1231 unsigned long ip, const char *fmt, va_list args)
1169{ 1232{
1170 static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED; 1233 static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED;
1171 static char trace_buf[TRACE_BUF_SIZE]; 1234 static char trace_buf[TRACE_BUF_SIZE];
1172 1235
1173 struct ftrace_event_call *call = &event_print; 1236 struct ftrace_event_call *call = &event_print;
1174 struct ring_buffer_event *event; 1237 struct ring_buffer_event *event;
1175 struct trace_array *tr = &global_trace; 1238 struct ring_buffer *buffer;
1176 struct trace_array_cpu *data; 1239 struct trace_array_cpu *data;
1177 int cpu, len = 0, size, pc; 1240 int cpu, len = 0, size, pc;
1178 struct print_entry *entry; 1241 struct print_entry *entry;
@@ -1200,7 +1263,9 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1200 trace_buf[len] = 0; 1263 trace_buf[len] = 0;
1201 1264
1202 size = sizeof(*entry) + len + 1; 1265 size = sizeof(*entry) + len + 1;
1203 event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc); 1266 buffer = tr->buffer;
1267 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1268 irq_flags, pc);
1204 if (!event) 1269 if (!event)
1205 goto out_unlock; 1270 goto out_unlock;
1206 entry = ring_buffer_event_data(event); 1271 entry = ring_buffer_event_data(event);
@@ -1208,8 +1273,8 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1208 1273
1209 memcpy(&entry->buf, trace_buf, len); 1274 memcpy(&entry->buf, trace_buf, len);
1210 entry->buf[len] = 0; 1275 entry->buf[len] = 0;
1211 if (!filter_check_discard(call, entry, tr->buffer, event)) 1276 if (!filter_check_discard(call, entry, buffer, event))
1212 ring_buffer_unlock_commit(tr->buffer, event); 1277 ring_buffer_unlock_commit(buffer, event);
1213 1278
1214 out_unlock: 1279 out_unlock:
1215 __raw_spin_unlock(&trace_buf_lock); 1280 __raw_spin_unlock(&trace_buf_lock);
@@ -1221,6 +1286,11 @@ int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1221 1286
1222 return len; 1287 return len;
1223} 1288}
1289
1290int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1291{
1292 return trace_array_printk(&global_trace, ip, fmt, args);
1293}
1224EXPORT_SYMBOL_GPL(trace_vprintk); 1294EXPORT_SYMBOL_GPL(trace_vprintk);
1225 1295
1226enum trace_file_type { 1296enum trace_file_type {
@@ -1360,6 +1430,37 @@ static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1360 return ent; 1430 return ent;
1361} 1431}
1362 1432
1433static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1434{
1435 struct trace_array *tr = iter->tr;
1436 struct ring_buffer_event *event;
1437 struct ring_buffer_iter *buf_iter;
1438 unsigned long entries = 0;
1439 u64 ts;
1440
1441 tr->data[cpu]->skipped_entries = 0;
1442
1443 if (!iter->buffer_iter[cpu])
1444 return;
1445
1446 buf_iter = iter->buffer_iter[cpu];
1447 ring_buffer_iter_reset(buf_iter);
1448
1449 /*
1450 * We could have the case with the max latency tracers
1451 * that a reset never took place on a cpu. This is evident
1452 * by the timestamp being before the start of the buffer.
1453 */
1454 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1455 if (ts >= iter->tr->time_start)
1456 break;
1457 entries++;
1458 ring_buffer_read(buf_iter, NULL);
1459 }
1460
1461 tr->data[cpu]->skipped_entries = entries;
1462}
1463
1363/* 1464/*
1364 * No necessary locking here. The worst thing which can 1465 * No necessary locking here. The worst thing which can
1365 * happen is loosing events consumed at the same time 1466 * happen is loosing events consumed at the same time
@@ -1398,10 +1499,9 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1398 1499
1399 if (cpu_file == TRACE_PIPE_ALL_CPU) { 1500 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1400 for_each_tracing_cpu(cpu) 1501 for_each_tracing_cpu(cpu)
1401 ring_buffer_iter_reset(iter->buffer_iter[cpu]); 1502 tracing_iter_reset(iter, cpu);
1402 } else 1503 } else
1403 ring_buffer_iter_reset(iter->buffer_iter[cpu_file]); 1504 tracing_iter_reset(iter, cpu_file);
1404
1405 1505
1406 ftrace_enable_cpu(); 1506 ftrace_enable_cpu();
1407 1507
@@ -1450,16 +1550,32 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1450 struct trace_array *tr = iter->tr; 1550 struct trace_array *tr = iter->tr;
1451 struct trace_array_cpu *data = tr->data[tr->cpu]; 1551 struct trace_array_cpu *data = tr->data[tr->cpu];
1452 struct tracer *type = current_trace; 1552 struct tracer *type = current_trace;
1453 unsigned long total; 1553 unsigned long entries = 0;
1454 unsigned long entries; 1554 unsigned long total = 0;
1555 unsigned long count;
1455 const char *name = "preemption"; 1556 const char *name = "preemption";
1557 int cpu;
1456 1558
1457 if (type) 1559 if (type)
1458 name = type->name; 1560 name = type->name;
1459 1561
1460 entries = ring_buffer_entries(iter->tr->buffer); 1562
1461 total = entries + 1563 for_each_tracing_cpu(cpu) {
1462 ring_buffer_overruns(iter->tr->buffer); 1564 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1565 /*
1566 * If this buffer has skipped entries, then we hold all
1567 * entries for the trace and we need to ignore the
1568 * ones before the time stamp.
1569 */
1570 if (tr->data[cpu]->skipped_entries) {
1571 count -= tr->data[cpu]->skipped_entries;
1572 /* total is the same as the entries */
1573 total += count;
1574 } else
1575 total += count +
1576 ring_buffer_overrun_cpu(tr->buffer, cpu);
1577 entries += count;
1578 }
1463 1579
1464 seq_printf(m, "# %s latency trace v1.1.5 on %s\n", 1580 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1465 name, UTS_RELEASE); 1581 name, UTS_RELEASE);
@@ -1501,7 +1617,7 @@ print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1501 seq_puts(m, "\n# => ended at: "); 1617 seq_puts(m, "\n# => ended at: ");
1502 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags); 1618 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1503 trace_print_seq(m, &iter->seq); 1619 trace_print_seq(m, &iter->seq);
1504 seq_puts(m, "#\n"); 1620 seq_puts(m, "\n#\n");
1505 } 1621 }
1506 1622
1507 seq_puts(m, "#\n"); 1623 seq_puts(m, "#\n");
@@ -1520,6 +1636,9 @@ static void test_cpu_buff_start(struct trace_iterator *iter)
1520 if (cpumask_test_cpu(iter->cpu, iter->started)) 1636 if (cpumask_test_cpu(iter->cpu, iter->started))
1521 return; 1637 return;
1522 1638
1639 if (iter->tr->data[iter->cpu]->skipped_entries)
1640 return;
1641
1523 cpumask_set_cpu(iter->cpu, iter->started); 1642 cpumask_set_cpu(iter->cpu, iter->started);
1524 1643
1525 /* Don't print started cpu buffer for the first entry of the trace */ 1644 /* Don't print started cpu buffer for the first entry of the trace */
@@ -1782,19 +1901,23 @@ __tracing_open(struct inode *inode, struct file *file)
1782 if (ring_buffer_overruns(iter->tr->buffer)) 1901 if (ring_buffer_overruns(iter->tr->buffer))
1783 iter->iter_flags |= TRACE_FILE_ANNOTATE; 1902 iter->iter_flags |= TRACE_FILE_ANNOTATE;
1784 1903
1904 /* stop the trace while dumping */
1905 tracing_stop();
1906
1785 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) { 1907 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1786 for_each_tracing_cpu(cpu) { 1908 for_each_tracing_cpu(cpu) {
1787 1909
1788 iter->buffer_iter[cpu] = 1910 iter->buffer_iter[cpu] =
1789 ring_buffer_read_start(iter->tr->buffer, cpu); 1911 ring_buffer_read_start(iter->tr->buffer, cpu);
1912 tracing_iter_reset(iter, cpu);
1790 } 1913 }
1791 } else { 1914 } else {
1792 cpu = iter->cpu_file; 1915 cpu = iter->cpu_file;
1793 iter->buffer_iter[cpu] = 1916 iter->buffer_iter[cpu] =
1794 ring_buffer_read_start(iter->tr->buffer, cpu); 1917 ring_buffer_read_start(iter->tr->buffer, cpu);
1918 tracing_iter_reset(iter, cpu);
1795 } 1919 }
1796 1920
1797 /* TODO stop tracer */
1798 ret = seq_open(file, &tracer_seq_ops); 1921 ret = seq_open(file, &tracer_seq_ops);
1799 if (ret < 0) { 1922 if (ret < 0) {
1800 fail_ret = ERR_PTR(ret); 1923 fail_ret = ERR_PTR(ret);
@@ -1804,9 +1927,6 @@ __tracing_open(struct inode *inode, struct file *file)
1804 m = file->private_data; 1927 m = file->private_data;
1805 m->private = iter; 1928 m->private = iter;
1806 1929
1807 /* stop the trace while dumping */
1808 tracing_stop();
1809
1810 mutex_unlock(&trace_types_lock); 1930 mutex_unlock(&trace_types_lock);
1811 1931
1812 return iter; 1932 return iter;
@@ -1817,6 +1937,7 @@ __tracing_open(struct inode *inode, struct file *file)
1817 ring_buffer_read_finish(iter->buffer_iter[cpu]); 1937 ring_buffer_read_finish(iter->buffer_iter[cpu]);
1818 } 1938 }
1819 free_cpumask_var(iter->started); 1939 free_cpumask_var(iter->started);
1940 tracing_start();
1820 fail: 1941 fail:
1821 mutex_unlock(&trace_types_lock); 1942 mutex_unlock(&trace_types_lock);
1822 kfree(iter->trace); 1943 kfree(iter->trace);
@@ -3780,17 +3901,9 @@ trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3780 if (ret < 0) 3901 if (ret < 0)
3781 return ret; 3902 return ret;
3782 3903
3783 switch (val) { 3904 if (val != 0 && val != 1)
3784 case 0:
3785 trace_flags &= ~(1 << index);
3786 break;
3787 case 1:
3788 trace_flags |= 1 << index;
3789 break;
3790
3791 default:
3792 return -EINVAL; 3905 return -EINVAL;
3793 } 3906 set_tracer_flags(1 << index, val);
3794 3907
3795 *ppos += cnt; 3908 *ppos += cnt;
3796 3909
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index e2c06b21dd82..fa1dccb579d5 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -234,9 +234,6 @@ struct trace_array_cpu {
234 atomic_t disabled; 234 atomic_t disabled;
235 void *buffer_page; /* ring buffer spare */ 235 void *buffer_page; /* ring buffer spare */
236 236
237 /* these fields get copied into max-trace: */
238 unsigned long trace_idx;
239 unsigned long overrun;
240 unsigned long saved_latency; 237 unsigned long saved_latency;
241 unsigned long critical_start; 238 unsigned long critical_start;
242 unsigned long critical_end; 239 unsigned long critical_end;
@@ -244,6 +241,7 @@ struct trace_array_cpu {
244 unsigned long nice; 241 unsigned long nice;
245 unsigned long policy; 242 unsigned long policy;
246 unsigned long rt_priority; 243 unsigned long rt_priority;
244 unsigned long skipped_entries;
247 cycle_t preempt_timestamp; 245 cycle_t preempt_timestamp;
248 pid_t pid; 246 pid_t pid;
249 uid_t uid; 247 uid_t uid;
@@ -417,12 +415,13 @@ void init_tracer_sysprof_debugfs(struct dentry *d_tracer);
417 415
418struct ring_buffer_event; 416struct ring_buffer_event;
419 417
420struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr, 418struct ring_buffer_event *
421 int type, 419trace_buffer_lock_reserve(struct ring_buffer *buffer,
422 unsigned long len, 420 int type,
423 unsigned long flags, 421 unsigned long len,
424 int pc); 422 unsigned long flags,
425void trace_buffer_unlock_commit(struct trace_array *tr, 423 int pc);
424void trace_buffer_unlock_commit(struct ring_buffer *buffer,
426 struct ring_buffer_event *event, 425 struct ring_buffer_event *event,
427 unsigned long flags, int pc); 426 unsigned long flags, int pc);
428 427
@@ -483,10 +482,10 @@ void update_max_tr_single(struct trace_array *tr,
483#endif /* CONFIG_TRACER_MAX_TRACE */ 482#endif /* CONFIG_TRACER_MAX_TRACE */
484 483
485#ifdef CONFIG_STACKTRACE 484#ifdef CONFIG_STACKTRACE
486void ftrace_trace_stack(struct trace_array *tr, unsigned long flags, 485void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
487 int skip, int pc); 486 int skip, int pc);
488 487
489void ftrace_trace_userstack(struct trace_array *tr, unsigned long flags, 488void ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags,
490 int pc); 489 int pc);
491 490
492void __trace_stack(struct trace_array *tr, unsigned long flags, int skip, 491void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
@@ -567,6 +566,11 @@ extern int
567trace_vbprintk(unsigned long ip, const char *fmt, va_list args); 566trace_vbprintk(unsigned long ip, const char *fmt, va_list args);
568extern int 567extern int
569trace_vprintk(unsigned long ip, const char *fmt, va_list args); 568trace_vprintk(unsigned long ip, const char *fmt, va_list args);
569extern int
570trace_array_vprintk(struct trace_array *tr,
571 unsigned long ip, const char *fmt, va_list args);
572int trace_array_printk(struct trace_array *tr,
573 unsigned long ip, const char *fmt, ...);
570 574
571extern unsigned long trace_flags; 575extern unsigned long trace_flags;
572 576
diff --git a/kernel/trace/trace_boot.c b/kernel/trace/trace_boot.c
index a29ef23ffb47..19bfc75d467e 100644
--- a/kernel/trace/trace_boot.c
+++ b/kernel/trace/trace_boot.c
@@ -41,14 +41,12 @@ void disable_boot_trace(void)
41 41
42static int boot_trace_init(struct trace_array *tr) 42static int boot_trace_init(struct trace_array *tr)
43{ 43{
44 int cpu;
45 boot_trace = tr; 44 boot_trace = tr;
46 45
47 if (!tr) 46 if (!tr)
48 return 0; 47 return 0;
49 48
50 for_each_cpu(cpu, cpu_possible_mask) 49 tracing_reset_online_cpus(tr);
51 tracing_reset(tr, cpu);
52 50
53 tracing_sched_switch_assign_trace(tr); 51 tracing_sched_switch_assign_trace(tr);
54 return 0; 52 return 0;
@@ -132,6 +130,7 @@ struct tracer boot_tracer __read_mostly =
132void trace_boot_call(struct boot_trace_call *bt, initcall_t fn) 130void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
133{ 131{
134 struct ring_buffer_event *event; 132 struct ring_buffer_event *event;
133 struct ring_buffer *buffer;
135 struct trace_boot_call *entry; 134 struct trace_boot_call *entry;
136 struct trace_array *tr = boot_trace; 135 struct trace_array *tr = boot_trace;
137 136
@@ -144,13 +143,14 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
144 sprint_symbol(bt->func, (unsigned long)fn); 143 sprint_symbol(bt->func, (unsigned long)fn);
145 preempt_disable(); 144 preempt_disable();
146 145
147 event = trace_buffer_lock_reserve(tr, TRACE_BOOT_CALL, 146 buffer = tr->buffer;
147 event = trace_buffer_lock_reserve(buffer, TRACE_BOOT_CALL,
148 sizeof(*entry), 0, 0); 148 sizeof(*entry), 0, 0);
149 if (!event) 149 if (!event)
150 goto out; 150 goto out;
151 entry = ring_buffer_event_data(event); 151 entry = ring_buffer_event_data(event);
152 entry->boot_call = *bt; 152 entry->boot_call = *bt;
153 trace_buffer_unlock_commit(tr, event, 0, 0); 153 trace_buffer_unlock_commit(buffer, event, 0, 0);
154 out: 154 out:
155 preempt_enable(); 155 preempt_enable();
156} 156}
@@ -158,6 +158,7 @@ void trace_boot_call(struct boot_trace_call *bt, initcall_t fn)
158void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn) 158void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
159{ 159{
160 struct ring_buffer_event *event; 160 struct ring_buffer_event *event;
161 struct ring_buffer *buffer;
161 struct trace_boot_ret *entry; 162 struct trace_boot_ret *entry;
162 struct trace_array *tr = boot_trace; 163 struct trace_array *tr = boot_trace;
163 164
@@ -167,13 +168,14 @@ void trace_boot_ret(struct boot_trace_ret *bt, initcall_t fn)
167 sprint_symbol(bt->func, (unsigned long)fn); 168 sprint_symbol(bt->func, (unsigned long)fn);
168 preempt_disable(); 169 preempt_disable();
169 170
170 event = trace_buffer_lock_reserve(tr, TRACE_BOOT_RET, 171 buffer = tr->buffer;
172 event = trace_buffer_lock_reserve(buffer, TRACE_BOOT_RET,
171 sizeof(*entry), 0, 0); 173 sizeof(*entry), 0, 0);
172 if (!event) 174 if (!event)
173 goto out; 175 goto out;
174 entry = ring_buffer_event_data(event); 176 entry = ring_buffer_event_data(event);
175 entry->boot_ret = *bt; 177 entry->boot_ret = *bt;
176 trace_buffer_unlock_commit(tr, event, 0, 0); 178 trace_buffer_unlock_commit(buffer, event, 0, 0);
177 out: 179 out:
178 preempt_enable(); 180 preempt_enable();
179} 181}
diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c
index d33bcdeffe69..78b1ed230177 100644
--- a/kernel/trace/trace_events.c
+++ b/kernel/trace/trace_events.c
@@ -1438,6 +1438,7 @@ static void
1438function_test_events_call(unsigned long ip, unsigned long parent_ip) 1438function_test_events_call(unsigned long ip, unsigned long parent_ip)
1439{ 1439{
1440 struct ring_buffer_event *event; 1440 struct ring_buffer_event *event;
1441 struct ring_buffer *buffer;
1441 struct ftrace_entry *entry; 1442 struct ftrace_entry *entry;
1442 unsigned long flags; 1443 unsigned long flags;
1443 long disabled; 1444 long disabled;
@@ -1455,7 +1456,8 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
1455 1456
1456 local_save_flags(flags); 1457 local_save_flags(flags);
1457 1458
1458 event = trace_current_buffer_lock_reserve(TRACE_FN, sizeof(*entry), 1459 event = trace_current_buffer_lock_reserve(&buffer,
1460 TRACE_FN, sizeof(*entry),
1459 flags, pc); 1461 flags, pc);
1460 if (!event) 1462 if (!event)
1461 goto out; 1463 goto out;
@@ -1463,7 +1465,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
1463 entry->ip = ip; 1465 entry->ip = ip;
1464 entry->parent_ip = parent_ip; 1466 entry->parent_ip = parent_ip;
1465 1467
1466 trace_nowake_buffer_unlock_commit(event, flags, pc); 1468 trace_nowake_buffer_unlock_commit(buffer, event, flags, pc);
1467 1469
1468 out: 1470 out:
1469 atomic_dec(&per_cpu(test_event_disable, cpu)); 1471 atomic_dec(&per_cpu(test_event_disable, cpu));
diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c
index 3f4a251b7d16..b3749a2c3132 100644
--- a/kernel/trace/trace_functions_graph.c
+++ b/kernel/trace/trace_functions_graph.c
@@ -173,19 +173,20 @@ static int __trace_graph_entry(struct trace_array *tr,
173{ 173{
174 struct ftrace_event_call *call = &event_funcgraph_entry; 174 struct ftrace_event_call *call = &event_funcgraph_entry;
175 struct ring_buffer_event *event; 175 struct ring_buffer_event *event;
176 struct ring_buffer *buffer = tr->buffer;
176 struct ftrace_graph_ent_entry *entry; 177 struct ftrace_graph_ent_entry *entry;
177 178
178 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 179 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
179 return 0; 180 return 0;
180 181
181 event = trace_buffer_lock_reserve(tr, TRACE_GRAPH_ENT, 182 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,
182 sizeof(*entry), flags, pc); 183 sizeof(*entry), flags, pc);
183 if (!event) 184 if (!event)
184 return 0; 185 return 0;
185 entry = ring_buffer_event_data(event); 186 entry = ring_buffer_event_data(event);
186 entry->graph_ent = *trace; 187 entry->graph_ent = *trace;
187 if (!filter_current_check_discard(call, entry, event)) 188 if (!filter_current_check_discard(buffer, call, entry, event))
188 ring_buffer_unlock_commit(tr->buffer, event); 189 ring_buffer_unlock_commit(buffer, event);
189 190
190 return 1; 191 return 1;
191} 192}
@@ -236,19 +237,20 @@ static void __trace_graph_return(struct trace_array *tr,
236{ 237{
237 struct ftrace_event_call *call = &event_funcgraph_exit; 238 struct ftrace_event_call *call = &event_funcgraph_exit;
238 struct ring_buffer_event *event; 239 struct ring_buffer_event *event;
240 struct ring_buffer *buffer = tr->buffer;
239 struct ftrace_graph_ret_entry *entry; 241 struct ftrace_graph_ret_entry *entry;
240 242
241 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled)))) 243 if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
242 return; 244 return;
243 245
244 event = trace_buffer_lock_reserve(tr, TRACE_GRAPH_RET, 246 event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,
245 sizeof(*entry), flags, pc); 247 sizeof(*entry), flags, pc);
246 if (!event) 248 if (!event)
247 return; 249 return;
248 entry = ring_buffer_event_data(event); 250 entry = ring_buffer_event_data(event);
249 entry->ret = *trace; 251 entry->ret = *trace;
250 if (!filter_current_check_discard(call, entry, event)) 252 if (!filter_current_check_discard(buffer, call, entry, event))
251 ring_buffer_unlock_commit(tr->buffer, event); 253 ring_buffer_unlock_commit(buffer, event);
252} 254}
253 255
254void trace_graph_return(struct ftrace_graph_ret *trace) 256void trace_graph_return(struct ftrace_graph_ret *trace)
diff --git a/kernel/trace/trace_irqsoff.c b/kernel/trace/trace_irqsoff.c
index b923d13e2fad..5555b75a0d12 100644
--- a/kernel/trace/trace_irqsoff.c
+++ b/kernel/trace/trace_irqsoff.c
@@ -178,7 +178,6 @@ out_unlock:
178out: 178out:
179 data->critical_sequence = max_sequence; 179 data->critical_sequence = max_sequence;
180 data->preempt_timestamp = ftrace_now(cpu); 180 data->preempt_timestamp = ftrace_now(cpu);
181 tracing_reset(tr, cpu);
182 trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc); 181 trace_function(tr, CALLER_ADDR0, parent_ip, flags, pc);
183} 182}
184 183
@@ -208,7 +207,6 @@ start_critical_timing(unsigned long ip, unsigned long parent_ip)
208 data->critical_sequence = max_sequence; 207 data->critical_sequence = max_sequence;
209 data->preempt_timestamp = ftrace_now(cpu); 208 data->preempt_timestamp = ftrace_now(cpu);
210 data->critical_start = parent_ip ? : ip; 209 data->critical_start = parent_ip ? : ip;
211 tracing_reset(tr, cpu);
212 210
213 local_save_flags(flags); 211 local_save_flags(flags);
214 212
@@ -379,6 +377,7 @@ static void __irqsoff_tracer_init(struct trace_array *tr)
379 irqsoff_trace = tr; 377 irqsoff_trace = tr;
380 /* make sure that the tracer is visible */ 378 /* make sure that the tracer is visible */
381 smp_wmb(); 379 smp_wmb();
380 tracing_reset_online_cpus(tr);
382 start_irqsoff_tracer(tr); 381 start_irqsoff_tracer(tr);
383} 382}
384 383
diff --git a/kernel/trace/trace_mmiotrace.c b/kernel/trace/trace_mmiotrace.c
index d53b45ed0806..c4c9bbda53d3 100644
--- a/kernel/trace/trace_mmiotrace.c
+++ b/kernel/trace/trace_mmiotrace.c
@@ -307,11 +307,12 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
307 struct trace_array_cpu *data, 307 struct trace_array_cpu *data,
308 struct mmiotrace_rw *rw) 308 struct mmiotrace_rw *rw)
309{ 309{
310 struct ring_buffer *buffer = tr->buffer;
310 struct ring_buffer_event *event; 311 struct ring_buffer_event *event;
311 struct trace_mmiotrace_rw *entry; 312 struct trace_mmiotrace_rw *entry;
312 int pc = preempt_count(); 313 int pc = preempt_count();
313 314
314 event = trace_buffer_lock_reserve(tr, TRACE_MMIO_RW, 315 event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_RW,
315 sizeof(*entry), 0, pc); 316 sizeof(*entry), 0, pc);
316 if (!event) { 317 if (!event) {
317 atomic_inc(&dropped_count); 318 atomic_inc(&dropped_count);
@@ -319,7 +320,7 @@ static void __trace_mmiotrace_rw(struct trace_array *tr,
319 } 320 }
320 entry = ring_buffer_event_data(event); 321 entry = ring_buffer_event_data(event);
321 entry->rw = *rw; 322 entry->rw = *rw;
322 trace_buffer_unlock_commit(tr, event, 0, pc); 323 trace_buffer_unlock_commit(buffer, event, 0, pc);
323} 324}
324 325
325void mmio_trace_rw(struct mmiotrace_rw *rw) 326void mmio_trace_rw(struct mmiotrace_rw *rw)
@@ -333,11 +334,12 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
333 struct trace_array_cpu *data, 334 struct trace_array_cpu *data,
334 struct mmiotrace_map *map) 335 struct mmiotrace_map *map)
335{ 336{
337 struct ring_buffer *buffer = tr->buffer;
336 struct ring_buffer_event *event; 338 struct ring_buffer_event *event;
337 struct trace_mmiotrace_map *entry; 339 struct trace_mmiotrace_map *entry;
338 int pc = preempt_count(); 340 int pc = preempt_count();
339 341
340 event = trace_buffer_lock_reserve(tr, TRACE_MMIO_MAP, 342 event = trace_buffer_lock_reserve(buffer, TRACE_MMIO_MAP,
341 sizeof(*entry), 0, pc); 343 sizeof(*entry), 0, pc);
342 if (!event) { 344 if (!event) {
343 atomic_inc(&dropped_count); 345 atomic_inc(&dropped_count);
@@ -345,7 +347,7 @@ static void __trace_mmiotrace_map(struct trace_array *tr,
345 } 347 }
346 entry = ring_buffer_event_data(event); 348 entry = ring_buffer_event_data(event);
347 entry->map = *map; 349 entry->map = *map;
348 trace_buffer_unlock_commit(tr, event, 0, pc); 350 trace_buffer_unlock_commit(buffer, event, 0, pc);
349} 351}
350 352
351void mmio_trace_mapping(struct mmiotrace_map *map) 353void mmio_trace_mapping(struct mmiotrace_map *map)
diff --git a/kernel/trace/trace_power.c b/kernel/trace/trace_power.c
index 8a30d9874cd4..fe1a00f1445a 100644
--- a/kernel/trace/trace_power.c
+++ b/kernel/trace/trace_power.c
@@ -38,6 +38,7 @@ static void probe_power_end(struct power_trace *it)
38{ 38{
39 struct ftrace_event_call *call = &event_power; 39 struct ftrace_event_call *call = &event_power;
40 struct ring_buffer_event *event; 40 struct ring_buffer_event *event;
41 struct ring_buffer *buffer;
41 struct trace_power *entry; 42 struct trace_power *entry;
42 struct trace_array_cpu *data; 43 struct trace_array_cpu *data;
43 struct trace_array *tr = power_trace; 44 struct trace_array *tr = power_trace;
@@ -45,18 +46,20 @@ static void probe_power_end(struct power_trace *it)
45 if (!trace_power_enabled) 46 if (!trace_power_enabled)
46 return; 47 return;
47 48
49 buffer = tr->buffer;
50
48 preempt_disable(); 51 preempt_disable();
49 it->end = ktime_get(); 52 it->end = ktime_get();
50 data = tr->data[smp_processor_id()]; 53 data = tr->data[smp_processor_id()];
51 54
52 event = trace_buffer_lock_reserve(tr, TRACE_POWER, 55 event = trace_buffer_lock_reserve(buffer, TRACE_POWER,
53 sizeof(*entry), 0, 0); 56 sizeof(*entry), 0, 0);
54 if (!event) 57 if (!event)
55 goto out; 58 goto out;
56 entry = ring_buffer_event_data(event); 59 entry = ring_buffer_event_data(event);
57 entry->state_data = *it; 60 entry->state_data = *it;
58 if (!filter_check_discard(call, entry, tr->buffer, event)) 61 if (!filter_check_discard(call, entry, buffer, event))
59 trace_buffer_unlock_commit(tr, event, 0, 0); 62 trace_buffer_unlock_commit(buffer, event, 0, 0);
60 out: 63 out:
61 preempt_enable(); 64 preempt_enable();
62} 65}
@@ -66,6 +69,7 @@ static void probe_power_mark(struct power_trace *it, unsigned int type,
66{ 69{
67 struct ftrace_event_call *call = &event_power; 70 struct ftrace_event_call *call = &event_power;
68 struct ring_buffer_event *event; 71 struct ring_buffer_event *event;
72 struct ring_buffer *buffer;
69 struct trace_power *entry; 73 struct trace_power *entry;
70 struct trace_array_cpu *data; 74 struct trace_array_cpu *data;
71 struct trace_array *tr = power_trace; 75 struct trace_array *tr = power_trace;
@@ -73,6 +77,8 @@ static void probe_power_mark(struct power_trace *it, unsigned int type,
73 if (!trace_power_enabled) 77 if (!trace_power_enabled)
74 return; 78 return;
75 79
80 buffer = tr->buffer;
81
76 memset(it, 0, sizeof(struct power_trace)); 82 memset(it, 0, sizeof(struct power_trace));
77 it->state = level; 83 it->state = level;
78 it->type = type; 84 it->type = type;
@@ -81,14 +87,14 @@ static void probe_power_mark(struct power_trace *it, unsigned int type,
81 it->end = it->stamp; 87 it->end = it->stamp;
82 data = tr->data[smp_processor_id()]; 88 data = tr->data[smp_processor_id()];
83 89
84 event = trace_buffer_lock_reserve(tr, TRACE_POWER, 90 event = trace_buffer_lock_reserve(buffer, TRACE_POWER,
85 sizeof(*entry), 0, 0); 91 sizeof(*entry), 0, 0);
86 if (!event) 92 if (!event)
87 goto out; 93 goto out;
88 entry = ring_buffer_event_data(event); 94 entry = ring_buffer_event_data(event);
89 entry->state_data = *it; 95 entry->state_data = *it;
90 if (!filter_check_discard(call, entry, tr->buffer, event)) 96 if (!filter_check_discard(call, entry, buffer, event))
91 trace_buffer_unlock_commit(tr, event, 0, 0); 97 trace_buffer_unlock_commit(buffer, event, 0, 0);
92 out: 98 out:
93 preempt_enable(); 99 preempt_enable();
94} 100}
@@ -144,14 +150,12 @@ static void power_trace_reset(struct trace_array *tr)
144 150
145static int power_trace_init(struct trace_array *tr) 151static int power_trace_init(struct trace_array *tr)
146{ 152{
147 int cpu;
148 power_trace = tr; 153 power_trace = tr;
149 154
150 trace_power_enabled = 1; 155 trace_power_enabled = 1;
151 tracing_power_register(); 156 tracing_power_register();
152 157
153 for_each_cpu(cpu, cpu_possible_mask) 158 tracing_reset_online_cpus(tr);
154 tracing_reset(tr, cpu);
155 return 0; 159 return 0;
156} 160}
157 161
diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c
index e1285d7b5488..5fca0f51fde4 100644
--- a/kernel/trace/trace_sched_switch.c
+++ b/kernel/trace/trace_sched_switch.c
@@ -28,10 +28,11 @@ tracing_sched_switch_trace(struct trace_array *tr,
28 unsigned long flags, int pc) 28 unsigned long flags, int pc)
29{ 29{
30 struct ftrace_event_call *call = &event_context_switch; 30 struct ftrace_event_call *call = &event_context_switch;
31 struct ring_buffer *buffer = tr->buffer;
31 struct ring_buffer_event *event; 32 struct ring_buffer_event *event;
32 struct ctx_switch_entry *entry; 33 struct ctx_switch_entry *entry;
33 34
34 event = trace_buffer_lock_reserve(tr, TRACE_CTX, 35 event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
35 sizeof(*entry), flags, pc); 36 sizeof(*entry), flags, pc);
36 if (!event) 37 if (!event)
37 return; 38 return;
@@ -44,8 +45,8 @@ tracing_sched_switch_trace(struct trace_array *tr,
44 entry->next_state = next->state; 45 entry->next_state = next->state;
45 entry->next_cpu = task_cpu(next); 46 entry->next_cpu = task_cpu(next);
46 47
47 if (!filter_check_discard(call, entry, tr->buffer, event)) 48 if (!filter_check_discard(call, entry, buffer, event))
48 trace_buffer_unlock_commit(tr, event, flags, pc); 49 trace_buffer_unlock_commit(buffer, event, flags, pc);
49} 50}
50 51
51static void 52static void
@@ -86,8 +87,9 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
86 struct ftrace_event_call *call = &event_wakeup; 87 struct ftrace_event_call *call = &event_wakeup;
87 struct ring_buffer_event *event; 88 struct ring_buffer_event *event;
88 struct ctx_switch_entry *entry; 89 struct ctx_switch_entry *entry;
90 struct ring_buffer *buffer = tr->buffer;
89 91
90 event = trace_buffer_lock_reserve(tr, TRACE_WAKE, 92 event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
91 sizeof(*entry), flags, pc); 93 sizeof(*entry), flags, pc);
92 if (!event) 94 if (!event)
93 return; 95 return;
@@ -100,10 +102,10 @@ tracing_sched_wakeup_trace(struct trace_array *tr,
100 entry->next_state = wakee->state; 102 entry->next_state = wakee->state;
101 entry->next_cpu = task_cpu(wakee); 103 entry->next_cpu = task_cpu(wakee);
102 104
103 if (!filter_check_discard(call, entry, tr->buffer, event)) 105 if (!filter_check_discard(call, entry, buffer, event))
104 ring_buffer_unlock_commit(tr->buffer, event); 106 ring_buffer_unlock_commit(buffer, event);
105 ftrace_trace_stack(tr, flags, 6, pc); 107 ftrace_trace_stack(tr->buffer, flags, 6, pc);
106 ftrace_trace_userstack(tr, flags, pc); 108 ftrace_trace_userstack(tr->buffer, flags, pc);
107} 109}
108 110
109static void 111static void
diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c
index eacb27225173..ad69f105a7c6 100644
--- a/kernel/trace/trace_sched_wakeup.c
+++ b/kernel/trace/trace_sched_wakeup.c
@@ -186,11 +186,6 @@ out:
186 186
187static void __wakeup_reset(struct trace_array *tr) 187static void __wakeup_reset(struct trace_array *tr)
188{ 188{
189 int cpu;
190
191 for_each_possible_cpu(cpu)
192 tracing_reset(tr, cpu);
193
194 wakeup_cpu = -1; 189 wakeup_cpu = -1;
195 wakeup_prio = -1; 190 wakeup_prio = -1;
196 191
@@ -204,6 +199,8 @@ static void wakeup_reset(struct trace_array *tr)
204{ 199{
205 unsigned long flags; 200 unsigned long flags;
206 201
202 tracing_reset_online_cpus(tr);
203
207 local_irq_save(flags); 204 local_irq_save(flags);
208 __raw_spin_lock(&wakeup_lock); 205 __raw_spin_lock(&wakeup_lock);
209 __wakeup_reset(tr); 206 __wakeup_reset(tr);
diff --git a/kernel/trace/trace_syscalls.c b/kernel/trace/trace_syscalls.c
index 4f5fae6fad90..8712ce3c6a0e 100644
--- a/kernel/trace/trace_syscalls.c
+++ b/kernel/trace/trace_syscalls.c
@@ -223,6 +223,7 @@ void ftrace_syscall_enter(struct pt_regs *regs, long id)
223 struct syscall_trace_enter *entry; 223 struct syscall_trace_enter *entry;
224 struct syscall_metadata *sys_data; 224 struct syscall_metadata *sys_data;
225 struct ring_buffer_event *event; 225 struct ring_buffer_event *event;
226 struct ring_buffer *buffer;
226 int size; 227 int size;
227 int syscall_nr; 228 int syscall_nr;
228 229
@@ -238,8 +239,8 @@ void ftrace_syscall_enter(struct pt_regs *regs, long id)
238 239
239 size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args; 240 size = sizeof(*entry) + sizeof(unsigned long) * sys_data->nb_args;
240 241
241 event = trace_current_buffer_lock_reserve(sys_data->enter_id, size, 242 event = trace_current_buffer_lock_reserve(&buffer, sys_data->enter_id,
242 0, 0); 243 size, 0, 0);
243 if (!event) 244 if (!event)
244 return; 245 return;
245 246
@@ -247,8 +248,9 @@ void ftrace_syscall_enter(struct pt_regs *regs, long id)
247 entry->nr = syscall_nr; 248 entry->nr = syscall_nr;
248 syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args); 249 syscall_get_arguments(current, regs, 0, sys_data->nb_args, entry->args);
249 250
250 if (!filter_current_check_discard(sys_data->enter_event, entry, event)) 251 if (!filter_current_check_discard(buffer, sys_data->enter_event,
251 trace_current_buffer_unlock_commit(event, 0, 0); 252 entry, event))
253 trace_current_buffer_unlock_commit(buffer, event, 0, 0);
252} 254}
253 255
254void ftrace_syscall_exit(struct pt_regs *regs, long ret) 256void ftrace_syscall_exit(struct pt_regs *regs, long ret)
@@ -256,6 +258,7 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret)
256 struct syscall_trace_exit *entry; 258 struct syscall_trace_exit *entry;
257 struct syscall_metadata *sys_data; 259 struct syscall_metadata *sys_data;
258 struct ring_buffer_event *event; 260 struct ring_buffer_event *event;
261 struct ring_buffer *buffer;
259 int syscall_nr; 262 int syscall_nr;
260 263
261 syscall_nr = syscall_get_nr(current, regs); 264 syscall_nr = syscall_get_nr(current, regs);
@@ -268,7 +271,7 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret)
268 if (!sys_data) 271 if (!sys_data)
269 return; 272 return;
270 273
271 event = trace_current_buffer_lock_reserve(sys_data->exit_id, 274 event = trace_current_buffer_lock_reserve(&buffer, sys_data->exit_id,
272 sizeof(*entry), 0, 0); 275 sizeof(*entry), 0, 0);
273 if (!event) 276 if (!event)
274 return; 277 return;
@@ -277,8 +280,9 @@ void ftrace_syscall_exit(struct pt_regs *regs, long ret)
277 entry->nr = syscall_nr; 280 entry->nr = syscall_nr;
278 entry->ret = syscall_get_return_value(current, regs); 281 entry->ret = syscall_get_return_value(current, regs);
279 282
280 if (!filter_current_check_discard(sys_data->exit_event, entry, event)) 283 if (!filter_current_check_discard(buffer, sys_data->exit_event,
281 trace_current_buffer_unlock_commit(event, 0, 0); 284 entry, event))
285 trace_current_buffer_unlock_commit(buffer, event, 0, 0);
282} 286}
283 287
284int reg_event_syscall_enter(void *ptr) 288int reg_event_syscall_enter(void *ptr)
diff --git a/kernel/wait.c b/kernel/wait.c
index ea7c3b4275cf..c4bd3d825f35 100644
--- a/kernel/wait.c
+++ b/kernel/wait.c
@@ -10,13 +10,14 @@
10#include <linux/wait.h> 10#include <linux/wait.h>
11#include <linux/hash.h> 11#include <linux/hash.h>
12 12
13void init_waitqueue_head(wait_queue_head_t *q) 13void __init_waitqueue_head(wait_queue_head_t *q, struct lock_class_key *key)
14{ 14{
15 spin_lock_init(&q->lock); 15 spin_lock_init(&q->lock);
16 lockdep_set_class(&q->lock, key);
16 INIT_LIST_HEAD(&q->task_list); 17 INIT_LIST_HEAD(&q->task_list);
17} 18}
18 19
19EXPORT_SYMBOL(init_waitqueue_head); 20EXPORT_SYMBOL(__init_waitqueue_head);
20 21
21void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait) 22void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait)
22{ 23{
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-02 10:49:23 -0500