aboutsummaryrefslogtreecommitdiffstats
path: root/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2012-05-22 21:18:55 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2012-05-22 21:18:55 -0400
commit2ff2b289a695807e291e1ed9f639d8a3ba5f4254 (patch)
treee4b7f44e5cc1582ba2be8aeba221f4841f4c86a6 /kernel
parent88d6ae8dc33af12fe1c7941b1fae2767374046fd (diff)
parent73787190d04a34e6da745da893b3ae8bedde418f (diff)
Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull perf changes from Ingo Molnar: "Lots of changes: - (much) improved assembly annotation support in perf report, with jump visualization, searching, navigation, visual output improvements and more. - kernel support for AMD IBS PMU hardware features. Notably 'perf record -e cycles:p' and 'perf top -e cycles:p' should work without skid now, like PEBS does on the Intel side, because it takes advantage of IBS transparently. - the libtracevents library: it is the first step towards unifying tracing tooling and perf, and it also gives a tracing library for external tools like powertop to rely on. - infrastructure: various improvements and refactoring of the UI modules and related code - infrastructure: cleanup and simplification of the profiling targets code (--uid, --pid, --tid, --cpu, --all-cpus, etc.) - tons of robustness fixes all around - various ftrace updates: speedups, cleanups, robustness improvements. - typing 'make' in tools/ will now give you a menu of projects to build and a short help text to explain what each does. - ... and lots of other changes I forgot to list. The perf record make bzImage + perf report regression you reported should be fixed." * 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (166 commits) tracing: Remove kernel_lock annotations tracing: Fix initial buffer_size_kb state ring-buffer: Merge separate resize loops perf evsel: Create events initially disabled -- again perf tools: Split term type into value type and term type perf hists: Fix callchain ip printf format perf target: Add uses_mmap field ftrace: Remove selecting FRAME_POINTER with FUNCTION_TRACER ftrace/x86: Have x86 ftrace use the ftrace_modify_all_code() ftrace: Make ftrace_modify_all_code() global for archs to use ftrace: Return record ip addr for ftrace_location() ftrace: Consolidate ftrace_location() and ftrace_text_reserved() ftrace: Speed up search by skipping pages by address ftrace: Remove extra helper functions ftrace: Sort all function addresses, not just per page tracing: change CPU ring buffer state from tracing_cpumask tracing: Check return value of tracing_dentry_percpu() ring-buffer: Reset head page before running self test ring-buffer: Add integrity check at end of iter read ring-buffer: Make addition of pages in ring buffer atomic ...
Diffstat (limited to 'kernel')
-rw-r--r--kernel/events/core.c25
-rw-r--r--kernel/sched/core.c9
-rw-r--r--kernel/trace/Kconfig1
-rw-r--r--kernel/trace/ftrace.c242
-rw-r--r--kernel/trace/ring_buffer.c585
-rw-r--r--kernel/trace/trace.c503
-rw-r--r--kernel/trace/trace.h4
-rw-r--r--kernel/trace/trace_printk.c4
8 files changed, 874 insertions, 499 deletions
diff --git a/kernel/events/core.c b/kernel/events/core.c
index fd126f82b57c..91a445925855 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2039,8 +2039,8 @@ static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2039 * accessing the event control register. If a NMI hits, then it will 2039 * accessing the event control register. If a NMI hits, then it will
2040 * not restart the event. 2040 * not restart the event.
2041 */ 2041 */
2042void __perf_event_task_sched_out(struct task_struct *task, 2042static void __perf_event_task_sched_out(struct task_struct *task,
2043 struct task_struct *next) 2043 struct task_struct *next)
2044{ 2044{
2045 int ctxn; 2045 int ctxn;
2046 2046
@@ -2279,8 +2279,8 @@ static void perf_branch_stack_sched_in(struct task_struct *prev,
2279 * accessing the event control register. If a NMI hits, then it will 2279 * accessing the event control register. If a NMI hits, then it will
2280 * keep the event running. 2280 * keep the event running.
2281 */ 2281 */
2282void __perf_event_task_sched_in(struct task_struct *prev, 2282static void __perf_event_task_sched_in(struct task_struct *prev,
2283 struct task_struct *task) 2283 struct task_struct *task)
2284{ 2284{
2285 struct perf_event_context *ctx; 2285 struct perf_event_context *ctx;
2286 int ctxn; 2286 int ctxn;
@@ -2305,6 +2305,12 @@ void __perf_event_task_sched_in(struct task_struct *prev,
2305 perf_branch_stack_sched_in(prev, task); 2305 perf_branch_stack_sched_in(prev, task);
2306} 2306}
2307 2307
2308void __perf_event_task_sched(struct task_struct *prev, struct task_struct *next)
2309{
2310 __perf_event_task_sched_out(prev, next);
2311 __perf_event_task_sched_in(prev, next);
2312}
2313
2308static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 2314static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2309{ 2315{
2310 u64 frequency = event->attr.sample_freq; 2316 u64 frequency = event->attr.sample_freq;
@@ -4957,7 +4963,7 @@ void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4957 if (rctx < 0) 4963 if (rctx < 0)
4958 return; 4964 return;
4959 4965
4960 perf_sample_data_init(&data, addr); 4966 perf_sample_data_init(&data, addr, 0);
4961 4967
4962 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 4968 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
4963 4969
@@ -5215,7 +5221,7 @@ void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
5215 .data = record, 5221 .data = record,
5216 }; 5222 };
5217 5223
5218 perf_sample_data_init(&data, addr); 5224 perf_sample_data_init(&data, addr, 0);
5219 data.raw = &raw; 5225 data.raw = &raw;
5220 5226
5221 hlist_for_each_entry_rcu(event, node, head, hlist_entry) { 5227 hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
@@ -5318,7 +5324,7 @@ void perf_bp_event(struct perf_event *bp, void *data)
5318 struct perf_sample_data sample; 5324 struct perf_sample_data sample;
5319 struct pt_regs *regs = data; 5325 struct pt_regs *regs = data;
5320 5326
5321 perf_sample_data_init(&sample, bp->attr.bp_addr); 5327 perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
5322 5328
5323 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 5329 if (!bp->hw.state && !perf_exclude_event(bp, regs))
5324 perf_swevent_event(bp, 1, &sample, regs); 5330 perf_swevent_event(bp, 1, &sample, regs);
@@ -5344,13 +5350,12 @@ static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5344 5350
5345 event->pmu->read(event); 5351 event->pmu->read(event);
5346 5352
5347 perf_sample_data_init(&data, 0); 5353 perf_sample_data_init(&data, 0, event->hw.last_period);
5348 data.period = event->hw.last_period;
5349 regs = get_irq_regs(); 5354 regs = get_irq_regs();
5350 5355
5351 if (regs && !perf_exclude_event(event, regs)) { 5356 if (regs && !perf_exclude_event(event, regs)) {
5352 if (!(event->attr.exclude_idle && is_idle_task(current))) 5357 if (!(event->attr.exclude_idle && is_idle_task(current)))
5353 if (perf_event_overflow(event, &data, regs)) 5358 if (__perf_event_overflow(event, 1, &data, regs))
5354 ret = HRTIMER_NORESTART; 5359 ret = HRTIMER_NORESTART;
5355 } 5360 }
5356 5361
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 03667c3fdb33..d2e2e173d8f7 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1914,7 +1914,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev,
1914 struct task_struct *next) 1914 struct task_struct *next)
1915{ 1915{
1916 sched_info_switch(prev, next); 1916 sched_info_switch(prev, next);
1917 perf_event_task_sched_out(prev, next); 1917 perf_event_task_sched(prev, next);
1918 fire_sched_out_preempt_notifiers(prev, next); 1918 fire_sched_out_preempt_notifiers(prev, next);
1919 prepare_lock_switch(rq, next); 1919 prepare_lock_switch(rq, next);
1920 prepare_arch_switch(next); 1920 prepare_arch_switch(next);
@@ -1957,13 +1957,6 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev)
1957 */ 1957 */
1958 prev_state = prev->state; 1958 prev_state = prev->state;
1959 finish_arch_switch(prev); 1959 finish_arch_switch(prev);
1960#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
1961 local_irq_disable();
1962#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
1963 perf_event_task_sched_in(prev, current);
1964#ifdef __ARCH_WANT_INTERRUPTS_ON_CTXSW
1965 local_irq_enable();
1966#endif /* __ARCH_WANT_INTERRUPTS_ON_CTXSW */
1967 finish_lock_switch(rq, prev); 1960 finish_lock_switch(rq, prev);
1968 finish_arch_post_lock_switch(); 1961 finish_arch_post_lock_switch();
1969 1962
diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig
index a1d2849f2473..d81a1a532994 100644
--- a/kernel/trace/Kconfig
+++ b/kernel/trace/Kconfig
@@ -141,7 +141,6 @@ if FTRACE
141config FUNCTION_TRACER 141config FUNCTION_TRACER
142 bool "Kernel Function Tracer" 142 bool "Kernel Function Tracer"
143 depends on HAVE_FUNCTION_TRACER 143 depends on HAVE_FUNCTION_TRACER
144 select FRAME_POINTER if !ARM_UNWIND && !PPC && !S390 && !MICROBLAZE
145 select KALLSYMS 144 select KALLSYMS
146 select GENERIC_TRACER 145 select GENERIC_TRACER
147 select CONTEXT_SWITCH_TRACER 146 select CONTEXT_SWITCH_TRACER
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 0fa92f677c92..a008663d86c8 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -1383,44 +1383,73 @@ ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip)
1383 1383
1384static int ftrace_cmp_recs(const void *a, const void *b) 1384static int ftrace_cmp_recs(const void *a, const void *b)
1385{ 1385{
1386 const struct dyn_ftrace *reca = a; 1386 const struct dyn_ftrace *key = a;
1387 const struct dyn_ftrace *recb = b; 1387 const struct dyn_ftrace *rec = b;
1388 1388
1389 if (reca->ip > recb->ip) 1389 if (key->flags < rec->ip)
1390 return 1;
1391 if (reca->ip < recb->ip)
1392 return -1; 1390 return -1;
1391 if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
1392 return 1;
1393 return 0; 1393 return 0;
1394} 1394}
1395 1395
1396/** 1396static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
1397 * ftrace_location - return true if the ip giving is a traced location
1398 * @ip: the instruction pointer to check
1399 *
1400 * Returns 1 if @ip given is a pointer to a ftrace location.
1401 * That is, the instruction that is either a NOP or call to
1402 * the function tracer. It checks the ftrace internal tables to
1403 * determine if the address belongs or not.
1404 */
1405int ftrace_location(unsigned long ip)
1406{ 1397{
1407 struct ftrace_page *pg; 1398 struct ftrace_page *pg;
1408 struct dyn_ftrace *rec; 1399 struct dyn_ftrace *rec;
1409 struct dyn_ftrace key; 1400 struct dyn_ftrace key;
1410 1401
1411 key.ip = ip; 1402 key.ip = start;
1403 key.flags = end; /* overload flags, as it is unsigned long */
1412 1404
1413 for (pg = ftrace_pages_start; pg; pg = pg->next) { 1405 for (pg = ftrace_pages_start; pg; pg = pg->next) {
1406 if (end < pg->records[0].ip ||
1407 start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
1408 continue;
1414 rec = bsearch(&key, pg->records, pg->index, 1409 rec = bsearch(&key, pg->records, pg->index,
1415 sizeof(struct dyn_ftrace), 1410 sizeof(struct dyn_ftrace),
1416 ftrace_cmp_recs); 1411 ftrace_cmp_recs);
1417 if (rec) 1412 if (rec)
1418 return 1; 1413 return rec->ip;
1419 } 1414 }
1420 1415
1421 return 0; 1416 return 0;
1422} 1417}
1423 1418
1419/**
1420 * ftrace_location - return true if the ip giving is a traced location
1421 * @ip: the instruction pointer to check
1422 *
1423 * Returns rec->ip if @ip given is a pointer to a ftrace location.
1424 * That is, the instruction that is either a NOP or call to
1425 * the function tracer. It checks the ftrace internal tables to
1426 * determine if the address belongs or not.
1427 */
1428unsigned long ftrace_location(unsigned long ip)
1429{
1430 return ftrace_location_range(ip, ip);
1431}
1432
1433/**
1434 * ftrace_text_reserved - return true if range contains an ftrace location
1435 * @start: start of range to search
1436 * @end: end of range to search (inclusive). @end points to the last byte to check.
1437 *
1438 * Returns 1 if @start and @end contains a ftrace location.
1439 * That is, the instruction that is either a NOP or call to
1440 * the function tracer. It checks the ftrace internal tables to
1441 * determine if the address belongs or not.
1442 */
1443int ftrace_text_reserved(void *start, void *end)
1444{
1445 unsigned long ret;
1446
1447 ret = ftrace_location_range((unsigned long)start,
1448 (unsigned long)end);
1449
1450 return (int)!!ret;
1451}
1452
1424static void __ftrace_hash_rec_update(struct ftrace_ops *ops, 1453static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
1425 int filter_hash, 1454 int filter_hash,
1426 bool inc) 1455 bool inc)
@@ -1520,35 +1549,6 @@ static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
1520 __ftrace_hash_rec_update(ops, filter_hash, 1); 1549 __ftrace_hash_rec_update(ops, filter_hash, 1);
1521} 1550}
1522 1551
1523static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
1524{
1525 if (ftrace_pages->index == ftrace_pages->size) {
1526 /* We should have allocated enough */
1527 if (WARN_ON(!ftrace_pages->next))
1528 return NULL;
1529 ftrace_pages = ftrace_pages->next;
1530 }
1531
1532 return &ftrace_pages->records[ftrace_pages->index++];
1533}
1534
1535static struct dyn_ftrace *
1536ftrace_record_ip(unsigned long ip)
1537{
1538 struct dyn_ftrace *rec;
1539
1540 if (ftrace_disabled)
1541 return NULL;
1542
1543 rec = ftrace_alloc_dyn_node(ip);
1544 if (!rec)
1545 return NULL;
1546
1547 rec->ip = ip;
1548
1549 return rec;
1550}
1551
1552static void print_ip_ins(const char *fmt, unsigned char *p) 1552static void print_ip_ins(const char *fmt, unsigned char *p)
1553{ 1553{
1554 int i; 1554 int i;
@@ -1598,21 +1598,6 @@ void ftrace_bug(int failed, unsigned long ip)
1598 } 1598 }
1599} 1599}
1600 1600
1601
1602/* Return 1 if the address range is reserved for ftrace */
1603int ftrace_text_reserved(void *start, void *end)
1604{
1605 struct dyn_ftrace *rec;
1606 struct ftrace_page *pg;
1607
1608 do_for_each_ftrace_rec(pg, rec) {
1609 if (rec->ip <= (unsigned long)end &&
1610 rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1611 return 1;
1612 } while_for_each_ftrace_rec();
1613 return 0;
1614}
1615
1616static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update) 1601static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
1617{ 1602{
1618 unsigned long flag = 0UL; 1603 unsigned long flag = 0UL;
@@ -1698,7 +1683,7 @@ __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1698 return -1; /* unknow ftrace bug */ 1683 return -1; /* unknow ftrace bug */
1699} 1684}
1700 1685
1701static void ftrace_replace_code(int update) 1686void __weak ftrace_replace_code(int enable)
1702{ 1687{
1703 struct dyn_ftrace *rec; 1688 struct dyn_ftrace *rec;
1704 struct ftrace_page *pg; 1689 struct ftrace_page *pg;
@@ -1708,7 +1693,7 @@ static void ftrace_replace_code(int update)
1708 return; 1693 return;
1709 1694
1710 do_for_each_ftrace_rec(pg, rec) { 1695 do_for_each_ftrace_rec(pg, rec) {
1711 failed = __ftrace_replace_code(rec, update); 1696 failed = __ftrace_replace_code(rec, enable);
1712 if (failed) { 1697 if (failed) {
1713 ftrace_bug(failed, rec->ip); 1698 ftrace_bug(failed, rec->ip);
1714 /* Stop processing */ 1699 /* Stop processing */
@@ -1826,22 +1811,27 @@ int __weak ftrace_arch_code_modify_post_process(void)
1826 return 0; 1811 return 0;
1827} 1812}
1828 1813
1829static int __ftrace_modify_code(void *data) 1814void ftrace_modify_all_code(int command)
1830{ 1815{
1831 int *command = data; 1816 if (command & FTRACE_UPDATE_CALLS)
1832
1833 if (*command & FTRACE_UPDATE_CALLS)
1834 ftrace_replace_code(1); 1817 ftrace_replace_code(1);
1835 else if (*command & FTRACE_DISABLE_CALLS) 1818 else if (command & FTRACE_DISABLE_CALLS)
1836 ftrace_replace_code(0); 1819 ftrace_replace_code(0);
1837 1820
1838 if (*command & FTRACE_UPDATE_TRACE_FUNC) 1821 if (command & FTRACE_UPDATE_TRACE_FUNC)
1839 ftrace_update_ftrace_func(ftrace_trace_function); 1822 ftrace_update_ftrace_func(ftrace_trace_function);
1840 1823
1841 if (*command & FTRACE_START_FUNC_RET) 1824 if (command & FTRACE_START_FUNC_RET)
1842 ftrace_enable_ftrace_graph_caller(); 1825 ftrace_enable_ftrace_graph_caller();
1843 else if (*command & FTRACE_STOP_FUNC_RET) 1826 else if (command & FTRACE_STOP_FUNC_RET)
1844 ftrace_disable_ftrace_graph_caller(); 1827 ftrace_disable_ftrace_graph_caller();
1828}
1829
1830static int __ftrace_modify_code(void *data)
1831{
1832 int *command = data;
1833
1834 ftrace_modify_all_code(*command);
1845 1835
1846 return 0; 1836 return 0;
1847} 1837}
@@ -2469,57 +2459,35 @@ static int
2469ftrace_avail_open(struct inode *inode, struct file *file) 2459ftrace_avail_open(struct inode *inode, struct file *file)
2470{ 2460{
2471 struct ftrace_iterator *iter; 2461 struct ftrace_iterator *iter;
2472 int ret;
2473 2462
2474 if (unlikely(ftrace_disabled)) 2463 if (unlikely(ftrace_disabled))
2475 return -ENODEV; 2464 return -ENODEV;
2476 2465
2477 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2466 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2478 if (!iter) 2467 if (iter) {
2479 return -ENOMEM; 2468 iter->pg = ftrace_pages_start;
2480 2469 iter->ops = &global_ops;
2481 iter->pg = ftrace_pages_start;
2482 iter->ops = &global_ops;
2483
2484 ret = seq_open(file, &show_ftrace_seq_ops);
2485 if (!ret) {
2486 struct seq_file *m = file->private_data;
2487
2488 m->private = iter;
2489 } else {
2490 kfree(iter);
2491 } 2470 }
2492 2471
2493 return ret; 2472 return iter ? 0 : -ENOMEM;
2494} 2473}
2495 2474
2496static int 2475static int
2497ftrace_enabled_open(struct inode *inode, struct file *file) 2476ftrace_enabled_open(struct inode *inode, struct file *file)
2498{ 2477{
2499 struct ftrace_iterator *iter; 2478 struct ftrace_iterator *iter;
2500 int ret;
2501 2479
2502 if (unlikely(ftrace_disabled)) 2480 if (unlikely(ftrace_disabled))
2503 return -ENODEV; 2481 return -ENODEV;
2504 2482
2505 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2483 iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
2506 if (!iter) 2484 if (iter) {
2507 return -ENOMEM; 2485 iter->pg = ftrace_pages_start;
2508 2486 iter->flags = FTRACE_ITER_ENABLED;
2509 iter->pg = ftrace_pages_start; 2487 iter->ops = &global_ops;
2510 iter->flags = FTRACE_ITER_ENABLED;
2511 iter->ops = &global_ops;
2512
2513 ret = seq_open(file, &show_ftrace_seq_ops);
2514 if (!ret) {
2515 struct seq_file *m = file->private_data;
2516
2517 m->private = iter;
2518 } else {
2519 kfree(iter);
2520 } 2488 }
2521 2489
2522 return ret; 2490 return iter ? 0 : -ENOMEM;
2523} 2491}
2524 2492
2525static void ftrace_filter_reset(struct ftrace_hash *hash) 2493static void ftrace_filter_reset(struct ftrace_hash *hash)
@@ -3688,22 +3656,36 @@ static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
3688 return 0; 3656 return 0;
3689} 3657}
3690 3658
3691static void ftrace_swap_recs(void *a, void *b, int size) 3659static int ftrace_cmp_ips(const void *a, const void *b)
3660{
3661 const unsigned long *ipa = a;
3662 const unsigned long *ipb = b;
3663
3664 if (*ipa > *ipb)
3665 return 1;
3666 if (*ipa < *ipb)
3667 return -1;
3668 return 0;
3669}
3670
3671static void ftrace_swap_ips(void *a, void *b, int size)
3692{ 3672{
3693 struct dyn_ftrace *reca = a; 3673 unsigned long *ipa = a;
3694 struct dyn_ftrace *recb = b; 3674 unsigned long *ipb = b;
3695 struct dyn_ftrace t; 3675 unsigned long t;
3696 3676
3697 t = *reca; 3677 t = *ipa;
3698 *reca = *recb; 3678 *ipa = *ipb;
3699 *recb = t; 3679 *ipb = t;
3700} 3680}
3701 3681
3702static int ftrace_process_locs(struct module *mod, 3682static int ftrace_process_locs(struct module *mod,
3703 unsigned long *start, 3683 unsigned long *start,
3704 unsigned long *end) 3684 unsigned long *end)
3705{ 3685{
3686 struct ftrace_page *start_pg;
3706 struct ftrace_page *pg; 3687 struct ftrace_page *pg;
3688 struct dyn_ftrace *rec;
3707 unsigned long count; 3689 unsigned long count;
3708 unsigned long *p; 3690 unsigned long *p;
3709 unsigned long addr; 3691 unsigned long addr;
@@ -3715,8 +3697,11 @@ static int ftrace_process_locs(struct module *mod,
3715 if (!count) 3697 if (!count)
3716 return 0; 3698 return 0;
3717 3699
3718 pg = ftrace_allocate_pages(count); 3700 sort(start, count, sizeof(*start),
3719 if (!pg) 3701 ftrace_cmp_ips, ftrace_swap_ips);
3702
3703 start_pg = ftrace_allocate_pages(count);
3704 if (!start_pg)
3720 return -ENOMEM; 3705 return -ENOMEM;
3721 3706
3722 mutex_lock(&ftrace_lock); 3707 mutex_lock(&ftrace_lock);
@@ -3729,7 +3714,7 @@ static int ftrace_process_locs(struct module *mod,
3729 if (!mod) { 3714 if (!mod) {
3730 WARN_ON(ftrace_pages || ftrace_pages_start); 3715 WARN_ON(ftrace_pages || ftrace_pages_start);
3731 /* First initialization */ 3716 /* First initialization */
3732 ftrace_pages = ftrace_pages_start = pg; 3717 ftrace_pages = ftrace_pages_start = start_pg;
3733 } else { 3718 } else {
3734 if (!ftrace_pages) 3719 if (!ftrace_pages)
3735 goto out; 3720 goto out;
@@ -3740,11 +3725,11 @@ static int ftrace_process_locs(struct module *mod,
3740 ftrace_pages = ftrace_pages->next; 3725 ftrace_pages = ftrace_pages->next;
3741 } 3726 }
3742 3727
3743 ftrace_pages->next = pg; 3728 ftrace_pages->next = start_pg;
3744 ftrace_pages = pg;
3745 } 3729 }
3746 3730
3747 p = start; 3731 p = start;
3732 pg = start_pg;
3748 while (p < end) { 3733 while (p < end) {
3749 addr = ftrace_call_adjust(*p++); 3734 addr = ftrace_call_adjust(*p++);
3750 /* 3735 /*
@@ -3755,17 +3740,26 @@ static int ftrace_process_locs(struct module *mod,
3755 */ 3740 */
3756 if (!addr) 3741 if (!addr)
3757 continue; 3742 continue;
3758 if (!ftrace_record_ip(addr)) 3743
3759 break; 3744 if (pg->index == pg->size) {
3745 /* We should have allocated enough */
3746 if (WARN_ON(!pg->next))
3747 break;
3748 pg = pg->next;
3749 }
3750
3751 rec = &pg->records[pg->index++];
3752 rec->ip = addr;
3760 } 3753 }
3761 3754
3762 /* These new locations need to be initialized */ 3755 /* We should have used all pages */
3763 ftrace_new_pgs = pg; 3756 WARN_ON(pg->next);
3757
3758 /* Assign the last page to ftrace_pages */
3759 ftrace_pages = pg;
3764 3760
3765 /* Make each individual set of pages sorted by ips */ 3761 /* These new locations need to be initialized */
3766 for (; pg; pg = pg->next) 3762 ftrace_new_pgs = start_pg;
3767 sort(pg->records, pg->index, sizeof(struct dyn_ftrace),
3768 ftrace_cmp_recs, ftrace_swap_recs);
3769 3763
3770 /* 3764 /*
3771 * We only need to disable interrupts on start up 3765 * We only need to disable interrupts on start up
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c
index cf8d11e91efd..6420cda62336 100644
--- a/kernel/trace/ring_buffer.c
+++ b/kernel/trace/ring_buffer.c
@@ -23,6 +23,8 @@
23#include <asm/local.h> 23#include <asm/local.h>
24#include "trace.h" 24#include "trace.h"
25 25
26static void update_pages_handler(struct work_struct *work);
27
26/* 28/*
27 * The ring buffer header is special. We must manually up keep it. 29 * The ring buffer header is special. We must manually up keep it.
28 */ 30 */
@@ -449,6 +451,7 @@ struct ring_buffer_per_cpu {
449 raw_spinlock_t reader_lock; /* serialize readers */ 451 raw_spinlock_t reader_lock; /* serialize readers */
450 arch_spinlock_t lock; 452 arch_spinlock_t lock;
451 struct lock_class_key lock_key; 453 struct lock_class_key lock_key;
454 unsigned int nr_pages;
452 struct list_head *pages; 455 struct list_head *pages;
453 struct buffer_page *head_page; /* read from head */ 456 struct buffer_page *head_page; /* read from head */
454 struct buffer_page *tail_page; /* write to tail */ 457 struct buffer_page *tail_page; /* write to tail */
@@ -466,13 +469,18 @@ struct ring_buffer_per_cpu {
466 unsigned long read_bytes; 469 unsigned long read_bytes;
467 u64 write_stamp; 470 u64 write_stamp;
468 u64 read_stamp; 471 u64 read_stamp;
472 /* ring buffer pages to update, > 0 to add, < 0 to remove */
473 int nr_pages_to_update;
474 struct list_head new_pages; /* new pages to add */
475 struct work_struct update_pages_work;
476 struct completion update_done;
469}; 477};
470 478
471struct ring_buffer { 479struct ring_buffer {
472 unsigned pages;
473 unsigned flags; 480 unsigned flags;
474 int cpus; 481 int cpus;
475 atomic_t record_disabled; 482 atomic_t record_disabled;
483 atomic_t resize_disabled;
476 cpumask_var_t cpumask; 484 cpumask_var_t cpumask;
477 485
478 struct lock_class_key *reader_lock_key; 486 struct lock_class_key *reader_lock_key;
@@ -937,6 +945,10 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
937 struct list_head *head = cpu_buffer->pages; 945 struct list_head *head = cpu_buffer->pages;
938 struct buffer_page *bpage, *tmp; 946 struct buffer_page *bpage, *tmp;
939 947
948 /* Reset the head page if it exists */
949 if (cpu_buffer->head_page)
950 rb_set_head_page(cpu_buffer);
951
940 rb_head_page_deactivate(cpu_buffer); 952 rb_head_page_deactivate(cpu_buffer);
941 953
942 if (RB_WARN_ON(cpu_buffer, head->next->prev != head)) 954 if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
@@ -963,14 +975,10 @@ static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
963 return 0; 975 return 0;
964} 976}
965 977
966static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, 978static int __rb_allocate_pages(int nr_pages, struct list_head *pages, int cpu)
967 unsigned nr_pages)
968{ 979{
980 int i;
969 struct buffer_page *bpage, *tmp; 981 struct buffer_page *bpage, *tmp;
970 LIST_HEAD(pages);
971 unsigned i;
972
973 WARN_ON(!nr_pages);
974 982
975 for (i = 0; i < nr_pages; i++) { 983 for (i = 0; i < nr_pages; i++) {
976 struct page *page; 984 struct page *page;
@@ -981,15 +989,13 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
981 */ 989 */
982 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), 990 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
983 GFP_KERNEL | __GFP_NORETRY, 991 GFP_KERNEL | __GFP_NORETRY,
984 cpu_to_node(cpu_buffer->cpu)); 992 cpu_to_node(cpu));
985 if (!bpage) 993 if (!bpage)
986 goto free_pages; 994 goto free_pages;
987 995
988 rb_check_bpage(cpu_buffer, bpage); 996 list_add(&bpage->list, pages);
989 997
990 list_add(&bpage->list, &pages); 998 page = alloc_pages_node(cpu_to_node(cpu),
991
992 page = alloc_pages_node(cpu_to_node(cpu_buffer->cpu),
993 GFP_KERNEL | __GFP_NORETRY, 0); 999 GFP_KERNEL | __GFP_NORETRY, 0);
994 if (!page) 1000 if (!page)
995 goto free_pages; 1001 goto free_pages;
@@ -997,6 +1003,27 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
997 rb_init_page(bpage->page); 1003 rb_init_page(bpage->page);
998 } 1004 }
999 1005
1006 return 0;
1007
1008free_pages:
1009 list_for_each_entry_safe(bpage, tmp, pages, list) {
1010 list_del_init(&bpage->list);
1011 free_buffer_page(bpage);
1012 }
1013
1014 return -ENOMEM;
1015}
1016
1017static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1018 unsigned nr_pages)
1019{
1020 LIST_HEAD(pages);
1021
1022 WARN_ON(!nr_pages);
1023
1024 if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))
1025 return -ENOMEM;
1026
1000 /* 1027 /*
1001 * The ring buffer page list is a circular list that does not 1028 * The ring buffer page list is a circular list that does not
1002 * start and end with a list head. All page list items point to 1029 * start and end with a list head. All page list items point to
@@ -1005,20 +1032,15 @@ static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1005 cpu_buffer->pages = pages.next; 1032 cpu_buffer->pages = pages.next;
1006 list_del(&pages); 1033 list_del(&pages);
1007 1034
1035 cpu_buffer->nr_pages = nr_pages;
1036
1008 rb_check_pages(cpu_buffer); 1037 rb_check_pages(cpu_buffer);
1009 1038
1010 return 0; 1039 return 0;
1011
1012 free_pages:
1013 list_for_each_entry_safe(bpage, tmp, &pages, list) {
1014 list_del_init(&bpage->list);
1015 free_buffer_page(bpage);
1016 }
1017 return -ENOMEM;
1018} 1040}
1019 1041
1020static struct ring_buffer_per_cpu * 1042static struct ring_buffer_per_cpu *
1021rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu) 1043rb_allocate_cpu_buffer(struct ring_buffer *buffer, int nr_pages, int cpu)
1022{ 1044{
1023 struct ring_buffer_per_cpu *cpu_buffer; 1045 struct ring_buffer_per_cpu *cpu_buffer;
1024 struct buffer_page *bpage; 1046 struct buffer_page *bpage;
@@ -1035,6 +1057,8 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
1035 raw_spin_lock_init(&cpu_buffer->reader_lock); 1057 raw_spin_lock_init(&cpu_buffer->reader_lock);
1036 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); 1058 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
1037 cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED; 1059 cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1060 INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);
1061 init_completion(&cpu_buffer->update_done);
1038 1062
1039 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()), 1063 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
1040 GFP_KERNEL, cpu_to_node(cpu)); 1064 GFP_KERNEL, cpu_to_node(cpu));
@@ -1052,7 +1076,7 @@ rb_allocate_cpu_buffer(struct ring_buffer *buffer, int cpu)
1052 1076
1053 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 1077 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
1054 1078
1055 ret = rb_allocate_pages(cpu_buffer, buffer->pages); 1079 ret = rb_allocate_pages(cpu_buffer, nr_pages);
1056 if (ret < 0) 1080 if (ret < 0)
1057 goto fail_free_reader; 1081 goto fail_free_reader;
1058 1082
@@ -1113,7 +1137,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1113{ 1137{
1114 struct ring_buffer *buffer; 1138 struct ring_buffer *buffer;
1115 int bsize; 1139 int bsize;
1116 int cpu; 1140 int cpu, nr_pages;
1117 1141
1118 /* keep it in its own cache line */ 1142 /* keep it in its own cache line */
1119 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), 1143 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
@@ -1124,14 +1148,14 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1124 if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) 1148 if (!alloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
1125 goto fail_free_buffer; 1149 goto fail_free_buffer;
1126 1150
1127 buffer->pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 1151 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1128 buffer->flags = flags; 1152 buffer->flags = flags;
1129 buffer->clock = trace_clock_local; 1153 buffer->clock = trace_clock_local;
1130 buffer->reader_lock_key = key; 1154 buffer->reader_lock_key = key;
1131 1155
1132 /* need at least two pages */ 1156 /* need at least two pages */
1133 if (buffer->pages < 2) 1157 if (nr_pages < 2)
1134 buffer->pages = 2; 1158 nr_pages = 2;
1135 1159
1136 /* 1160 /*
1137 * In case of non-hotplug cpu, if the ring-buffer is allocated 1161 * In case of non-hotplug cpu, if the ring-buffer is allocated
@@ -1154,7 +1178,7 @@ struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1154 1178
1155 for_each_buffer_cpu(buffer, cpu) { 1179 for_each_buffer_cpu(buffer, cpu) {
1156 buffer->buffers[cpu] = 1180 buffer->buffers[cpu] =
1157 rb_allocate_cpu_buffer(buffer, cpu); 1181 rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
1158 if (!buffer->buffers[cpu]) 1182 if (!buffer->buffers[cpu])
1159 goto fail_free_buffers; 1183 goto fail_free_buffers;
1160 } 1184 }
@@ -1222,58 +1246,222 @@ void ring_buffer_set_clock(struct ring_buffer *buffer,
1222 1246
1223static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer); 1247static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
1224 1248
1225static void 1249static inline unsigned long rb_page_entries(struct buffer_page *bpage)
1226rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned nr_pages)
1227{ 1250{
1228 struct buffer_page *bpage; 1251 return local_read(&bpage->entries) & RB_WRITE_MASK;
1229 struct list_head *p; 1252}
1230 unsigned i; 1253
1254static inline unsigned long rb_page_write(struct buffer_page *bpage)
1255{
1256 return local_read(&bpage->write) & RB_WRITE_MASK;
1257}
1258
1259static int
1260rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned int nr_pages)
1261{
1262 struct list_head *tail_page, *to_remove, *next_page;
1263 struct buffer_page *to_remove_page, *tmp_iter_page;
1264 struct buffer_page *last_page, *first_page;
1265 unsigned int nr_removed;
1266 unsigned long head_bit;
1267 int page_entries;
1268
1269 head_bit = 0;
1231 1270
1232 raw_spin_lock_irq(&cpu_buffer->reader_lock); 1271 raw_spin_lock_irq(&cpu_buffer->reader_lock);
1233 rb_head_page_deactivate(cpu_buffer); 1272 atomic_inc(&cpu_buffer->record_disabled);
1273 /*
1274 * We don't race with the readers since we have acquired the reader
1275 * lock. We also don't race with writers after disabling recording.
1276 * This makes it easy to figure out the first and the last page to be
1277 * removed from the list. We unlink all the pages in between including
1278 * the first and last pages. This is done in a busy loop so that we
1279 * lose the least number of traces.
1280 * The pages are freed after we restart recording and unlock readers.
1281 */
1282 tail_page = &cpu_buffer->tail_page->list;
1234 1283
1235 for (i = 0; i < nr_pages; i++) { 1284 /*
1236 if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages))) 1285 * tail page might be on reader page, we remove the next page
1237 goto out; 1286 * from the ring buffer
1238 p = cpu_buffer->pages->next; 1287 */
1239 bpage = list_entry(p, struct buffer_page, list); 1288 if (cpu_buffer->tail_page == cpu_buffer->reader_page)
1240 list_del_init(&bpage->list); 1289 tail_page = rb_list_head(tail_page->next);
1241 free_buffer_page(bpage); 1290 to_remove = tail_page;
1291
1292 /* start of pages to remove */
1293 first_page = list_entry(rb_list_head(to_remove->next),
1294 struct buffer_page, list);
1295
1296 for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) {
1297 to_remove = rb_list_head(to_remove)->next;
1298 head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD;
1242 } 1299 }
1243 if (RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages)))
1244 goto out;
1245 1300
1246 rb_reset_cpu(cpu_buffer); 1301 next_page = rb_list_head(to_remove)->next;
1247 rb_check_pages(cpu_buffer);
1248 1302
1249out: 1303 /*
1304 * Now we remove all pages between tail_page and next_page.
1305 * Make sure that we have head_bit value preserved for the
1306 * next page
1307 */
1308 tail_page->next = (struct list_head *)((unsigned long)next_page |
1309 head_bit);
1310 next_page = rb_list_head(next_page);
1311 next_page->prev = tail_page;
1312
1313 /* make sure pages points to a valid page in the ring buffer */
1314 cpu_buffer->pages = next_page;
1315
1316 /* update head page */
1317 if (head_bit)
1318 cpu_buffer->head_page = list_entry(next_page,
1319 struct buffer_page, list);
1320
1321 /*
1322 * change read pointer to make sure any read iterators reset
1323 * themselves
1324 */
1325 cpu_buffer->read = 0;
1326
1327 /* pages are removed, resume tracing and then free the pages */
1328 atomic_dec(&cpu_buffer->record_disabled);
1250 raw_spin_unlock_irq(&cpu_buffer->reader_lock); 1329 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1330
1331 RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages));
1332
1333 /* last buffer page to remove */
1334 last_page = list_entry(rb_list_head(to_remove), struct buffer_page,
1335 list);
1336 tmp_iter_page = first_page;
1337
1338 do {
1339 to_remove_page = tmp_iter_page;
1340 rb_inc_page(cpu_buffer, &tmp_iter_page);
1341
1342 /* update the counters */
1343 page_entries = rb_page_entries(to_remove_page);
1344 if (page_entries) {
1345 /*
1346 * If something was added to this page, it was full
1347 * since it is not the tail page. So we deduct the
1348 * bytes consumed in ring buffer from here.
1349 * No need to update overruns, since this page is
1350 * deleted from ring buffer and its entries are
1351 * already accounted for.
1352 */
1353 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
1354 }
1355
1356 /*
1357 * We have already removed references to this list item, just
1358 * free up the buffer_page and its page
1359 */
1360 free_buffer_page(to_remove_page);
1361 nr_removed--;
1362
1363 } while (to_remove_page != last_page);
1364
1365 RB_WARN_ON(cpu_buffer, nr_removed);
1366
1367 return nr_removed == 0;
1251} 1368}
1252 1369
1253static void 1370static int
1254rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer, 1371rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer)
1255 struct list_head *pages, unsigned nr_pages)
1256{ 1372{
1257 struct buffer_page *bpage; 1373 struct list_head *pages = &cpu_buffer->new_pages;
1258 struct list_head *p; 1374 int retries, success;
1259 unsigned i;
1260 1375
1261 raw_spin_lock_irq(&cpu_buffer->reader_lock); 1376 raw_spin_lock_irq(&cpu_buffer->reader_lock);
1262 rb_head_page_deactivate(cpu_buffer); 1377 /*
1378 * We are holding the reader lock, so the reader page won't be swapped
1379 * in the ring buffer. Now we are racing with the writer trying to
1380 * move head page and the tail page.
1381 * We are going to adapt the reader page update process where:
1382 * 1. We first splice the start and end of list of new pages between
1383 * the head page and its previous page.
1384 * 2. We cmpxchg the prev_page->next to point from head page to the
1385 * start of new pages list.
1386 * 3. Finally, we update the head->prev to the end of new list.
1387 *
1388 * We will try this process 10 times, to make sure that we don't keep
1389 * spinning.
1390 */
1391 retries = 10;
1392 success = 0;
1393 while (retries--) {
1394 struct list_head *head_page, *prev_page, *r;
1395 struct list_head *last_page, *first_page;
1396 struct list_head *head_page_with_bit;
1263 1397
1264 for (i = 0; i < nr_pages; i++) { 1398 head_page = &rb_set_head_page(cpu_buffer)->list;
1265 if (RB_WARN_ON(cpu_buffer, list_empty(pages))) 1399 prev_page = head_page->prev;
1266 goto out; 1400
1267 p = pages->next; 1401 first_page = pages->next;
1268 bpage = list_entry(p, struct buffer_page, list); 1402 last_page = pages->prev;
1269 list_del_init(&bpage->list); 1403
1270 list_add_tail(&bpage->list, cpu_buffer->pages); 1404 head_page_with_bit = (struct list_head *)
1405 ((unsigned long)head_page | RB_PAGE_HEAD);
1406
1407 last_page->next = head_page_with_bit;
1408 first_page->prev = prev_page;
1409
1410 r = cmpxchg(&prev_page->next, head_page_with_bit, first_page);
1411
1412 if (r == head_page_with_bit) {
1413 /*
1414 * yay, we replaced the page pointer to our new list,
1415 * now, we just have to update to head page's prev
1416 * pointer to point to end of list
1417 */
1418 head_page->prev = last_page;
1419 success = 1;
1420 break;
1421 }
1271 } 1422 }
1272 rb_reset_cpu(cpu_buffer);
1273 rb_check_pages(cpu_buffer);
1274 1423
1275out: 1424 if (success)
1425 INIT_LIST_HEAD(pages);
1426 /*
1427 * If we weren't successful in adding in new pages, warn and stop
1428 * tracing
1429 */
1430 RB_WARN_ON(cpu_buffer, !success);
1276 raw_spin_unlock_irq(&cpu_buffer->reader_lock); 1431 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1432
1433 /* free pages if they weren't inserted */
1434 if (!success) {
1435 struct buffer_page *bpage, *tmp;
1436 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
1437 list) {
1438 list_del_init(&bpage->list);
1439 free_buffer_page(bpage);
1440 }
1441 }
1442 return success;
1443}
1444
1445static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer)
1446{
1447 int success;
1448
1449 if (cpu_buffer->nr_pages_to_update > 0)
1450 success = rb_insert_pages(cpu_buffer);
1451 else
1452 success = rb_remove_pages(cpu_buffer,
1453 -cpu_buffer->nr_pages_to_update);
1454
1455 if (success)
1456 cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
1457}
1458
1459static void update_pages_handler(struct work_struct *work)
1460{
1461 struct ring_buffer_per_cpu *cpu_buffer = container_of(work,
1462 struct ring_buffer_per_cpu, update_pages_work);
1463 rb_update_pages(cpu_buffer);
1464 complete(&cpu_buffer->update_done);
1277} 1465}
1278 1466
1279/** 1467/**
@@ -1283,16 +1471,14 @@ out:
1283 * 1471 *
1284 * Minimum size is 2 * BUF_PAGE_SIZE. 1472 * Minimum size is 2 * BUF_PAGE_SIZE.
1285 * 1473 *
1286 * Returns -1 on failure. 1474 * Returns 0 on success and < 0 on failure.
1287 */ 1475 */
1288int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size) 1476int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
1477 int cpu_id)
1289{ 1478{
1290 struct ring_buffer_per_cpu *cpu_buffer; 1479 struct ring_buffer_per_cpu *cpu_buffer;
1291 unsigned nr_pages, rm_pages, new_pages; 1480 unsigned nr_pages;
1292 struct buffer_page *bpage, *tmp; 1481 int cpu, err = 0;
1293 unsigned long buffer_size;
1294 LIST_HEAD(pages);
1295 int i, cpu;
1296 1482
1297 /* 1483 /*
1298 * Always succeed at resizing a non-existent buffer: 1484 * Always succeed at resizing a non-existent buffer:
@@ -1302,113 +1488,154 @@ int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size)
1302 1488
1303 size = DIV_ROUND_UP(size, BUF_PAGE_SIZE); 1489 size = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1304 size *= BUF_PAGE_SIZE; 1490 size *= BUF_PAGE_SIZE;
1305 buffer_size = buffer->pages * BUF_PAGE_SIZE;
1306 1491
1307 /* we need a minimum of two pages */ 1492 /* we need a minimum of two pages */
1308 if (size < BUF_PAGE_SIZE * 2) 1493 if (size < BUF_PAGE_SIZE * 2)
1309 size = BUF_PAGE_SIZE * 2; 1494 size = BUF_PAGE_SIZE * 2;
1310 1495
1311 if (size == buffer_size) 1496 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1312 return size;
1313
1314 atomic_inc(&buffer->record_disabled);
1315 1497
1316 /* Make sure all writers are done with this buffer. */ 1498 /*
1317 synchronize_sched(); 1499 * Don't succeed if resizing is disabled, as a reader might be
1500 * manipulating the ring buffer and is expecting a sane state while
1501 * this is true.
1502 */
1503 if (atomic_read(&buffer->resize_disabled))
1504 return -EBUSY;
1318 1505
1506 /* prevent another thread from changing buffer sizes */
1319 mutex_lock(&buffer->mutex); 1507 mutex_lock(&buffer->mutex);
1320 get_online_cpus();
1321
1322 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1323 1508
1324 if (size < buffer_size) { 1509 if (cpu_id == RING_BUFFER_ALL_CPUS) {
1510 /* calculate the pages to update */
1511 for_each_buffer_cpu(buffer, cpu) {
1512 cpu_buffer = buffer->buffers[cpu];
1325 1513
1326 /* easy case, just free pages */ 1514 cpu_buffer->nr_pages_to_update = nr_pages -
1327 if (RB_WARN_ON(buffer, nr_pages >= buffer->pages)) 1515 cpu_buffer->nr_pages;
1328 goto out_fail; 1516 /*
1517 * nothing more to do for removing pages or no update
1518 */
1519 if (cpu_buffer->nr_pages_to_update <= 0)
1520 continue;
1521 /*
1522 * to add pages, make sure all new pages can be
1523 * allocated without receiving ENOMEM
1524 */
1525 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1526 if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1527 &cpu_buffer->new_pages, cpu)) {
1528 /* not enough memory for new pages */
1529 err = -ENOMEM;
1530 goto out_err;
1531 }
1532 }
1329 1533
1330 rm_pages = buffer->pages - nr_pages; 1534 get_online_cpus();
1535 /*
1536 * Fire off all the required work handlers
1537 * We can't schedule on offline CPUs, but it's not necessary
1538 * since we can change their buffer sizes without any race.
1539 */
1540 for_each_buffer_cpu(buffer, cpu) {
1541 cpu_buffer = buffer->buffers[cpu];
1542 if (!cpu_buffer->nr_pages_to_update)
1543 continue;
1544
1545 if (cpu_online(cpu))
1546 schedule_work_on(cpu,
1547 &cpu_buffer->update_pages_work);
1548 else
1549 rb_update_pages(cpu_buffer);
1550 }
1331 1551
1552 /* wait for all the updates to complete */
1332 for_each_buffer_cpu(buffer, cpu) { 1553 for_each_buffer_cpu(buffer, cpu) {
1333 cpu_buffer = buffer->buffers[cpu]; 1554 cpu_buffer = buffer->buffers[cpu];
1334 rb_remove_pages(cpu_buffer, rm_pages); 1555 if (!cpu_buffer->nr_pages_to_update)
1556 continue;
1557
1558 if (cpu_online(cpu))
1559 wait_for_completion(&cpu_buffer->update_done);
1560 cpu_buffer->nr_pages_to_update = 0;
1335 } 1561 }
1336 goto out;
1337 }
1338 1562
1339 /* 1563 put_online_cpus();
1340 * This is a bit more difficult. We only want to add pages 1564 } else {
1341 * when we can allocate enough for all CPUs. We do this 1565 cpu_buffer = buffer->buffers[cpu_id];
1342 * by allocating all the pages and storing them on a local
1343 * link list. If we succeed in our allocation, then we
1344 * add these pages to the cpu_buffers. Otherwise we just free
1345 * them all and return -ENOMEM;
1346 */
1347 if (RB_WARN_ON(buffer, nr_pages <= buffer->pages))
1348 goto out_fail;
1349 1566
1350 new_pages = nr_pages - buffer->pages; 1567 if (nr_pages == cpu_buffer->nr_pages)
1568 goto out;
1351 1569
1352 for_each_buffer_cpu(buffer, cpu) { 1570 cpu_buffer->nr_pages_to_update = nr_pages -
1353 for (i = 0; i < new_pages; i++) { 1571 cpu_buffer->nr_pages;
1354 struct page *page; 1572
1355 /* 1573 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1356 * __GFP_NORETRY flag makes sure that the allocation 1574 if (cpu_buffer->nr_pages_to_update > 0 &&
1357 * fails gracefully without invoking oom-killer and 1575 __rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1358 * the system is not destabilized. 1576 &cpu_buffer->new_pages, cpu_id)) {
1359 */ 1577 err = -ENOMEM;
1360 bpage = kzalloc_node(ALIGN(sizeof(*bpage), 1578 goto out_err;
1361 cache_line_size()),
1362 GFP_KERNEL | __GFP_NORETRY,
1363 cpu_to_node(cpu));
1364 if (!bpage)
1365 goto free_pages;
1366 list_add(&bpage->list, &pages);
1367 page = alloc_pages_node(cpu_to_node(cpu),
1368 GFP_KERNEL | __GFP_NORETRY, 0);
1369 if (!page)
1370 goto free_pages;
1371 bpage->page = page_address(page);
1372 rb_init_page(bpage->page);
1373 } 1579 }
1374 }
1375 1580
1376 for_each_buffer_cpu(buffer, cpu) { 1581 get_online_cpus();
1377 cpu_buffer = buffer->buffers[cpu];
1378 rb_insert_pages(cpu_buffer, &pages, new_pages);
1379 }
1380 1582
1381 if (RB_WARN_ON(buffer, !list_empty(&pages))) 1583 if (cpu_online(cpu_id)) {
1382 goto out_fail; 1584 schedule_work_on(cpu_id,
1585 &cpu_buffer->update_pages_work);
1586 wait_for_completion(&cpu_buffer->update_done);
1587 } else
1588 rb_update_pages(cpu_buffer);
1589
1590 cpu_buffer->nr_pages_to_update = 0;
1591 put_online_cpus();
1592 }
1383 1593
1384 out: 1594 out:
1385 buffer->pages = nr_pages; 1595 /*
1386 put_online_cpus(); 1596 * The ring buffer resize can happen with the ring buffer
1597 * enabled, so that the update disturbs the tracing as little
1598 * as possible. But if the buffer is disabled, we do not need
1599 * to worry about that, and we can take the time to verify
1600 * that the buffer is not corrupt.
1601 */
1602 if (atomic_read(&buffer->record_disabled)) {
1603 atomic_inc(&buffer->record_disabled);
1604 /*
1605 * Even though the buffer was disabled, we must make sure
1606 * that it is truly disabled before calling rb_check_pages.
1607 * There could have been a race between checking
1608 * record_disable and incrementing it.
1609 */
1610 synchronize_sched();
1611 for_each_buffer_cpu(buffer, cpu) {
1612 cpu_buffer = buffer->buffers[cpu];
1613 rb_check_pages(cpu_buffer);
1614 }
1615 atomic_dec(&buffer->record_disabled);
1616 }
1617
1387 mutex_unlock(&buffer->mutex); 1618 mutex_unlock(&buffer->mutex);
1619 return size;
1388 1620
1389 atomic_dec(&buffer->record_disabled); 1621 out_err:
1622 for_each_buffer_cpu(buffer, cpu) {
1623 struct buffer_page *bpage, *tmp;
1390 1624
1391 return size; 1625 cpu_buffer = buffer->buffers[cpu];
1626 cpu_buffer->nr_pages_to_update = 0;
1392 1627
1393 free_pages: 1628 if (list_empty(&cpu_buffer->new_pages))
1394 list_for_each_entry_safe(bpage, tmp, &pages, list) { 1629 continue;
1395 list_del_init(&bpage->list);
1396 free_buffer_page(bpage);
1397 }
1398 put_online_cpus();
1399 mutex_unlock(&buffer->mutex);
1400 atomic_dec(&buffer->record_disabled);
1401 return -ENOMEM;
1402 1630
1403 /* 1631 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
1404 * Something went totally wrong, and we are too paranoid 1632 list) {
1405 * to even clean up the mess. 1633 list_del_init(&bpage->list);
1406 */ 1634 free_buffer_page(bpage);
1407 out_fail: 1635 }
1408 put_online_cpus(); 1636 }
1409 mutex_unlock(&buffer->mutex); 1637 mutex_unlock(&buffer->mutex);
1410 atomic_dec(&buffer->record_disabled); 1638 return err;
1411 return -1;
1412} 1639}
1413EXPORT_SYMBOL_GPL(ring_buffer_resize); 1640EXPORT_SYMBOL_GPL(ring_buffer_resize);
1414 1641
@@ -1447,21 +1674,11 @@ rb_iter_head_event(struct ring_buffer_iter *iter)
1447 return __rb_page_index(iter->head_page, iter->head); 1674 return __rb_page_index(iter->head_page, iter->head);
1448} 1675}
1449 1676
1450static inline unsigned long rb_page_write(struct buffer_page *bpage)
1451{
1452 return local_read(&bpage->write) & RB_WRITE_MASK;
1453}
1454
1455static inline unsigned rb_page_commit(struct buffer_page *bpage) 1677static inline unsigned rb_page_commit(struct buffer_page *bpage)
1456{ 1678{
1457 return local_read(&bpage->page->commit); 1679 return local_read(&bpage->page->commit);
1458} 1680}
1459 1681
1460static inline unsigned long rb_page_entries(struct buffer_page *bpage)
1461{
1462 return local_read(&bpage->entries) & RB_WRITE_MASK;
1463}
1464
1465/* Size is determined by what has been committed */ 1682/* Size is determined by what has been committed */
1466static inline unsigned rb_page_size(struct buffer_page *bpage) 1683static inline unsigned rb_page_size(struct buffer_page *bpage)
1467{ 1684{
@@ -1510,7 +1727,7 @@ rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
1510 * assign the commit to the tail. 1727 * assign the commit to the tail.
1511 */ 1728 */
1512 again: 1729 again:
1513 max_count = cpu_buffer->buffer->pages * 100; 1730 max_count = cpu_buffer->nr_pages * 100;
1514 1731
1515 while (cpu_buffer->commit_page != cpu_buffer->tail_page) { 1732 while (cpu_buffer->commit_page != cpu_buffer->tail_page) {
1516 if (RB_WARN_ON(cpu_buffer, !(--max_count))) 1733 if (RB_WARN_ON(cpu_buffer, !(--max_count)))
@@ -3486,6 +3703,7 @@ ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)
3486 3703
3487 iter->cpu_buffer = cpu_buffer; 3704 iter->cpu_buffer = cpu_buffer;
3488 3705
3706 atomic_inc(&buffer->resize_disabled);
3489 atomic_inc(&cpu_buffer->record_disabled); 3707 atomic_inc(&cpu_buffer->record_disabled);
3490 3708
3491 return iter; 3709 return iter;
@@ -3548,7 +3766,14 @@ ring_buffer_read_finish(struct ring_buffer_iter *iter)
3548{ 3766{
3549 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer; 3767 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
3550 3768
3769 /*
3770 * Ring buffer is disabled from recording, here's a good place
3771 * to check the integrity of the ring buffer.
3772 */
3773 rb_check_pages(cpu_buffer);
3774
3551 atomic_dec(&cpu_buffer->record_disabled); 3775 atomic_dec(&cpu_buffer->record_disabled);
3776 atomic_dec(&cpu_buffer->buffer->resize_disabled);
3552 kfree(iter); 3777 kfree(iter);
3553} 3778}
3554EXPORT_SYMBOL_GPL(ring_buffer_read_finish); 3779EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
@@ -3588,9 +3813,18 @@ EXPORT_SYMBOL_GPL(ring_buffer_read);
3588 * ring_buffer_size - return the size of the ring buffer (in bytes) 3813 * ring_buffer_size - return the size of the ring buffer (in bytes)
3589 * @buffer: The ring buffer. 3814 * @buffer: The ring buffer.
3590 */ 3815 */
3591unsigned long ring_buffer_size(struct ring_buffer *buffer) 3816unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
3592{ 3817{
3593 return BUF_PAGE_SIZE * buffer->pages; 3818 /*
3819 * Earlier, this method returned
3820 * BUF_PAGE_SIZE * buffer->nr_pages
3821 * Since the nr_pages field is now removed, we have converted this to
3822 * return the per cpu buffer value.
3823 */
3824 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3825 return 0;
3826
3827 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
3594} 3828}
3595EXPORT_SYMBOL_GPL(ring_buffer_size); 3829EXPORT_SYMBOL_GPL(ring_buffer_size);
3596 3830
@@ -3611,6 +3845,7 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
3611 cpu_buffer->commit_page = cpu_buffer->head_page; 3845 cpu_buffer->commit_page = cpu_buffer->head_page;
3612 3846
3613 INIT_LIST_HEAD(&cpu_buffer->reader_page->list); 3847 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
3848 INIT_LIST_HEAD(&cpu_buffer->new_pages);
3614 local_set(&cpu_buffer->reader_page->write, 0); 3849 local_set(&cpu_buffer->reader_page->write, 0);
3615 local_set(&cpu_buffer->reader_page->entries, 0); 3850 local_set(&cpu_buffer->reader_page->entries, 0);
3616 local_set(&cpu_buffer->reader_page->page->commit, 0); 3851 local_set(&cpu_buffer->reader_page->page->commit, 0);
@@ -3647,8 +3882,12 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
3647 if (!cpumask_test_cpu(cpu, buffer->cpumask)) 3882 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3648 return; 3883 return;
3649 3884
3885 atomic_inc(&buffer->resize_disabled);
3650 atomic_inc(&cpu_buffer->record_disabled); 3886 atomic_inc(&cpu_buffer->record_disabled);
3651 3887
3888 /* Make sure all commits have finished */
3889 synchronize_sched();
3890
3652 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); 3891 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3653 3892
3654 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing))) 3893 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
@@ -3664,6 +3903,7 @@ void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
3664 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); 3903 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3665 3904
3666 atomic_dec(&cpu_buffer->record_disabled); 3905 atomic_dec(&cpu_buffer->record_disabled);
3906 atomic_dec(&buffer->resize_disabled);
3667} 3907}
3668EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu); 3908EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
3669 3909
@@ -3765,8 +4005,11 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
3765 !cpumask_test_cpu(cpu, buffer_b->cpumask)) 4005 !cpumask_test_cpu(cpu, buffer_b->cpumask))
3766 goto out; 4006 goto out;
3767 4007
4008 cpu_buffer_a = buffer_a->buffers[cpu];
4009 cpu_buffer_b = buffer_b->buffers[cpu];
4010
3768 /* At least make sure the two buffers are somewhat the same */ 4011 /* At least make sure the two buffers are somewhat the same */
3769 if (buffer_a->pages != buffer_b->pages) 4012 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
3770 goto out; 4013 goto out;
3771 4014
3772 ret = -EAGAIN; 4015 ret = -EAGAIN;
@@ -3780,9 +4023,6 @@ int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
3780 if (atomic_read(&buffer_b->record_disabled)) 4023 if (atomic_read(&buffer_b->record_disabled))
3781 goto out; 4024 goto out;
3782 4025
3783 cpu_buffer_a = buffer_a->buffers[cpu];
3784 cpu_buffer_b = buffer_b->buffers[cpu];
3785
3786 if (atomic_read(&cpu_buffer_a->record_disabled)) 4026 if (atomic_read(&cpu_buffer_a->record_disabled))
3787 goto out; 4027 goto out;
3788 4028
@@ -4071,6 +4311,8 @@ static int rb_cpu_notify(struct notifier_block *self,
4071 struct ring_buffer *buffer = 4311 struct ring_buffer *buffer =
4072 container_of(self, struct ring_buffer, cpu_notify); 4312 container_of(self, struct ring_buffer, cpu_notify);
4073 long cpu = (long)hcpu; 4313 long cpu = (long)hcpu;
4314 int cpu_i, nr_pages_same;
4315 unsigned int nr_pages;
4074 4316
4075 switch (action) { 4317 switch (action) {
4076 case CPU_UP_PREPARE: 4318 case CPU_UP_PREPARE:
@@ -4078,8 +4320,23 @@ static int rb_cpu_notify(struct notifier_block *self,
4078 if (cpumask_test_cpu(cpu, buffer->cpumask)) 4320 if (cpumask_test_cpu(cpu, buffer->cpumask))
4079 return NOTIFY_OK; 4321 return NOTIFY_OK;
4080 4322
4323 nr_pages = 0;
4324 nr_pages_same = 1;
4325 /* check if all cpu sizes are same */
4326 for_each_buffer_cpu(buffer, cpu_i) {
4327 /* fill in the size from first enabled cpu */
4328 if (nr_pages == 0)
4329 nr_pages = buffer->buffers[cpu_i]->nr_pages;
4330 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
4331 nr_pages_same = 0;
4332 break;
4333 }
4334 }
4335 /* allocate minimum pages, user can later expand it */
4336 if (!nr_pages_same)
4337 nr_pages = 2;
4081 buffer->buffers[cpu] = 4338 buffer->buffers[cpu] =
4082 rb_allocate_cpu_buffer(buffer, cpu); 4339 rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
4083 if (!buffer->buffers[cpu]) { 4340 if (!buffer->buffers[cpu]) {
4084 WARN(1, "failed to allocate ring buffer on CPU %ld\n", 4341 WARN(1, "failed to allocate ring buffer on CPU %ld\n",
4085 cpu); 4342 cpu);
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index 2a22255c1010..68032c6177db 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -87,18 +87,6 @@ static int tracing_disabled = 1;
87 87
88DEFINE_PER_CPU(int, ftrace_cpu_disabled); 88DEFINE_PER_CPU(int, ftrace_cpu_disabled);
89 89
90static inline void ftrace_disable_cpu(void)
91{
92 preempt_disable();
93 __this_cpu_inc(ftrace_cpu_disabled);
94}
95
96static inline void ftrace_enable_cpu(void)
97{
98 __this_cpu_dec(ftrace_cpu_disabled);
99 preempt_enable();
100}
101
102cpumask_var_t __read_mostly tracing_buffer_mask; 90cpumask_var_t __read_mostly tracing_buffer_mask;
103 91
104/* 92/*
@@ -629,7 +617,6 @@ ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
629static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt) 617static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
630{ 618{
631 int len; 619 int len;
632 void *ret;
633 620
634 if (s->len <= s->readpos) 621 if (s->len <= s->readpos)
635 return -EBUSY; 622 return -EBUSY;
@@ -637,9 +624,7 @@ static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
637 len = s->len - s->readpos; 624 len = s->len - s->readpos;
638 if (cnt > len) 625 if (cnt > len)
639 cnt = len; 626 cnt = len;
640 ret = memcpy(buf, s->buffer + s->readpos, cnt); 627 memcpy(buf, s->buffer + s->readpos, cnt);
641 if (!ret)
642 return -EFAULT;
643 628
644 s->readpos += cnt; 629 s->readpos += cnt;
645 return cnt; 630 return cnt;
@@ -751,8 +736,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
751 736
752 arch_spin_lock(&ftrace_max_lock); 737 arch_spin_lock(&ftrace_max_lock);
753 738
754 ftrace_disable_cpu();
755
756 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu); 739 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
757 740
758 if (ret == -EBUSY) { 741 if (ret == -EBUSY) {
@@ -766,8 +749,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
766 "Failed to swap buffers due to commit in progress\n"); 749 "Failed to swap buffers due to commit in progress\n");
767 } 750 }
768 751
769 ftrace_enable_cpu();
770
771 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY); 752 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
772 753
773 __update_max_tr(tr, tsk, cpu); 754 __update_max_tr(tr, tsk, cpu);
@@ -782,8 +763,6 @@ update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
782 * Register a new plugin tracer. 763 * Register a new plugin tracer.
783 */ 764 */
784int register_tracer(struct tracer *type) 765int register_tracer(struct tracer *type)
785__releases(kernel_lock)
786__acquires(kernel_lock)
787{ 766{
788 struct tracer *t; 767 struct tracer *t;
789 int ret = 0; 768 int ret = 0;
@@ -841,7 +820,8 @@ __acquires(kernel_lock)
841 820
842 /* If we expanded the buffers, make sure the max is expanded too */ 821 /* If we expanded the buffers, make sure the max is expanded too */
843 if (ring_buffer_expanded && type->use_max_tr) 822 if (ring_buffer_expanded && type->use_max_tr)
844 ring_buffer_resize(max_tr.buffer, trace_buf_size); 823 ring_buffer_resize(max_tr.buffer, trace_buf_size,
824 RING_BUFFER_ALL_CPUS);
845 825
846 /* the test is responsible for initializing and enabling */ 826 /* the test is responsible for initializing and enabling */
847 pr_info("Testing tracer %s: ", type->name); 827 pr_info("Testing tracer %s: ", type->name);
@@ -857,7 +837,8 @@ __acquires(kernel_lock)
857 837
858 /* Shrink the max buffer again */ 838 /* Shrink the max buffer again */
859 if (ring_buffer_expanded && type->use_max_tr) 839 if (ring_buffer_expanded && type->use_max_tr)
860 ring_buffer_resize(max_tr.buffer, 1); 840 ring_buffer_resize(max_tr.buffer, 1,
841 RING_BUFFER_ALL_CPUS);
861 842
862 printk(KERN_CONT "PASSED\n"); 843 printk(KERN_CONT "PASSED\n");
863 } 844 }
@@ -917,13 +898,6 @@ out:
917 mutex_unlock(&trace_types_lock); 898 mutex_unlock(&trace_types_lock);
918} 899}
919 900
920static void __tracing_reset(struct ring_buffer *buffer, int cpu)
921{
922 ftrace_disable_cpu();
923 ring_buffer_reset_cpu(buffer, cpu);
924 ftrace_enable_cpu();
925}
926
927void tracing_reset(struct trace_array *tr, int cpu) 901void tracing_reset(struct trace_array *tr, int cpu)
928{ 902{
929 struct ring_buffer *buffer = tr->buffer; 903 struct ring_buffer *buffer = tr->buffer;
@@ -932,7 +906,7 @@ void tracing_reset(struct trace_array *tr, int cpu)
932 906
933 /* Make sure all commits have finished */ 907 /* Make sure all commits have finished */
934 synchronize_sched(); 908 synchronize_sched();
935 __tracing_reset(buffer, cpu); 909 ring_buffer_reset_cpu(buffer, cpu);
936 910
937 ring_buffer_record_enable(buffer); 911 ring_buffer_record_enable(buffer);
938} 912}
@@ -950,7 +924,7 @@ void tracing_reset_online_cpus(struct trace_array *tr)
950 tr->time_start = ftrace_now(tr->cpu); 924 tr->time_start = ftrace_now(tr->cpu);
951 925
952 for_each_online_cpu(cpu) 926 for_each_online_cpu(cpu)
953 __tracing_reset(buffer, cpu); 927 ring_buffer_reset_cpu(buffer, cpu);
954 928
955 ring_buffer_record_enable(buffer); 929 ring_buffer_record_enable(buffer);
956} 930}
@@ -1498,25 +1472,119 @@ static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1498 1472
1499#endif /* CONFIG_STACKTRACE */ 1473#endif /* CONFIG_STACKTRACE */
1500 1474
1475/* created for use with alloc_percpu */
1476struct trace_buffer_struct {
1477 char buffer[TRACE_BUF_SIZE];
1478};
1479
1480static struct trace_buffer_struct *trace_percpu_buffer;
1481static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1482static struct trace_buffer_struct *trace_percpu_irq_buffer;
1483static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1484
1485/*
1486 * The buffer used is dependent on the context. There is a per cpu
1487 * buffer for normal context, softirq contex, hard irq context and
1488 * for NMI context. Thise allows for lockless recording.
1489 *
1490 * Note, if the buffers failed to be allocated, then this returns NULL
1491 */
1492static char *get_trace_buf(void)
1493{
1494 struct trace_buffer_struct *percpu_buffer;
1495 struct trace_buffer_struct *buffer;
1496
1497 /*
1498 * If we have allocated per cpu buffers, then we do not
1499 * need to do any locking.
1500 */
1501 if (in_nmi())
1502 percpu_buffer = trace_percpu_nmi_buffer;
1503 else if (in_irq())
1504 percpu_buffer = trace_percpu_irq_buffer;
1505 else if (in_softirq())
1506 percpu_buffer = trace_percpu_sirq_buffer;
1507 else
1508 percpu_buffer = trace_percpu_buffer;
1509
1510 if (!percpu_buffer)
1511 return NULL;
1512
1513 buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1514
1515 return buffer->buffer;
1516}
1517
1518static int alloc_percpu_trace_buffer(void)
1519{
1520 struct trace_buffer_struct *buffers;
1521 struct trace_buffer_struct *sirq_buffers;
1522 struct trace_buffer_struct *irq_buffers;
1523 struct trace_buffer_struct *nmi_buffers;
1524
1525 buffers = alloc_percpu(struct trace_buffer_struct);
1526 if (!buffers)
1527 goto err_warn;
1528
1529 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1530 if (!sirq_buffers)
1531 goto err_sirq;
1532
1533 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1534 if (!irq_buffers)
1535 goto err_irq;
1536
1537 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1538 if (!nmi_buffers)
1539 goto err_nmi;
1540
1541 trace_percpu_buffer = buffers;
1542 trace_percpu_sirq_buffer = sirq_buffers;
1543 trace_percpu_irq_buffer = irq_buffers;
1544 trace_percpu_nmi_buffer = nmi_buffers;
1545
1546 return 0;
1547
1548 err_nmi:
1549 free_percpu(irq_buffers);
1550 err_irq:
1551 free_percpu(sirq_buffers);
1552 err_sirq:
1553 free_percpu(buffers);
1554 err_warn:
1555 WARN(1, "Could not allocate percpu trace_printk buffer");
1556 return -ENOMEM;
1557}
1558
1559void trace_printk_init_buffers(void)
1560{
1561 static int buffers_allocated;
1562
1563 if (buffers_allocated)
1564 return;
1565
1566 if (alloc_percpu_trace_buffer())
1567 return;
1568
1569 pr_info("ftrace: Allocated trace_printk buffers\n");
1570
1571 buffers_allocated = 1;
1572}
1573
1501/** 1574/**
1502 * trace_vbprintk - write binary msg to tracing buffer 1575 * trace_vbprintk - write binary msg to tracing buffer
1503 * 1576 *
1504 */ 1577 */
1505int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) 1578int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1506{ 1579{
1507 static arch_spinlock_t trace_buf_lock =
1508 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1509 static u32 trace_buf[TRACE_BUF_SIZE];
1510
1511 struct ftrace_event_call *call = &event_bprint; 1580 struct ftrace_event_call *call = &event_bprint;
1512 struct ring_buffer_event *event; 1581 struct ring_buffer_event *event;
1513 struct ring_buffer *buffer; 1582 struct ring_buffer *buffer;
1514 struct trace_array *tr = &global_trace; 1583 struct trace_array *tr = &global_trace;
1515 struct trace_array_cpu *data;
1516 struct bprint_entry *entry; 1584 struct bprint_entry *entry;
1517 unsigned long flags; 1585 unsigned long flags;
1518 int disable; 1586 char *tbuffer;
1519 int cpu, len = 0, size, pc; 1587 int len = 0, size, pc;
1520 1588
1521 if (unlikely(tracing_selftest_running || tracing_disabled)) 1589 if (unlikely(tracing_selftest_running || tracing_disabled))
1522 return 0; 1590 return 0;
@@ -1526,43 +1594,36 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1526 1594
1527 pc = preempt_count(); 1595 pc = preempt_count();
1528 preempt_disable_notrace(); 1596 preempt_disable_notrace();
1529 cpu = raw_smp_processor_id();
1530 data = tr->data[cpu];
1531 1597
1532 disable = atomic_inc_return(&data->disabled); 1598 tbuffer = get_trace_buf();
1533 if (unlikely(disable != 1)) 1599 if (!tbuffer) {
1600 len = 0;
1534 goto out; 1601 goto out;
1602 }
1535 1603
1536 /* Lockdep uses trace_printk for lock tracing */ 1604 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1537 local_irq_save(flags);
1538 arch_spin_lock(&trace_buf_lock);
1539 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1540 1605
1541 if (len > TRACE_BUF_SIZE || len < 0) 1606 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1542 goto out_unlock; 1607 goto out;
1543 1608
1609 local_save_flags(flags);
1544 size = sizeof(*entry) + sizeof(u32) * len; 1610 size = sizeof(*entry) + sizeof(u32) * len;
1545 buffer = tr->buffer; 1611 buffer = tr->buffer;
1546 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, 1612 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1547 flags, pc); 1613 flags, pc);
1548 if (!event) 1614 if (!event)
1549 goto out_unlock; 1615 goto out;
1550 entry = ring_buffer_event_data(event); 1616 entry = ring_buffer_event_data(event);
1551 entry->ip = ip; 1617 entry->ip = ip;
1552 entry->fmt = fmt; 1618 entry->fmt = fmt;
1553 1619
1554 memcpy(entry->buf, trace_buf, sizeof(u32) * len); 1620 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1555 if (!filter_check_discard(call, entry, buffer, event)) { 1621 if (!filter_check_discard(call, entry, buffer, event)) {
1556 ring_buffer_unlock_commit(buffer, event); 1622 ring_buffer_unlock_commit(buffer, event);
1557 ftrace_trace_stack(buffer, flags, 6, pc); 1623 ftrace_trace_stack(buffer, flags, 6, pc);
1558 } 1624 }
1559 1625
1560out_unlock:
1561 arch_spin_unlock(&trace_buf_lock);
1562 local_irq_restore(flags);
1563
1564out: 1626out:
1565 atomic_dec_return(&data->disabled);
1566 preempt_enable_notrace(); 1627 preempt_enable_notrace();
1567 unpause_graph_tracing(); 1628 unpause_graph_tracing();
1568 1629
@@ -1588,58 +1649,53 @@ int trace_array_printk(struct trace_array *tr,
1588int trace_array_vprintk(struct trace_array *tr, 1649int trace_array_vprintk(struct trace_array *tr,
1589 unsigned long ip, const char *fmt, va_list args) 1650 unsigned long ip, const char *fmt, va_list args)
1590{ 1651{
1591 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1592 static char trace_buf[TRACE_BUF_SIZE];
1593
1594 struct ftrace_event_call *call = &event_print; 1652 struct ftrace_event_call *call = &event_print;
1595 struct ring_buffer_event *event; 1653 struct ring_buffer_event *event;
1596 struct ring_buffer *buffer; 1654 struct ring_buffer *buffer;
1597 struct trace_array_cpu *data; 1655 int len = 0, size, pc;
1598 int cpu, len = 0, size, pc;
1599 struct print_entry *entry; 1656 struct print_entry *entry;
1600 unsigned long irq_flags; 1657 unsigned long flags;
1601 int disable; 1658 char *tbuffer;
1602 1659
1603 if (tracing_disabled || tracing_selftest_running) 1660 if (tracing_disabled || tracing_selftest_running)
1604 return 0; 1661 return 0;
1605 1662
1663 /* Don't pollute graph traces with trace_vprintk internals */
1664 pause_graph_tracing();
1665
1606 pc = preempt_count(); 1666 pc = preempt_count();
1607 preempt_disable_notrace(); 1667 preempt_disable_notrace();
1608 cpu = raw_smp_processor_id();
1609 data = tr->data[cpu];
1610 1668
1611 disable = atomic_inc_return(&data->disabled); 1669
1612 if (unlikely(disable != 1)) 1670 tbuffer = get_trace_buf();
1671 if (!tbuffer) {
1672 len = 0;
1613 goto out; 1673 goto out;
1674 }
1614 1675
1615 pause_graph_tracing(); 1676 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1616 raw_local_irq_save(irq_flags); 1677 if (len > TRACE_BUF_SIZE)
1617 arch_spin_lock(&trace_buf_lock); 1678 goto out;
1618 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1619 1679
1680 local_save_flags(flags);
1620 size = sizeof(*entry) + len + 1; 1681 size = sizeof(*entry) + len + 1;
1621 buffer = tr->buffer; 1682 buffer = tr->buffer;
1622 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size, 1683 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1623 irq_flags, pc); 1684 flags, pc);
1624 if (!event) 1685 if (!event)
1625 goto out_unlock; 1686 goto out;
1626 entry = ring_buffer_event_data(event); 1687 entry = ring_buffer_event_data(event);
1627 entry->ip = ip; 1688 entry->ip = ip;
1628 1689
1629 memcpy(&entry->buf, trace_buf, len); 1690 memcpy(&entry->buf, tbuffer, len);
1630 entry->buf[len] = '\0'; 1691 entry->buf[len] = '\0';
1631 if (!filter_check_discard(call, entry, buffer, event)) { 1692 if (!filter_check_discard(call, entry, buffer, event)) {
1632 ring_buffer_unlock_commit(buffer, event); 1693 ring_buffer_unlock_commit(buffer, event);
1633 ftrace_trace_stack(buffer, irq_flags, 6, pc); 1694 ftrace_trace_stack(buffer, flags, 6, pc);
1634 } 1695 }
1635
1636 out_unlock:
1637 arch_spin_unlock(&trace_buf_lock);
1638 raw_local_irq_restore(irq_flags);
1639 unpause_graph_tracing();
1640 out: 1696 out:
1641 atomic_dec_return(&data->disabled);
1642 preempt_enable_notrace(); 1697 preempt_enable_notrace();
1698 unpause_graph_tracing();
1643 1699
1644 return len; 1700 return len;
1645} 1701}
@@ -1652,14 +1708,9 @@ EXPORT_SYMBOL_GPL(trace_vprintk);
1652 1708
1653static void trace_iterator_increment(struct trace_iterator *iter) 1709static void trace_iterator_increment(struct trace_iterator *iter)
1654{ 1710{
1655 /* Don't allow ftrace to trace into the ring buffers */
1656 ftrace_disable_cpu();
1657
1658 iter->idx++; 1711 iter->idx++;
1659 if (iter->buffer_iter[iter->cpu]) 1712 if (iter->buffer_iter[iter->cpu])
1660 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL); 1713 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1661
1662 ftrace_enable_cpu();
1663} 1714}
1664 1715
1665static struct trace_entry * 1716static struct trace_entry *
@@ -1669,17 +1720,12 @@ peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1669 struct ring_buffer_event *event; 1720 struct ring_buffer_event *event;
1670 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu]; 1721 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1671 1722
1672 /* Don't allow ftrace to trace into the ring buffers */
1673 ftrace_disable_cpu();
1674
1675 if (buf_iter) 1723 if (buf_iter)
1676 event = ring_buffer_iter_peek(buf_iter, ts); 1724 event = ring_buffer_iter_peek(buf_iter, ts);
1677 else 1725 else
1678 event = ring_buffer_peek(iter->tr->buffer, cpu, ts, 1726 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1679 lost_events); 1727 lost_events);
1680 1728
1681 ftrace_enable_cpu();
1682
1683 if (event) { 1729 if (event) {
1684 iter->ent_size = ring_buffer_event_length(event); 1730 iter->ent_size = ring_buffer_event_length(event);
1685 return ring_buffer_event_data(event); 1731 return ring_buffer_event_data(event);
@@ -1769,11 +1815,8 @@ void *trace_find_next_entry_inc(struct trace_iterator *iter)
1769 1815
1770static void trace_consume(struct trace_iterator *iter) 1816static void trace_consume(struct trace_iterator *iter)
1771{ 1817{
1772 /* Don't allow ftrace to trace into the ring buffers */
1773 ftrace_disable_cpu();
1774 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts, 1818 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1775 &iter->lost_events); 1819 &iter->lost_events);
1776 ftrace_enable_cpu();
1777} 1820}
1778 1821
1779static void *s_next(struct seq_file *m, void *v, loff_t *pos) 1822static void *s_next(struct seq_file *m, void *v, loff_t *pos)
@@ -1862,16 +1905,12 @@ static void *s_start(struct seq_file *m, loff_t *pos)
1862 iter->cpu = 0; 1905 iter->cpu = 0;
1863 iter->idx = -1; 1906 iter->idx = -1;
1864 1907
1865 ftrace_disable_cpu();
1866
1867 if (cpu_file == TRACE_PIPE_ALL_CPU) { 1908 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1868 for_each_tracing_cpu(cpu) 1909 for_each_tracing_cpu(cpu)
1869 tracing_iter_reset(iter, cpu); 1910 tracing_iter_reset(iter, cpu);
1870 } else 1911 } else
1871 tracing_iter_reset(iter, cpu_file); 1912 tracing_iter_reset(iter, cpu_file);
1872 1913
1873 ftrace_enable_cpu();
1874
1875 iter->leftover = 0; 1914 iter->leftover = 0;
1876 for (p = iter; p && l < *pos; p = s_next(m, p, &l)) 1915 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1877 ; 1916 ;
@@ -2332,15 +2371,13 @@ static struct trace_iterator *
2332__tracing_open(struct inode *inode, struct file *file) 2371__tracing_open(struct inode *inode, struct file *file)
2333{ 2372{
2334 long cpu_file = (long) inode->i_private; 2373 long cpu_file = (long) inode->i_private;
2335 void *fail_ret = ERR_PTR(-ENOMEM);
2336 struct trace_iterator *iter; 2374 struct trace_iterator *iter;
2337 struct seq_file *m; 2375 int cpu;
2338 int cpu, ret;
2339 2376
2340 if (tracing_disabled) 2377 if (tracing_disabled)
2341 return ERR_PTR(-ENODEV); 2378 return ERR_PTR(-ENODEV);
2342 2379
2343 iter = kzalloc(sizeof(*iter), GFP_KERNEL); 2380 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2344 if (!iter) 2381 if (!iter)
2345 return ERR_PTR(-ENOMEM); 2382 return ERR_PTR(-ENOMEM);
2346 2383
@@ -2397,32 +2434,15 @@ __tracing_open(struct inode *inode, struct file *file)
2397 tracing_iter_reset(iter, cpu); 2434 tracing_iter_reset(iter, cpu);
2398 } 2435 }
2399 2436
2400 ret = seq_open(file, &tracer_seq_ops);
2401 if (ret < 0) {
2402 fail_ret = ERR_PTR(ret);
2403 goto fail_buffer;
2404 }
2405
2406 m = file->private_data;
2407 m->private = iter;
2408
2409 mutex_unlock(&trace_types_lock); 2437 mutex_unlock(&trace_types_lock);
2410 2438
2411 return iter; 2439 return iter;
2412 2440
2413 fail_buffer:
2414 for_each_tracing_cpu(cpu) {
2415 if (iter->buffer_iter[cpu])
2416 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2417 }
2418 free_cpumask_var(iter->started);
2419 tracing_start();
2420 fail: 2441 fail:
2421 mutex_unlock(&trace_types_lock); 2442 mutex_unlock(&trace_types_lock);
2422 kfree(iter->trace); 2443 kfree(iter->trace);
2423 kfree(iter); 2444 seq_release_private(inode, file);
2424 2445 return ERR_PTR(-ENOMEM);
2425 return fail_ret;
2426} 2446}
2427 2447
2428int tracing_open_generic(struct inode *inode, struct file *filp) 2448int tracing_open_generic(struct inode *inode, struct file *filp)
@@ -2458,11 +2478,10 @@ static int tracing_release(struct inode *inode, struct file *file)
2458 tracing_start(); 2478 tracing_start();
2459 mutex_unlock(&trace_types_lock); 2479 mutex_unlock(&trace_types_lock);
2460 2480
2461 seq_release(inode, file);
2462 mutex_destroy(&iter->mutex); 2481 mutex_destroy(&iter->mutex);
2463 free_cpumask_var(iter->started); 2482 free_cpumask_var(iter->started);
2464 kfree(iter->trace); 2483 kfree(iter->trace);
2465 kfree(iter); 2484 seq_release_private(inode, file);
2466 return 0; 2485 return 0;
2467} 2486}
2468 2487
@@ -2648,10 +2667,12 @@ tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2648 if (cpumask_test_cpu(cpu, tracing_cpumask) && 2667 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2649 !cpumask_test_cpu(cpu, tracing_cpumask_new)) { 2668 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2650 atomic_inc(&global_trace.data[cpu]->disabled); 2669 atomic_inc(&global_trace.data[cpu]->disabled);
2670 ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2651 } 2671 }
2652 if (!cpumask_test_cpu(cpu, tracing_cpumask) && 2672 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2653 cpumask_test_cpu(cpu, tracing_cpumask_new)) { 2673 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2654 atomic_dec(&global_trace.data[cpu]->disabled); 2674 atomic_dec(&global_trace.data[cpu]->disabled);
2675 ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2655 } 2676 }
2656 } 2677 }
2657 arch_spin_unlock(&ftrace_max_lock); 2678 arch_spin_unlock(&ftrace_max_lock);
@@ -2974,7 +2995,14 @@ int tracer_init(struct tracer *t, struct trace_array *tr)
2974 return t->init(tr); 2995 return t->init(tr);
2975} 2996}
2976 2997
2977static int __tracing_resize_ring_buffer(unsigned long size) 2998static void set_buffer_entries(struct trace_array *tr, unsigned long val)
2999{
3000 int cpu;
3001 for_each_tracing_cpu(cpu)
3002 tr->data[cpu]->entries = val;
3003}
3004
3005static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
2978{ 3006{
2979 int ret; 3007 int ret;
2980 3008
@@ -2985,19 +3013,32 @@ static int __tracing_resize_ring_buffer(unsigned long size)
2985 */ 3013 */
2986 ring_buffer_expanded = 1; 3014 ring_buffer_expanded = 1;
2987 3015
2988 ret = ring_buffer_resize(global_trace.buffer, size); 3016 ret = ring_buffer_resize(global_trace.buffer, size, cpu);
2989 if (ret < 0) 3017 if (ret < 0)
2990 return ret; 3018 return ret;
2991 3019
2992 if (!current_trace->use_max_tr) 3020 if (!current_trace->use_max_tr)
2993 goto out; 3021 goto out;
2994 3022
2995 ret = ring_buffer_resize(max_tr.buffer, size); 3023 ret = ring_buffer_resize(max_tr.buffer, size, cpu);
2996 if (ret < 0) { 3024 if (ret < 0) {
2997 int r; 3025 int r = 0;
3026
3027 if (cpu == RING_BUFFER_ALL_CPUS) {
3028 int i;
3029 for_each_tracing_cpu(i) {
3030 r = ring_buffer_resize(global_trace.buffer,
3031 global_trace.data[i]->entries,
3032 i);
3033 if (r < 0)
3034 break;
3035 }
3036 } else {
3037 r = ring_buffer_resize(global_trace.buffer,
3038 global_trace.data[cpu]->entries,
3039 cpu);
3040 }
2998 3041
2999 r = ring_buffer_resize(global_trace.buffer,
3000 global_trace.entries);
3001 if (r < 0) { 3042 if (r < 0) {
3002 /* 3043 /*
3003 * AARGH! We are left with different 3044 * AARGH! We are left with different
@@ -3019,43 +3060,39 @@ static int __tracing_resize_ring_buffer(unsigned long size)
3019 return ret; 3060 return ret;
3020 } 3061 }
3021 3062
3022 max_tr.entries = size; 3063 if (cpu == RING_BUFFER_ALL_CPUS)
3064 set_buffer_entries(&max_tr, size);
3065 else
3066 max_tr.data[cpu]->entries = size;
3067
3023 out: 3068 out:
3024 global_trace.entries = size; 3069 if (cpu == RING_BUFFER_ALL_CPUS)
3070 set_buffer_entries(&global_trace, size);
3071 else
3072 global_trace.data[cpu]->entries = size;
3025 3073
3026 return ret; 3074 return ret;
3027} 3075}
3028 3076
3029static ssize_t tracing_resize_ring_buffer(unsigned long size) 3077static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3030{ 3078{
3031 int cpu, ret = size; 3079 int ret = size;
3032 3080
3033 mutex_lock(&trace_types_lock); 3081 mutex_lock(&trace_types_lock);
3034 3082
3035 tracing_stop(); 3083 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3036 3084 /* make sure, this cpu is enabled in the mask */
3037 /* disable all cpu buffers */ 3085 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3038 for_each_tracing_cpu(cpu) { 3086 ret = -EINVAL;
3039 if (global_trace.data[cpu]) 3087 goto out;
3040 atomic_inc(&global_trace.data[cpu]->disabled); 3088 }
3041 if (max_tr.data[cpu])
3042 atomic_inc(&max_tr.data[cpu]->disabled);
3043 } 3089 }
3044 3090
3045 if (size != global_trace.entries) 3091 ret = __tracing_resize_ring_buffer(size, cpu_id);
3046 ret = __tracing_resize_ring_buffer(size);
3047
3048 if (ret < 0) 3092 if (ret < 0)
3049 ret = -ENOMEM; 3093 ret = -ENOMEM;
3050 3094
3051 for_each_tracing_cpu(cpu) { 3095out:
3052 if (global_trace.data[cpu])
3053 atomic_dec(&global_trace.data[cpu]->disabled);
3054 if (max_tr.data[cpu])
3055 atomic_dec(&max_tr.data[cpu]->disabled);
3056 }
3057
3058 tracing_start();
3059 mutex_unlock(&trace_types_lock); 3096 mutex_unlock(&trace_types_lock);
3060 3097
3061 return ret; 3098 return ret;
@@ -3078,7 +3115,8 @@ int tracing_update_buffers(void)
3078 3115
3079 mutex_lock(&trace_types_lock); 3116 mutex_lock(&trace_types_lock);
3080 if (!ring_buffer_expanded) 3117 if (!ring_buffer_expanded)
3081 ret = __tracing_resize_ring_buffer(trace_buf_size); 3118 ret = __tracing_resize_ring_buffer(trace_buf_size,
3119 RING_BUFFER_ALL_CPUS);
3082 mutex_unlock(&trace_types_lock); 3120 mutex_unlock(&trace_types_lock);
3083 3121
3084 return ret; 3122 return ret;
@@ -3102,7 +3140,8 @@ static int tracing_set_tracer(const char *buf)
3102 mutex_lock(&trace_types_lock); 3140 mutex_lock(&trace_types_lock);
3103 3141
3104 if (!ring_buffer_expanded) { 3142 if (!ring_buffer_expanded) {
3105 ret = __tracing_resize_ring_buffer(trace_buf_size); 3143 ret = __tracing_resize_ring_buffer(trace_buf_size,
3144 RING_BUFFER_ALL_CPUS);
3106 if (ret < 0) 3145 if (ret < 0)
3107 goto out; 3146 goto out;
3108 ret = 0; 3147 ret = 0;
@@ -3128,8 +3167,8 @@ static int tracing_set_tracer(const char *buf)
3128 * The max_tr ring buffer has some state (e.g. ring->clock) and 3167 * The max_tr ring buffer has some state (e.g. ring->clock) and
3129 * we want preserve it. 3168 * we want preserve it.
3130 */ 3169 */
3131 ring_buffer_resize(max_tr.buffer, 1); 3170 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3132 max_tr.entries = 1; 3171 set_buffer_entries(&max_tr, 1);
3133 } 3172 }
3134 destroy_trace_option_files(topts); 3173 destroy_trace_option_files(topts);
3135 3174
@@ -3137,10 +3176,17 @@ static int tracing_set_tracer(const char *buf)
3137 3176
3138 topts = create_trace_option_files(current_trace); 3177 topts = create_trace_option_files(current_trace);
3139 if (current_trace->use_max_tr) { 3178 if (current_trace->use_max_tr) {
3140 ret = ring_buffer_resize(max_tr.buffer, global_trace.entries); 3179 int cpu;
3141 if (ret < 0) 3180 /* we need to make per cpu buffer sizes equivalent */
3142 goto out; 3181 for_each_tracing_cpu(cpu) {
3143 max_tr.entries = global_trace.entries; 3182 ret = ring_buffer_resize(max_tr.buffer,
3183 global_trace.data[cpu]->entries,
3184 cpu);
3185 if (ret < 0)
3186 goto out;
3187 max_tr.data[cpu]->entries =
3188 global_trace.data[cpu]->entries;
3189 }
3144 } 3190 }
3145 3191
3146 if (t->init) { 3192 if (t->init) {
@@ -3642,30 +3688,82 @@ out_err:
3642 goto out; 3688 goto out;
3643} 3689}
3644 3690
3691struct ftrace_entries_info {
3692 struct trace_array *tr;
3693 int cpu;
3694};
3695
3696static int tracing_entries_open(struct inode *inode, struct file *filp)
3697{
3698 struct ftrace_entries_info *info;
3699
3700 if (tracing_disabled)
3701 return -ENODEV;
3702
3703 info = kzalloc(sizeof(*info), GFP_KERNEL);
3704 if (!info)
3705 return -ENOMEM;
3706
3707 info->tr = &global_trace;
3708 info->cpu = (unsigned long)inode->i_private;
3709
3710 filp->private_data = info;
3711
3712 return 0;
3713}
3714
3645static ssize_t 3715static ssize_t
3646tracing_entries_read(struct file *filp, char __user *ubuf, 3716tracing_entries_read(struct file *filp, char __user *ubuf,
3647 size_t cnt, loff_t *ppos) 3717 size_t cnt, loff_t *ppos)
3648{ 3718{
3649 struct trace_array *tr = filp->private_data; 3719 struct ftrace_entries_info *info = filp->private_data;
3650 char buf[96]; 3720 struct trace_array *tr = info->tr;
3651 int r; 3721 char buf[64];
3722 int r = 0;
3723 ssize_t ret;
3652 3724
3653 mutex_lock(&trace_types_lock); 3725 mutex_lock(&trace_types_lock);
3654 if (!ring_buffer_expanded) 3726
3655 r = sprintf(buf, "%lu (expanded: %lu)\n", 3727 if (info->cpu == RING_BUFFER_ALL_CPUS) {
3656 tr->entries >> 10, 3728 int cpu, buf_size_same;
3657 trace_buf_size >> 10); 3729 unsigned long size;
3658 else 3730
3659 r = sprintf(buf, "%lu\n", tr->entries >> 10); 3731 size = 0;
3732 buf_size_same = 1;
3733 /* check if all cpu sizes are same */
3734 for_each_tracing_cpu(cpu) {
3735 /* fill in the size from first enabled cpu */
3736 if (size == 0)
3737 size = tr->data[cpu]->entries;
3738 if (size != tr->data[cpu]->entries) {
3739 buf_size_same = 0;
3740 break;
3741 }
3742 }
3743
3744 if (buf_size_same) {
3745 if (!ring_buffer_expanded)
3746 r = sprintf(buf, "%lu (expanded: %lu)\n",
3747 size >> 10,
3748 trace_buf_size >> 10);
3749 else
3750 r = sprintf(buf, "%lu\n", size >> 10);
3751 } else
3752 r = sprintf(buf, "X\n");
3753 } else
3754 r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3755
3660 mutex_unlock(&trace_types_lock); 3756 mutex_unlock(&trace_types_lock);
3661 3757
3662 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); 3758 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3759 return ret;
3663} 3760}
3664 3761
3665static ssize_t 3762static ssize_t
3666tracing_entries_write(struct file *filp, const char __user *ubuf, 3763tracing_entries_write(struct file *filp, const char __user *ubuf,
3667 size_t cnt, loff_t *ppos) 3764 size_t cnt, loff_t *ppos)
3668{ 3765{
3766 struct ftrace_entries_info *info = filp->private_data;
3669 unsigned long val; 3767 unsigned long val;
3670 int ret; 3768 int ret;
3671 3769
@@ -3680,7 +3778,7 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
3680 /* value is in KB */ 3778 /* value is in KB */
3681 val <<= 10; 3779 val <<= 10;
3682 3780
3683 ret = tracing_resize_ring_buffer(val); 3781 ret = tracing_resize_ring_buffer(val, info->cpu);
3684 if (ret < 0) 3782 if (ret < 0)
3685 return ret; 3783 return ret;
3686 3784
@@ -3689,6 +3787,16 @@ tracing_entries_write(struct file *filp, const char __user *ubuf,
3689 return cnt; 3787 return cnt;
3690} 3788}
3691 3789
3790static int
3791tracing_entries_release(struct inode *inode, struct file *filp)
3792{
3793 struct ftrace_entries_info *info = filp->private_data;
3794
3795 kfree(info);
3796
3797 return 0;
3798}
3799
3692static ssize_t 3800static ssize_t
3693tracing_total_entries_read(struct file *filp, char __user *ubuf, 3801tracing_total_entries_read(struct file *filp, char __user *ubuf,
3694 size_t cnt, loff_t *ppos) 3802 size_t cnt, loff_t *ppos)
@@ -3700,7 +3808,7 @@ tracing_total_entries_read(struct file *filp, char __user *ubuf,
3700 3808
3701 mutex_lock(&trace_types_lock); 3809 mutex_lock(&trace_types_lock);
3702 for_each_tracing_cpu(cpu) { 3810 for_each_tracing_cpu(cpu) {
3703 size += tr->entries >> 10; 3811 size += tr->data[cpu]->entries >> 10;
3704 if (!ring_buffer_expanded) 3812 if (!ring_buffer_expanded)
3705 expanded_size += trace_buf_size >> 10; 3813 expanded_size += trace_buf_size >> 10;
3706 } 3814 }
@@ -3734,7 +3842,7 @@ tracing_free_buffer_release(struct inode *inode, struct file *filp)
3734 if (trace_flags & TRACE_ITER_STOP_ON_FREE) 3842 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3735 tracing_off(); 3843 tracing_off();
3736 /* resize the ring buffer to 0 */ 3844 /* resize the ring buffer to 0 */
3737 tracing_resize_ring_buffer(0); 3845 tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3738 3846
3739 return 0; 3847 return 0;
3740} 3848}
@@ -3749,14 +3857,14 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
3749 struct print_entry *entry; 3857 struct print_entry *entry;
3750 unsigned long irq_flags; 3858 unsigned long irq_flags;
3751 struct page *pages[2]; 3859 struct page *pages[2];
3860 void *map_page[2];
3752 int nr_pages = 1; 3861 int nr_pages = 1;
3753 ssize_t written; 3862 ssize_t written;
3754 void *page1;
3755 void *page2;
3756 int offset; 3863 int offset;
3757 int size; 3864 int size;
3758 int len; 3865 int len;
3759 int ret; 3866 int ret;
3867 int i;
3760 3868
3761 if (tracing_disabled) 3869 if (tracing_disabled)
3762 return -EINVAL; 3870 return -EINVAL;
@@ -3795,9 +3903,8 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
3795 goto out; 3903 goto out;
3796 } 3904 }
3797 3905
3798 page1 = kmap_atomic(pages[0]); 3906 for (i = 0; i < nr_pages; i++)
3799 if (nr_pages == 2) 3907 map_page[i] = kmap_atomic(pages[i]);
3800 page2 = kmap_atomic(pages[1]);
3801 3908
3802 local_save_flags(irq_flags); 3909 local_save_flags(irq_flags);
3803 size = sizeof(*entry) + cnt + 2; /* possible \n added */ 3910 size = sizeof(*entry) + cnt + 2; /* possible \n added */
@@ -3815,10 +3922,10 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
3815 3922
3816 if (nr_pages == 2) { 3923 if (nr_pages == 2) {
3817 len = PAGE_SIZE - offset; 3924 len = PAGE_SIZE - offset;
3818 memcpy(&entry->buf, page1 + offset, len); 3925 memcpy(&entry->buf, map_page[0] + offset, len);
3819 memcpy(&entry->buf[len], page2, cnt - len); 3926 memcpy(&entry->buf[len], map_page[1], cnt - len);
3820 } else 3927 } else
3821 memcpy(&entry->buf, page1 + offset, cnt); 3928 memcpy(&entry->buf, map_page[0] + offset, cnt);
3822 3929
3823 if (entry->buf[cnt - 1] != '\n') { 3930 if (entry->buf[cnt - 1] != '\n') {
3824 entry->buf[cnt] = '\n'; 3931 entry->buf[cnt] = '\n';
@@ -3833,11 +3940,10 @@ tracing_mark_write(struct file *filp, const char __user *ubuf,
3833 *fpos += written; 3940 *fpos += written;
3834 3941
3835 out_unlock: 3942 out_unlock:
3836 if (nr_pages == 2) 3943 for (i = 0; i < nr_pages; i++){
3837 kunmap_atomic(page2); 3944 kunmap_atomic(map_page[i]);
3838 kunmap_atomic(page1); 3945 put_page(pages[i]);
3839 while (nr_pages > 0) 3946 }
3840 put_page(pages[--nr_pages]);
3841 out: 3947 out:
3842 return written; 3948 return written;
3843} 3949}
@@ -3933,9 +4039,10 @@ static const struct file_operations tracing_pipe_fops = {
3933}; 4039};
3934 4040
3935static const struct file_operations tracing_entries_fops = { 4041static const struct file_operations tracing_entries_fops = {
3936 .open = tracing_open_generic, 4042 .open = tracing_entries_open,
3937 .read = tracing_entries_read, 4043 .read = tracing_entries_read,
3938 .write = tracing_entries_write, 4044 .write = tracing_entries_write,
4045 .release = tracing_entries_release,
3939 .llseek = generic_file_llseek, 4046 .llseek = generic_file_llseek,
3940}; 4047};
3941 4048
@@ -4367,6 +4474,9 @@ static void tracing_init_debugfs_percpu(long cpu)
4367 struct dentry *d_cpu; 4474 struct dentry *d_cpu;
4368 char cpu_dir[30]; /* 30 characters should be more than enough */ 4475 char cpu_dir[30]; /* 30 characters should be more than enough */
4369 4476
4477 if (!d_percpu)
4478 return;
4479
4370 snprintf(cpu_dir, 30, "cpu%ld", cpu); 4480 snprintf(cpu_dir, 30, "cpu%ld", cpu);
4371 d_cpu = debugfs_create_dir(cpu_dir, d_percpu); 4481 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4372 if (!d_cpu) { 4482 if (!d_cpu) {
@@ -4387,6 +4497,9 @@ static void tracing_init_debugfs_percpu(long cpu)
4387 4497
4388 trace_create_file("stats", 0444, d_cpu, 4498 trace_create_file("stats", 0444, d_cpu,
4389 (void *) cpu, &tracing_stats_fops); 4499 (void *) cpu, &tracing_stats_fops);
4500
4501 trace_create_file("buffer_size_kb", 0444, d_cpu,
4502 (void *) cpu, &tracing_entries_fops);
4390} 4503}
4391 4504
4392#ifdef CONFIG_FTRACE_SELFTEST 4505#ifdef CONFIG_FTRACE_SELFTEST
@@ -4718,7 +4831,7 @@ static __init int tracer_init_debugfs(void)
4718 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops); 4831 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4719 4832
4720 trace_create_file("buffer_size_kb", 0644, d_tracer, 4833 trace_create_file("buffer_size_kb", 0644, d_tracer,
4721 &global_trace, &tracing_entries_fops); 4834 (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
4722 4835
4723 trace_create_file("buffer_total_size_kb", 0444, d_tracer, 4836 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4724 &global_trace, &tracing_total_entries_fops); 4837 &global_trace, &tracing_total_entries_fops);
@@ -4957,6 +5070,10 @@ __init static int tracer_alloc_buffers(void)
4957 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL)) 5070 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4958 goto out_free_buffer_mask; 5071 goto out_free_buffer_mask;
4959 5072
5073 /* Only allocate trace_printk buffers if a trace_printk exists */
5074 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5075 trace_printk_init_buffers();
5076
4960 /* To save memory, keep the ring buffer size to its minimum */ 5077 /* To save memory, keep the ring buffer size to its minimum */
4961 if (ring_buffer_expanded) 5078 if (ring_buffer_expanded)
4962 ring_buf_size = trace_buf_size; 5079 ring_buf_size = trace_buf_size;
@@ -4975,7 +5092,6 @@ __init static int tracer_alloc_buffers(void)
4975 WARN_ON(1); 5092 WARN_ON(1);
4976 goto out_free_cpumask; 5093 goto out_free_cpumask;
4977 } 5094 }
4978 global_trace.entries = ring_buffer_size(global_trace.buffer);
4979 if (global_trace.buffer_disabled) 5095 if (global_trace.buffer_disabled)
4980 tracing_off(); 5096 tracing_off();
4981 5097
@@ -4988,7 +5104,6 @@ __init static int tracer_alloc_buffers(void)
4988 ring_buffer_free(global_trace.buffer); 5104 ring_buffer_free(global_trace.buffer);
4989 goto out_free_cpumask; 5105 goto out_free_cpumask;
4990 } 5106 }
4991 max_tr.entries = 1;
4992#endif 5107#endif
4993 5108
4994 /* Allocate the first page for all buffers */ 5109 /* Allocate the first page for all buffers */
@@ -4997,6 +5112,12 @@ __init static int tracer_alloc_buffers(void)
4997 max_tr.data[i] = &per_cpu(max_tr_data, i); 5112 max_tr.data[i] = &per_cpu(max_tr_data, i);
4998 } 5113 }
4999 5114
5115 set_buffer_entries(&global_trace,
5116 ring_buffer_size(global_trace.buffer, 0));
5117#ifdef CONFIG_TRACER_MAX_TRACE
5118 set_buffer_entries(&max_tr, 1);
5119#endif
5120
5000 trace_init_cmdlines(); 5121 trace_init_cmdlines();
5001 5122
5002 register_tracer(&nop_trace); 5123 register_tracer(&nop_trace);
diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h
index f95d65da6db8..6c6f7933eede 100644
--- a/kernel/trace/trace.h
+++ b/kernel/trace/trace.h
@@ -131,6 +131,7 @@ struct trace_array_cpu {
131 atomic_t disabled; 131 atomic_t disabled;
132 void *buffer_page; /* ring buffer spare */ 132 void *buffer_page; /* ring buffer spare */
133 133
134 unsigned long entries;
134 unsigned long saved_latency; 135 unsigned long saved_latency;
135 unsigned long critical_start; 136 unsigned long critical_start;
136 unsigned long critical_end; 137 unsigned long critical_end;
@@ -152,7 +153,6 @@ struct trace_array_cpu {
152 */ 153 */
153struct trace_array { 154struct trace_array {
154 struct ring_buffer *buffer; 155 struct ring_buffer *buffer;
155 unsigned long entries;
156 int cpu; 156 int cpu;
157 int buffer_disabled; 157 int buffer_disabled;
158 cycle_t time_start; 158 cycle_t time_start;
@@ -826,6 +826,8 @@ extern struct list_head ftrace_events;
826extern const char *__start___trace_bprintk_fmt[]; 826extern const char *__start___trace_bprintk_fmt[];
827extern const char *__stop___trace_bprintk_fmt[]; 827extern const char *__stop___trace_bprintk_fmt[];
828 828
829void trace_printk_init_buffers(void);
830
829#undef FTRACE_ENTRY 831#undef FTRACE_ENTRY
830#define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \ 832#define FTRACE_ENTRY(call, struct_name, id, tstruct, print, filter) \
831 extern struct ftrace_event_call \ 833 extern struct ftrace_event_call \
diff --git a/kernel/trace/trace_printk.c b/kernel/trace/trace_printk.c
index 6fd4ffd042f9..a9077c1b4ad3 100644
--- a/kernel/trace/trace_printk.c
+++ b/kernel/trace/trace_printk.c
@@ -51,6 +51,10 @@ void hold_module_trace_bprintk_format(const char **start, const char **end)
51 const char **iter; 51 const char **iter;
52 char *fmt; 52 char *fmt;
53 53
54 /* allocate the trace_printk per cpu buffers */
55 if (start != end)
56 trace_printk_init_buffers();
57
54 mutex_lock(&btrace_mutex); 58 mutex_lock(&btrace_mutex);
55 for (iter = start; iter < end; iter++) { 59 for (iter = start; iter < end; iter++) {
56 struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter); 60 struct trace_bprintk_fmt *tb_fmt = lookup_format(*iter);
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-12 15:14:57 -0400