aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/oprofile
diff options
context:
space:
mode:
authorIngo Molnar <mingo@elte.hu>2009-01-10 22:45:50 -0500
committerIngo Molnar <mingo@elte.hu>2009-01-10 22:45:50 -0500
commite8b722f487589a1f60ca27adc695494f188d404e (patch)
treebe3897dceb9b7c0949a8917ab11eea2752375e3b /drivers/oprofile
parent01d07820a0df6b6134c1bb75b1e84c9d0cdab3be (diff)
parentc59765042f53a79a7a65585042ff463b69cb248c (diff)
Merge commit 'v2.6.29-rc1' into irq/urgent
Diffstat (limited to 'drivers/oprofile')
-rw-r--r--drivers/oprofile/buffer_sync.c188
-rw-r--r--drivers/oprofile/cpu_buffer.c316
-rw-r--r--drivers/oprofile/cpu_buffer.h89
-rw-r--r--drivers/oprofile/event_buffer.c4
-rw-r--r--drivers/oprofile/oprof.c4
-rw-r--r--drivers/oprofile/oprof.h8
-rw-r--r--drivers/oprofile/oprofile_files.c27
7 files changed, 340 insertions, 296 deletions
diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c
index 65e8294a9e29..9da5a4b81133 100644
--- a/drivers/oprofile/buffer_sync.c
+++ b/drivers/oprofile/buffer_sync.c
@@ -1,11 +1,12 @@
1/** 1/**
2 * @file buffer_sync.c 2 * @file buffer_sync.c
3 * 3 *
4 * @remark Copyright 2002 OProfile authors 4 * @remark Copyright 2002-2009 OProfile authors
5 * @remark Read the file COPYING 5 * @remark Read the file COPYING
6 * 6 *
7 * @author John Levon <levon@movementarian.org> 7 * @author John Levon <levon@movementarian.org>
8 * @author Barry Kasindorf 8 * @author Barry Kasindorf
9 * @author Robert Richter <robert.richter@amd.com>
9 * 10 *
10 * This is the core of the buffer management. Each 11 * This is the core of the buffer management. Each
11 * CPU buffer is processed and entered into the 12 * CPU buffer is processed and entered into the
@@ -315,88 +316,73 @@ static void add_trace_begin(void)
315 add_event_entry(TRACE_BEGIN_CODE); 316 add_event_entry(TRACE_BEGIN_CODE);
316} 317}
317 318
318#ifdef CONFIG_OPROFILE_IBS 319static void add_data(struct op_entry *entry, struct mm_struct *mm)
319
320#define IBS_FETCH_CODE_SIZE 2
321#define IBS_OP_CODE_SIZE 5
322
323/*
324 * Add IBS fetch and op entries to event buffer
325 */
326static void add_ibs_begin(int cpu, int code, struct mm_struct *mm)
327{ 320{
328 unsigned long rip; 321 unsigned long code, pc, val;
329 int i, count; 322 unsigned long cookie;
330 unsigned long ibs_cookie = 0;
331 off_t offset; 323 off_t offset;
332 struct op_sample *sample;
333
334 sample = cpu_buffer_read_entry(cpu);
335 if (!sample)
336 goto Error;
337 rip = sample->eip;
338 324
339#ifdef __LP64__ 325 if (!op_cpu_buffer_get_data(entry, &code))
340 rip += sample->event << 32; 326 return;
341#endif 327 if (!op_cpu_buffer_get_data(entry, &pc))
328 return;
329 if (!op_cpu_buffer_get_size(entry))
330 return;
342 331
343 if (mm) { 332 if (mm) {
344 ibs_cookie = lookup_dcookie(mm, rip, &offset); 333 cookie = lookup_dcookie(mm, pc, &offset);
345 334
346 if (ibs_cookie == NO_COOKIE) 335 if (cookie == NO_COOKIE)
347 offset = rip; 336 offset = pc;
348 if (ibs_cookie == INVALID_COOKIE) { 337 if (cookie == INVALID_COOKIE) {
349 atomic_inc(&oprofile_stats.sample_lost_no_mapping); 338 atomic_inc(&oprofile_stats.sample_lost_no_mapping);
350 offset = rip; 339 offset = pc;
351 } 340 }
352 if (ibs_cookie != last_cookie) { 341 if (cookie != last_cookie) {
353 add_cookie_switch(ibs_cookie); 342 add_cookie_switch(cookie);
354 last_cookie = ibs_cookie; 343 last_cookie = cookie;
355 } 344 }
356 } else 345 } else
357 offset = rip; 346 offset = pc;
358 347
359 add_event_entry(ESCAPE_CODE); 348 add_event_entry(ESCAPE_CODE);
360 add_event_entry(code); 349 add_event_entry(code);
361 add_event_entry(offset); /* Offset from Dcookie */ 350 add_event_entry(offset); /* Offset from Dcookie */
362 351
363 /* we send the Dcookie offset, but send the raw Linear Add also*/ 352 while (op_cpu_buffer_get_data(entry, &val))
364 add_event_entry(sample->eip); 353 add_event_entry(val);
365 add_event_entry(sample->event);
366
367 if (code == IBS_FETCH_CODE)
368 count = IBS_FETCH_CODE_SIZE; /*IBS FETCH is 2 int64s*/
369 else
370 count = IBS_OP_CODE_SIZE; /*IBS OP is 5 int64s*/
371
372 for (i = 0; i < count; i++) {
373 sample = cpu_buffer_read_entry(cpu);
374 if (!sample)
375 goto Error;
376 add_event_entry(sample->eip);
377 add_event_entry(sample->event);
378 }
379
380 return;
381
382Error:
383 return;
384} 354}
385 355
386#endif 356static inline void add_sample_entry(unsigned long offset, unsigned long event)
387
388static void add_sample_entry(unsigned long offset, unsigned long event)
389{ 357{
390 add_event_entry(offset); 358 add_event_entry(offset);
391 add_event_entry(event); 359 add_event_entry(event);
392} 360}
393 361
394 362
395static int add_us_sample(struct mm_struct *mm, struct op_sample *s) 363/*
364 * Add a sample to the global event buffer. If possible the
365 * sample is converted into a persistent dentry/offset pair
366 * for later lookup from userspace. Return 0 on failure.
367 */
368static int
369add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel)
396{ 370{
397 unsigned long cookie; 371 unsigned long cookie;
398 off_t offset; 372 off_t offset;
399 373
374 if (in_kernel) {
375 add_sample_entry(s->eip, s->event);
376 return 1;
377 }
378
379 /* add userspace sample */
380
381 if (!mm) {
382 atomic_inc(&oprofile_stats.sample_lost_no_mm);
383 return 0;
384 }
385
400 cookie = lookup_dcookie(mm, s->eip, &offset); 386 cookie = lookup_dcookie(mm, s->eip, &offset);
401 387
402 if (cookie == INVALID_COOKIE) { 388 if (cookie == INVALID_COOKIE) {
@@ -415,25 +401,6 @@ static int add_us_sample(struct mm_struct *mm, struct op_sample *s)
415} 401}
416 402
417 403
418/* Add a sample to the global event buffer. If possible the
419 * sample is converted into a persistent dentry/offset pair
420 * for later lookup from userspace.
421 */
422static int
423add_sample(struct mm_struct *mm, struct op_sample *s, int in_kernel)
424{
425 if (in_kernel) {
426 add_sample_entry(s->eip, s->event);
427 return 1;
428 } else if (mm) {
429 return add_us_sample(mm, s);
430 } else {
431 atomic_inc(&oprofile_stats.sample_lost_no_mm);
432 }
433 return 0;
434}
435
436
437static void release_mm(struct mm_struct *mm) 404static void release_mm(struct mm_struct *mm)
438{ 405{
439 if (!mm) 406 if (!mm)
@@ -526,66 +493,69 @@ void sync_buffer(int cpu)
526{ 493{
527 struct mm_struct *mm = NULL; 494 struct mm_struct *mm = NULL;
528 struct mm_struct *oldmm; 495 struct mm_struct *oldmm;
496 unsigned long val;
529 struct task_struct *new; 497 struct task_struct *new;
530 unsigned long cookie = 0; 498 unsigned long cookie = 0;
531 int in_kernel = 1; 499 int in_kernel = 1;
532 sync_buffer_state state = sb_buffer_start; 500 sync_buffer_state state = sb_buffer_start;
533 unsigned int i; 501 unsigned int i;
534 unsigned long available; 502 unsigned long available;
503 unsigned long flags;
504 struct op_entry entry;
505 struct op_sample *sample;
535 506
536 mutex_lock(&buffer_mutex); 507 mutex_lock(&buffer_mutex);
537 508
538 add_cpu_switch(cpu); 509 add_cpu_switch(cpu);
539 510
540 cpu_buffer_reset(cpu); 511 op_cpu_buffer_reset(cpu);
541 available = cpu_buffer_entries(cpu); 512 available = op_cpu_buffer_entries(cpu);
542 513
543 for (i = 0; i < available; ++i) { 514 for (i = 0; i < available; ++i) {
544 struct op_sample *s = cpu_buffer_read_entry(cpu); 515 sample = op_cpu_buffer_read_entry(&entry, cpu);
545 if (!s) 516 if (!sample)
546 break; 517 break;
547 518
548 if (is_code(s->eip)) { 519 if (is_code(sample->eip)) {
549 switch (s->event) { 520 flags = sample->event;
550 case 0: 521 if (flags & TRACE_BEGIN) {
551 case CPU_IS_KERNEL: 522 state = sb_bt_start;
523 add_trace_begin();
524 }
525 if (flags & KERNEL_CTX_SWITCH) {
552 /* kernel/userspace switch */ 526 /* kernel/userspace switch */
553 in_kernel = s->event; 527 in_kernel = flags & IS_KERNEL;
554 if (state == sb_buffer_start) 528 if (state == sb_buffer_start)
555 state = sb_sample_start; 529 state = sb_sample_start;
556 add_kernel_ctx_switch(s->event); 530 add_kernel_ctx_switch(flags & IS_KERNEL);
557 break; 531 }
558 case CPU_TRACE_BEGIN: 532 if (flags & USER_CTX_SWITCH
559 state = sb_bt_start; 533 && op_cpu_buffer_get_data(&entry, &val)) {
560 add_trace_begin();
561 break;
562#ifdef CONFIG_OPROFILE_IBS
563 case IBS_FETCH_BEGIN:
564 state = sb_bt_start;
565 add_ibs_begin(cpu, IBS_FETCH_CODE, mm);
566 break;
567 case IBS_OP_BEGIN:
568 state = sb_bt_start;
569 add_ibs_begin(cpu, IBS_OP_CODE, mm);
570 break;
571#endif
572 default:
573 /* userspace context switch */ 534 /* userspace context switch */
535 new = (struct task_struct *)val;
574 oldmm = mm; 536 oldmm = mm;
575 new = (struct task_struct *)s->event;
576 release_mm(oldmm); 537 release_mm(oldmm);
577 mm = take_tasks_mm(new); 538 mm = take_tasks_mm(new);
578 if (mm != oldmm) 539 if (mm != oldmm)
579 cookie = get_exec_dcookie(mm); 540 cookie = get_exec_dcookie(mm);
580 add_user_ctx_switch(new, cookie); 541 add_user_ctx_switch(new, cookie);
581 break;
582 }
583 } else if (state >= sb_bt_start &&
584 !add_sample(mm, s, in_kernel)) {
585 if (state == sb_bt_start) {
586 state = sb_bt_ignore;
587 atomic_inc(&oprofile_stats.bt_lost_no_mapping);
588 } 542 }
543 if (op_cpu_buffer_get_size(&entry))
544 add_data(&entry, mm);
545 continue;
546 }
547
548 if (state < sb_bt_start)
549 /* ignore sample */
550 continue;
551
552 if (add_sample(mm, sample, in_kernel))
553 continue;
554
555 /* ignore backtraces if failed to add a sample */
556 if (state == sb_bt_start) {
557 state = sb_bt_ignore;
558 atomic_inc(&oprofile_stats.bt_lost_no_mapping);
589 } 559 }
590 } 560 }
591 release_mm(mm); 561 release_mm(mm);
diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c
index 61090969158f..2e03b6d796d3 100644
--- a/drivers/oprofile/cpu_buffer.c
+++ b/drivers/oprofile/cpu_buffer.c
@@ -1,11 +1,12 @@
1/** 1/**
2 * @file cpu_buffer.c 2 * @file cpu_buffer.c
3 * 3 *
4 * @remark Copyright 2002 OProfile authors 4 * @remark Copyright 2002-2009 OProfile authors
5 * @remark Read the file COPYING 5 * @remark Read the file COPYING
6 * 6 *
7 * @author John Levon <levon@movementarian.org> 7 * @author John Levon <levon@movementarian.org>
8 * @author Barry Kasindorf <barry.kasindorf@amd.com> 8 * @author Barry Kasindorf <barry.kasindorf@amd.com>
9 * @author Robert Richter <robert.richter@amd.com>
9 * 10 *
10 * Each CPU has a local buffer that stores PC value/event 11 * Each CPU has a local buffer that stores PC value/event
11 * pairs. We also log context switches when we notice them. 12 * pairs. We also log context switches when we notice them.
@@ -45,8 +46,8 @@
45 * can be changed to a single buffer solution when the ring buffer 46 * can be changed to a single buffer solution when the ring buffer
46 * access is implemented as non-locking atomic code. 47 * access is implemented as non-locking atomic code.
47 */ 48 */
48struct ring_buffer *op_ring_buffer_read; 49static struct ring_buffer *op_ring_buffer_read;
49struct ring_buffer *op_ring_buffer_write; 50static struct ring_buffer *op_ring_buffer_write;
50DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); 51DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer);
51 52
52static void wq_sync_buffer(struct work_struct *work); 53static void wq_sync_buffer(struct work_struct *work);
@@ -54,19 +55,9 @@ static void wq_sync_buffer(struct work_struct *work);
54#define DEFAULT_TIMER_EXPIRE (HZ / 10) 55#define DEFAULT_TIMER_EXPIRE (HZ / 10)
55static int work_enabled; 56static int work_enabled;
56 57
57void free_cpu_buffers(void)
58{
59 if (op_ring_buffer_read)
60 ring_buffer_free(op_ring_buffer_read);
61 op_ring_buffer_read = NULL;
62 if (op_ring_buffer_write)
63 ring_buffer_free(op_ring_buffer_write);
64 op_ring_buffer_write = NULL;
65}
66
67unsigned long oprofile_get_cpu_buffer_size(void) 58unsigned long oprofile_get_cpu_buffer_size(void)
68{ 59{
69 return fs_cpu_buffer_size; 60 return oprofile_cpu_buffer_size;
70} 61}
71 62
72void oprofile_cpu_buffer_inc_smpl_lost(void) 63void oprofile_cpu_buffer_inc_smpl_lost(void)
@@ -77,11 +68,21 @@ void oprofile_cpu_buffer_inc_smpl_lost(void)
77 cpu_buf->sample_lost_overflow++; 68 cpu_buf->sample_lost_overflow++;
78} 69}
79 70
71void free_cpu_buffers(void)
72{
73 if (op_ring_buffer_read)
74 ring_buffer_free(op_ring_buffer_read);
75 op_ring_buffer_read = NULL;
76 if (op_ring_buffer_write)
77 ring_buffer_free(op_ring_buffer_write);
78 op_ring_buffer_write = NULL;
79}
80
80int alloc_cpu_buffers(void) 81int alloc_cpu_buffers(void)
81{ 82{
82 int i; 83 int i;
83 84
84 unsigned long buffer_size = fs_cpu_buffer_size; 85 unsigned long buffer_size = oprofile_cpu_buffer_size;
85 86
86 op_ring_buffer_read = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS); 87 op_ring_buffer_read = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS);
87 if (!op_ring_buffer_read) 88 if (!op_ring_buffer_read)
@@ -97,8 +98,6 @@ int alloc_cpu_buffers(void)
97 b->last_is_kernel = -1; 98 b->last_is_kernel = -1;
98 b->tracing = 0; 99 b->tracing = 0;
99 b->buffer_size = buffer_size; 100 b->buffer_size = buffer_size;
100 b->tail_pos = 0;
101 b->head_pos = 0;
102 b->sample_received = 0; 101 b->sample_received = 0;
103 b->sample_lost_overflow = 0; 102 b->sample_lost_overflow = 0;
104 b->backtrace_aborted = 0; 103 b->backtrace_aborted = 0;
@@ -145,47 +144,156 @@ void end_cpu_work(void)
145 flush_scheduled_work(); 144 flush_scheduled_work();
146} 145}
147 146
148static inline int 147/*
149add_sample(struct oprofile_cpu_buffer *cpu_buf, 148 * This function prepares the cpu buffer to write a sample.
150 unsigned long pc, unsigned long event) 149 *
150 * Struct op_entry is used during operations on the ring buffer while
151 * struct op_sample contains the data that is stored in the ring
152 * buffer. Struct entry can be uninitialized. The function reserves a
153 * data array that is specified by size. Use
154 * op_cpu_buffer_write_commit() after preparing the sample. In case of
155 * errors a null pointer is returned, otherwise the pointer to the
156 * sample.
157 *
158 */
159struct op_sample
160*op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size)
161{
162 entry->event = ring_buffer_lock_reserve
163 (op_ring_buffer_write, sizeof(struct op_sample) +
164 size * sizeof(entry->sample->data[0]), &entry->irq_flags);
165 if (entry->event)
166 entry->sample = ring_buffer_event_data(entry->event);
167 else
168 entry->sample = NULL;
169
170 if (!entry->sample)
171 return NULL;
172
173 entry->size = size;
174 entry->data = entry->sample->data;
175
176 return entry->sample;
177}
178
179int op_cpu_buffer_write_commit(struct op_entry *entry)
180{
181 return ring_buffer_unlock_commit(op_ring_buffer_write, entry->event,
182 entry->irq_flags);
183}
184
185struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu)
186{
187 struct ring_buffer_event *e;
188 e = ring_buffer_consume(op_ring_buffer_read, cpu, NULL);
189 if (e)
190 goto event;
191 if (ring_buffer_swap_cpu(op_ring_buffer_read,
192 op_ring_buffer_write,
193 cpu))
194 return NULL;
195 e = ring_buffer_consume(op_ring_buffer_read, cpu, NULL);
196 if (e)
197 goto event;
198 return NULL;
199
200event:
201 entry->event = e;
202 entry->sample = ring_buffer_event_data(e);
203 entry->size = (ring_buffer_event_length(e) - sizeof(struct op_sample))
204 / sizeof(entry->sample->data[0]);
205 entry->data = entry->sample->data;
206 return entry->sample;
207}
208
209unsigned long op_cpu_buffer_entries(int cpu)
210{
211 return ring_buffer_entries_cpu(op_ring_buffer_read, cpu)
212 + ring_buffer_entries_cpu(op_ring_buffer_write, cpu);
213}
214
215static int
216op_add_code(struct oprofile_cpu_buffer *cpu_buf, unsigned long backtrace,
217 int is_kernel, struct task_struct *task)
151{ 218{
152 struct op_entry entry; 219 struct op_entry entry;
153 int ret; 220 struct op_sample *sample;
221 unsigned long flags;
222 int size;
223
224 flags = 0;
225
226 if (backtrace)
227 flags |= TRACE_BEGIN;
228
229 /* notice a switch from user->kernel or vice versa */
230 is_kernel = !!is_kernel;
231 if (cpu_buf->last_is_kernel != is_kernel) {
232 cpu_buf->last_is_kernel = is_kernel;
233 flags |= KERNEL_CTX_SWITCH;
234 if (is_kernel)
235 flags |= IS_KERNEL;
236 }
237
238 /* notice a task switch */
239 if (cpu_buf->last_task != task) {
240 cpu_buf->last_task = task;
241 flags |= USER_CTX_SWITCH;
242 }
243
244 if (!flags)
245 /* nothing to do */
246 return 0;
247
248 if (flags & USER_CTX_SWITCH)
249 size = 1;
250 else
251 size = 0;
252
253 sample = op_cpu_buffer_write_reserve(&entry, size);
254 if (!sample)
255 return -ENOMEM;
154 256
155 ret = cpu_buffer_write_entry(&entry); 257 sample->eip = ESCAPE_CODE;
156 if (ret) 258 sample->event = flags;
157 return ret;
158 259
159 entry.sample->eip = pc; 260 if (size)
160 entry.sample->event = event; 261 op_cpu_buffer_add_data(&entry, (unsigned long)task);
161 262
162 ret = cpu_buffer_write_commit(&entry); 263 op_cpu_buffer_write_commit(&entry);
163 if (ret)
164 return ret;
165 264
166 return 0; 265 return 0;
167} 266}
168 267
169static inline int 268static inline int
170add_code(struct oprofile_cpu_buffer *buffer, unsigned long value) 269op_add_sample(struct oprofile_cpu_buffer *cpu_buf,
270 unsigned long pc, unsigned long event)
171{ 271{
172 return add_sample(buffer, ESCAPE_CODE, value); 272 struct op_entry entry;
273 struct op_sample *sample;
274
275 sample = op_cpu_buffer_write_reserve(&entry, 0);
276 if (!sample)
277 return -ENOMEM;
278
279 sample->eip = pc;
280 sample->event = event;
281
282 return op_cpu_buffer_write_commit(&entry);
173} 283}
174 284
175/* This must be safe from any context. It's safe writing here 285/*
176 * because of the head/tail separation of the writer and reader 286 * This must be safe from any context.
177 * of the CPU buffer.
178 * 287 *
179 * is_kernel is needed because on some architectures you cannot 288 * is_kernel is needed because on some architectures you cannot
180 * tell if you are in kernel or user space simply by looking at 289 * tell if you are in kernel or user space simply by looking at
181 * pc. We tag this in the buffer by generating kernel enter/exit 290 * pc. We tag this in the buffer by generating kernel enter/exit
182 * events whenever is_kernel changes 291 * events whenever is_kernel changes
183 */ 292 */
184static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, 293static int
185 int is_kernel, unsigned long event) 294log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
295 unsigned long backtrace, int is_kernel, unsigned long event)
186{ 296{
187 struct task_struct *task;
188
189 cpu_buf->sample_received++; 297 cpu_buf->sample_received++;
190 298
191 if (pc == ESCAPE_CODE) { 299 if (pc == ESCAPE_CODE) {
@@ -193,25 +301,10 @@ static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc,
193 return 0; 301 return 0;
194 } 302 }
195 303
196 is_kernel = !!is_kernel; 304 if (op_add_code(cpu_buf, backtrace, is_kernel, current))
197 305 goto fail;
198 task = current;
199
200 /* notice a switch from user->kernel or vice versa */
201 if (cpu_buf->last_is_kernel != is_kernel) {
202 cpu_buf->last_is_kernel = is_kernel;
203 if (add_code(cpu_buf, is_kernel))
204 goto fail;
205 }
206
207 /* notice a task switch */
208 if (cpu_buf->last_task != task) {
209 cpu_buf->last_task = task;
210 if (add_code(cpu_buf, (unsigned long)task))
211 goto fail;
212 }
213 306
214 if (add_sample(cpu_buf, pc, event)) 307 if (op_add_sample(cpu_buf, pc, event))
215 goto fail; 308 goto fail;
216 309
217 return 1; 310 return 1;
@@ -221,109 +314,102 @@ fail:
221 return 0; 314 return 0;
222} 315}
223 316
224static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf) 317static inline void oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf)
225{ 318{
226 add_code(cpu_buf, CPU_TRACE_BEGIN);
227 cpu_buf->tracing = 1; 319 cpu_buf->tracing = 1;
228 return 1;
229} 320}
230 321
231static void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf) 322static inline void oprofile_end_trace(struct oprofile_cpu_buffer *cpu_buf)
232{ 323{
233 cpu_buf->tracing = 0; 324 cpu_buf->tracing = 0;
234} 325}
235 326
236void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs, 327static inline void
237 unsigned long event, int is_kernel) 328__oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
329 unsigned long event, int is_kernel)
238{ 330{
239 struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); 331 struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
240 332 unsigned long backtrace = oprofile_backtrace_depth;
241 if (!backtrace_depth) {
242 log_sample(cpu_buf, pc, is_kernel, event);
243 return;
244 }
245
246 if (!oprofile_begin_trace(cpu_buf))
247 return;
248 333
249 /* 334 /*
250 * if log_sample() fail we can't backtrace since we lost the 335 * if log_sample() fail we can't backtrace since we lost the
251 * source of this event 336 * source of this event
252 */ 337 */
253 if (log_sample(cpu_buf, pc, is_kernel, event)) 338 if (!log_sample(cpu_buf, pc, backtrace, is_kernel, event))
254 oprofile_ops.backtrace(regs, backtrace_depth); 339 /* failed */
340 return;
341
342 if (!backtrace)
343 return;
344
345 oprofile_begin_trace(cpu_buf);
346 oprofile_ops.backtrace(regs, backtrace);
255 oprofile_end_trace(cpu_buf); 347 oprofile_end_trace(cpu_buf);
256} 348}
257 349
350void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs,
351 unsigned long event, int is_kernel)
352{
353 __oprofile_add_ext_sample(pc, regs, event, is_kernel);
354}
355
258void oprofile_add_sample(struct pt_regs * const regs, unsigned long event) 356void oprofile_add_sample(struct pt_regs * const regs, unsigned long event)
259{ 357{
260 int is_kernel = !user_mode(regs); 358 int is_kernel = !user_mode(regs);
261 unsigned long pc = profile_pc(regs); 359 unsigned long pc = profile_pc(regs);
262 360
263 oprofile_add_ext_sample(pc, regs, event, is_kernel); 361 __oprofile_add_ext_sample(pc, regs, event, is_kernel);
264} 362}
265 363
266#ifdef CONFIG_OPROFILE_IBS 364/*
267 365 * Add samples with data to the ring buffer.
268#define MAX_IBS_SAMPLE_SIZE 14 366 *
269 367 * Use oprofile_add_data(&entry, val) to add data and
270void oprofile_add_ibs_sample(struct pt_regs * const regs, 368 * oprofile_write_commit(&entry) to commit the sample.
271 unsigned int * const ibs_sample, int ibs_code) 369 */
370void
371oprofile_write_reserve(struct op_entry *entry, struct pt_regs * const regs,
372 unsigned long pc, int code, int size)
272{ 373{
374 struct op_sample *sample;
273 int is_kernel = !user_mode(regs); 375 int is_kernel = !user_mode(regs);
274 struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); 376 struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
275 struct task_struct *task;
276 int fail = 0;
277 377
278 cpu_buf->sample_received++; 378 cpu_buf->sample_received++;
279 379
280 /* notice a switch from user->kernel or vice versa */ 380 /* no backtraces for samples with data */
281 if (cpu_buf->last_is_kernel != is_kernel) { 381 if (op_add_code(cpu_buf, 0, is_kernel, current))
282 if (add_code(cpu_buf, is_kernel)) 382 goto fail;
283 goto fail;
284 cpu_buf->last_is_kernel = is_kernel;
285 }
286
287 /* notice a task switch */
288 if (!is_kernel) {
289 task = current;
290 if (cpu_buf->last_task != task) {
291 if (add_code(cpu_buf, (unsigned long)task))
292 goto fail;
293 cpu_buf->last_task = task;
294 }
295 }
296
297 fail = fail || add_code(cpu_buf, ibs_code);
298 fail = fail || add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]);
299 fail = fail || add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]);
300 fail = fail || add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]);
301
302 if (ibs_code == IBS_OP_BEGIN) {
303 fail = fail || add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]);
304 fail = fail || add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]);
305 fail = fail || add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]);
306 }
307 383
308 if (fail) 384 sample = op_cpu_buffer_write_reserve(entry, size + 2);
385 if (!sample)
309 goto fail; 386 goto fail;
387 sample->eip = ESCAPE_CODE;
388 sample->event = 0; /* no flags */
310 389
311 if (backtrace_depth) 390 op_cpu_buffer_add_data(entry, code);
312 oprofile_ops.backtrace(regs, backtrace_depth); 391 op_cpu_buffer_add_data(entry, pc);
313 392
314 return; 393 return;
315 394
316fail: 395fail:
317 cpu_buf->sample_lost_overflow++; 396 cpu_buf->sample_lost_overflow++;
318 return;
319} 397}
320 398
321#endif 399int oprofile_add_data(struct op_entry *entry, unsigned long val)
400{
401 return op_cpu_buffer_add_data(entry, val);
402}
403
404int oprofile_write_commit(struct op_entry *entry)
405{
406 return op_cpu_buffer_write_commit(entry);
407}
322 408
323void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event) 409void oprofile_add_pc(unsigned long pc, int is_kernel, unsigned long event)
324{ 410{
325 struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); 411 struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer);
326 log_sample(cpu_buf, pc, is_kernel, event); 412 log_sample(cpu_buf, pc, 0, is_kernel, event);
327} 413}
328 414
329void oprofile_add_trace(unsigned long pc) 415void oprofile_add_trace(unsigned long pc)
@@ -340,7 +426,7 @@ void oprofile_add_trace(unsigned long pc)
340 if (pc == ESCAPE_CODE) 426 if (pc == ESCAPE_CODE)
341 goto fail; 427 goto fail;
342 428
343 if (add_sample(cpu_buf, pc, 0)) 429 if (op_add_sample(cpu_buf, pc, 0))
344 goto fail; 430 goto fail;
345 431
346 return; 432 return;
diff --git a/drivers/oprofile/cpu_buffer.h b/drivers/oprofile/cpu_buffer.h
index aacb0f0bc566..63f81c44846a 100644
--- a/drivers/oprofile/cpu_buffer.h
+++ b/drivers/oprofile/cpu_buffer.h
@@ -1,10 +1,11 @@
1/** 1/**
2 * @file cpu_buffer.h 2 * @file cpu_buffer.h
3 * 3 *
4 * @remark Copyright 2002 OProfile authors 4 * @remark Copyright 2002-2009 OProfile authors
5 * @remark Read the file COPYING 5 * @remark Read the file COPYING
6 * 6 *
7 * @author John Levon <levon@movementarian.org> 7 * @author John Levon <levon@movementarian.org>
8 * @author Robert Richter <robert.richter@amd.com>
8 */ 9 */
9 10
10#ifndef OPROFILE_CPU_BUFFER_H 11#ifndef OPROFILE_CPU_BUFFER_H
@@ -31,17 +32,12 @@ void end_cpu_work(void);
31struct op_sample { 32struct op_sample {
32 unsigned long eip; 33 unsigned long eip;
33 unsigned long event; 34 unsigned long event;
35 unsigned long data[0];
34}; 36};
35 37
36struct op_entry { 38struct op_entry;
37 struct ring_buffer_event *event;
38 struct op_sample *sample;
39 unsigned long irq_flags;
40};
41 39
42struct oprofile_cpu_buffer { 40struct oprofile_cpu_buffer {
43 volatile unsigned long head_pos;
44 volatile unsigned long tail_pos;
45 unsigned long buffer_size; 41 unsigned long buffer_size;
46 struct task_struct *last_task; 42 struct task_struct *last_task;
47 int last_is_kernel; 43 int last_is_kernel;
@@ -54,8 +50,6 @@ struct oprofile_cpu_buffer {
54 struct delayed_work work; 50 struct delayed_work work;
55}; 51};
56 52
57extern struct ring_buffer *op_ring_buffer_read;
58extern struct ring_buffer *op_ring_buffer_write;
59DECLARE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); 53DECLARE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer);
60 54
61/* 55/*
@@ -64,7 +58,7 @@ DECLARE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer);
64 * reset these to invalid values; the next sample collected will 58 * reset these to invalid values; the next sample collected will
65 * populate the buffer with proper values to initialize the buffer 59 * populate the buffer with proper values to initialize the buffer
66 */ 60 */
67static inline void cpu_buffer_reset(int cpu) 61static inline void op_cpu_buffer_reset(int cpu)
68{ 62{
69 struct oprofile_cpu_buffer *cpu_buf = &per_cpu(cpu_buffer, cpu); 63 struct oprofile_cpu_buffer *cpu_buf = &per_cpu(cpu_buffer, cpu);
70 64
@@ -72,55 +66,48 @@ static inline void cpu_buffer_reset(int cpu)
72 cpu_buf->last_task = NULL; 66 cpu_buf->last_task = NULL;
73} 67}
74 68
75static inline int cpu_buffer_write_entry(struct op_entry *entry) 69struct op_sample
76{ 70*op_cpu_buffer_write_reserve(struct op_entry *entry, unsigned long size);
77 entry->event = ring_buffer_lock_reserve(op_ring_buffer_write, 71int op_cpu_buffer_write_commit(struct op_entry *entry);
78 sizeof(struct op_sample), 72struct op_sample *op_cpu_buffer_read_entry(struct op_entry *entry, int cpu);
79 &entry->irq_flags); 73unsigned long op_cpu_buffer_entries(int cpu);
80 if (entry->event)
81 entry->sample = ring_buffer_event_data(entry->event);
82 else
83 entry->sample = NULL;
84
85 if (!entry->sample)
86 return -ENOMEM;
87
88 return 0;
89}
90 74
91static inline int cpu_buffer_write_commit(struct op_entry *entry) 75/* returns the remaining free size of data in the entry */
76static inline
77int op_cpu_buffer_add_data(struct op_entry *entry, unsigned long val)
92{ 78{
93 return ring_buffer_unlock_commit(op_ring_buffer_write, entry->event, 79 if (!entry->size)
94 entry->irq_flags); 80 return 0;
81 *entry->data = val;
82 entry->size--;
83 entry->data++;
84 return entry->size;
95} 85}
96 86
97static inline struct op_sample *cpu_buffer_read_entry(int cpu) 87/* returns the size of data in the entry */
88static inline
89int op_cpu_buffer_get_size(struct op_entry *entry)
98{ 90{
99 struct ring_buffer_event *e; 91 return entry->size;
100 e = ring_buffer_consume(op_ring_buffer_read, cpu, NULL);
101 if (e)
102 return ring_buffer_event_data(e);
103 if (ring_buffer_swap_cpu(op_ring_buffer_read,
104 op_ring_buffer_write,
105 cpu))
106 return NULL;
107 e = ring_buffer_consume(op_ring_buffer_read, cpu, NULL);
108 if (e)
109 return ring_buffer_event_data(e);
110 return NULL;
111} 92}
112 93
113/* "acquire" as many cpu buffer slots as we can */ 94/* returns 0 if empty or the size of data including the current value */
114static inline unsigned long cpu_buffer_entries(int cpu) 95static inline
96int op_cpu_buffer_get_data(struct op_entry *entry, unsigned long *val)
115{ 97{
116 return ring_buffer_entries_cpu(op_ring_buffer_read, cpu) 98 int size = entry->size;
117 + ring_buffer_entries_cpu(op_ring_buffer_write, cpu); 99 if (!size)
100 return 0;
101 *val = *entry->data;
102 entry->size--;
103 entry->data++;
104 return size;
118} 105}
119 106
120/* transient events for the CPU buffer -> event buffer */ 107/* extra data flags */
121#define CPU_IS_KERNEL 1 108#define KERNEL_CTX_SWITCH (1UL << 0)
122#define CPU_TRACE_BEGIN 2 109#define IS_KERNEL (1UL << 1)
123#define IBS_FETCH_BEGIN 3 110#define TRACE_BEGIN (1UL << 2)
124#define IBS_OP_BEGIN 4 111#define USER_CTX_SWITCH (1UL << 3)
125 112
126#endif /* OPROFILE_CPU_BUFFER_H */ 113#endif /* OPROFILE_CPU_BUFFER_H */
diff --git a/drivers/oprofile/event_buffer.c b/drivers/oprofile/event_buffer.c
index 191a3202cecc..2b7ae366ceb1 100644
--- a/drivers/oprofile/event_buffer.c
+++ b/drivers/oprofile/event_buffer.c
@@ -73,8 +73,8 @@ int alloc_event_buffer(void)
73 unsigned long flags; 73 unsigned long flags;
74 74
75 spin_lock_irqsave(&oprofilefs_lock, flags); 75 spin_lock_irqsave(&oprofilefs_lock, flags);
76 buffer_size = fs_buffer_size; 76 buffer_size = oprofile_buffer_size;
77 buffer_watershed = fs_buffer_watershed; 77 buffer_watershed = oprofile_buffer_watershed;
78 spin_unlock_irqrestore(&oprofilefs_lock, flags); 78 spin_unlock_irqrestore(&oprofilefs_lock, flags);
79 79
80 if (buffer_watershed >= buffer_size) 80 if (buffer_watershed >= buffer_size)
diff --git a/drivers/oprofile/oprof.c b/drivers/oprofile/oprof.c
index cd375907f26f..3cffce90f82a 100644
--- a/drivers/oprofile/oprof.c
+++ b/drivers/oprofile/oprof.c
@@ -23,7 +23,7 @@
23struct oprofile_operations oprofile_ops; 23struct oprofile_operations oprofile_ops;
24 24
25unsigned long oprofile_started; 25unsigned long oprofile_started;
26unsigned long backtrace_depth; 26unsigned long oprofile_backtrace_depth;
27static unsigned long is_setup; 27static unsigned long is_setup;
28static DEFINE_MUTEX(start_mutex); 28static DEFINE_MUTEX(start_mutex);
29 29
@@ -172,7 +172,7 @@ int oprofile_set_backtrace(unsigned long val)
172 goto out; 172 goto out;
173 } 173 }
174 174
175 backtrace_depth = val; 175 oprofile_backtrace_depth = val;
176 176
177out: 177out:
178 mutex_unlock(&start_mutex); 178 mutex_unlock(&start_mutex);
diff --git a/drivers/oprofile/oprof.h b/drivers/oprofile/oprof.h
index 5df0c21a608f..c288d3c24b50 100644
--- a/drivers/oprofile/oprof.h
+++ b/drivers/oprofile/oprof.h
@@ -21,12 +21,12 @@ void oprofile_stop(void);
21 21
22struct oprofile_operations; 22struct oprofile_operations;
23 23
24extern unsigned long fs_buffer_size; 24extern unsigned long oprofile_buffer_size;
25extern unsigned long fs_cpu_buffer_size; 25extern unsigned long oprofile_cpu_buffer_size;
26extern unsigned long fs_buffer_watershed; 26extern unsigned long oprofile_buffer_watershed;
27extern struct oprofile_operations oprofile_ops; 27extern struct oprofile_operations oprofile_ops;
28extern unsigned long oprofile_started; 28extern unsigned long oprofile_started;
29extern unsigned long backtrace_depth; 29extern unsigned long oprofile_backtrace_depth;
30 30
31struct super_block; 31struct super_block;
32struct dentry; 32struct dentry;
diff --git a/drivers/oprofile/oprofile_files.c b/drivers/oprofile/oprofile_files.c
index d8201998b0b7..5d36ffc30dd5 100644
--- a/drivers/oprofile/oprofile_files.c
+++ b/drivers/oprofile/oprofile_files.c
@@ -14,17 +14,18 @@
14#include "oprofile_stats.h" 14#include "oprofile_stats.h"
15#include "oprof.h" 15#include "oprof.h"
16 16
17#define FS_BUFFER_SIZE_DEFAULT 131072 17#define BUFFER_SIZE_DEFAULT 131072
18#define FS_CPU_BUFFER_SIZE_DEFAULT 8192 18#define CPU_BUFFER_SIZE_DEFAULT 8192
19#define FS_BUFFER_WATERSHED_DEFAULT 32768 /* FIXME: tune */ 19#define BUFFER_WATERSHED_DEFAULT 32768 /* FIXME: tune */
20 20
21unsigned long fs_buffer_size; 21unsigned long oprofile_buffer_size;
22unsigned long fs_cpu_buffer_size; 22unsigned long oprofile_cpu_buffer_size;
23unsigned long fs_buffer_watershed; 23unsigned long oprofile_buffer_watershed;
24 24
25static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) 25static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
26{ 26{
27 return oprofilefs_ulong_to_user(backtrace_depth, buf, count, offset); 27 return oprofilefs_ulong_to_user(oprofile_backtrace_depth, buf, count,
28 offset);
28} 29}
29 30
30 31
@@ -125,16 +126,16 @@ static const struct file_operations dump_fops = {
125void oprofile_create_files(struct super_block *sb, struct dentry *root) 126void oprofile_create_files(struct super_block *sb, struct dentry *root)
126{ 127{
127 /* reinitialize default values */ 128 /* reinitialize default values */
128 fs_buffer_size = FS_BUFFER_SIZE_DEFAULT; 129 oprofile_buffer_size = BUFFER_SIZE_DEFAULT;
129 fs_cpu_buffer_size = FS_CPU_BUFFER_SIZE_DEFAULT; 130 oprofile_cpu_buffer_size = CPU_BUFFER_SIZE_DEFAULT;
130 fs_buffer_watershed = FS_BUFFER_WATERSHED_DEFAULT; 131 oprofile_buffer_watershed = BUFFER_WATERSHED_DEFAULT;
131 132
132 oprofilefs_create_file(sb, root, "enable", &enable_fops); 133 oprofilefs_create_file(sb, root, "enable", &enable_fops);
133 oprofilefs_create_file_perm(sb, root, "dump", &dump_fops, 0666); 134 oprofilefs_create_file_perm(sb, root, "dump", &dump_fops, 0666);
134 oprofilefs_create_file(sb, root, "buffer", &event_buffer_fops); 135 oprofilefs_create_file(sb, root, "buffer", &event_buffer_fops);
135 oprofilefs_create_ulong(sb, root, "buffer_size", &fs_buffer_size); 136 oprofilefs_create_ulong(sb, root, "buffer_size", &oprofile_buffer_size);
136 oprofilefs_create_ulong(sb, root, "buffer_watershed", &fs_buffer_watershed); 137 oprofilefs_create_ulong(sb, root, "buffer_watershed", &oprofile_buffer_watershed);
137 oprofilefs_create_ulong(sb, root, "cpu_buffer_size", &fs_cpu_buffer_size); 138 oprofilefs_create_ulong(sb, root, "cpu_buffer_size", &oprofile_cpu_buffer_size);
138 oprofilefs_create_file(sb, root, "cpu_type", &cpu_type_fops); 139 oprofilefs_create_file(sb, root, "cpu_type", &cpu_type_fops);
139 oprofilefs_create_file(sb, root, "backtrace_depth", &depth_fops); 140 oprofilefs_create_file(sb, root, "backtrace_depth", &depth_fops);
140 oprofilefs_create_file(sb, root, "pointer_size", &pointer_size_fops); 141 oprofilefs_create_file(sb, root, "pointer_size", &pointer_size_fops);