diff options
author | David Vrabel <david.vrabel@csr.com> | 2009-01-02 08:17:13 -0500 |
---|---|---|
committer | David Vrabel <david.vrabel@csr.com> | 2009-01-02 08:17:13 -0500 |
commit | b21a207141d83a06abc5f492b80204602e02ca44 (patch) | |
tree | f0152cde543008c72d7eb5c12c18095ad92785e6 /drivers/oprofile | |
parent | 3af373021fa32f8f787bfbdcc1a9277a287bde4e (diff) | |
parent | b58602a4bac012b5f4fc12fe6b46ab237b610d5d (diff) |
Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6 into for-upstream
Conflicts:
drivers/uwb/wlp/eda.c
Diffstat (limited to 'drivers/oprofile')
-rw-r--r-- | drivers/oprofile/buffer_sync.c | 119 | ||||
-rw-r--r-- | drivers/oprofile/cpu_buffer.c | 197 | ||||
-rw-r--r-- | drivers/oprofile/cpu_buffer.h | 69 | ||||
-rw-r--r-- | drivers/oprofile/oprofile_files.c | 15 |
4 files changed, 219 insertions, 181 deletions
diff --git a/drivers/oprofile/buffer_sync.c b/drivers/oprofile/buffer_sync.c index b55cd23ffdef..65e8294a9e29 100644 --- a/drivers/oprofile/buffer_sync.c +++ b/drivers/oprofile/buffer_sync.c | |||
@@ -200,7 +200,7 @@ static inline unsigned long fast_get_dcookie(struct path *path) | |||
200 | { | 200 | { |
201 | unsigned long cookie; | 201 | unsigned long cookie; |
202 | 202 | ||
203 | if (path->dentry->d_cookie) | 203 | if (path->dentry->d_flags & DCACHE_COOKIE) |
204 | return (unsigned long)path->dentry; | 204 | return (unsigned long)path->dentry; |
205 | get_dcookie(path, &cookie); | 205 | get_dcookie(path, &cookie); |
206 | return cookie; | 206 | return cookie; |
@@ -268,18 +268,6 @@ lookup_dcookie(struct mm_struct *mm, unsigned long addr, off_t *offset) | |||
268 | return cookie; | 268 | return cookie; |
269 | } | 269 | } |
270 | 270 | ||
271 | static void increment_tail(struct oprofile_cpu_buffer *b) | ||
272 | { | ||
273 | unsigned long new_tail = b->tail_pos + 1; | ||
274 | |||
275 | rmb(); /* be sure fifo pointers are synchromized */ | ||
276 | |||
277 | if (new_tail < b->buffer_size) | ||
278 | b->tail_pos = new_tail; | ||
279 | else | ||
280 | b->tail_pos = 0; | ||
281 | } | ||
282 | |||
283 | static unsigned long last_cookie = INVALID_COOKIE; | 271 | static unsigned long last_cookie = INVALID_COOKIE; |
284 | 272 | ||
285 | static void add_cpu_switch(int i) | 273 | static void add_cpu_switch(int i) |
@@ -331,28 +319,25 @@ static void add_trace_begin(void) | |||
331 | 319 | ||
332 | #define IBS_FETCH_CODE_SIZE 2 | 320 | #define IBS_FETCH_CODE_SIZE 2 |
333 | #define IBS_OP_CODE_SIZE 5 | 321 | #define IBS_OP_CODE_SIZE 5 |
334 | #define IBS_EIP(offset) \ | ||
335 | (((struct op_sample *)&cpu_buf->buffer[(offset)])->eip) | ||
336 | #define IBS_EVENT(offset) \ | ||
337 | (((struct op_sample *)&cpu_buf->buffer[(offset)])->event) | ||
338 | 322 | ||
339 | /* | 323 | /* |
340 | * Add IBS fetch and op entries to event buffer | 324 | * Add IBS fetch and op entries to event buffer |
341 | */ | 325 | */ |
342 | static void add_ibs_begin(struct oprofile_cpu_buffer *cpu_buf, int code, | 326 | static void add_ibs_begin(int cpu, int code, struct mm_struct *mm) |
343 | struct mm_struct *mm) | ||
344 | { | 327 | { |
345 | unsigned long rip; | 328 | unsigned long rip; |
346 | int i, count; | 329 | int i, count; |
347 | unsigned long ibs_cookie = 0; | 330 | unsigned long ibs_cookie = 0; |
348 | off_t offset; | 331 | off_t offset; |
332 | struct op_sample *sample; | ||
349 | 333 | ||
350 | increment_tail(cpu_buf); /* move to RIP entry */ | 334 | sample = cpu_buffer_read_entry(cpu); |
351 | 335 | if (!sample) | |
352 | rip = IBS_EIP(cpu_buf->tail_pos); | 336 | goto Error; |
337 | rip = sample->eip; | ||
353 | 338 | ||
354 | #ifdef __LP64__ | 339 | #ifdef __LP64__ |
355 | rip += IBS_EVENT(cpu_buf->tail_pos) << 32; | 340 | rip += sample->event << 32; |
356 | #endif | 341 | #endif |
357 | 342 | ||
358 | if (mm) { | 343 | if (mm) { |
@@ -376,8 +361,8 @@ static void add_ibs_begin(struct oprofile_cpu_buffer *cpu_buf, int code, | |||
376 | add_event_entry(offset); /* Offset from Dcookie */ | 361 | add_event_entry(offset); /* Offset from Dcookie */ |
377 | 362 | ||
378 | /* we send the Dcookie offset, but send the raw Linear Add also*/ | 363 | /* we send the Dcookie offset, but send the raw Linear Add also*/ |
379 | add_event_entry(IBS_EIP(cpu_buf->tail_pos)); | 364 | add_event_entry(sample->eip); |
380 | add_event_entry(IBS_EVENT(cpu_buf->tail_pos)); | 365 | add_event_entry(sample->event); |
381 | 366 | ||
382 | if (code == IBS_FETCH_CODE) | 367 | if (code == IBS_FETCH_CODE) |
383 | count = IBS_FETCH_CODE_SIZE; /*IBS FETCH is 2 int64s*/ | 368 | count = IBS_FETCH_CODE_SIZE; /*IBS FETCH is 2 int64s*/ |
@@ -385,10 +370,17 @@ static void add_ibs_begin(struct oprofile_cpu_buffer *cpu_buf, int code, | |||
385 | count = IBS_OP_CODE_SIZE; /*IBS OP is 5 int64s*/ | 370 | count = IBS_OP_CODE_SIZE; /*IBS OP is 5 int64s*/ |
386 | 371 | ||
387 | for (i = 0; i < count; i++) { | 372 | for (i = 0; i < count; i++) { |
388 | increment_tail(cpu_buf); | 373 | sample = cpu_buffer_read_entry(cpu); |
389 | add_event_entry(IBS_EIP(cpu_buf->tail_pos)); | 374 | if (!sample) |
390 | add_event_entry(IBS_EVENT(cpu_buf->tail_pos)); | 375 | goto Error; |
376 | add_event_entry(sample->eip); | ||
377 | add_event_entry(sample->event); | ||
391 | } | 378 | } |
379 | |||
380 | return; | ||
381 | |||
382 | Error: | ||
383 | return; | ||
392 | } | 384 | } |
393 | 385 | ||
394 | #endif | 386 | #endif |
@@ -466,33 +458,6 @@ static inline int is_code(unsigned long val) | |||
466 | } | 458 | } |
467 | 459 | ||
468 | 460 | ||
469 | /* "acquire" as many cpu buffer slots as we can */ | ||
470 | static unsigned long get_slots(struct oprofile_cpu_buffer *b) | ||
471 | { | ||
472 | unsigned long head = b->head_pos; | ||
473 | unsigned long tail = b->tail_pos; | ||
474 | |||
475 | /* | ||
476 | * Subtle. This resets the persistent last_task | ||
477 | * and in_kernel values used for switching notes. | ||
478 | * BUT, there is a small window between reading | ||
479 | * head_pos, and this call, that means samples | ||
480 | * can appear at the new head position, but not | ||
481 | * be prefixed with the notes for switching | ||
482 | * kernel mode or a task switch. This small hole | ||
483 | * can lead to mis-attribution or samples where | ||
484 | * we don't know if it's in the kernel or not, | ||
485 | * at the start of an event buffer. | ||
486 | */ | ||
487 | cpu_buffer_reset(b); | ||
488 | |||
489 | if (head >= tail) | ||
490 | return head - tail; | ||
491 | |||
492 | return head + (b->buffer_size - tail); | ||
493 | } | ||
494 | |||
495 | |||
496 | /* Move tasks along towards death. Any tasks on dead_tasks | 461 | /* Move tasks along towards death. Any tasks on dead_tasks |
497 | * will definitely have no remaining references in any | 462 | * will definitely have no remaining references in any |
498 | * CPU buffers at this point, because we use two lists, | 463 | * CPU buffers at this point, because we use two lists, |
@@ -559,61 +524,61 @@ typedef enum { | |||
559 | */ | 524 | */ |
560 | void sync_buffer(int cpu) | 525 | void sync_buffer(int cpu) |
561 | { | 526 | { |
562 | struct oprofile_cpu_buffer *cpu_buf = &per_cpu(cpu_buffer, cpu); | ||
563 | struct mm_struct *mm = NULL; | 527 | struct mm_struct *mm = NULL; |
528 | struct mm_struct *oldmm; | ||
564 | struct task_struct *new; | 529 | struct task_struct *new; |
565 | unsigned long cookie = 0; | 530 | unsigned long cookie = 0; |
566 | int in_kernel = 1; | 531 | int in_kernel = 1; |
567 | sync_buffer_state state = sb_buffer_start; | 532 | sync_buffer_state state = sb_buffer_start; |
568 | #ifndef CONFIG_OPROFILE_IBS | ||
569 | unsigned int i; | 533 | unsigned int i; |
570 | unsigned long available; | 534 | unsigned long available; |
571 | #endif | ||
572 | 535 | ||
573 | mutex_lock(&buffer_mutex); | 536 | mutex_lock(&buffer_mutex); |
574 | 537 | ||
575 | add_cpu_switch(cpu); | 538 | add_cpu_switch(cpu); |
576 | 539 | ||
577 | /* Remember, only we can modify tail_pos */ | 540 | cpu_buffer_reset(cpu); |
578 | 541 | available = cpu_buffer_entries(cpu); | |
579 | #ifndef CONFIG_OPROFILE_IBS | ||
580 | available = get_slots(cpu_buf); | ||
581 | 542 | ||
582 | for (i = 0; i < available; ++i) { | 543 | for (i = 0; i < available; ++i) { |
583 | #else | 544 | struct op_sample *s = cpu_buffer_read_entry(cpu); |
584 | while (get_slots(cpu_buf)) { | 545 | if (!s) |
585 | #endif | 546 | break; |
586 | struct op_sample *s = &cpu_buf->buffer[cpu_buf->tail_pos]; | ||
587 | 547 | ||
588 | if (is_code(s->eip)) { | 548 | if (is_code(s->eip)) { |
589 | if (s->event <= CPU_IS_KERNEL) { | 549 | switch (s->event) { |
550 | case 0: | ||
551 | case CPU_IS_KERNEL: | ||
590 | /* kernel/userspace switch */ | 552 | /* kernel/userspace switch */ |
591 | in_kernel = s->event; | 553 | in_kernel = s->event; |
592 | if (state == sb_buffer_start) | 554 | if (state == sb_buffer_start) |
593 | state = sb_sample_start; | 555 | state = sb_sample_start; |
594 | add_kernel_ctx_switch(s->event); | 556 | add_kernel_ctx_switch(s->event); |
595 | } else if (s->event == CPU_TRACE_BEGIN) { | 557 | break; |
558 | case CPU_TRACE_BEGIN: | ||
596 | state = sb_bt_start; | 559 | state = sb_bt_start; |
597 | add_trace_begin(); | 560 | add_trace_begin(); |
561 | break; | ||
598 | #ifdef CONFIG_OPROFILE_IBS | 562 | #ifdef CONFIG_OPROFILE_IBS |
599 | } else if (s->event == IBS_FETCH_BEGIN) { | 563 | case IBS_FETCH_BEGIN: |
600 | state = sb_bt_start; | 564 | state = sb_bt_start; |
601 | add_ibs_begin(cpu_buf, IBS_FETCH_CODE, mm); | 565 | add_ibs_begin(cpu, IBS_FETCH_CODE, mm); |
602 | } else if (s->event == IBS_OP_BEGIN) { | 566 | break; |
567 | case IBS_OP_BEGIN: | ||
603 | state = sb_bt_start; | 568 | state = sb_bt_start; |
604 | add_ibs_begin(cpu_buf, IBS_OP_CODE, mm); | 569 | add_ibs_begin(cpu, IBS_OP_CODE, mm); |
570 | break; | ||
605 | #endif | 571 | #endif |
606 | } else { | 572 | default: |
607 | struct mm_struct *oldmm = mm; | ||
608 | |||
609 | /* userspace context switch */ | 573 | /* userspace context switch */ |
574 | oldmm = mm; | ||
610 | new = (struct task_struct *)s->event; | 575 | new = (struct task_struct *)s->event; |
611 | |||
612 | release_mm(oldmm); | 576 | release_mm(oldmm); |
613 | mm = take_tasks_mm(new); | 577 | mm = take_tasks_mm(new); |
614 | if (mm != oldmm) | 578 | if (mm != oldmm) |
615 | cookie = get_exec_dcookie(mm); | 579 | cookie = get_exec_dcookie(mm); |
616 | add_user_ctx_switch(new, cookie); | 580 | add_user_ctx_switch(new, cookie); |
581 | break; | ||
617 | } | 582 | } |
618 | } else if (state >= sb_bt_start && | 583 | } else if (state >= sb_bt_start && |
619 | !add_sample(mm, s, in_kernel)) { | 584 | !add_sample(mm, s, in_kernel)) { |
@@ -622,8 +587,6 @@ void sync_buffer(int cpu) | |||
622 | atomic_inc(&oprofile_stats.bt_lost_no_mapping); | 587 | atomic_inc(&oprofile_stats.bt_lost_no_mapping); |
623 | } | 588 | } |
624 | } | 589 | } |
625 | |||
626 | increment_tail(cpu_buf); | ||
627 | } | 590 | } |
628 | release_mm(mm); | 591 | release_mm(mm); |
629 | 592 | ||
diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 01d38e78cde1..61090969158f 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c | |||
@@ -28,6 +28,25 @@ | |||
28 | #include "buffer_sync.h" | 28 | #include "buffer_sync.h" |
29 | #include "oprof.h" | 29 | #include "oprof.h" |
30 | 30 | ||
31 | #define OP_BUFFER_FLAGS 0 | ||
32 | |||
33 | /* | ||
34 | * Read and write access is using spin locking. Thus, writing to the | ||
35 | * buffer by NMI handler (x86) could occur also during critical | ||
36 | * sections when reading the buffer. To avoid this, there are 2 | ||
37 | * buffers for independent read and write access. Read access is in | ||
38 | * process context only, write access only in the NMI handler. If the | ||
39 | * read buffer runs empty, both buffers are swapped atomically. There | ||
40 | * is potentially a small window during swapping where the buffers are | ||
41 | * disabled and samples could be lost. | ||
42 | * | ||
43 | * Using 2 buffers is a little bit overhead, but the solution is clear | ||
44 | * and does not require changes in the ring buffer implementation. It | ||
45 | * can be changed to a single buffer solution when the ring buffer | ||
46 | * access is implemented as non-locking atomic code. | ||
47 | */ | ||
48 | struct ring_buffer *op_ring_buffer_read; | ||
49 | struct ring_buffer *op_ring_buffer_write; | ||
31 | DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); | 50 | DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); |
32 | 51 | ||
33 | static void wq_sync_buffer(struct work_struct *work); | 52 | static void wq_sync_buffer(struct work_struct *work); |
@@ -37,12 +56,12 @@ static int work_enabled; | |||
37 | 56 | ||
38 | void free_cpu_buffers(void) | 57 | void free_cpu_buffers(void) |
39 | { | 58 | { |
40 | int i; | 59 | if (op_ring_buffer_read) |
41 | 60 | ring_buffer_free(op_ring_buffer_read); | |
42 | for_each_possible_cpu(i) { | 61 | op_ring_buffer_read = NULL; |
43 | vfree(per_cpu(cpu_buffer, i).buffer); | 62 | if (op_ring_buffer_write) |
44 | per_cpu(cpu_buffer, i).buffer = NULL; | 63 | ring_buffer_free(op_ring_buffer_write); |
45 | } | 64 | op_ring_buffer_write = NULL; |
46 | } | 65 | } |
47 | 66 | ||
48 | unsigned long oprofile_get_cpu_buffer_size(void) | 67 | unsigned long oprofile_get_cpu_buffer_size(void) |
@@ -64,14 +83,16 @@ int alloc_cpu_buffers(void) | |||
64 | 83 | ||
65 | unsigned long buffer_size = fs_cpu_buffer_size; | 84 | unsigned long buffer_size = fs_cpu_buffer_size; |
66 | 85 | ||
86 | op_ring_buffer_read = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS); | ||
87 | if (!op_ring_buffer_read) | ||
88 | goto fail; | ||
89 | op_ring_buffer_write = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS); | ||
90 | if (!op_ring_buffer_write) | ||
91 | goto fail; | ||
92 | |||
67 | for_each_possible_cpu(i) { | 93 | for_each_possible_cpu(i) { |
68 | struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i); | 94 | struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i); |
69 | 95 | ||
70 | b->buffer = vmalloc_node(sizeof(struct op_sample) * buffer_size, | ||
71 | cpu_to_node(i)); | ||
72 | if (!b->buffer) | ||
73 | goto fail; | ||
74 | |||
75 | b->last_task = NULL; | 96 | b->last_task = NULL; |
76 | b->last_is_kernel = -1; | 97 | b->last_is_kernel = -1; |
77 | b->tracing = 0; | 98 | b->tracing = 0; |
@@ -124,57 +145,31 @@ void end_cpu_work(void) | |||
124 | flush_scheduled_work(); | 145 | flush_scheduled_work(); |
125 | } | 146 | } |
126 | 147 | ||
127 | /* Resets the cpu buffer to a sane state. */ | 148 | static inline int |
128 | void cpu_buffer_reset(struct oprofile_cpu_buffer *cpu_buf) | 149 | add_sample(struct oprofile_cpu_buffer *cpu_buf, |
129 | { | 150 | unsigned long pc, unsigned long event) |
130 | /* reset these to invalid values; the next sample | ||
131 | * collected will populate the buffer with proper | ||
132 | * values to initialize the buffer | ||
133 | */ | ||
134 | cpu_buf->last_is_kernel = -1; | ||
135 | cpu_buf->last_task = NULL; | ||
136 | } | ||
137 | |||
138 | /* compute number of available slots in cpu_buffer queue */ | ||
139 | static unsigned long nr_available_slots(struct oprofile_cpu_buffer const *b) | ||
140 | { | 151 | { |
141 | unsigned long head = b->head_pos; | 152 | struct op_entry entry; |
142 | unsigned long tail = b->tail_pos; | 153 | int ret; |
143 | 154 | ||
144 | if (tail > head) | 155 | ret = cpu_buffer_write_entry(&entry); |
145 | return (tail - head) - 1; | 156 | if (ret) |
157 | return ret; | ||
146 | 158 | ||
147 | return tail + (b->buffer_size - head) - 1; | 159 | entry.sample->eip = pc; |
148 | } | 160 | entry.sample->event = event; |
149 | 161 | ||
150 | static void increment_head(struct oprofile_cpu_buffer *b) | 162 | ret = cpu_buffer_write_commit(&entry); |
151 | { | 163 | if (ret) |
152 | unsigned long new_head = b->head_pos + 1; | 164 | return ret; |
153 | |||
154 | /* Ensure anything written to the slot before we | ||
155 | * increment is visible */ | ||
156 | wmb(); | ||
157 | |||
158 | if (new_head < b->buffer_size) | ||
159 | b->head_pos = new_head; | ||
160 | else | ||
161 | b->head_pos = 0; | ||
162 | } | ||
163 | 165 | ||
164 | static inline void | 166 | return 0; |
165 | add_sample(struct oprofile_cpu_buffer *cpu_buf, | ||
166 | unsigned long pc, unsigned long event) | ||
167 | { | ||
168 | struct op_sample *entry = &cpu_buf->buffer[cpu_buf->head_pos]; | ||
169 | entry->eip = pc; | ||
170 | entry->event = event; | ||
171 | increment_head(cpu_buf); | ||
172 | } | 167 | } |
173 | 168 | ||
174 | static inline void | 169 | static inline int |
175 | add_code(struct oprofile_cpu_buffer *buffer, unsigned long value) | 170 | add_code(struct oprofile_cpu_buffer *buffer, unsigned long value) |
176 | { | 171 | { |
177 | add_sample(buffer, ESCAPE_CODE, value); | 172 | return add_sample(buffer, ESCAPE_CODE, value); |
178 | } | 173 | } |
179 | 174 | ||
180 | /* This must be safe from any context. It's safe writing here | 175 | /* This must be safe from any context. It's safe writing here |
@@ -198,11 +193,6 @@ static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, | |||
198 | return 0; | 193 | return 0; |
199 | } | 194 | } |
200 | 195 | ||
201 | if (nr_available_slots(cpu_buf) < 3) { | ||
202 | cpu_buf->sample_lost_overflow++; | ||
203 | return 0; | ||
204 | } | ||
205 | |||
206 | is_kernel = !!is_kernel; | 196 | is_kernel = !!is_kernel; |
207 | 197 | ||
208 | task = current; | 198 | task = current; |
@@ -210,26 +200,29 @@ static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, | |||
210 | /* notice a switch from user->kernel or vice versa */ | 200 | /* notice a switch from user->kernel or vice versa */ |
211 | if (cpu_buf->last_is_kernel != is_kernel) { | 201 | if (cpu_buf->last_is_kernel != is_kernel) { |
212 | cpu_buf->last_is_kernel = is_kernel; | 202 | cpu_buf->last_is_kernel = is_kernel; |
213 | add_code(cpu_buf, is_kernel); | 203 | if (add_code(cpu_buf, is_kernel)) |
204 | goto fail; | ||
214 | } | 205 | } |
215 | 206 | ||
216 | /* notice a task switch */ | 207 | /* notice a task switch */ |
217 | if (cpu_buf->last_task != task) { | 208 | if (cpu_buf->last_task != task) { |
218 | cpu_buf->last_task = task; | 209 | cpu_buf->last_task = task; |
219 | add_code(cpu_buf, (unsigned long)task); | 210 | if (add_code(cpu_buf, (unsigned long)task)) |
211 | goto fail; | ||
220 | } | 212 | } |
221 | 213 | ||
222 | add_sample(cpu_buf, pc, event); | 214 | if (add_sample(cpu_buf, pc, event)) |
215 | goto fail; | ||
216 | |||
223 | return 1; | 217 | return 1; |
218 | |||
219 | fail: | ||
220 | cpu_buf->sample_lost_overflow++; | ||
221 | return 0; | ||
224 | } | 222 | } |
225 | 223 | ||
226 | static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf) | 224 | static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf) |
227 | { | 225 | { |
228 | if (nr_available_slots(cpu_buf) < 4) { | ||
229 | cpu_buf->sample_lost_overflow++; | ||
230 | return 0; | ||
231 | } | ||
232 | |||
233 | add_code(cpu_buf, CPU_TRACE_BEGIN); | 226 | add_code(cpu_buf, CPU_TRACE_BEGIN); |
234 | cpu_buf->tracing = 1; | 227 | cpu_buf->tracing = 1; |
235 | return 1; | 228 | return 1; |
@@ -253,8 +246,10 @@ void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs, | |||
253 | if (!oprofile_begin_trace(cpu_buf)) | 246 | if (!oprofile_begin_trace(cpu_buf)) |
254 | return; | 247 | return; |
255 | 248 | ||
256 | /* if log_sample() fail we can't backtrace since we lost the source | 249 | /* |
257 | * of this event */ | 250 | * if log_sample() fail we can't backtrace since we lost the |
251 | * source of this event | ||
252 | */ | ||
258 | if (log_sample(cpu_buf, pc, is_kernel, event)) | 253 | if (log_sample(cpu_buf, pc, is_kernel, event)) |
259 | oprofile_ops.backtrace(regs, backtrace_depth); | 254 | oprofile_ops.backtrace(regs, backtrace_depth); |
260 | oprofile_end_trace(cpu_buf); | 255 | oprofile_end_trace(cpu_buf); |
@@ -272,49 +267,55 @@ void oprofile_add_sample(struct pt_regs * const regs, unsigned long event) | |||
272 | 267 | ||
273 | #define MAX_IBS_SAMPLE_SIZE 14 | 268 | #define MAX_IBS_SAMPLE_SIZE 14 |
274 | 269 | ||
275 | void oprofile_add_ibs_sample(struct pt_regs *const regs, | 270 | void oprofile_add_ibs_sample(struct pt_regs * const regs, |
276 | unsigned int *const ibs_sample, int ibs_code) | 271 | unsigned int * const ibs_sample, int ibs_code) |
277 | { | 272 | { |
278 | int is_kernel = !user_mode(regs); | 273 | int is_kernel = !user_mode(regs); |
279 | struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); | 274 | struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); |
280 | struct task_struct *task; | 275 | struct task_struct *task; |
276 | int fail = 0; | ||
281 | 277 | ||
282 | cpu_buf->sample_received++; | 278 | cpu_buf->sample_received++; |
283 | 279 | ||
284 | if (nr_available_slots(cpu_buf) < MAX_IBS_SAMPLE_SIZE) { | ||
285 | /* we can't backtrace since we lost the source of this event */ | ||
286 | cpu_buf->sample_lost_overflow++; | ||
287 | return; | ||
288 | } | ||
289 | |||
290 | /* notice a switch from user->kernel or vice versa */ | 280 | /* notice a switch from user->kernel or vice versa */ |
291 | if (cpu_buf->last_is_kernel != is_kernel) { | 281 | if (cpu_buf->last_is_kernel != is_kernel) { |
282 | if (add_code(cpu_buf, is_kernel)) | ||
283 | goto fail; | ||
292 | cpu_buf->last_is_kernel = is_kernel; | 284 | cpu_buf->last_is_kernel = is_kernel; |
293 | add_code(cpu_buf, is_kernel); | ||
294 | } | 285 | } |
295 | 286 | ||
296 | /* notice a task switch */ | 287 | /* notice a task switch */ |
297 | if (!is_kernel) { | 288 | if (!is_kernel) { |
298 | task = current; | 289 | task = current; |
299 | if (cpu_buf->last_task != task) { | 290 | if (cpu_buf->last_task != task) { |
291 | if (add_code(cpu_buf, (unsigned long)task)) | ||
292 | goto fail; | ||
300 | cpu_buf->last_task = task; | 293 | cpu_buf->last_task = task; |
301 | add_code(cpu_buf, (unsigned long)task); | ||
302 | } | 294 | } |
303 | } | 295 | } |
304 | 296 | ||
305 | add_code(cpu_buf, ibs_code); | 297 | fail = fail || add_code(cpu_buf, ibs_code); |
306 | add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]); | 298 | fail = fail || add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]); |
307 | add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]); | 299 | fail = fail || add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]); |
308 | add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]); | 300 | fail = fail || add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]); |
309 | 301 | ||
310 | if (ibs_code == IBS_OP_BEGIN) { | 302 | if (ibs_code == IBS_OP_BEGIN) { |
311 | add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]); | 303 | fail = fail || add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]); |
312 | add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]); | 304 | fail = fail || add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]); |
313 | add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]); | 305 | fail = fail || add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]); |
314 | } | 306 | } |
315 | 307 | ||
308 | if (fail) | ||
309 | goto fail; | ||
310 | |||
316 | if (backtrace_depth) | 311 | if (backtrace_depth) |
317 | oprofile_ops.backtrace(regs, backtrace_depth); | 312 | oprofile_ops.backtrace(regs, backtrace_depth); |
313 | |||
314 | return; | ||
315 | |||
316 | fail: | ||
317 | cpu_buf->sample_lost_overflow++; | ||
318 | return; | ||
318 | } | 319 | } |
319 | 320 | ||
320 | #endif | 321 | #endif |
@@ -332,21 +333,21 @@ void oprofile_add_trace(unsigned long pc) | |||
332 | if (!cpu_buf->tracing) | 333 | if (!cpu_buf->tracing) |
333 | return; | 334 | return; |
334 | 335 | ||
335 | if (nr_available_slots(cpu_buf) < 1) { | 336 | /* |
336 | cpu_buf->tracing = 0; | 337 | * broken frame can give an eip with the same value as an |
337 | cpu_buf->sample_lost_overflow++; | 338 | * escape code, abort the trace if we get it |
338 | return; | 339 | */ |
339 | } | 340 | if (pc == ESCAPE_CODE) |
341 | goto fail; | ||
340 | 342 | ||
341 | /* broken frame can give an eip with the same value as an escape code, | 343 | if (add_sample(cpu_buf, pc, 0)) |
342 | * abort the trace if we get it */ | 344 | goto fail; |
343 | if (pc == ESCAPE_CODE) { | ||
344 | cpu_buf->tracing = 0; | ||
345 | cpu_buf->backtrace_aborted++; | ||
346 | return; | ||
347 | } | ||
348 | 345 | ||
349 | add_sample(cpu_buf, pc, 0); | 346 | return; |
347 | fail: | ||
348 | cpu_buf->tracing = 0; | ||
349 | cpu_buf->backtrace_aborted++; | ||
350 | return; | ||
350 | } | 351 | } |
351 | 352 | ||
352 | /* | 353 | /* |
diff --git a/drivers/oprofile/cpu_buffer.h b/drivers/oprofile/cpu_buffer.h index d3cc26264db5..aacb0f0bc566 100644 --- a/drivers/oprofile/cpu_buffer.h +++ b/drivers/oprofile/cpu_buffer.h | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <linux/workqueue.h> | 15 | #include <linux/workqueue.h> |
16 | #include <linux/cache.h> | 16 | #include <linux/cache.h> |
17 | #include <linux/sched.h> | 17 | #include <linux/sched.h> |
18 | #include <linux/ring_buffer.h> | ||
18 | 19 | ||
19 | struct task_struct; | 20 | struct task_struct; |
20 | 21 | ||
@@ -32,6 +33,12 @@ struct op_sample { | |||
32 | unsigned long event; | 33 | unsigned long event; |
33 | }; | 34 | }; |
34 | 35 | ||
36 | struct op_entry { | ||
37 | struct ring_buffer_event *event; | ||
38 | struct op_sample *sample; | ||
39 | unsigned long irq_flags; | ||
40 | }; | ||
41 | |||
35 | struct oprofile_cpu_buffer { | 42 | struct oprofile_cpu_buffer { |
36 | volatile unsigned long head_pos; | 43 | volatile unsigned long head_pos; |
37 | volatile unsigned long tail_pos; | 44 | volatile unsigned long tail_pos; |
@@ -39,7 +46,6 @@ struct oprofile_cpu_buffer { | |||
39 | struct task_struct *last_task; | 46 | struct task_struct *last_task; |
40 | int last_is_kernel; | 47 | int last_is_kernel; |
41 | int tracing; | 48 | int tracing; |
42 | struct op_sample *buffer; | ||
43 | unsigned long sample_received; | 49 | unsigned long sample_received; |
44 | unsigned long sample_lost_overflow; | 50 | unsigned long sample_lost_overflow; |
45 | unsigned long backtrace_aborted; | 51 | unsigned long backtrace_aborted; |
@@ -48,9 +54,68 @@ struct oprofile_cpu_buffer { | |||
48 | struct delayed_work work; | 54 | struct delayed_work work; |
49 | }; | 55 | }; |
50 | 56 | ||
57 | extern struct ring_buffer *op_ring_buffer_read; | ||
58 | extern struct ring_buffer *op_ring_buffer_write; | ||
51 | DECLARE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); | 59 | DECLARE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); |
52 | 60 | ||
53 | void cpu_buffer_reset(struct oprofile_cpu_buffer *cpu_buf); | 61 | /* |
62 | * Resets the cpu buffer to a sane state. | ||
63 | * | ||
64 | * reset these to invalid values; the next sample collected will | ||
65 | * populate the buffer with proper values to initialize the buffer | ||
66 | */ | ||
67 | static inline void cpu_buffer_reset(int cpu) | ||
68 | { | ||
69 | struct oprofile_cpu_buffer *cpu_buf = &per_cpu(cpu_buffer, cpu); | ||
70 | |||
71 | cpu_buf->last_is_kernel = -1; | ||
72 | cpu_buf->last_task = NULL; | ||
73 | } | ||
74 | |||
75 | static inline int cpu_buffer_write_entry(struct op_entry *entry) | ||
76 | { | ||
77 | entry->event = ring_buffer_lock_reserve(op_ring_buffer_write, | ||
78 | sizeof(struct op_sample), | ||
79 | &entry->irq_flags); | ||
80 | if (entry->event) | ||
81 | entry->sample = ring_buffer_event_data(entry->event); | ||
82 | else | ||
83 | entry->sample = NULL; | ||
84 | |||
85 | if (!entry->sample) | ||
86 | return -ENOMEM; | ||
87 | |||
88 | return 0; | ||
89 | } | ||
90 | |||
91 | static inline int cpu_buffer_write_commit(struct op_entry *entry) | ||
92 | { | ||
93 | return ring_buffer_unlock_commit(op_ring_buffer_write, entry->event, | ||
94 | entry->irq_flags); | ||
95 | } | ||
96 | |||
97 | static inline struct op_sample *cpu_buffer_read_entry(int cpu) | ||
98 | { | ||
99 | struct ring_buffer_event *e; | ||
100 | e = ring_buffer_consume(op_ring_buffer_read, cpu, NULL); | ||
101 | if (e) | ||
102 | return ring_buffer_event_data(e); | ||
103 | if (ring_buffer_swap_cpu(op_ring_buffer_read, | ||
104 | op_ring_buffer_write, | ||
105 | cpu)) | ||
106 | return NULL; | ||
107 | e = ring_buffer_consume(op_ring_buffer_read, cpu, NULL); | ||
108 | if (e) | ||
109 | return ring_buffer_event_data(e); | ||
110 | return NULL; | ||
111 | } | ||
112 | |||
113 | /* "acquire" as many cpu buffer slots as we can */ | ||
114 | static inline unsigned long cpu_buffer_entries(int cpu) | ||
115 | { | ||
116 | return ring_buffer_entries_cpu(op_ring_buffer_read, cpu) | ||
117 | + ring_buffer_entries_cpu(op_ring_buffer_write, cpu); | ||
118 | } | ||
54 | 119 | ||
55 | /* transient events for the CPU buffer -> event buffer */ | 120 | /* transient events for the CPU buffer -> event buffer */ |
56 | #define CPU_IS_KERNEL 1 | 121 | #define CPU_IS_KERNEL 1 |
diff --git a/drivers/oprofile/oprofile_files.c b/drivers/oprofile/oprofile_files.c index cc106d503ace..d8201998b0b7 100644 --- a/drivers/oprofile/oprofile_files.c +++ b/drivers/oprofile/oprofile_files.c | |||
@@ -14,9 +14,13 @@ | |||
14 | #include "oprofile_stats.h" | 14 | #include "oprofile_stats.h" |
15 | #include "oprof.h" | 15 | #include "oprof.h" |
16 | 16 | ||
17 | unsigned long fs_buffer_size = 131072; | 17 | #define FS_BUFFER_SIZE_DEFAULT 131072 |
18 | unsigned long fs_cpu_buffer_size = 8192; | 18 | #define FS_CPU_BUFFER_SIZE_DEFAULT 8192 |
19 | unsigned long fs_buffer_watershed = 32768; /* FIXME: tune */ | 19 | #define FS_BUFFER_WATERSHED_DEFAULT 32768 /* FIXME: tune */ |
20 | |||
21 | unsigned long fs_buffer_size; | ||
22 | unsigned long fs_cpu_buffer_size; | ||
23 | unsigned long fs_buffer_watershed; | ||
20 | 24 | ||
21 | static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) | 25 | static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset) |
22 | { | 26 | { |
@@ -120,6 +124,11 @@ static const struct file_operations dump_fops = { | |||
120 | 124 | ||
121 | void oprofile_create_files(struct super_block *sb, struct dentry *root) | 125 | void oprofile_create_files(struct super_block *sb, struct dentry *root) |
122 | { | 126 | { |
127 | /* reinitialize default values */ | ||
128 | fs_buffer_size = FS_BUFFER_SIZE_DEFAULT; | ||
129 | fs_cpu_buffer_size = FS_CPU_BUFFER_SIZE_DEFAULT; | ||
130 | fs_buffer_watershed = FS_BUFFER_WATERSHED_DEFAULT; | ||
131 | |||
123 | oprofilefs_create_file(sb, root, "enable", &enable_fops); | 132 | oprofilefs_create_file(sb, root, "enable", &enable_fops); |
124 | oprofilefs_create_file_perm(sb, root, "dump", &dump_fops, 0666); | 133 | oprofilefs_create_file_perm(sb, root, "dump", &dump_fops, 0666); |
125 | oprofilefs_create_file(sb, root, "buffer", &event_buffer_fops); | 134 | oprofilefs_create_file(sb, root, "buffer", &event_buffer_fops); |