diff options
author | Ingo Molnar <mingo@elte.hu> | 2009-06-06 09:19:13 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2009-06-06 12:58:30 -0400 |
commit | 8035e4288078cb806e7dd6bafe4d3e54d44cab3f (patch) | |
tree | 7b54e827020e9a9dbc2823009cb2815fc3e35fc9 /Documentation/perf_counter/builtin-annotate.c | |
parent | 6e6b754ffdb6415723686c733f13275397e44422 (diff) |
perf_counter tools: Prepare for 'perf annotate'
Prepare for the 'perf annotate' implementation by splitting off
builtin-annotate.c from builtin-report.c.
( We keep this commit separate to ease the later librarization
of the facilities that perf-report and perf-annotate shares. )
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
LKML-Reference: <new-submission>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'Documentation/perf_counter/builtin-annotate.c')
-rw-r--r-- | Documentation/perf_counter/builtin-annotate.c | 1291 |
1 files changed, 1291 insertions, 0 deletions
diff --git a/Documentation/perf_counter/builtin-annotate.c b/Documentation/perf_counter/builtin-annotate.c new file mode 100644 index 000000000000..d656484ec983 --- /dev/null +++ b/Documentation/perf_counter/builtin-annotate.c | |||
@@ -0,0 +1,1291 @@ | |||
1 | /* | ||
2 | * builtin-annotate.c | ||
3 | * | ||
4 | * Builtin annotate command: Analyze the perf.data input file, | ||
5 | * look up and read DSOs and symbol information and display | ||
6 | * a histogram of results, along various sorting keys. | ||
7 | */ | ||
8 | #include "builtin.h" | ||
9 | |||
10 | #include "util/util.h" | ||
11 | |||
12 | #include "util/color.h" | ||
13 | #include "util/list.h" | ||
14 | #include "util/cache.h" | ||
15 | #include "util/rbtree.h" | ||
16 | #include "util/symbol.h" | ||
17 | #include "util/string.h" | ||
18 | |||
19 | #include "perf.h" | ||
20 | |||
21 | #include "util/parse-options.h" | ||
22 | #include "util/parse-events.h" | ||
23 | |||
24 | #define SHOW_KERNEL 1 | ||
25 | #define SHOW_USER 2 | ||
26 | #define SHOW_HV 4 | ||
27 | |||
28 | static char const *input_name = "perf.data"; | ||
29 | static char *vmlinux = NULL; | ||
30 | |||
31 | static char default_sort_order[] = "comm,dso"; | ||
32 | static char *sort_order = default_sort_order; | ||
33 | |||
34 | static int input; | ||
35 | static int show_mask = SHOW_KERNEL | SHOW_USER | SHOW_HV; | ||
36 | |||
37 | static int dump_trace = 0; | ||
38 | #define dprintf(x...) do { if (dump_trace) printf(x); } while (0) | ||
39 | |||
40 | static int verbose; | ||
41 | static int full_paths; | ||
42 | |||
43 | static unsigned long page_size; | ||
44 | static unsigned long mmap_window = 32; | ||
45 | |||
46 | struct ip_event { | ||
47 | struct perf_event_header header; | ||
48 | __u64 ip; | ||
49 | __u32 pid, tid; | ||
50 | }; | ||
51 | |||
52 | struct mmap_event { | ||
53 | struct perf_event_header header; | ||
54 | __u32 pid, tid; | ||
55 | __u64 start; | ||
56 | __u64 len; | ||
57 | __u64 pgoff; | ||
58 | char filename[PATH_MAX]; | ||
59 | }; | ||
60 | |||
61 | struct comm_event { | ||
62 | struct perf_event_header header; | ||
63 | __u32 pid, tid; | ||
64 | char comm[16]; | ||
65 | }; | ||
66 | |||
67 | struct fork_event { | ||
68 | struct perf_event_header header; | ||
69 | __u32 pid, ppid; | ||
70 | }; | ||
71 | |||
72 | struct period_event { | ||
73 | struct perf_event_header header; | ||
74 | __u64 time; | ||
75 | __u64 id; | ||
76 | __u64 sample_period; | ||
77 | }; | ||
78 | |||
79 | typedef union event_union { | ||
80 | struct perf_event_header header; | ||
81 | struct ip_event ip; | ||
82 | struct mmap_event mmap; | ||
83 | struct comm_event comm; | ||
84 | struct fork_event fork; | ||
85 | struct period_event period; | ||
86 | } event_t; | ||
87 | |||
88 | static LIST_HEAD(dsos); | ||
89 | static struct dso *kernel_dso; | ||
90 | static struct dso *vdso; | ||
91 | |||
92 | static void dsos__add(struct dso *dso) | ||
93 | { | ||
94 | list_add_tail(&dso->node, &dsos); | ||
95 | } | ||
96 | |||
97 | static struct dso *dsos__find(const char *name) | ||
98 | { | ||
99 | struct dso *pos; | ||
100 | |||
101 | list_for_each_entry(pos, &dsos, node) | ||
102 | if (strcmp(pos->name, name) == 0) | ||
103 | return pos; | ||
104 | return NULL; | ||
105 | } | ||
106 | |||
107 | static struct dso *dsos__findnew(const char *name) | ||
108 | { | ||
109 | struct dso *dso = dsos__find(name); | ||
110 | int nr; | ||
111 | |||
112 | if (dso) | ||
113 | return dso; | ||
114 | |||
115 | dso = dso__new(name, 0); | ||
116 | if (!dso) | ||
117 | goto out_delete_dso; | ||
118 | |||
119 | nr = dso__load(dso, NULL, verbose); | ||
120 | if (nr < 0) { | ||
121 | if (verbose) | ||
122 | fprintf(stderr, "Failed to open: %s\n", name); | ||
123 | goto out_delete_dso; | ||
124 | } | ||
125 | if (!nr && verbose) { | ||
126 | fprintf(stderr, | ||
127 | "No symbols found in: %s, maybe install a debug package?\n", | ||
128 | name); | ||
129 | } | ||
130 | |||
131 | dsos__add(dso); | ||
132 | |||
133 | return dso; | ||
134 | |||
135 | out_delete_dso: | ||
136 | dso__delete(dso); | ||
137 | return NULL; | ||
138 | } | ||
139 | |||
140 | static void dsos__fprintf(FILE *fp) | ||
141 | { | ||
142 | struct dso *pos; | ||
143 | |||
144 | list_for_each_entry(pos, &dsos, node) | ||
145 | dso__fprintf(pos, fp); | ||
146 | } | ||
147 | |||
148 | static struct symbol *vdso__find_symbol(struct dso *dso, uint64_t ip) | ||
149 | { | ||
150 | return dso__find_symbol(kernel_dso, ip); | ||
151 | } | ||
152 | |||
153 | static int load_kernel(void) | ||
154 | { | ||
155 | int err; | ||
156 | |||
157 | kernel_dso = dso__new("[kernel]", 0); | ||
158 | if (!kernel_dso) | ||
159 | return -1; | ||
160 | |||
161 | err = dso__load_kernel(kernel_dso, vmlinux, NULL, verbose); | ||
162 | if (err) { | ||
163 | dso__delete(kernel_dso); | ||
164 | kernel_dso = NULL; | ||
165 | } else | ||
166 | dsos__add(kernel_dso); | ||
167 | |||
168 | vdso = dso__new("[vdso]", 0); | ||
169 | if (!vdso) | ||
170 | return -1; | ||
171 | |||
172 | vdso->find_symbol = vdso__find_symbol; | ||
173 | |||
174 | dsos__add(vdso); | ||
175 | |||
176 | return err; | ||
177 | } | ||
178 | |||
179 | static char __cwd[PATH_MAX]; | ||
180 | static char *cwd = __cwd; | ||
181 | static int cwdlen; | ||
182 | |||
183 | static int strcommon(const char *pathname) | ||
184 | { | ||
185 | int n = 0; | ||
186 | |||
187 | while (pathname[n] == cwd[n] && n < cwdlen) | ||
188 | ++n; | ||
189 | |||
190 | return n; | ||
191 | } | ||
192 | |||
193 | struct map { | ||
194 | struct list_head node; | ||
195 | uint64_t start; | ||
196 | uint64_t end; | ||
197 | uint64_t pgoff; | ||
198 | uint64_t (*map_ip)(struct map *, uint64_t); | ||
199 | struct dso *dso; | ||
200 | }; | ||
201 | |||
202 | static uint64_t map__map_ip(struct map *map, uint64_t ip) | ||
203 | { | ||
204 | return ip - map->start + map->pgoff; | ||
205 | } | ||
206 | |||
207 | static uint64_t vdso__map_ip(struct map *map, uint64_t ip) | ||
208 | { | ||
209 | return ip; | ||
210 | } | ||
211 | |||
212 | static struct map *map__new(struct mmap_event *event) | ||
213 | { | ||
214 | struct map *self = malloc(sizeof(*self)); | ||
215 | |||
216 | if (self != NULL) { | ||
217 | const char *filename = event->filename; | ||
218 | char newfilename[PATH_MAX]; | ||
219 | |||
220 | if (cwd) { | ||
221 | int n = strcommon(filename); | ||
222 | |||
223 | if (n == cwdlen) { | ||
224 | snprintf(newfilename, sizeof(newfilename), | ||
225 | ".%s", filename + n); | ||
226 | filename = newfilename; | ||
227 | } | ||
228 | } | ||
229 | |||
230 | self->start = event->start; | ||
231 | self->end = event->start + event->len; | ||
232 | self->pgoff = event->pgoff; | ||
233 | |||
234 | self->dso = dsos__findnew(filename); | ||
235 | if (self->dso == NULL) | ||
236 | goto out_delete; | ||
237 | |||
238 | if (self->dso == vdso) | ||
239 | self->map_ip = vdso__map_ip; | ||
240 | else | ||
241 | self->map_ip = map__map_ip; | ||
242 | } | ||
243 | return self; | ||
244 | out_delete: | ||
245 | free(self); | ||
246 | return NULL; | ||
247 | } | ||
248 | |||
249 | static struct map *map__clone(struct map *self) | ||
250 | { | ||
251 | struct map *map = malloc(sizeof(*self)); | ||
252 | |||
253 | if (!map) | ||
254 | return NULL; | ||
255 | |||
256 | memcpy(map, self, sizeof(*self)); | ||
257 | |||
258 | return map; | ||
259 | } | ||
260 | |||
261 | static int map__overlap(struct map *l, struct map *r) | ||
262 | { | ||
263 | if (l->start > r->start) { | ||
264 | struct map *t = l; | ||
265 | l = r; | ||
266 | r = t; | ||
267 | } | ||
268 | |||
269 | if (l->end > r->start) | ||
270 | return 1; | ||
271 | |||
272 | return 0; | ||
273 | } | ||
274 | |||
275 | static size_t map__fprintf(struct map *self, FILE *fp) | ||
276 | { | ||
277 | return fprintf(fp, " %"PRIx64"-%"PRIx64" %"PRIx64" %s\n", | ||
278 | self->start, self->end, self->pgoff, self->dso->name); | ||
279 | } | ||
280 | |||
281 | |||
282 | struct thread { | ||
283 | struct rb_node rb_node; | ||
284 | struct list_head maps; | ||
285 | pid_t pid; | ||
286 | char *comm; | ||
287 | }; | ||
288 | |||
289 | static struct thread *thread__new(pid_t pid) | ||
290 | { | ||
291 | struct thread *self = malloc(sizeof(*self)); | ||
292 | |||
293 | if (self != NULL) { | ||
294 | self->pid = pid; | ||
295 | self->comm = malloc(32); | ||
296 | if (self->comm) | ||
297 | snprintf(self->comm, 32, ":%d", self->pid); | ||
298 | INIT_LIST_HEAD(&self->maps); | ||
299 | } | ||
300 | |||
301 | return self; | ||
302 | } | ||
303 | |||
304 | static int thread__set_comm(struct thread *self, const char *comm) | ||
305 | { | ||
306 | if (self->comm) | ||
307 | free(self->comm); | ||
308 | self->comm = strdup(comm); | ||
309 | return self->comm ? 0 : -ENOMEM; | ||
310 | } | ||
311 | |||
312 | static size_t thread__fprintf(struct thread *self, FILE *fp) | ||
313 | { | ||
314 | struct map *pos; | ||
315 | size_t ret = fprintf(fp, "Thread %d %s\n", self->pid, self->comm); | ||
316 | |||
317 | list_for_each_entry(pos, &self->maps, node) | ||
318 | ret += map__fprintf(pos, fp); | ||
319 | |||
320 | return ret; | ||
321 | } | ||
322 | |||
323 | |||
324 | static struct rb_root threads; | ||
325 | static struct thread *last_match; | ||
326 | |||
327 | static struct thread *threads__findnew(pid_t pid) | ||
328 | { | ||
329 | struct rb_node **p = &threads.rb_node; | ||
330 | struct rb_node *parent = NULL; | ||
331 | struct thread *th; | ||
332 | |||
333 | /* | ||
334 | * Font-end cache - PID lookups come in blocks, | ||
335 | * so most of the time we dont have to look up | ||
336 | * the full rbtree: | ||
337 | */ | ||
338 | if (last_match && last_match->pid == pid) | ||
339 | return last_match; | ||
340 | |||
341 | while (*p != NULL) { | ||
342 | parent = *p; | ||
343 | th = rb_entry(parent, struct thread, rb_node); | ||
344 | |||
345 | if (th->pid == pid) { | ||
346 | last_match = th; | ||
347 | return th; | ||
348 | } | ||
349 | |||
350 | if (pid < th->pid) | ||
351 | p = &(*p)->rb_left; | ||
352 | else | ||
353 | p = &(*p)->rb_right; | ||
354 | } | ||
355 | |||
356 | th = thread__new(pid); | ||
357 | if (th != NULL) { | ||
358 | rb_link_node(&th->rb_node, parent, p); | ||
359 | rb_insert_color(&th->rb_node, &threads); | ||
360 | last_match = th; | ||
361 | } | ||
362 | |||
363 | return th; | ||
364 | } | ||
365 | |||
366 | static void thread__insert_map(struct thread *self, struct map *map) | ||
367 | { | ||
368 | struct map *pos, *tmp; | ||
369 | |||
370 | list_for_each_entry_safe(pos, tmp, &self->maps, node) { | ||
371 | if (map__overlap(pos, map)) { | ||
372 | list_del_init(&pos->node); | ||
373 | /* XXX leaks dsos */ | ||
374 | free(pos); | ||
375 | } | ||
376 | } | ||
377 | |||
378 | list_add_tail(&map->node, &self->maps); | ||
379 | } | ||
380 | |||
381 | static int thread__fork(struct thread *self, struct thread *parent) | ||
382 | { | ||
383 | struct map *map; | ||
384 | |||
385 | if (self->comm) | ||
386 | free(self->comm); | ||
387 | self->comm = strdup(parent->comm); | ||
388 | if (!self->comm) | ||
389 | return -ENOMEM; | ||
390 | |||
391 | list_for_each_entry(map, &parent->maps, node) { | ||
392 | struct map *new = map__clone(map); | ||
393 | if (!new) | ||
394 | return -ENOMEM; | ||
395 | thread__insert_map(self, new); | ||
396 | } | ||
397 | |||
398 | return 0; | ||
399 | } | ||
400 | |||
401 | static struct map *thread__find_map(struct thread *self, uint64_t ip) | ||
402 | { | ||
403 | struct map *pos; | ||
404 | |||
405 | if (self == NULL) | ||
406 | return NULL; | ||
407 | |||
408 | list_for_each_entry(pos, &self->maps, node) | ||
409 | if (ip >= pos->start && ip <= pos->end) | ||
410 | return pos; | ||
411 | |||
412 | return NULL; | ||
413 | } | ||
414 | |||
415 | static size_t threads__fprintf(FILE *fp) | ||
416 | { | ||
417 | size_t ret = 0; | ||
418 | struct rb_node *nd; | ||
419 | |||
420 | for (nd = rb_first(&threads); nd; nd = rb_next(nd)) { | ||
421 | struct thread *pos = rb_entry(nd, struct thread, rb_node); | ||
422 | |||
423 | ret += thread__fprintf(pos, fp); | ||
424 | } | ||
425 | |||
426 | return ret; | ||
427 | } | ||
428 | |||
429 | /* | ||
430 | * histogram, sorted on item, collects counts | ||
431 | */ | ||
432 | |||
433 | static struct rb_root hist; | ||
434 | |||
435 | struct hist_entry { | ||
436 | struct rb_node rb_node; | ||
437 | |||
438 | struct thread *thread; | ||
439 | struct map *map; | ||
440 | struct dso *dso; | ||
441 | struct symbol *sym; | ||
442 | uint64_t ip; | ||
443 | char level; | ||
444 | |||
445 | uint32_t count; | ||
446 | }; | ||
447 | |||
448 | /* | ||
449 | * configurable sorting bits | ||
450 | */ | ||
451 | |||
452 | struct sort_entry { | ||
453 | struct list_head list; | ||
454 | |||
455 | char *header; | ||
456 | |||
457 | int64_t (*cmp)(struct hist_entry *, struct hist_entry *); | ||
458 | int64_t (*collapse)(struct hist_entry *, struct hist_entry *); | ||
459 | size_t (*print)(FILE *fp, struct hist_entry *); | ||
460 | }; | ||
461 | |||
462 | /* --sort pid */ | ||
463 | |||
464 | static int64_t | ||
465 | sort__thread_cmp(struct hist_entry *left, struct hist_entry *right) | ||
466 | { | ||
467 | return right->thread->pid - left->thread->pid; | ||
468 | } | ||
469 | |||
470 | static size_t | ||
471 | sort__thread_print(FILE *fp, struct hist_entry *self) | ||
472 | { | ||
473 | return fprintf(fp, "%16s:%5d", self->thread->comm ?: "", self->thread->pid); | ||
474 | } | ||
475 | |||
476 | static struct sort_entry sort_thread = { | ||
477 | .header = " Command: Pid", | ||
478 | .cmp = sort__thread_cmp, | ||
479 | .print = sort__thread_print, | ||
480 | }; | ||
481 | |||
482 | /* --sort comm */ | ||
483 | |||
484 | static int64_t | ||
485 | sort__comm_cmp(struct hist_entry *left, struct hist_entry *right) | ||
486 | { | ||
487 | return right->thread->pid - left->thread->pid; | ||
488 | } | ||
489 | |||
490 | static int64_t | ||
491 | sort__comm_collapse(struct hist_entry *left, struct hist_entry *right) | ||
492 | { | ||
493 | char *comm_l = left->thread->comm; | ||
494 | char *comm_r = right->thread->comm; | ||
495 | |||
496 | if (!comm_l || !comm_r) { | ||
497 | if (!comm_l && !comm_r) | ||
498 | return 0; | ||
499 | else if (!comm_l) | ||
500 | return -1; | ||
501 | else | ||
502 | return 1; | ||
503 | } | ||
504 | |||
505 | return strcmp(comm_l, comm_r); | ||
506 | } | ||
507 | |||
508 | static size_t | ||
509 | sort__comm_print(FILE *fp, struct hist_entry *self) | ||
510 | { | ||
511 | return fprintf(fp, "%16s", self->thread->comm); | ||
512 | } | ||
513 | |||
514 | static struct sort_entry sort_comm = { | ||
515 | .header = " Command", | ||
516 | .cmp = sort__comm_cmp, | ||
517 | .collapse = sort__comm_collapse, | ||
518 | .print = sort__comm_print, | ||
519 | }; | ||
520 | |||
521 | /* --sort dso */ | ||
522 | |||
523 | static int64_t | ||
524 | sort__dso_cmp(struct hist_entry *left, struct hist_entry *right) | ||
525 | { | ||
526 | struct dso *dso_l = left->dso; | ||
527 | struct dso *dso_r = right->dso; | ||
528 | |||
529 | if (!dso_l || !dso_r) { | ||
530 | if (!dso_l && !dso_r) | ||
531 | return 0; | ||
532 | else if (!dso_l) | ||
533 | return -1; | ||
534 | else | ||
535 | return 1; | ||
536 | } | ||
537 | |||
538 | return strcmp(dso_l->name, dso_r->name); | ||
539 | } | ||
540 | |||
541 | static size_t | ||
542 | sort__dso_print(FILE *fp, struct hist_entry *self) | ||
543 | { | ||
544 | if (self->dso) | ||
545 | return fprintf(fp, "%-25s", self->dso->name); | ||
546 | |||
547 | return fprintf(fp, "%016llx ", (__u64)self->ip); | ||
548 | } | ||
549 | |||
550 | static struct sort_entry sort_dso = { | ||
551 | .header = "Shared Object ", | ||
552 | .cmp = sort__dso_cmp, | ||
553 | .print = sort__dso_print, | ||
554 | }; | ||
555 | |||
556 | /* --sort symbol */ | ||
557 | |||
558 | static int64_t | ||
559 | sort__sym_cmp(struct hist_entry *left, struct hist_entry *right) | ||
560 | { | ||
561 | uint64_t ip_l, ip_r; | ||
562 | |||
563 | if (left->sym == right->sym) | ||
564 | return 0; | ||
565 | |||
566 | ip_l = left->sym ? left->sym->start : left->ip; | ||
567 | ip_r = right->sym ? right->sym->start : right->ip; | ||
568 | |||
569 | return (int64_t)(ip_r - ip_l); | ||
570 | } | ||
571 | |||
572 | static size_t | ||
573 | sort__sym_print(FILE *fp, struct hist_entry *self) | ||
574 | { | ||
575 | size_t ret = 0; | ||
576 | |||
577 | if (verbose) | ||
578 | ret += fprintf(fp, "%#018llx ", (__u64)self->ip); | ||
579 | |||
580 | if (self->sym) { | ||
581 | ret += fprintf(fp, "[%c] %s", | ||
582 | self->dso == kernel_dso ? 'k' : '.', self->sym->name); | ||
583 | } else { | ||
584 | ret += fprintf(fp, "%#016llx", (__u64)self->ip); | ||
585 | } | ||
586 | |||
587 | return ret; | ||
588 | } | ||
589 | |||
590 | static struct sort_entry sort_sym = { | ||
591 | .header = "Symbol", | ||
592 | .cmp = sort__sym_cmp, | ||
593 | .print = sort__sym_print, | ||
594 | }; | ||
595 | |||
596 | static int sort__need_collapse = 0; | ||
597 | |||
598 | struct sort_dimension { | ||
599 | char *name; | ||
600 | struct sort_entry *entry; | ||
601 | int taken; | ||
602 | }; | ||
603 | |||
604 | static struct sort_dimension sort_dimensions[] = { | ||
605 | { .name = "pid", .entry = &sort_thread, }, | ||
606 | { .name = "comm", .entry = &sort_comm, }, | ||
607 | { .name = "dso", .entry = &sort_dso, }, | ||
608 | { .name = "symbol", .entry = &sort_sym, }, | ||
609 | }; | ||
610 | |||
611 | static LIST_HEAD(hist_entry__sort_list); | ||
612 | |||
613 | static int sort_dimension__add(char *tok) | ||
614 | { | ||
615 | int i; | ||
616 | |||
617 | for (i = 0; i < ARRAY_SIZE(sort_dimensions); i++) { | ||
618 | struct sort_dimension *sd = &sort_dimensions[i]; | ||
619 | |||
620 | if (sd->taken) | ||
621 | continue; | ||
622 | |||
623 | if (strncasecmp(tok, sd->name, strlen(tok))) | ||
624 | continue; | ||
625 | |||
626 | if (sd->entry->collapse) | ||
627 | sort__need_collapse = 1; | ||
628 | |||
629 | list_add_tail(&sd->entry->list, &hist_entry__sort_list); | ||
630 | sd->taken = 1; | ||
631 | |||
632 | return 0; | ||
633 | } | ||
634 | |||
635 | return -ESRCH; | ||
636 | } | ||
637 | |||
638 | static int64_t | ||
639 | hist_entry__cmp(struct hist_entry *left, struct hist_entry *right) | ||
640 | { | ||
641 | struct sort_entry *se; | ||
642 | int64_t cmp = 0; | ||
643 | |||
644 | list_for_each_entry(se, &hist_entry__sort_list, list) { | ||
645 | cmp = se->cmp(left, right); | ||
646 | if (cmp) | ||
647 | break; | ||
648 | } | ||
649 | |||
650 | return cmp; | ||
651 | } | ||
652 | |||
653 | static int64_t | ||
654 | hist_entry__collapse(struct hist_entry *left, struct hist_entry *right) | ||
655 | { | ||
656 | struct sort_entry *se; | ||
657 | int64_t cmp = 0; | ||
658 | |||
659 | list_for_each_entry(se, &hist_entry__sort_list, list) { | ||
660 | int64_t (*f)(struct hist_entry *, struct hist_entry *); | ||
661 | |||
662 | f = se->collapse ?: se->cmp; | ||
663 | |||
664 | cmp = f(left, right); | ||
665 | if (cmp) | ||
666 | break; | ||
667 | } | ||
668 | |||
669 | return cmp; | ||
670 | } | ||
671 | |||
672 | static size_t | ||
673 | hist_entry__fprintf(FILE *fp, struct hist_entry *self, uint64_t total_samples) | ||
674 | { | ||
675 | struct sort_entry *se; | ||
676 | size_t ret; | ||
677 | |||
678 | if (total_samples) { | ||
679 | double percent = self->count * 100.0 / total_samples; | ||
680 | char *color = PERF_COLOR_NORMAL; | ||
681 | |||
682 | /* | ||
683 | * We color high-overhead entries in red, low-overhead | ||
684 | * entries in green - and keep the middle ground normal: | ||
685 | */ | ||
686 | if (percent >= 5.0) | ||
687 | color = PERF_COLOR_RED; | ||
688 | if (percent < 0.5) | ||
689 | color = PERF_COLOR_GREEN; | ||
690 | |||
691 | ret = color_fprintf(fp, color, " %6.2f%%", | ||
692 | (self->count * 100.0) / total_samples); | ||
693 | } else | ||
694 | ret = fprintf(fp, "%12d ", self->count); | ||
695 | |||
696 | list_for_each_entry(se, &hist_entry__sort_list, list) { | ||
697 | fprintf(fp, " "); | ||
698 | ret += se->print(fp, self); | ||
699 | } | ||
700 | |||
701 | ret += fprintf(fp, "\n"); | ||
702 | |||
703 | return ret; | ||
704 | } | ||
705 | |||
706 | /* | ||
707 | * collect histogram counts | ||
708 | */ | ||
709 | |||
710 | static int | ||
711 | hist_entry__add(struct thread *thread, struct map *map, struct dso *dso, | ||
712 | struct symbol *sym, uint64_t ip, char level) | ||
713 | { | ||
714 | struct rb_node **p = &hist.rb_node; | ||
715 | struct rb_node *parent = NULL; | ||
716 | struct hist_entry *he; | ||
717 | struct hist_entry entry = { | ||
718 | .thread = thread, | ||
719 | .map = map, | ||
720 | .dso = dso, | ||
721 | .sym = sym, | ||
722 | .ip = ip, | ||
723 | .level = level, | ||
724 | .count = 1, | ||
725 | }; | ||
726 | int cmp; | ||
727 | |||
728 | while (*p != NULL) { | ||
729 | parent = *p; | ||
730 | he = rb_entry(parent, struct hist_entry, rb_node); | ||
731 | |||
732 | cmp = hist_entry__cmp(&entry, he); | ||
733 | |||
734 | if (!cmp) { | ||
735 | he->count++; | ||
736 | return 0; | ||
737 | } | ||
738 | |||
739 | if (cmp < 0) | ||
740 | p = &(*p)->rb_left; | ||
741 | else | ||
742 | p = &(*p)->rb_right; | ||
743 | } | ||
744 | |||
745 | he = malloc(sizeof(*he)); | ||
746 | if (!he) | ||
747 | return -ENOMEM; | ||
748 | *he = entry; | ||
749 | rb_link_node(&he->rb_node, parent, p); | ||
750 | rb_insert_color(&he->rb_node, &hist); | ||
751 | |||
752 | return 0; | ||
753 | } | ||
754 | |||
755 | static void hist_entry__free(struct hist_entry *he) | ||
756 | { | ||
757 | free(he); | ||
758 | } | ||
759 | |||
760 | /* | ||
761 | * collapse the histogram | ||
762 | */ | ||
763 | |||
764 | static struct rb_root collapse_hists; | ||
765 | |||
766 | static void collapse__insert_entry(struct hist_entry *he) | ||
767 | { | ||
768 | struct rb_node **p = &collapse_hists.rb_node; | ||
769 | struct rb_node *parent = NULL; | ||
770 | struct hist_entry *iter; | ||
771 | int64_t cmp; | ||
772 | |||
773 | while (*p != NULL) { | ||
774 | parent = *p; | ||
775 | iter = rb_entry(parent, struct hist_entry, rb_node); | ||
776 | |||
777 | cmp = hist_entry__collapse(iter, he); | ||
778 | |||
779 | if (!cmp) { | ||
780 | iter->count += he->count; | ||
781 | hist_entry__free(he); | ||
782 | return; | ||
783 | } | ||
784 | |||
785 | if (cmp < 0) | ||
786 | p = &(*p)->rb_left; | ||
787 | else | ||
788 | p = &(*p)->rb_right; | ||
789 | } | ||
790 | |||
791 | rb_link_node(&he->rb_node, parent, p); | ||
792 | rb_insert_color(&he->rb_node, &collapse_hists); | ||
793 | } | ||
794 | |||
795 | static void collapse__resort(void) | ||
796 | { | ||
797 | struct rb_node *next; | ||
798 | struct hist_entry *n; | ||
799 | |||
800 | if (!sort__need_collapse) | ||
801 | return; | ||
802 | |||
803 | next = rb_first(&hist); | ||
804 | while (next) { | ||
805 | n = rb_entry(next, struct hist_entry, rb_node); | ||
806 | next = rb_next(&n->rb_node); | ||
807 | |||
808 | rb_erase(&n->rb_node, &hist); | ||
809 | collapse__insert_entry(n); | ||
810 | } | ||
811 | } | ||
812 | |||
813 | /* | ||
814 | * reverse the map, sort on count. | ||
815 | */ | ||
816 | |||
817 | static struct rb_root output_hists; | ||
818 | |||
819 | static void output__insert_entry(struct hist_entry *he) | ||
820 | { | ||
821 | struct rb_node **p = &output_hists.rb_node; | ||
822 | struct rb_node *parent = NULL; | ||
823 | struct hist_entry *iter; | ||
824 | |||
825 | while (*p != NULL) { | ||
826 | parent = *p; | ||
827 | iter = rb_entry(parent, struct hist_entry, rb_node); | ||
828 | |||
829 | if (he->count > iter->count) | ||
830 | p = &(*p)->rb_left; | ||
831 | else | ||
832 | p = &(*p)->rb_right; | ||
833 | } | ||
834 | |||
835 | rb_link_node(&he->rb_node, parent, p); | ||
836 | rb_insert_color(&he->rb_node, &output_hists); | ||
837 | } | ||
838 | |||
839 | static void output__resort(void) | ||
840 | { | ||
841 | struct rb_node *next; | ||
842 | struct hist_entry *n; | ||
843 | struct rb_root *tree = &hist; | ||
844 | |||
845 | if (sort__need_collapse) | ||
846 | tree = &collapse_hists; | ||
847 | |||
848 | next = rb_first(tree); | ||
849 | |||
850 | while (next) { | ||
851 | n = rb_entry(next, struct hist_entry, rb_node); | ||
852 | next = rb_next(&n->rb_node); | ||
853 | |||
854 | rb_erase(&n->rb_node, tree); | ||
855 | output__insert_entry(n); | ||
856 | } | ||
857 | } | ||
858 | |||
859 | static size_t output__fprintf(FILE *fp, uint64_t total_samples) | ||
860 | { | ||
861 | struct hist_entry *pos; | ||
862 | struct sort_entry *se; | ||
863 | struct rb_node *nd; | ||
864 | size_t ret = 0; | ||
865 | |||
866 | fprintf(fp, "\n"); | ||
867 | fprintf(fp, "#\n"); | ||
868 | fprintf(fp, "# (%Ld samples)\n", (__u64)total_samples); | ||
869 | fprintf(fp, "#\n"); | ||
870 | |||
871 | fprintf(fp, "# Overhead"); | ||
872 | list_for_each_entry(se, &hist_entry__sort_list, list) | ||
873 | fprintf(fp, " %s", se->header); | ||
874 | fprintf(fp, "\n"); | ||
875 | |||
876 | fprintf(fp, "# ........"); | ||
877 | list_for_each_entry(se, &hist_entry__sort_list, list) { | ||
878 | int i; | ||
879 | |||
880 | fprintf(fp, " "); | ||
881 | for (i = 0; i < strlen(se->header); i++) | ||
882 | fprintf(fp, "."); | ||
883 | } | ||
884 | fprintf(fp, "\n"); | ||
885 | |||
886 | fprintf(fp, "#\n"); | ||
887 | |||
888 | for (nd = rb_first(&output_hists); nd; nd = rb_next(nd)) { | ||
889 | pos = rb_entry(nd, struct hist_entry, rb_node); | ||
890 | ret += hist_entry__fprintf(fp, pos, total_samples); | ||
891 | } | ||
892 | |||
893 | if (!strcmp(sort_order, default_sort_order)) { | ||
894 | fprintf(fp, "#\n"); | ||
895 | fprintf(fp, "# (For more details, try: perf annotate --sort comm,dso,symbol)\n"); | ||
896 | fprintf(fp, "#\n"); | ||
897 | } | ||
898 | fprintf(fp, "\n"); | ||
899 | |||
900 | return ret; | ||
901 | } | ||
902 | |||
903 | static void register_idle_thread(void) | ||
904 | { | ||
905 | struct thread *thread = threads__findnew(0); | ||
906 | |||
907 | if (thread == NULL || | ||
908 | thread__set_comm(thread, "[idle]")) { | ||
909 | fprintf(stderr, "problem inserting idle task.\n"); | ||
910 | exit(-1); | ||
911 | } | ||
912 | } | ||
913 | |||
914 | static unsigned long total = 0, | ||
915 | total_mmap = 0, | ||
916 | total_comm = 0, | ||
917 | total_fork = 0, | ||
918 | total_unknown = 0; | ||
919 | |||
920 | static int | ||
921 | process_overflow_event(event_t *event, unsigned long offset, unsigned long head) | ||
922 | { | ||
923 | char level; | ||
924 | int show = 0; | ||
925 | struct dso *dso = NULL; | ||
926 | struct thread *thread = threads__findnew(event->ip.pid); | ||
927 | uint64_t ip = event->ip.ip; | ||
928 | struct map *map = NULL; | ||
929 | |||
930 | dprintf("%p [%p]: PERF_EVENT (IP, %d): %d: %p\n", | ||
931 | (void *)(offset + head), | ||
932 | (void *)(long)(event->header.size), | ||
933 | event->header.misc, | ||
934 | event->ip.pid, | ||
935 | (void *)(long)ip); | ||
936 | |||
937 | dprintf(" ... thread: %s:%d\n", thread->comm, thread->pid); | ||
938 | |||
939 | if (thread == NULL) { | ||
940 | fprintf(stderr, "problem processing %d event, skipping it.\n", | ||
941 | event->header.type); | ||
942 | return -1; | ||
943 | } | ||
944 | |||
945 | if (event->header.misc & PERF_EVENT_MISC_KERNEL) { | ||
946 | show = SHOW_KERNEL; | ||
947 | level = 'k'; | ||
948 | |||
949 | dso = kernel_dso; | ||
950 | |||
951 | dprintf(" ...... dso: %s\n", dso->name); | ||
952 | |||
953 | } else if (event->header.misc & PERF_EVENT_MISC_USER) { | ||
954 | |||
955 | show = SHOW_USER; | ||
956 | level = '.'; | ||
957 | |||
958 | map = thread__find_map(thread, ip); | ||
959 | if (map != NULL) { | ||
960 | ip = map->map_ip(map, ip); | ||
961 | dso = map->dso; | ||
962 | } else { | ||
963 | /* | ||
964 | * If this is outside of all known maps, | ||
965 | * and is a negative address, try to look it | ||
966 | * up in the kernel dso, as it might be a | ||
967 | * vsyscall (which executes in user-mode): | ||
968 | */ | ||
969 | if ((long long)ip < 0) | ||
970 | dso = kernel_dso; | ||
971 | } | ||
972 | dprintf(" ...... dso: %s\n", dso ? dso->name : "<not found>"); | ||
973 | |||
974 | } else { | ||
975 | show = SHOW_HV; | ||
976 | level = 'H'; | ||
977 | dprintf(" ...... dso: [hypervisor]\n"); | ||
978 | } | ||
979 | |||
980 | if (show & show_mask) { | ||
981 | struct symbol *sym = NULL; | ||
982 | |||
983 | if (dso) | ||
984 | sym = dso->find_symbol(dso, ip); | ||
985 | |||
986 | if (hist_entry__add(thread, map, dso, sym, ip, level)) { | ||
987 | fprintf(stderr, | ||
988 | "problem incrementing symbol count, skipping event\n"); | ||
989 | return -1; | ||
990 | } | ||
991 | } | ||
992 | total++; | ||
993 | |||
994 | return 0; | ||
995 | } | ||
996 | |||
997 | static int | ||
998 | process_mmap_event(event_t *event, unsigned long offset, unsigned long head) | ||
999 | { | ||
1000 | struct thread *thread = threads__findnew(event->mmap.pid); | ||
1001 | struct map *map = map__new(&event->mmap); | ||
1002 | |||
1003 | dprintf("%p [%p]: PERF_EVENT_MMAP %d: [%p(%p) @ %p]: %s\n", | ||
1004 | (void *)(offset + head), | ||
1005 | (void *)(long)(event->header.size), | ||
1006 | event->mmap.pid, | ||
1007 | (void *)(long)event->mmap.start, | ||
1008 | (void *)(long)event->mmap.len, | ||
1009 | (void *)(long)event->mmap.pgoff, | ||
1010 | event->mmap.filename); | ||
1011 | |||
1012 | if (thread == NULL || map == NULL) { | ||
1013 | dprintf("problem processing PERF_EVENT_MMAP, skipping event.\n"); | ||
1014 | return 0; | ||
1015 | } | ||
1016 | |||
1017 | thread__insert_map(thread, map); | ||
1018 | total_mmap++; | ||
1019 | |||
1020 | return 0; | ||
1021 | } | ||
1022 | |||
1023 | static int | ||
1024 | process_comm_event(event_t *event, unsigned long offset, unsigned long head) | ||
1025 | { | ||
1026 | struct thread *thread = threads__findnew(event->comm.pid); | ||
1027 | |||
1028 | dprintf("%p [%p]: PERF_EVENT_COMM: %s:%d\n", | ||
1029 | (void *)(offset + head), | ||
1030 | (void *)(long)(event->header.size), | ||
1031 | event->comm.comm, event->comm.pid); | ||
1032 | |||
1033 | if (thread == NULL || | ||
1034 | thread__set_comm(thread, event->comm.comm)) { | ||
1035 | dprintf("problem processing PERF_EVENT_COMM, skipping event.\n"); | ||
1036 | return -1; | ||
1037 | } | ||
1038 | total_comm++; | ||
1039 | |||
1040 | return 0; | ||
1041 | } | ||
1042 | |||
1043 | static int | ||
1044 | process_fork_event(event_t *event, unsigned long offset, unsigned long head) | ||
1045 | { | ||
1046 | struct thread *thread = threads__findnew(event->fork.pid); | ||
1047 | struct thread *parent = threads__findnew(event->fork.ppid); | ||
1048 | |||
1049 | dprintf("%p [%p]: PERF_EVENT_FORK: %d:%d\n", | ||
1050 | (void *)(offset + head), | ||
1051 | (void *)(long)(event->header.size), | ||
1052 | event->fork.pid, event->fork.ppid); | ||
1053 | |||
1054 | if (!thread || !parent || thread__fork(thread, parent)) { | ||
1055 | dprintf("problem processing PERF_EVENT_FORK, skipping event.\n"); | ||
1056 | return -1; | ||
1057 | } | ||
1058 | total_fork++; | ||
1059 | |||
1060 | return 0; | ||
1061 | } | ||
1062 | |||
1063 | static int | ||
1064 | process_period_event(event_t *event, unsigned long offset, unsigned long head) | ||
1065 | { | ||
1066 | dprintf("%p [%p]: PERF_EVENT_PERIOD: time:%Ld, id:%Ld: period:%Ld\n", | ||
1067 | (void *)(offset + head), | ||
1068 | (void *)(long)(event->header.size), | ||
1069 | event->period.time, | ||
1070 | event->period.id, | ||
1071 | event->period.sample_period); | ||
1072 | |||
1073 | return 0; | ||
1074 | } | ||
1075 | |||
1076 | static int | ||
1077 | process_event(event_t *event, unsigned long offset, unsigned long head) | ||
1078 | { | ||
1079 | if (event->header.misc & PERF_EVENT_MISC_OVERFLOW) | ||
1080 | return process_overflow_event(event, offset, head); | ||
1081 | |||
1082 | switch (event->header.type) { | ||
1083 | case PERF_EVENT_MMAP: | ||
1084 | return process_mmap_event(event, offset, head); | ||
1085 | |||
1086 | case PERF_EVENT_COMM: | ||
1087 | return process_comm_event(event, offset, head); | ||
1088 | |||
1089 | case PERF_EVENT_FORK: | ||
1090 | return process_fork_event(event, offset, head); | ||
1091 | |||
1092 | case PERF_EVENT_PERIOD: | ||
1093 | return process_period_event(event, offset, head); | ||
1094 | /* | ||
1095 | * We dont process them right now but they are fine: | ||
1096 | */ | ||
1097 | |||
1098 | case PERF_EVENT_THROTTLE: | ||
1099 | case PERF_EVENT_UNTHROTTLE: | ||
1100 | return 0; | ||
1101 | |||
1102 | default: | ||
1103 | return -1; | ||
1104 | } | ||
1105 | |||
1106 | return 0; | ||
1107 | } | ||
1108 | |||
1109 | static int __cmd_annotate(void) | ||
1110 | { | ||
1111 | int ret, rc = EXIT_FAILURE; | ||
1112 | unsigned long offset = 0; | ||
1113 | unsigned long head = 0; | ||
1114 | struct stat stat; | ||
1115 | event_t *event; | ||
1116 | uint32_t size; | ||
1117 | char *buf; | ||
1118 | |||
1119 | register_idle_thread(); | ||
1120 | |||
1121 | input = open(input_name, O_RDONLY); | ||
1122 | if (input < 0) { | ||
1123 | perror("failed to open file"); | ||
1124 | exit(-1); | ||
1125 | } | ||
1126 | |||
1127 | ret = fstat(input, &stat); | ||
1128 | if (ret < 0) { | ||
1129 | perror("failed to stat file"); | ||
1130 | exit(-1); | ||
1131 | } | ||
1132 | |||
1133 | if (!stat.st_size) { | ||
1134 | fprintf(stderr, "zero-sized file, nothing to do!\n"); | ||
1135 | exit(0); | ||
1136 | } | ||
1137 | |||
1138 | if (load_kernel() < 0) { | ||
1139 | perror("failed to load kernel symbols"); | ||
1140 | return EXIT_FAILURE; | ||
1141 | } | ||
1142 | |||
1143 | if (!full_paths) { | ||
1144 | if (getcwd(__cwd, sizeof(__cwd)) == NULL) { | ||
1145 | perror("failed to get the current directory"); | ||
1146 | return EXIT_FAILURE; | ||
1147 | } | ||
1148 | cwdlen = strlen(cwd); | ||
1149 | } else { | ||
1150 | cwd = NULL; | ||
1151 | cwdlen = 0; | ||
1152 | } | ||
1153 | remap: | ||
1154 | buf = (char *)mmap(NULL, page_size * mmap_window, PROT_READ, | ||
1155 | MAP_SHARED, input, offset); | ||
1156 | if (buf == MAP_FAILED) { | ||
1157 | perror("failed to mmap file"); | ||
1158 | exit(-1); | ||
1159 | } | ||
1160 | |||
1161 | more: | ||
1162 | event = (event_t *)(buf + head); | ||
1163 | |||
1164 | size = event->header.size; | ||
1165 | if (!size) | ||
1166 | size = 8; | ||
1167 | |||
1168 | if (head + event->header.size >= page_size * mmap_window) { | ||
1169 | unsigned long shift = page_size * (head / page_size); | ||
1170 | int ret; | ||
1171 | |||
1172 | ret = munmap(buf, page_size * mmap_window); | ||
1173 | assert(ret == 0); | ||
1174 | |||
1175 | offset += shift; | ||
1176 | head -= shift; | ||
1177 | goto remap; | ||
1178 | } | ||
1179 | |||
1180 | size = event->header.size; | ||
1181 | |||
1182 | dprintf("%p [%p]: event: %d\n", | ||
1183 | (void *)(offset + head), | ||
1184 | (void *)(long)event->header.size, | ||
1185 | event->header.type); | ||
1186 | |||
1187 | if (!size || process_event(event, offset, head) < 0) { | ||
1188 | |||
1189 | dprintf("%p [%p]: skipping unknown header type: %d\n", | ||
1190 | (void *)(offset + head), | ||
1191 | (void *)(long)(event->header.size), | ||
1192 | event->header.type); | ||
1193 | |||
1194 | total_unknown++; | ||
1195 | |||
1196 | /* | ||
1197 | * assume we lost track of the stream, check alignment, and | ||
1198 | * increment a single u64 in the hope to catch on again 'soon'. | ||
1199 | */ | ||
1200 | |||
1201 | if (unlikely(head & 7)) | ||
1202 | head &= ~7ULL; | ||
1203 | |||
1204 | size = 8; | ||
1205 | } | ||
1206 | |||
1207 | head += size; | ||
1208 | |||
1209 | if (offset + head < stat.st_size) | ||
1210 | goto more; | ||
1211 | |||
1212 | rc = EXIT_SUCCESS; | ||
1213 | close(input); | ||
1214 | |||
1215 | dprintf(" IP events: %10ld\n", total); | ||
1216 | dprintf(" mmap events: %10ld\n", total_mmap); | ||
1217 | dprintf(" comm events: %10ld\n", total_comm); | ||
1218 | dprintf(" fork events: %10ld\n", total_fork); | ||
1219 | dprintf(" unknown events: %10ld\n", total_unknown); | ||
1220 | |||
1221 | if (dump_trace) | ||
1222 | return 0; | ||
1223 | |||
1224 | if (verbose >= 3) | ||
1225 | threads__fprintf(stdout); | ||
1226 | |||
1227 | if (verbose >= 2) | ||
1228 | dsos__fprintf(stdout); | ||
1229 | |||
1230 | collapse__resort(); | ||
1231 | output__resort(); | ||
1232 | output__fprintf(stdout, total); | ||
1233 | |||
1234 | return rc; | ||
1235 | } | ||
1236 | |||
1237 | static const char * const annotate_usage[] = { | ||
1238 | "perf annotate [<options>] <command>", | ||
1239 | NULL | ||
1240 | }; | ||
1241 | |||
1242 | static const struct option options[] = { | ||
1243 | OPT_STRING('i', "input", &input_name, "file", | ||
1244 | "input file name"), | ||
1245 | OPT_BOOLEAN('v', "verbose", &verbose, | ||
1246 | "be more verbose (show symbol address, etc)"), | ||
1247 | OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace, | ||
1248 | "dump raw trace in ASCII"), | ||
1249 | OPT_STRING('k', "vmlinux", &vmlinux, "file", "vmlinux pathname"), | ||
1250 | OPT_STRING('s', "sort", &sort_order, "key[,key2...]", | ||
1251 | "sort by key(s): pid, comm, dso, symbol. Default: pid,symbol"), | ||
1252 | OPT_BOOLEAN('P', "full-paths", &full_paths, | ||
1253 | "Don't shorten the pathnames taking into account the cwd"), | ||
1254 | OPT_END() | ||
1255 | }; | ||
1256 | |||
1257 | static void setup_sorting(void) | ||
1258 | { | ||
1259 | char *tmp, *tok, *str = strdup(sort_order); | ||
1260 | |||
1261 | for (tok = strtok_r(str, ", ", &tmp); | ||
1262 | tok; tok = strtok_r(NULL, ", ", &tmp)) { | ||
1263 | if (sort_dimension__add(tok) < 0) { | ||
1264 | error("Unknown --sort key: `%s'", tok); | ||
1265 | usage_with_options(annotate_usage, options); | ||
1266 | } | ||
1267 | } | ||
1268 | |||
1269 | free(str); | ||
1270 | } | ||
1271 | |||
1272 | int cmd_annotate(int argc, const char **argv, const char *prefix) | ||
1273 | { | ||
1274 | symbol__init(); | ||
1275 | |||
1276 | page_size = getpagesize(); | ||
1277 | |||
1278 | argc = parse_options(argc, argv, options, annotate_usage, 0); | ||
1279 | |||
1280 | setup_sorting(); | ||
1281 | |||
1282 | /* | ||
1283 | * Any (unrecognized) arguments left? | ||
1284 | */ | ||
1285 | if (argc) | ||
1286 | usage_with_options(annotate_usage, options); | ||
1287 | |||
1288 | setup_pager(); | ||
1289 | |||
1290 | return __cmd_annotate(); | ||
1291 | } | ||