aboutsummaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/perf/util/intel-pt-decoder/Build2
-rw-r--r--tools/perf/util/intel-pt-decoder/intel-pt-decoder.c1816
-rw-r--r--tools/perf/util/intel-pt-decoder/intel-pt-decoder.h104
3 files changed, 1921 insertions, 1 deletions
diff --git a/tools/perf/util/intel-pt-decoder/Build b/tools/perf/util/intel-pt-decoder/Build
index 3c717b420c89..240730d682c1 100644
--- a/tools/perf/util/intel-pt-decoder/Build
+++ b/tools/perf/util/intel-pt-decoder/Build
@@ -1,4 +1,4 @@
1libperf-$(CONFIG_AUXTRACE) += intel-pt-pkt-decoder.o intel-pt-insn-decoder.o intel-pt-log.o 1libperf-$(CONFIG_AUXTRACE) += intel-pt-pkt-decoder.o intel-pt-insn-decoder.o intel-pt-log.o intel-pt-decoder.o
2 2
3inat_tables_script = util/intel-pt-decoder/gen-insn-attr-x86.awk 3inat_tables_script = util/intel-pt-decoder/gen-insn-attr-x86.awk
4inat_tables_maps = util/intel-pt-decoder/x86-opcode-map.txt 4inat_tables_maps = util/intel-pt-decoder/x86-opcode-map.txt
diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c
new file mode 100644
index 000000000000..f8ac462fec1a
--- /dev/null
+++ b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c
@@ -0,0 +1,1816 @@
1/*
2 * intel_pt_decoder.c: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16#ifndef _GNU_SOURCE
17#define _GNU_SOURCE
18#endif
19#include <stdlib.h>
20#include <stdbool.h>
21#include <string.h>
22#include <errno.h>
23#include <stdint.h>
24#include <inttypes.h>
25
26#include "../cache.h"
27#include "../util.h"
28
29#include "intel-pt-insn-decoder.h"
30#include "intel-pt-pkt-decoder.h"
31#include "intel-pt-decoder.h"
32#include "intel-pt-log.h"
33
34#define INTEL_PT_BLK_SIZE 1024
35
36#define BIT63 (((uint64_t)1 << 63))
37
38#define INTEL_PT_RETURN 1
39
40/* Maximum number of loops with no packets consumed i.e. stuck in a loop */
41#define INTEL_PT_MAX_LOOPS 10000
42
43struct intel_pt_blk {
44 struct intel_pt_blk *prev;
45 uint64_t ip[INTEL_PT_BLK_SIZE];
46};
47
48struct intel_pt_stack {
49 struct intel_pt_blk *blk;
50 struct intel_pt_blk *spare;
51 int pos;
52};
53
54enum intel_pt_pkt_state {
55 INTEL_PT_STATE_NO_PSB,
56 INTEL_PT_STATE_NO_IP,
57 INTEL_PT_STATE_ERR_RESYNC,
58 INTEL_PT_STATE_IN_SYNC,
59 INTEL_PT_STATE_TNT,
60 INTEL_PT_STATE_TIP,
61 INTEL_PT_STATE_TIP_PGD,
62 INTEL_PT_STATE_FUP,
63 INTEL_PT_STATE_FUP_NO_TIP,
64};
65
66#ifdef INTEL_PT_STRICT
67#define INTEL_PT_STATE_ERR1 INTEL_PT_STATE_NO_PSB
68#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_PSB
69#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_NO_PSB
70#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_NO_PSB
71#else
72#define INTEL_PT_STATE_ERR1 (decoder->pkt_state)
73#define INTEL_PT_STATE_ERR2 INTEL_PT_STATE_NO_IP
74#define INTEL_PT_STATE_ERR3 INTEL_PT_STATE_ERR_RESYNC
75#define INTEL_PT_STATE_ERR4 INTEL_PT_STATE_IN_SYNC
76#endif
77
78struct intel_pt_decoder {
79 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
80 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
81 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
82 uint64_t max_insn_cnt, void *data);
83 void *data;
84 struct intel_pt_state state;
85 const unsigned char *buf;
86 size_t len;
87 bool return_compression;
88 bool pge;
89 uint64_t pos;
90 uint64_t last_ip;
91 uint64_t ip;
92 uint64_t cr3;
93 uint64_t timestamp;
94 uint64_t tsc_timestamp;
95 uint64_t ref_timestamp;
96 uint64_t ret_addr;
97 struct intel_pt_stack stack;
98 enum intel_pt_pkt_state pkt_state;
99 struct intel_pt_pkt packet;
100 struct intel_pt_pkt tnt;
101 int pkt_step;
102 int pkt_len;
103 unsigned int cbr;
104 unsigned int max_non_turbo_ratio;
105 int exec_mode;
106 unsigned int insn_bytes;
107 uint64_t sign_bit;
108 uint64_t sign_bits;
109 uint64_t period;
110 enum intel_pt_period_type period_type;
111 uint64_t period_insn_cnt;
112 uint64_t period_mask;
113 uint64_t period_ticks;
114 uint64_t last_masked_timestamp;
115 bool continuous_period;
116 bool overflow;
117 bool set_fup_tx_flags;
118 unsigned int fup_tx_flags;
119 unsigned int tx_flags;
120 uint64_t timestamp_insn_cnt;
121 uint64_t stuck_ip;
122 int no_progress;
123 int stuck_ip_prd;
124 int stuck_ip_cnt;
125 const unsigned char *next_buf;
126 size_t next_len;
127 unsigned char temp_buf[INTEL_PT_PKT_MAX_SZ];
128};
129
130static uint64_t intel_pt_lower_power_of_2(uint64_t x)
131{
132 int i;
133
134 for (i = 0; x != 1; i++)
135 x >>= 1;
136
137 return x << i;
138}
139
140static void intel_pt_setup_period(struct intel_pt_decoder *decoder)
141{
142 if (decoder->period_type == INTEL_PT_PERIOD_TICKS) {
143 uint64_t period;
144
145 period = intel_pt_lower_power_of_2(decoder->period);
146 decoder->period_mask = ~(period - 1);
147 decoder->period_ticks = period;
148 }
149}
150
151struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params)
152{
153 struct intel_pt_decoder *decoder;
154
155 if (!params->get_trace || !params->walk_insn)
156 return NULL;
157
158 decoder = zalloc(sizeof(struct intel_pt_decoder));
159 if (!decoder)
160 return NULL;
161
162 decoder->get_trace = params->get_trace;
163 decoder->walk_insn = params->walk_insn;
164 decoder->data = params->data;
165 decoder->return_compression = params->return_compression;
166
167 decoder->sign_bit = (uint64_t)1 << 47;
168 decoder->sign_bits = ~(((uint64_t)1 << 48) - 1);
169
170 decoder->period = params->period;
171 decoder->period_type = params->period_type;
172
173 decoder->max_non_turbo_ratio = params->max_non_turbo_ratio;
174
175 intel_pt_setup_period(decoder);
176
177 return decoder;
178}
179
180static void intel_pt_pop_blk(struct intel_pt_stack *stack)
181{
182 struct intel_pt_blk *blk = stack->blk;
183
184 stack->blk = blk->prev;
185 if (!stack->spare)
186 stack->spare = blk;
187 else
188 free(blk);
189}
190
191static uint64_t intel_pt_pop(struct intel_pt_stack *stack)
192{
193 if (!stack->pos) {
194 if (!stack->blk)
195 return 0;
196 intel_pt_pop_blk(stack);
197 if (!stack->blk)
198 return 0;
199 stack->pos = INTEL_PT_BLK_SIZE;
200 }
201 return stack->blk->ip[--stack->pos];
202}
203
204static int intel_pt_alloc_blk(struct intel_pt_stack *stack)
205{
206 struct intel_pt_blk *blk;
207
208 if (stack->spare) {
209 blk = stack->spare;
210 stack->spare = NULL;
211 } else {
212 blk = malloc(sizeof(struct intel_pt_blk));
213 if (!blk)
214 return -ENOMEM;
215 }
216
217 blk->prev = stack->blk;
218 stack->blk = blk;
219 stack->pos = 0;
220 return 0;
221}
222
223static int intel_pt_push(struct intel_pt_stack *stack, uint64_t ip)
224{
225 int err;
226
227 if (!stack->blk || stack->pos == INTEL_PT_BLK_SIZE) {
228 err = intel_pt_alloc_blk(stack);
229 if (err)
230 return err;
231 }
232
233 stack->blk->ip[stack->pos++] = ip;
234 return 0;
235}
236
237static void intel_pt_clear_stack(struct intel_pt_stack *stack)
238{
239 while (stack->blk)
240 intel_pt_pop_blk(stack);
241 stack->pos = 0;
242}
243
244static void intel_pt_free_stack(struct intel_pt_stack *stack)
245{
246 intel_pt_clear_stack(stack);
247 zfree(&stack->blk);
248 zfree(&stack->spare);
249}
250
251void intel_pt_decoder_free(struct intel_pt_decoder *decoder)
252{
253 intel_pt_free_stack(&decoder->stack);
254 free(decoder);
255}
256
257static int intel_pt_ext_err(int code)
258{
259 switch (code) {
260 case -ENOMEM:
261 return INTEL_PT_ERR_NOMEM;
262 case -ENOSYS:
263 return INTEL_PT_ERR_INTERN;
264 case -EBADMSG:
265 return INTEL_PT_ERR_BADPKT;
266 case -ENODATA:
267 return INTEL_PT_ERR_NODATA;
268 case -EILSEQ:
269 return INTEL_PT_ERR_NOINSN;
270 case -ENOENT:
271 return INTEL_PT_ERR_MISMAT;
272 case -EOVERFLOW:
273 return INTEL_PT_ERR_OVR;
274 case -ENOSPC:
275 return INTEL_PT_ERR_LOST;
276 case -ELOOP:
277 return INTEL_PT_ERR_NELOOP;
278 default:
279 return INTEL_PT_ERR_UNK;
280 }
281}
282
283static const char *intel_pt_err_msgs[] = {
284 [INTEL_PT_ERR_NOMEM] = "Memory allocation failed",
285 [INTEL_PT_ERR_INTERN] = "Internal error",
286 [INTEL_PT_ERR_BADPKT] = "Bad packet",
287 [INTEL_PT_ERR_NODATA] = "No more data",
288 [INTEL_PT_ERR_NOINSN] = "Failed to get instruction",
289 [INTEL_PT_ERR_MISMAT] = "Trace doesn't match instruction",
290 [INTEL_PT_ERR_OVR] = "Overflow packet",
291 [INTEL_PT_ERR_LOST] = "Lost trace data",
292 [INTEL_PT_ERR_UNK] = "Unknown error!",
293 [INTEL_PT_ERR_NELOOP] = "Never-ending loop",
294};
295
296int intel_pt__strerror(int code, char *buf, size_t buflen)
297{
298 if (code < 1 || code > INTEL_PT_ERR_MAX)
299 code = INTEL_PT_ERR_UNK;
300 strlcpy(buf, intel_pt_err_msgs[code], buflen);
301 return 0;
302}
303
304static uint64_t intel_pt_calc_ip(struct intel_pt_decoder *decoder,
305 const struct intel_pt_pkt *packet,
306 uint64_t last_ip)
307{
308 uint64_t ip;
309
310 switch (packet->count) {
311 case 2:
312 ip = (last_ip & (uint64_t)0xffffffffffff0000ULL) |
313 packet->payload;
314 break;
315 case 4:
316 ip = (last_ip & (uint64_t)0xffffffff00000000ULL) |
317 packet->payload;
318 break;
319 case 6:
320 ip = packet->payload;
321 break;
322 default:
323 return 0;
324 }
325
326 if (ip & decoder->sign_bit)
327 return ip | decoder->sign_bits;
328
329 return ip;
330}
331
332static inline void intel_pt_set_last_ip(struct intel_pt_decoder *decoder)
333{
334 decoder->last_ip = intel_pt_calc_ip(decoder, &decoder->packet,
335 decoder->last_ip);
336}
337
338static inline void intel_pt_set_ip(struct intel_pt_decoder *decoder)
339{
340 intel_pt_set_last_ip(decoder);
341 decoder->ip = decoder->last_ip;
342}
343
344static void intel_pt_decoder_log_packet(struct intel_pt_decoder *decoder)
345{
346 intel_pt_log_packet(&decoder->packet, decoder->pkt_len, decoder->pos,
347 decoder->buf);
348}
349
350static int intel_pt_bug(struct intel_pt_decoder *decoder)
351{
352 intel_pt_log("ERROR: Internal error\n");
353 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
354 return -ENOSYS;
355}
356
357static inline void intel_pt_clear_tx_flags(struct intel_pt_decoder *decoder)
358{
359 decoder->tx_flags = 0;
360}
361
362static inline void intel_pt_update_in_tx(struct intel_pt_decoder *decoder)
363{
364 decoder->tx_flags = decoder->packet.payload & INTEL_PT_IN_TX;
365}
366
367static int intel_pt_bad_packet(struct intel_pt_decoder *decoder)
368{
369 intel_pt_clear_tx_flags(decoder);
370 decoder->pkt_len = 1;
371 decoder->pkt_step = 1;
372 intel_pt_decoder_log_packet(decoder);
373 if (decoder->pkt_state != INTEL_PT_STATE_NO_PSB) {
374 intel_pt_log("ERROR: Bad packet\n");
375 decoder->pkt_state = INTEL_PT_STATE_ERR1;
376 }
377 return -EBADMSG;
378}
379
380static int intel_pt_get_data(struct intel_pt_decoder *decoder)
381{
382 struct intel_pt_buffer buffer = { .buf = 0, };
383 int ret;
384
385 decoder->pkt_step = 0;
386
387 intel_pt_log("Getting more data\n");
388 ret = decoder->get_trace(&buffer, decoder->data);
389 if (ret)
390 return ret;
391 decoder->buf = buffer.buf;
392 decoder->len = buffer.len;
393 if (!decoder->len) {
394 intel_pt_log("No more data\n");
395 return -ENODATA;
396 }
397 if (!buffer.consecutive) {
398 decoder->ip = 0;
399 decoder->pkt_state = INTEL_PT_STATE_NO_PSB;
400 decoder->ref_timestamp = buffer.ref_timestamp;
401 decoder->timestamp = 0;
402 decoder->state.trace_nr = buffer.trace_nr;
403 intel_pt_log("Reference timestamp 0x%" PRIx64 "\n",
404 decoder->ref_timestamp);
405 return -ENOLINK;
406 }
407
408 return 0;
409}
410
411static int intel_pt_get_next_data(struct intel_pt_decoder *decoder)
412{
413 if (!decoder->next_buf)
414 return intel_pt_get_data(decoder);
415
416 decoder->buf = decoder->next_buf;
417 decoder->len = decoder->next_len;
418 decoder->next_buf = 0;
419 decoder->next_len = 0;
420 return 0;
421}
422
423static int intel_pt_get_split_packet(struct intel_pt_decoder *decoder)
424{
425 unsigned char *buf = decoder->temp_buf;
426 size_t old_len, len, n;
427 int ret;
428
429 old_len = decoder->len;
430 len = decoder->len;
431 memcpy(buf, decoder->buf, len);
432
433 ret = intel_pt_get_data(decoder);
434 if (ret) {
435 decoder->pos += old_len;
436 return ret < 0 ? ret : -EINVAL;
437 }
438
439 n = INTEL_PT_PKT_MAX_SZ - len;
440 if (n > decoder->len)
441 n = decoder->len;
442 memcpy(buf + len, decoder->buf, n);
443 len += n;
444
445 ret = intel_pt_get_packet(buf, len, &decoder->packet);
446 if (ret < (int)old_len) {
447 decoder->next_buf = decoder->buf;
448 decoder->next_len = decoder->len;
449 decoder->buf = buf;
450 decoder->len = old_len;
451 return intel_pt_bad_packet(decoder);
452 }
453
454 decoder->next_buf = decoder->buf + (ret - old_len);
455 decoder->next_len = decoder->len - (ret - old_len);
456
457 decoder->buf = buf;
458 decoder->len = ret;
459
460 return ret;
461}
462
463static int intel_pt_get_next_packet(struct intel_pt_decoder *decoder)
464{
465 int ret;
466
467 do {
468 decoder->pos += decoder->pkt_step;
469 decoder->buf += decoder->pkt_step;
470 decoder->len -= decoder->pkt_step;
471
472 if (!decoder->len) {
473 ret = intel_pt_get_next_data(decoder);
474 if (ret)
475 return ret;
476 }
477
478 ret = intel_pt_get_packet(decoder->buf, decoder->len,
479 &decoder->packet);
480 if (ret == INTEL_PT_NEED_MORE_BYTES &&
481 decoder->len < INTEL_PT_PKT_MAX_SZ && !decoder->next_buf) {
482 ret = intel_pt_get_split_packet(decoder);
483 if (ret < 0)
484 return ret;
485 }
486 if (ret <= 0)
487 return intel_pt_bad_packet(decoder);
488
489 decoder->pkt_len = ret;
490 decoder->pkt_step = ret;
491 intel_pt_decoder_log_packet(decoder);
492 } while (decoder->packet.type == INTEL_PT_PAD);
493
494 return 0;
495}
496
497static uint64_t intel_pt_next_period(struct intel_pt_decoder *decoder)
498{
499 uint64_t timestamp, masked_timestamp;
500
501 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
502 masked_timestamp = timestamp & decoder->period_mask;
503 if (decoder->continuous_period) {
504 if (masked_timestamp != decoder->last_masked_timestamp)
505 return 1;
506 } else {
507 timestamp += 1;
508 masked_timestamp = timestamp & decoder->period_mask;
509 if (masked_timestamp != decoder->last_masked_timestamp) {
510 decoder->last_masked_timestamp = masked_timestamp;
511 decoder->continuous_period = true;
512 }
513 }
514 return decoder->period_ticks - (timestamp - masked_timestamp);
515}
516
517static uint64_t intel_pt_next_sample(struct intel_pt_decoder *decoder)
518{
519 switch (decoder->period_type) {
520 case INTEL_PT_PERIOD_INSTRUCTIONS:
521 return decoder->period - decoder->period_insn_cnt;
522 case INTEL_PT_PERIOD_TICKS:
523 return intel_pt_next_period(decoder);
524 case INTEL_PT_PERIOD_NONE:
525 default:
526 return 0;
527 }
528}
529
530static void intel_pt_sample_insn(struct intel_pt_decoder *decoder)
531{
532 uint64_t timestamp, masked_timestamp;
533
534 switch (decoder->period_type) {
535 case INTEL_PT_PERIOD_INSTRUCTIONS:
536 decoder->period_insn_cnt = 0;
537 break;
538 case INTEL_PT_PERIOD_TICKS:
539 timestamp = decoder->timestamp + decoder->timestamp_insn_cnt;
540 masked_timestamp = timestamp & decoder->period_mask;
541 decoder->last_masked_timestamp = masked_timestamp;
542 break;
543 case INTEL_PT_PERIOD_NONE:
544 default:
545 break;
546 }
547
548 decoder->state.type |= INTEL_PT_INSTRUCTION;
549}
550
551static int intel_pt_walk_insn(struct intel_pt_decoder *decoder,
552 struct intel_pt_insn *intel_pt_insn, uint64_t ip)
553{
554 uint64_t max_insn_cnt, insn_cnt = 0;
555 int err;
556
557 max_insn_cnt = intel_pt_next_sample(decoder);
558
559 err = decoder->walk_insn(intel_pt_insn, &insn_cnt, &decoder->ip, ip,
560 max_insn_cnt, decoder->data);
561
562 decoder->timestamp_insn_cnt += insn_cnt;
563 decoder->period_insn_cnt += insn_cnt;
564
565 if (err) {
566 decoder->no_progress = 0;
567 decoder->pkt_state = INTEL_PT_STATE_ERR2;
568 intel_pt_log_at("ERROR: Failed to get instruction",
569 decoder->ip);
570 if (err == -ENOENT)
571 return -ENOLINK;
572 return -EILSEQ;
573 }
574
575 if (ip && decoder->ip == ip) {
576 err = -EAGAIN;
577 goto out;
578 }
579
580 if (max_insn_cnt && insn_cnt >= max_insn_cnt)
581 intel_pt_sample_insn(decoder);
582
583 if (intel_pt_insn->branch == INTEL_PT_BR_NO_BRANCH) {
584 decoder->state.type = INTEL_PT_INSTRUCTION;
585 decoder->state.from_ip = decoder->ip;
586 decoder->state.to_ip = 0;
587 decoder->ip += intel_pt_insn->length;
588 err = INTEL_PT_RETURN;
589 goto out;
590 }
591
592 if (intel_pt_insn->op == INTEL_PT_OP_CALL) {
593 /* Zero-length calls are excluded */
594 if (intel_pt_insn->branch != INTEL_PT_BR_UNCONDITIONAL ||
595 intel_pt_insn->rel) {
596 err = intel_pt_push(&decoder->stack, decoder->ip +
597 intel_pt_insn->length);
598 if (err)
599 goto out;
600 }
601 } else if (intel_pt_insn->op == INTEL_PT_OP_RET) {
602 decoder->ret_addr = intel_pt_pop(&decoder->stack);
603 }
604
605 if (intel_pt_insn->branch == INTEL_PT_BR_UNCONDITIONAL) {
606 int cnt = decoder->no_progress++;
607
608 decoder->state.from_ip = decoder->ip;
609 decoder->ip += intel_pt_insn->length +
610 intel_pt_insn->rel;
611 decoder->state.to_ip = decoder->ip;
612 err = INTEL_PT_RETURN;
613
614 /*
615 * Check for being stuck in a loop. This can happen if a
616 * decoder error results in the decoder erroneously setting the
617 * ip to an address that is itself in an infinite loop that
618 * consumes no packets. When that happens, there must be an
619 * unconditional branch.
620 */
621 if (cnt) {
622 if (cnt == 1) {
623 decoder->stuck_ip = decoder->state.to_ip;
624 decoder->stuck_ip_prd = 1;
625 decoder->stuck_ip_cnt = 1;
626 } else if (cnt > INTEL_PT_MAX_LOOPS ||
627 decoder->state.to_ip == decoder->stuck_ip) {
628 intel_pt_log_at("ERROR: Never-ending loop",
629 decoder->state.to_ip);
630 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
631 err = -ELOOP;
632 goto out;
633 } else if (!--decoder->stuck_ip_cnt) {
634 decoder->stuck_ip_prd += 1;
635 decoder->stuck_ip_cnt = decoder->stuck_ip_prd;
636 decoder->stuck_ip = decoder->state.to_ip;
637 }
638 }
639 goto out_no_progress;
640 }
641out:
642 decoder->no_progress = 0;
643out_no_progress:
644 decoder->state.insn_op = intel_pt_insn->op;
645 decoder->state.insn_len = intel_pt_insn->length;
646
647 if (decoder->tx_flags & INTEL_PT_IN_TX)
648 decoder->state.flags |= INTEL_PT_IN_TX;
649
650 return err;
651}
652
653static int intel_pt_walk_fup(struct intel_pt_decoder *decoder)
654{
655 struct intel_pt_insn intel_pt_insn;
656 uint64_t ip;
657 int err;
658
659 ip = decoder->last_ip;
660
661 while (1) {
662 err = intel_pt_walk_insn(decoder, &intel_pt_insn, ip);
663 if (err == INTEL_PT_RETURN)
664 return 0;
665 if (err == -EAGAIN) {
666 if (decoder->set_fup_tx_flags) {
667 decoder->set_fup_tx_flags = false;
668 decoder->tx_flags = decoder->fup_tx_flags;
669 decoder->state.type = INTEL_PT_TRANSACTION;
670 decoder->state.from_ip = decoder->ip;
671 decoder->state.to_ip = 0;
672 decoder->state.flags = decoder->fup_tx_flags;
673 return 0;
674 }
675 return err;
676 }
677 decoder->set_fup_tx_flags = false;
678 if (err)
679 return err;
680
681 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
682 intel_pt_log_at("ERROR: Unexpected indirect branch",
683 decoder->ip);
684 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
685 return -ENOENT;
686 }
687
688 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
689 intel_pt_log_at("ERROR: Unexpected conditional branch",
690 decoder->ip);
691 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
692 return -ENOENT;
693 }
694
695 intel_pt_bug(decoder);
696 }
697}
698
699static int intel_pt_walk_tip(struct intel_pt_decoder *decoder)
700{
701 struct intel_pt_insn intel_pt_insn;
702 int err;
703
704 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
705 if (err == INTEL_PT_RETURN)
706 return 0;
707 if (err)
708 return err;
709
710 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
711 if (decoder->pkt_state == INTEL_PT_STATE_TIP_PGD) {
712 decoder->pge = false;
713 decoder->continuous_period = false;
714 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
715 decoder->state.from_ip = decoder->ip;
716 decoder->state.to_ip = 0;
717 if (decoder->packet.count != 0)
718 decoder->ip = decoder->last_ip;
719 } else {
720 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
721 decoder->state.from_ip = decoder->ip;
722 if (decoder->packet.count == 0) {
723 decoder->state.to_ip = 0;
724 } else {
725 decoder->state.to_ip = decoder->last_ip;
726 decoder->ip = decoder->last_ip;
727 }
728 }
729 return 0;
730 }
731
732 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
733 intel_pt_log_at("ERROR: Conditional branch when expecting indirect branch",
734 decoder->ip);
735 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
736 return -ENOENT;
737 }
738
739 return intel_pt_bug(decoder);
740}
741
742static int intel_pt_walk_tnt(struct intel_pt_decoder *decoder)
743{
744 struct intel_pt_insn intel_pt_insn;
745 int err;
746
747 while (1) {
748 err = intel_pt_walk_insn(decoder, &intel_pt_insn, 0);
749 if (err == INTEL_PT_RETURN)
750 return 0;
751 if (err)
752 return err;
753
754 if (intel_pt_insn.op == INTEL_PT_OP_RET) {
755 if (!decoder->return_compression) {
756 intel_pt_log_at("ERROR: RET when expecting conditional branch",
757 decoder->ip);
758 decoder->pkt_state = INTEL_PT_STATE_ERR3;
759 return -ENOENT;
760 }
761 if (!decoder->ret_addr) {
762 intel_pt_log_at("ERROR: Bad RET compression (stack empty)",
763 decoder->ip);
764 decoder->pkt_state = INTEL_PT_STATE_ERR3;
765 return -ENOENT;
766 }
767 if (!(decoder->tnt.payload & BIT63)) {
768 intel_pt_log_at("ERROR: Bad RET compression (TNT=N)",
769 decoder->ip);
770 decoder->pkt_state = INTEL_PT_STATE_ERR3;
771 return -ENOENT;
772 }
773 decoder->tnt.count -= 1;
774 if (!decoder->tnt.count)
775 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
776 decoder->tnt.payload <<= 1;
777 decoder->state.from_ip = decoder->ip;
778 decoder->ip = decoder->ret_addr;
779 decoder->state.to_ip = decoder->ip;
780 return 0;
781 }
782
783 if (intel_pt_insn.branch == INTEL_PT_BR_INDIRECT) {
784 /* Handle deferred TIPs */
785 err = intel_pt_get_next_packet(decoder);
786 if (err)
787 return err;
788 if (decoder->packet.type != INTEL_PT_TIP ||
789 decoder->packet.count == 0) {
790 intel_pt_log_at("ERROR: Missing deferred TIP for indirect branch",
791 decoder->ip);
792 decoder->pkt_state = INTEL_PT_STATE_ERR3;
793 decoder->pkt_step = 0;
794 return -ENOENT;
795 }
796 intel_pt_set_last_ip(decoder);
797 decoder->state.from_ip = decoder->ip;
798 decoder->state.to_ip = decoder->last_ip;
799 decoder->ip = decoder->last_ip;
800 return 0;
801 }
802
803 if (intel_pt_insn.branch == INTEL_PT_BR_CONDITIONAL) {
804 decoder->tnt.count -= 1;
805 if (!decoder->tnt.count)
806 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
807 if (decoder->tnt.payload & BIT63) {
808 decoder->tnt.payload <<= 1;
809 decoder->state.from_ip = decoder->ip;
810 decoder->ip += intel_pt_insn.length +
811 intel_pt_insn.rel;
812 decoder->state.to_ip = decoder->ip;
813 return 0;
814 }
815 /* Instruction sample for a non-taken branch */
816 if (decoder->state.type & INTEL_PT_INSTRUCTION) {
817 decoder->tnt.payload <<= 1;
818 decoder->state.type = INTEL_PT_INSTRUCTION;
819 decoder->state.from_ip = decoder->ip;
820 decoder->state.to_ip = 0;
821 decoder->ip += intel_pt_insn.length;
822 return 0;
823 }
824 decoder->ip += intel_pt_insn.length;
825 if (!decoder->tnt.count)
826 return -EAGAIN;
827 decoder->tnt.payload <<= 1;
828 continue;
829 }
830
831 return intel_pt_bug(decoder);
832 }
833}
834
835static int intel_pt_mode_tsx(struct intel_pt_decoder *decoder, bool *no_tip)
836{
837 unsigned int fup_tx_flags;
838 int err;
839
840 fup_tx_flags = decoder->packet.payload &
841 (INTEL_PT_IN_TX | INTEL_PT_ABORT_TX);
842 err = intel_pt_get_next_packet(decoder);
843 if (err)
844 return err;
845 if (decoder->packet.type == INTEL_PT_FUP) {
846 decoder->fup_tx_flags = fup_tx_flags;
847 decoder->set_fup_tx_flags = true;
848 if (!(decoder->fup_tx_flags & INTEL_PT_ABORT_TX))
849 *no_tip = true;
850 } else {
851 intel_pt_log_at("ERROR: Missing FUP after MODE.TSX",
852 decoder->pos);
853 intel_pt_update_in_tx(decoder);
854 }
855 return 0;
856}
857
858static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder)
859{
860 uint64_t timestamp;
861
862 if (decoder->ref_timestamp) {
863 timestamp = decoder->packet.payload |
864 (decoder->ref_timestamp & (0xffULL << 56));
865 if (timestamp < decoder->ref_timestamp) {
866 if (decoder->ref_timestamp - timestamp > (1ULL << 55))
867 timestamp += (1ULL << 56);
868 } else {
869 if (timestamp - decoder->ref_timestamp > (1ULL << 55))
870 timestamp -= (1ULL << 56);
871 }
872 decoder->tsc_timestamp = timestamp;
873 decoder->timestamp = timestamp;
874 decoder->ref_timestamp = 0;
875 decoder->timestamp_insn_cnt = 0;
876 } else if (decoder->timestamp) {
877 timestamp = decoder->packet.payload |
878 (decoder->timestamp & (0xffULL << 56));
879 if (timestamp < decoder->timestamp &&
880 decoder->timestamp - timestamp < 0x100) {
881 intel_pt_log_to("ERROR: Suppressing backwards timestamp",
882 timestamp);
883 timestamp = decoder->timestamp;
884 }
885 while (timestamp < decoder->timestamp) {
886 intel_pt_log_to("Wraparound timestamp", timestamp);
887 timestamp += (1ULL << 56);
888 }
889 decoder->tsc_timestamp = timestamp;
890 decoder->timestamp = timestamp;
891 decoder->timestamp_insn_cnt = 0;
892 }
893
894 intel_pt_log_to("Setting timestamp", decoder->timestamp);
895}
896
897static int intel_pt_overflow(struct intel_pt_decoder *decoder)
898{
899 intel_pt_log("ERROR: Buffer overflow\n");
900 intel_pt_clear_tx_flags(decoder);
901 decoder->pkt_state = INTEL_PT_STATE_ERR_RESYNC;
902 decoder->overflow = true;
903 return -EOVERFLOW;
904}
905
906/* Walk PSB+ packets when already in sync. */
907static int intel_pt_walk_psbend(struct intel_pt_decoder *decoder)
908{
909 int err;
910
911 while (1) {
912 err = intel_pt_get_next_packet(decoder);
913 if (err)
914 return err;
915
916 switch (decoder->packet.type) {
917 case INTEL_PT_PSBEND:
918 return 0;
919
920 case INTEL_PT_TIP_PGD:
921 case INTEL_PT_TIP_PGE:
922 case INTEL_PT_TIP:
923 case INTEL_PT_TNT:
924 case INTEL_PT_BAD:
925 case INTEL_PT_PSB:
926 intel_pt_log("ERROR: Unexpected packet\n");
927 return -EAGAIN;
928
929 case INTEL_PT_OVF:
930 return intel_pt_overflow(decoder);
931
932 case INTEL_PT_TSC:
933 intel_pt_calc_tsc_timestamp(decoder);
934 break;
935
936 case INTEL_PT_CBR:
937 decoder->cbr = decoder->packet.payload;
938 break;
939
940 case INTEL_PT_MODE_EXEC:
941 decoder->exec_mode = decoder->packet.payload;
942 break;
943
944 case INTEL_PT_PIP:
945 decoder->cr3 = decoder->packet.payload;
946 break;
947
948 case INTEL_PT_FUP:
949 decoder->pge = true;
950 intel_pt_set_last_ip(decoder);
951 break;
952
953 case INTEL_PT_MODE_TSX:
954 intel_pt_update_in_tx(decoder);
955 break;
956
957 case INTEL_PT_PAD:
958 default:
959 break;
960 }
961 }
962}
963
964static int intel_pt_walk_fup_tip(struct intel_pt_decoder *decoder)
965{
966 int err;
967
968 if (decoder->tx_flags & INTEL_PT_ABORT_TX) {
969 decoder->tx_flags = 0;
970 decoder->state.flags &= ~INTEL_PT_IN_TX;
971 decoder->state.flags |= INTEL_PT_ABORT_TX;
972 } else {
973 decoder->state.flags |= INTEL_PT_ASYNC;
974 }
975
976 while (1) {
977 err = intel_pt_get_next_packet(decoder);
978 if (err)
979 return err;
980
981 switch (decoder->packet.type) {
982 case INTEL_PT_TNT:
983 case INTEL_PT_FUP:
984 case INTEL_PT_PSB:
985 case INTEL_PT_TSC:
986 case INTEL_PT_CBR:
987 case INTEL_PT_MODE_TSX:
988 case INTEL_PT_BAD:
989 case INTEL_PT_PSBEND:
990 intel_pt_log("ERROR: Missing TIP after FUP\n");
991 decoder->pkt_state = INTEL_PT_STATE_ERR3;
992 return -ENOENT;
993
994 case INTEL_PT_OVF:
995 return intel_pt_overflow(decoder);
996
997 case INTEL_PT_TIP_PGD:
998 decoder->state.from_ip = decoder->ip;
999 decoder->state.to_ip = 0;
1000 if (decoder->packet.count != 0) {
1001 intel_pt_set_ip(decoder);
1002 intel_pt_log("Omitting PGD ip " x64_fmt "\n",
1003 decoder->ip);
1004 }
1005 decoder->pge = false;
1006 decoder->continuous_period = false;
1007 return 0;
1008
1009 case INTEL_PT_TIP_PGE:
1010 decoder->pge = true;
1011 intel_pt_log("Omitting PGE ip " x64_fmt "\n",
1012 decoder->ip);
1013 decoder->state.from_ip = 0;
1014 if (decoder->packet.count == 0) {
1015 decoder->state.to_ip = 0;
1016 } else {
1017 intel_pt_set_ip(decoder);
1018 decoder->state.to_ip = decoder->ip;
1019 }
1020 return 0;
1021
1022 case INTEL_PT_TIP:
1023 decoder->state.from_ip = decoder->ip;
1024 if (decoder->packet.count == 0) {
1025 decoder->state.to_ip = 0;
1026 } else {
1027 intel_pt_set_ip(decoder);
1028 decoder->state.to_ip = decoder->ip;
1029 }
1030 return 0;
1031
1032 case INTEL_PT_PIP:
1033 decoder->cr3 = decoder->packet.payload;
1034 break;
1035
1036 case INTEL_PT_MODE_EXEC:
1037 decoder->exec_mode = decoder->packet.payload;
1038 break;
1039
1040 case INTEL_PT_PAD:
1041 break;
1042
1043 default:
1044 return intel_pt_bug(decoder);
1045 }
1046 }
1047}
1048
1049static int intel_pt_walk_trace(struct intel_pt_decoder *decoder)
1050{
1051 bool no_tip = false;
1052 int err;
1053
1054 while (1) {
1055 err = intel_pt_get_next_packet(decoder);
1056 if (err)
1057 return err;
1058next:
1059 switch (decoder->packet.type) {
1060 case INTEL_PT_TNT:
1061 if (!decoder->packet.count)
1062 break;
1063 decoder->tnt = decoder->packet;
1064 decoder->pkt_state = INTEL_PT_STATE_TNT;
1065 err = intel_pt_walk_tnt(decoder);
1066 if (err == -EAGAIN)
1067 break;
1068 return err;
1069
1070 case INTEL_PT_TIP_PGD:
1071 if (decoder->packet.count != 0)
1072 intel_pt_set_last_ip(decoder);
1073 decoder->pkt_state = INTEL_PT_STATE_TIP_PGD;
1074 return intel_pt_walk_tip(decoder);
1075
1076 case INTEL_PT_TIP_PGE: {
1077 decoder->pge = true;
1078 if (decoder->packet.count == 0) {
1079 intel_pt_log_at("Skipping zero TIP.PGE",
1080 decoder->pos);
1081 break;
1082 }
1083 intel_pt_set_ip(decoder);
1084 decoder->state.from_ip = 0;
1085 decoder->state.to_ip = decoder->ip;
1086 return 0;
1087 }
1088
1089 case INTEL_PT_OVF:
1090 return intel_pt_overflow(decoder);
1091
1092 case INTEL_PT_TIP:
1093 if (decoder->packet.count != 0)
1094 intel_pt_set_last_ip(decoder);
1095 decoder->pkt_state = INTEL_PT_STATE_TIP;
1096 return intel_pt_walk_tip(decoder);
1097
1098 case INTEL_PT_FUP:
1099 if (decoder->packet.count == 0) {
1100 intel_pt_log_at("Skipping zero FUP",
1101 decoder->pos);
1102 no_tip = false;
1103 break;
1104 }
1105 intel_pt_set_last_ip(decoder);
1106 err = intel_pt_walk_fup(decoder);
1107 if (err != -EAGAIN) {
1108 if (err)
1109 return err;
1110 if (no_tip)
1111 decoder->pkt_state =
1112 INTEL_PT_STATE_FUP_NO_TIP;
1113 else
1114 decoder->pkt_state = INTEL_PT_STATE_FUP;
1115 return 0;
1116 }
1117 if (no_tip) {
1118 no_tip = false;
1119 break;
1120 }
1121 return intel_pt_walk_fup_tip(decoder);
1122
1123 case INTEL_PT_PSB:
1124 intel_pt_clear_stack(&decoder->stack);
1125 err = intel_pt_walk_psbend(decoder);
1126 if (err == -EAGAIN)
1127 goto next;
1128 if (err)
1129 return err;
1130 break;
1131
1132 case INTEL_PT_PIP:
1133 decoder->cr3 = decoder->packet.payload;
1134 break;
1135
1136 case INTEL_PT_TSC:
1137 intel_pt_calc_tsc_timestamp(decoder);
1138 break;
1139
1140 case INTEL_PT_CBR:
1141 decoder->cbr = decoder->packet.payload;
1142 break;
1143
1144 case INTEL_PT_MODE_EXEC:
1145 decoder->exec_mode = decoder->packet.payload;
1146 break;
1147
1148 case INTEL_PT_MODE_TSX:
1149 /* MODE_TSX need not be followed by FUP */
1150 if (!decoder->pge) {
1151 intel_pt_update_in_tx(decoder);
1152 break;
1153 }
1154 err = intel_pt_mode_tsx(decoder, &no_tip);
1155 if (err)
1156 return err;
1157 goto next;
1158
1159 case INTEL_PT_BAD: /* Does not happen */
1160 return intel_pt_bug(decoder);
1161
1162 case INTEL_PT_PSBEND:
1163 case INTEL_PT_PAD:
1164 break;
1165
1166 default:
1167 return intel_pt_bug(decoder);
1168 }
1169 }
1170}
1171
1172/* Walk PSB+ packets to get in sync. */
1173static int intel_pt_walk_psb(struct intel_pt_decoder *decoder)
1174{
1175 int err;
1176
1177 while (1) {
1178 err = intel_pt_get_next_packet(decoder);
1179 if (err)
1180 return err;
1181
1182 switch (decoder->packet.type) {
1183 case INTEL_PT_TIP_PGD:
1184 decoder->continuous_period = false;
1185 case INTEL_PT_TIP_PGE:
1186 case INTEL_PT_TIP:
1187 intel_pt_log("ERROR: Unexpected packet\n");
1188 return -ENOENT;
1189
1190 case INTEL_PT_FUP:
1191 decoder->pge = true;
1192 if (decoder->last_ip || decoder->packet.count == 6 ||
1193 decoder->packet.count == 0) {
1194 uint64_t current_ip = decoder->ip;
1195
1196 intel_pt_set_ip(decoder);
1197 if (current_ip)
1198 intel_pt_log_to("Setting IP",
1199 decoder->ip);
1200 }
1201 break;
1202
1203 case INTEL_PT_TSC:
1204 intel_pt_calc_tsc_timestamp(decoder);
1205 break;
1206
1207 case INTEL_PT_CBR:
1208 decoder->cbr = decoder->packet.payload;
1209 break;
1210
1211 case INTEL_PT_PIP:
1212 decoder->cr3 = decoder->packet.payload;
1213 break;
1214
1215 case INTEL_PT_MODE_EXEC:
1216 decoder->exec_mode = decoder->packet.payload;
1217 break;
1218
1219 case INTEL_PT_MODE_TSX:
1220 intel_pt_update_in_tx(decoder);
1221 break;
1222
1223 case INTEL_PT_TNT:
1224 intel_pt_log("ERROR: Unexpected packet\n");
1225 if (decoder->ip)
1226 decoder->pkt_state = INTEL_PT_STATE_ERR4;
1227 else
1228 decoder->pkt_state = INTEL_PT_STATE_ERR3;
1229 return -ENOENT;
1230
1231 case INTEL_PT_BAD: /* Does not happen */
1232 return intel_pt_bug(decoder);
1233
1234 case INTEL_PT_OVF:
1235 return intel_pt_overflow(decoder);
1236
1237 case INTEL_PT_PSBEND:
1238 return 0;
1239
1240 case INTEL_PT_PSB:
1241 case INTEL_PT_PAD:
1242 default:
1243 break;
1244 }
1245 }
1246}
1247
1248static int intel_pt_walk_to_ip(struct intel_pt_decoder *decoder)
1249{
1250 int err;
1251
1252 while (1) {
1253 err = intel_pt_get_next_packet(decoder);
1254 if (err)
1255 return err;
1256
1257 switch (decoder->packet.type) {
1258 case INTEL_PT_TIP_PGD:
1259 decoder->continuous_period = false;
1260 case INTEL_PT_TIP_PGE:
1261 case INTEL_PT_TIP:
1262 decoder->pge = decoder->packet.type != INTEL_PT_TIP_PGD;
1263 if (decoder->last_ip || decoder->packet.count == 6 ||
1264 decoder->packet.count == 0)
1265 intel_pt_set_ip(decoder);
1266 if (decoder->ip)
1267 return 0;
1268 break;
1269
1270 case INTEL_PT_FUP:
1271 if (decoder->overflow) {
1272 if (decoder->last_ip ||
1273 decoder->packet.count == 6 ||
1274 decoder->packet.count == 0)
1275 intel_pt_set_ip(decoder);
1276 if (decoder->ip)
1277 return 0;
1278 }
1279 if (decoder->packet.count)
1280 intel_pt_set_last_ip(decoder);
1281 break;
1282
1283 case INTEL_PT_TSC:
1284 intel_pt_calc_tsc_timestamp(decoder);
1285 break;
1286
1287 case INTEL_PT_CBR:
1288 decoder->cbr = decoder->packet.payload;
1289 break;
1290
1291 case INTEL_PT_PIP:
1292 decoder->cr3 = decoder->packet.payload;
1293 break;
1294
1295 case INTEL_PT_MODE_EXEC:
1296 decoder->exec_mode = decoder->packet.payload;
1297 break;
1298
1299 case INTEL_PT_MODE_TSX:
1300 intel_pt_update_in_tx(decoder);
1301 break;
1302
1303 case INTEL_PT_OVF:
1304 return intel_pt_overflow(decoder);
1305
1306 case INTEL_PT_BAD: /* Does not happen */
1307 return intel_pt_bug(decoder);
1308
1309 case INTEL_PT_PSB:
1310 err = intel_pt_walk_psb(decoder);
1311 if (err)
1312 return err;
1313 if (decoder->ip) {
1314 /* Do not have a sample */
1315 decoder->state.type = 0;
1316 return 0;
1317 }
1318 break;
1319
1320 case INTEL_PT_TNT:
1321 case INTEL_PT_PSBEND:
1322 case INTEL_PT_PAD:
1323 default:
1324 break;
1325 }
1326 }
1327}
1328
1329static int intel_pt_sync_ip(struct intel_pt_decoder *decoder)
1330{
1331 int err;
1332
1333 intel_pt_log("Scanning for full IP\n");
1334 err = intel_pt_walk_to_ip(decoder);
1335 if (err)
1336 return err;
1337
1338 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1339 decoder->overflow = false;
1340
1341 decoder->state.from_ip = 0;
1342 decoder->state.to_ip = decoder->ip;
1343 intel_pt_log_to("Setting IP", decoder->ip);
1344
1345 return 0;
1346}
1347
1348static int intel_pt_part_psb(struct intel_pt_decoder *decoder)
1349{
1350 const unsigned char *end = decoder->buf + decoder->len;
1351 size_t i;
1352
1353 for (i = INTEL_PT_PSB_LEN - 1; i; i--) {
1354 if (i > decoder->len)
1355 continue;
1356 if (!memcmp(end - i, INTEL_PT_PSB_STR, i))
1357 return i;
1358 }
1359 return 0;
1360}
1361
1362static int intel_pt_rest_psb(struct intel_pt_decoder *decoder, int part_psb)
1363{
1364 size_t rest_psb = INTEL_PT_PSB_LEN - part_psb;
1365 const char *psb = INTEL_PT_PSB_STR;
1366
1367 if (rest_psb > decoder->len ||
1368 memcmp(decoder->buf, psb + part_psb, rest_psb))
1369 return 0;
1370
1371 return rest_psb;
1372}
1373
1374static int intel_pt_get_split_psb(struct intel_pt_decoder *decoder,
1375 int part_psb)
1376{
1377 int rest_psb, ret;
1378
1379 decoder->pos += decoder->len;
1380 decoder->len = 0;
1381
1382 ret = intel_pt_get_next_data(decoder);
1383 if (ret)
1384 return ret;
1385
1386 rest_psb = intel_pt_rest_psb(decoder, part_psb);
1387 if (!rest_psb)
1388 return 0;
1389
1390 decoder->pos -= part_psb;
1391 decoder->next_buf = decoder->buf + rest_psb;
1392 decoder->next_len = decoder->len - rest_psb;
1393 memcpy(decoder->temp_buf, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1394 decoder->buf = decoder->temp_buf;
1395 decoder->len = INTEL_PT_PSB_LEN;
1396
1397 return 0;
1398}
1399
1400static int intel_pt_scan_for_psb(struct intel_pt_decoder *decoder)
1401{
1402 unsigned char *next;
1403 int ret;
1404
1405 intel_pt_log("Scanning for PSB\n");
1406 while (1) {
1407 if (!decoder->len) {
1408 ret = intel_pt_get_next_data(decoder);
1409 if (ret)
1410 return ret;
1411 }
1412
1413 next = memmem(decoder->buf, decoder->len, INTEL_PT_PSB_STR,
1414 INTEL_PT_PSB_LEN);
1415 if (!next) {
1416 int part_psb;
1417
1418 part_psb = intel_pt_part_psb(decoder);
1419 if (part_psb) {
1420 ret = intel_pt_get_split_psb(decoder, part_psb);
1421 if (ret)
1422 return ret;
1423 } else {
1424 decoder->pos += decoder->len;
1425 decoder->len = 0;
1426 }
1427 continue;
1428 }
1429
1430 decoder->pkt_step = next - decoder->buf;
1431 return intel_pt_get_next_packet(decoder);
1432 }
1433}
1434
1435static int intel_pt_sync(struct intel_pt_decoder *decoder)
1436{
1437 int err;
1438
1439 decoder->pge = false;
1440 decoder->continuous_period = false;
1441 decoder->last_ip = 0;
1442 decoder->ip = 0;
1443 intel_pt_clear_stack(&decoder->stack);
1444
1445 err = intel_pt_scan_for_psb(decoder);
1446 if (err)
1447 return err;
1448
1449 decoder->pkt_state = INTEL_PT_STATE_NO_IP;
1450
1451 err = intel_pt_walk_psb(decoder);
1452 if (err)
1453 return err;
1454
1455 if (decoder->ip) {
1456 decoder->state.type = 0; /* Do not have a sample */
1457 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1458 } else {
1459 return intel_pt_sync_ip(decoder);
1460 }
1461
1462 return 0;
1463}
1464
1465static uint64_t intel_pt_est_timestamp(struct intel_pt_decoder *decoder)
1466{
1467 uint64_t est = decoder->timestamp_insn_cnt << 1;
1468
1469 if (!decoder->cbr || !decoder->max_non_turbo_ratio)
1470 goto out;
1471
1472 est *= decoder->max_non_turbo_ratio;
1473 est /= decoder->cbr;
1474out:
1475 return decoder->timestamp + est;
1476}
1477
1478const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder)
1479{
1480 int err;
1481
1482 do {
1483 decoder->state.type = INTEL_PT_BRANCH;
1484 decoder->state.flags = 0;
1485
1486 switch (decoder->pkt_state) {
1487 case INTEL_PT_STATE_NO_PSB:
1488 err = intel_pt_sync(decoder);
1489 break;
1490 case INTEL_PT_STATE_NO_IP:
1491 decoder->last_ip = 0;
1492 /* Fall through */
1493 case INTEL_PT_STATE_ERR_RESYNC:
1494 err = intel_pt_sync_ip(decoder);
1495 break;
1496 case INTEL_PT_STATE_IN_SYNC:
1497 err = intel_pt_walk_trace(decoder);
1498 break;
1499 case INTEL_PT_STATE_TNT:
1500 err = intel_pt_walk_tnt(decoder);
1501 if (err == -EAGAIN)
1502 err = intel_pt_walk_trace(decoder);
1503 break;
1504 case INTEL_PT_STATE_TIP:
1505 case INTEL_PT_STATE_TIP_PGD:
1506 err = intel_pt_walk_tip(decoder);
1507 break;
1508 case INTEL_PT_STATE_FUP:
1509 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1510 err = intel_pt_walk_fup(decoder);
1511 if (err == -EAGAIN)
1512 err = intel_pt_walk_fup_tip(decoder);
1513 else if (!err)
1514 decoder->pkt_state = INTEL_PT_STATE_FUP;
1515 break;
1516 case INTEL_PT_STATE_FUP_NO_TIP:
1517 decoder->pkt_state = INTEL_PT_STATE_IN_SYNC;
1518 err = intel_pt_walk_fup(decoder);
1519 if (err == -EAGAIN)
1520 err = intel_pt_walk_trace(decoder);
1521 break;
1522 default:
1523 err = intel_pt_bug(decoder);
1524 break;
1525 }
1526 } while (err == -ENOLINK);
1527
1528 decoder->state.err = err ? intel_pt_ext_err(err) : 0;
1529 decoder->state.timestamp = decoder->timestamp;
1530 decoder->state.est_timestamp = intel_pt_est_timestamp(decoder);
1531 decoder->state.cr3 = decoder->cr3;
1532
1533 if (err)
1534 decoder->state.from_ip = decoder->ip;
1535
1536 return &decoder->state;
1537}
1538
1539static bool intel_pt_at_psb(unsigned char *buf, size_t len)
1540{
1541 if (len < INTEL_PT_PSB_LEN)
1542 return false;
1543 return memmem(buf, INTEL_PT_PSB_LEN, INTEL_PT_PSB_STR,
1544 INTEL_PT_PSB_LEN);
1545}
1546
1547/**
1548 * intel_pt_next_psb - move buffer pointer to the start of the next PSB packet.
1549 * @buf: pointer to buffer pointer
1550 * @len: size of buffer
1551 *
1552 * Updates the buffer pointer to point to the start of the next PSB packet if
1553 * there is one, otherwise the buffer pointer is unchanged. If @buf is updated,
1554 * @len is adjusted accordingly.
1555 *
1556 * Return: %true if a PSB packet is found, %false otherwise.
1557 */
1558static bool intel_pt_next_psb(unsigned char **buf, size_t *len)
1559{
1560 unsigned char *next;
1561
1562 next = memmem(*buf, *len, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1563 if (next) {
1564 *len -= next - *buf;
1565 *buf = next;
1566 return true;
1567 }
1568 return false;
1569}
1570
1571/**
1572 * intel_pt_step_psb - move buffer pointer to the start of the following PSB
1573 * packet.
1574 * @buf: pointer to buffer pointer
1575 * @len: size of buffer
1576 *
1577 * Updates the buffer pointer to point to the start of the following PSB packet
1578 * (skipping the PSB at @buf itself) if there is one, otherwise the buffer
1579 * pointer is unchanged. If @buf is updated, @len is adjusted accordingly.
1580 *
1581 * Return: %true if a PSB packet is found, %false otherwise.
1582 */
1583static bool intel_pt_step_psb(unsigned char **buf, size_t *len)
1584{
1585 unsigned char *next;
1586
1587 if (!*len)
1588 return false;
1589
1590 next = memmem(*buf + 1, *len - 1, INTEL_PT_PSB_STR, INTEL_PT_PSB_LEN);
1591 if (next) {
1592 *len -= next - *buf;
1593 *buf = next;
1594 return true;
1595 }
1596 return false;
1597}
1598
1599/**
1600 * intel_pt_last_psb - find the last PSB packet in a buffer.
1601 * @buf: buffer
1602 * @len: size of buffer
1603 *
1604 * This function finds the last PSB in a buffer.
1605 *
1606 * Return: A pointer to the last PSB in @buf if found, %NULL otherwise.
1607 */
1608static unsigned char *intel_pt_last_psb(unsigned char *buf, size_t len)
1609{
1610 const char *n = INTEL_PT_PSB_STR;
1611 unsigned char *p;
1612 size_t k;
1613
1614 if (len < INTEL_PT_PSB_LEN)
1615 return NULL;
1616
1617 k = len - INTEL_PT_PSB_LEN + 1;
1618 while (1) {
1619 p = memrchr(buf, n[0], k);
1620 if (!p)
1621 return NULL;
1622 if (!memcmp(p + 1, n + 1, INTEL_PT_PSB_LEN - 1))
1623 return p;
1624 k = p - buf;
1625 if (!k)
1626 return NULL;
1627 }
1628}
1629
1630/**
1631 * intel_pt_next_tsc - find and return next TSC.
1632 * @buf: buffer
1633 * @len: size of buffer
1634 * @tsc: TSC value returned
1635 *
1636 * Find a TSC packet in @buf and return the TSC value. This function assumes
1637 * that @buf starts at a PSB and that PSB+ will contain TSC and so stops if a
1638 * PSBEND packet is found.
1639 *
1640 * Return: %true if TSC is found, false otherwise.
1641 */
1642static bool intel_pt_next_tsc(unsigned char *buf, size_t len, uint64_t *tsc)
1643{
1644 struct intel_pt_pkt packet;
1645 int ret;
1646
1647 while (len) {
1648 ret = intel_pt_get_packet(buf, len, &packet);
1649 if (ret <= 0)
1650 return false;
1651 if (packet.type == INTEL_PT_TSC) {
1652 *tsc = packet.payload;
1653 return true;
1654 }
1655 if (packet.type == INTEL_PT_PSBEND)
1656 return false;
1657 buf += ret;
1658 len -= ret;
1659 }
1660 return false;
1661}
1662
1663/**
1664 * intel_pt_tsc_cmp - compare 7-byte TSCs.
1665 * @tsc1: first TSC to compare
1666 * @tsc2: second TSC to compare
1667 *
1668 * This function compares 7-byte TSC values allowing for the possibility that
1669 * TSC wrapped around. Generally it is not possible to know if TSC has wrapped
1670 * around so for that purpose this function assumes the absolute difference is
1671 * less than half the maximum difference.
1672 *
1673 * Return: %-1 if @tsc1 is before @tsc2, %0 if @tsc1 == @tsc2, %1 if @tsc1 is
1674 * after @tsc2.
1675 */
1676static int intel_pt_tsc_cmp(uint64_t tsc1, uint64_t tsc2)
1677{
1678 const uint64_t halfway = (1ULL << 55);
1679
1680 if (tsc1 == tsc2)
1681 return 0;
1682
1683 if (tsc1 < tsc2) {
1684 if (tsc2 - tsc1 < halfway)
1685 return -1;
1686 else
1687 return 1;
1688 } else {
1689 if (tsc1 - tsc2 < halfway)
1690 return 1;
1691 else
1692 return -1;
1693 }
1694}
1695
1696/**
1697 * intel_pt_find_overlap_tsc - determine start of non-overlapped trace data
1698 * using TSC.
1699 * @buf_a: first buffer
1700 * @len_a: size of first buffer
1701 * @buf_b: second buffer
1702 * @len_b: size of second buffer
1703 *
1704 * If the trace contains TSC we can look at the last TSC of @buf_a and the
1705 * first TSC of @buf_b in order to determine if the buffers overlap, and then
1706 * walk forward in @buf_b until a later TSC is found. A precondition is that
1707 * @buf_a and @buf_b are positioned at a PSB.
1708 *
1709 * Return: A pointer into @buf_b from where non-overlapped data starts, or
1710 * @buf_b + @len_b if there is no non-overlapped data.
1711 */
1712static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a,
1713 size_t len_a,
1714 unsigned char *buf_b,
1715 size_t len_b)
1716{
1717 uint64_t tsc_a, tsc_b;
1718 unsigned char *p;
1719 size_t len;
1720
1721 p = intel_pt_last_psb(buf_a, len_a);
1722 if (!p)
1723 return buf_b; /* No PSB in buf_a => no overlap */
1724
1725 len = len_a - (p - buf_a);
1726 if (!intel_pt_next_tsc(p, len, &tsc_a)) {
1727 /* The last PSB+ in buf_a is incomplete, so go back one more */
1728 len_a -= len;
1729 p = intel_pt_last_psb(buf_a, len_a);
1730 if (!p)
1731 return buf_b; /* No full PSB+ => assume no overlap */
1732 len = len_a - (p - buf_a);
1733 if (!intel_pt_next_tsc(p, len, &tsc_a))
1734 return buf_b; /* No TSC in buf_a => assume no overlap */
1735 }
1736
1737 while (1) {
1738 /* Ignore PSB+ with no TSC */
1739 if (intel_pt_next_tsc(buf_b, len_b, &tsc_b) &&
1740 intel_pt_tsc_cmp(tsc_a, tsc_b) < 0)
1741 return buf_b; /* tsc_a < tsc_b => no overlap */
1742
1743 if (!intel_pt_step_psb(&buf_b, &len_b))
1744 return buf_b + len_b; /* No PSB in buf_b => no data */
1745 }
1746}
1747
1748/**
1749 * intel_pt_find_overlap - determine start of non-overlapped trace data.
1750 * @buf_a: first buffer
1751 * @len_a: size of first buffer
1752 * @buf_b: second buffer
1753 * @len_b: size of second buffer
1754 * @have_tsc: can use TSC packets to detect overlap
1755 *
1756 * When trace samples or snapshots are recorded there is the possibility that
1757 * the data overlaps. Note that, for the purposes of decoding, data is only
1758 * useful if it begins with a PSB packet.
1759 *
1760 * Return: A pointer into @buf_b from where non-overlapped data starts, or
1761 * @buf_b + @len_b if there is no non-overlapped data.
1762 */
1763unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
1764 unsigned char *buf_b, size_t len_b,
1765 bool have_tsc)
1766{
1767 unsigned char *found;
1768
1769 /* Buffer 'b' must start at PSB so throw away everything before that */
1770 if (!intel_pt_next_psb(&buf_b, &len_b))
1771 return buf_b + len_b; /* No PSB */
1772
1773 if (!intel_pt_next_psb(&buf_a, &len_a))
1774 return buf_b; /* No overlap */
1775
1776 if (have_tsc) {
1777 found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b);
1778 if (found)
1779 return found;
1780 }
1781
1782 /*
1783 * Buffer 'b' cannot end within buffer 'a' so, for comparison purposes,
1784 * we can ignore the first part of buffer 'a'.
1785 */
1786 while (len_b < len_a) {
1787 if (!intel_pt_step_psb(&buf_a, &len_a))
1788 return buf_b; /* No overlap */
1789 }
1790
1791 /* Now len_b >= len_a */
1792 if (len_b > len_a) {
1793 /* The leftover buffer 'b' must start at a PSB */
1794 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
1795 if (!intel_pt_step_psb(&buf_a, &len_a))
1796 return buf_b; /* No overlap */
1797 }
1798 }
1799
1800 while (1) {
1801 /* Potential overlap so check the bytes */
1802 found = memmem(buf_a, len_a, buf_b, len_a);
1803 if (found)
1804 return buf_b + len_a;
1805
1806 /* Try again at next PSB in buffer 'a' */
1807 if (!intel_pt_step_psb(&buf_a, &len_a))
1808 return buf_b; /* No overlap */
1809
1810 /* The leftover buffer 'b' must start at a PSB */
1811 while (!intel_pt_at_psb(buf_b + len_a, len_b - len_a)) {
1812 if (!intel_pt_step_psb(&buf_a, &len_a))
1813 return buf_b; /* No overlap */
1814 }
1815 }
1816}
diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h
new file mode 100644
index 000000000000..4c4880230cc9
--- /dev/null
+++ b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h
@@ -0,0 +1,104 @@
1/*
2 * intel_pt_decoder.h: Intel Processor Trace support
3 * Copyright (c) 2013-2014, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 */
15
16#ifndef INCLUDE__INTEL_PT_DECODER_H__
17#define INCLUDE__INTEL_PT_DECODER_H__
18
19#include <stdint.h>
20#include <stddef.h>
21#include <stdbool.h>
22
23#include "intel-pt-insn-decoder.h"
24
25#define INTEL_PT_IN_TX (1 << 0)
26#define INTEL_PT_ABORT_TX (1 << 1)
27#define INTEL_PT_ASYNC (1 << 2)
28
29enum intel_pt_sample_type {
30 INTEL_PT_BRANCH = 1 << 0,
31 INTEL_PT_INSTRUCTION = 1 << 1,
32 INTEL_PT_TRANSACTION = 1 << 2,
33};
34
35enum intel_pt_period_type {
36 INTEL_PT_PERIOD_NONE,
37 INTEL_PT_PERIOD_INSTRUCTIONS,
38 INTEL_PT_PERIOD_TICKS,
39};
40
41enum {
42 INTEL_PT_ERR_NOMEM = 1,
43 INTEL_PT_ERR_INTERN,
44 INTEL_PT_ERR_BADPKT,
45 INTEL_PT_ERR_NODATA,
46 INTEL_PT_ERR_NOINSN,
47 INTEL_PT_ERR_MISMAT,
48 INTEL_PT_ERR_OVR,
49 INTEL_PT_ERR_LOST,
50 INTEL_PT_ERR_UNK,
51 INTEL_PT_ERR_NELOOP,
52 INTEL_PT_ERR_MAX,
53};
54
55struct intel_pt_state {
56 enum intel_pt_sample_type type;
57 int err;
58 uint64_t from_ip;
59 uint64_t to_ip;
60 uint64_t cr3;
61 uint64_t timestamp;
62 uint64_t est_timestamp;
63 uint64_t trace_nr;
64 uint32_t flags;
65 enum intel_pt_insn_op insn_op;
66 int insn_len;
67};
68
69struct intel_pt_insn;
70
71struct intel_pt_buffer {
72 const unsigned char *buf;
73 size_t len;
74 bool consecutive;
75 uint64_t ref_timestamp;
76 uint64_t trace_nr;
77};
78
79struct intel_pt_params {
80 int (*get_trace)(struct intel_pt_buffer *buffer, void *data);
81 int (*walk_insn)(struct intel_pt_insn *intel_pt_insn,
82 uint64_t *insn_cnt_ptr, uint64_t *ip, uint64_t to_ip,
83 uint64_t max_insn_cnt, void *data);
84 void *data;
85 bool return_compression;
86 uint64_t period;
87 enum intel_pt_period_type period_type;
88 unsigned max_non_turbo_ratio;
89};
90
91struct intel_pt_decoder;
92
93struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params);
94void intel_pt_decoder_free(struct intel_pt_decoder *decoder);
95
96const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder);
97
98unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a,
99 unsigned char *buf_b, size_t len_b,
100 bool have_tsc);
101
102int intel_pt__strerror(int code, char *buf, size_t buflen);
103
104#endif
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-30 22:08:40 -0400