diff options
Diffstat (limited to 'arch/x86/kernel/cpu/perf_event_amd_ibs.c')
-rw-r--r-- | arch/x86/kernel/cpu/perf_event_amd_ibs.c | 204 |
1 files changed, 183 insertions, 21 deletions
diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c index 40a6d9d5dd2..573d2487345 100644 --- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c +++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c | |||
@@ -44,9 +44,11 @@ struct perf_ibs { | |||
44 | u64 cnt_mask; | 44 | u64 cnt_mask; |
45 | u64 enable_mask; | 45 | u64 enable_mask; |
46 | u64 valid_mask; | 46 | u64 valid_mask; |
47 | u64 max_period; | ||
47 | unsigned long offset_mask[1]; | 48 | unsigned long offset_mask[1]; |
48 | int offset_max; | 49 | int offset_max; |
49 | struct cpu_perf_ibs __percpu *pcpu; | 50 | struct cpu_perf_ibs __percpu *pcpu; |
51 | u64 (*get_count)(u64 config); | ||
50 | }; | 52 | }; |
51 | 53 | ||
52 | struct perf_ibs_data { | 54 | struct perf_ibs_data { |
@@ -58,6 +60,78 @@ struct perf_ibs_data { | |||
58 | u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; | 60 | u64 regs[MSR_AMD64_IBS_REG_COUNT_MAX]; |
59 | }; | 61 | }; |
60 | 62 | ||
63 | static int | ||
64 | perf_event_set_period(struct hw_perf_event *hwc, u64 min, u64 max, u64 *count) | ||
65 | { | ||
66 | s64 left = local64_read(&hwc->period_left); | ||
67 | s64 period = hwc->sample_period; | ||
68 | int overflow = 0; | ||
69 | |||
70 | /* | ||
71 | * If we are way outside a reasonable range then just skip forward: | ||
72 | */ | ||
73 | if (unlikely(left <= -period)) { | ||
74 | left = period; | ||
75 | local64_set(&hwc->period_left, left); | ||
76 | hwc->last_period = period; | ||
77 | overflow = 1; | ||
78 | } | ||
79 | |||
80 | if (unlikely(left <= 0)) { | ||
81 | left += period; | ||
82 | local64_set(&hwc->period_left, left); | ||
83 | hwc->last_period = period; | ||
84 | overflow = 1; | ||
85 | } | ||
86 | |||
87 | if (unlikely(left < min)) | ||
88 | left = min; | ||
89 | |||
90 | if (left > max) | ||
91 | left = max; | ||
92 | |||
93 | *count = (u64)left; | ||
94 | |||
95 | return overflow; | ||
96 | } | ||
97 | |||
98 | static int | ||
99 | perf_event_try_update(struct perf_event *event, u64 new_raw_count, int width) | ||
100 | { | ||
101 | struct hw_perf_event *hwc = &event->hw; | ||
102 | int shift = 64 - width; | ||
103 | u64 prev_raw_count; | ||
104 | u64 delta; | ||
105 | |||
106 | /* | ||
107 | * Careful: an NMI might modify the previous event value. | ||
108 | * | ||
109 | * Our tactic to handle this is to first atomically read and | ||
110 | * exchange a new raw count - then add that new-prev delta | ||
111 | * count to the generic event atomically: | ||
112 | */ | ||
113 | prev_raw_count = local64_read(&hwc->prev_count); | ||
114 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | ||
115 | new_raw_count) != prev_raw_count) | ||
116 | return 0; | ||
117 | |||
118 | /* | ||
119 | * Now we have the new raw value and have updated the prev | ||
120 | * timestamp already. We can now calculate the elapsed delta | ||
121 | * (event-)time and add that to the generic event. | ||
122 | * | ||
123 | * Careful, not all hw sign-extends above the physical width | ||
124 | * of the count. | ||
125 | */ | ||
126 | delta = (new_raw_count << shift) - (prev_raw_count << shift); | ||
127 | delta >>= shift; | ||
128 | |||
129 | local64_add(delta, &event->count); | ||
130 | local64_sub(delta, &hwc->period_left); | ||
131 | |||
132 | return 1; | ||
133 | } | ||
134 | |||
61 | static struct perf_ibs perf_ibs_fetch; | 135 | static struct perf_ibs perf_ibs_fetch; |
62 | static struct perf_ibs perf_ibs_op; | 136 | static struct perf_ibs perf_ibs_op; |
63 | 137 | ||
@@ -91,18 +165,14 @@ static int perf_ibs_init(struct perf_event *event) | |||
91 | if (hwc->sample_period & 0x0f) | 165 | if (hwc->sample_period & 0x0f) |
92 | /* lower 4 bits can not be set in ibs max cnt */ | 166 | /* lower 4 bits can not be set in ibs max cnt */ |
93 | return -EINVAL; | 167 | return -EINVAL; |
94 | max_cnt = hwc->sample_period >> 4; | ||
95 | if (max_cnt & ~perf_ibs->cnt_mask) | ||
96 | /* out of range */ | ||
97 | return -EINVAL; | ||
98 | config |= max_cnt; | ||
99 | } else { | 168 | } else { |
100 | max_cnt = config & perf_ibs->cnt_mask; | 169 | max_cnt = config & perf_ibs->cnt_mask; |
170 | config &= ~perf_ibs->cnt_mask; | ||
101 | event->attr.sample_period = max_cnt << 4; | 171 | event->attr.sample_period = max_cnt << 4; |
102 | hwc->sample_period = event->attr.sample_period; | 172 | hwc->sample_period = event->attr.sample_period; |
103 | } | 173 | } |
104 | 174 | ||
105 | if (!max_cnt) | 175 | if (!hwc->sample_period) |
106 | return -EINVAL; | 176 | return -EINVAL; |
107 | 177 | ||
108 | hwc->config_base = perf_ibs->msr; | 178 | hwc->config_base = perf_ibs->msr; |
@@ -111,16 +181,71 @@ static int perf_ibs_init(struct perf_event *event) | |||
111 | return 0; | 181 | return 0; |
112 | } | 182 | } |
113 | 183 | ||
184 | static int perf_ibs_set_period(struct perf_ibs *perf_ibs, | ||
185 | struct hw_perf_event *hwc, u64 *period) | ||
186 | { | ||
187 | int ret; | ||
188 | |||
189 | /* ignore lower 4 bits in min count: */ | ||
190 | ret = perf_event_set_period(hwc, 1<<4, perf_ibs->max_period, period); | ||
191 | local64_set(&hwc->prev_count, 0); | ||
192 | |||
193 | return ret; | ||
194 | } | ||
195 | |||
196 | static u64 get_ibs_fetch_count(u64 config) | ||
197 | { | ||
198 | return (config & IBS_FETCH_CNT) >> 12; | ||
199 | } | ||
200 | |||
201 | static u64 get_ibs_op_count(u64 config) | ||
202 | { | ||
203 | return (config & IBS_OP_CUR_CNT) >> 32; | ||
204 | } | ||
205 | |||
206 | static void | ||
207 | perf_ibs_event_update(struct perf_ibs *perf_ibs, struct perf_event *event, | ||
208 | u64 config) | ||
209 | { | ||
210 | u64 count = perf_ibs->get_count(config); | ||
211 | |||
212 | while (!perf_event_try_update(event, count, 20)) { | ||
213 | rdmsrl(event->hw.config_base, config); | ||
214 | count = perf_ibs->get_count(config); | ||
215 | } | ||
216 | } | ||
217 | |||
218 | /* Note: The enable mask must be encoded in the config argument. */ | ||
219 | static inline void perf_ibs_enable_event(struct hw_perf_event *hwc, u64 config) | ||
220 | { | ||
221 | wrmsrl(hwc->config_base, hwc->config | config); | ||
222 | } | ||
223 | |||
224 | /* | ||
225 | * We cannot restore the ibs pmu state, so we always needs to update | ||
226 | * the event while stopping it and then reset the state when starting | ||
227 | * again. Thus, ignoring PERF_EF_RELOAD and PERF_EF_UPDATE flags in | ||
228 | * perf_ibs_start()/perf_ibs_stop() and instead always do it. | ||
229 | */ | ||
114 | static void perf_ibs_start(struct perf_event *event, int flags) | 230 | static void perf_ibs_start(struct perf_event *event, int flags) |
115 | { | 231 | { |
116 | struct hw_perf_event *hwc = &event->hw; | 232 | struct hw_perf_event *hwc = &event->hw; |
117 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); | 233 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); |
118 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | 234 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); |
235 | u64 config; | ||
119 | 236 | ||
120 | if (test_and_set_bit(IBS_STARTED, pcpu->state)) | 237 | if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) |
121 | return; | 238 | return; |
122 | 239 | ||
123 | wrmsrl(hwc->config_base, hwc->config | perf_ibs->enable_mask); | 240 | WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); |
241 | hwc->state = 0; | ||
242 | |||
243 | perf_ibs_set_period(perf_ibs, hwc, &config); | ||
244 | config = (config >> 4) | perf_ibs->enable_mask; | ||
245 | set_bit(IBS_STARTED, pcpu->state); | ||
246 | perf_ibs_enable_event(hwc, config); | ||
247 | |||
248 | perf_event_update_userpage(event); | ||
124 | } | 249 | } |
125 | 250 | ||
126 | static void perf_ibs_stop(struct perf_event *event, int flags) | 251 | static void perf_ibs_stop(struct perf_event *event, int flags) |
@@ -129,15 +254,28 @@ static void perf_ibs_stop(struct perf_event *event, int flags) | |||
129 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); | 254 | struct perf_ibs *perf_ibs = container_of(event->pmu, struct perf_ibs, pmu); |
130 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); | 255 | struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu); |
131 | u64 val; | 256 | u64 val; |
257 | int stopping; | ||
132 | 258 | ||
133 | if (!test_and_clear_bit(IBS_STARTED, pcpu->state)) | 259 | stopping = test_and_clear_bit(IBS_STARTED, pcpu->state); |
134 | return; | ||
135 | 260 | ||
136 | set_bit(IBS_STOPPING, pcpu->state); | 261 | if (!stopping && (hwc->state & PERF_HES_UPTODATE)) |
262 | return; | ||
137 | 263 | ||
138 | rdmsrl(hwc->config_base, val); | 264 | rdmsrl(hwc->config_base, val); |
139 | val &= ~perf_ibs->enable_mask; | 265 | |
140 | wrmsrl(hwc->config_base, val); | 266 | if (stopping) { |
267 | set_bit(IBS_STOPPING, pcpu->state); | ||
268 | val &= ~perf_ibs->enable_mask; | ||
269 | wrmsrl(hwc->config_base, val); | ||
270 | WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); | ||
271 | hwc->state |= PERF_HES_STOPPED; | ||
272 | } | ||
273 | |||
274 | if (hwc->state & PERF_HES_UPTODATE) | ||
275 | return; | ||
276 | |||
277 | perf_ibs_event_update(perf_ibs, event, val); | ||
278 | hwc->state |= PERF_HES_UPTODATE; | ||
141 | } | 279 | } |
142 | 280 | ||
143 | static int perf_ibs_add(struct perf_event *event, int flags) | 281 | static int perf_ibs_add(struct perf_event *event, int flags) |
@@ -148,6 +286,8 @@ static int perf_ibs_add(struct perf_event *event, int flags) | |||
148 | if (test_and_set_bit(IBS_ENABLED, pcpu->state)) | 286 | if (test_and_set_bit(IBS_ENABLED, pcpu->state)) |
149 | return -ENOSPC; | 287 | return -ENOSPC; |
150 | 288 | ||
289 | event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; | ||
290 | |||
151 | pcpu->event = event; | 291 | pcpu->event = event; |
152 | 292 | ||
153 | if (flags & PERF_EF_START) | 293 | if (flags & PERF_EF_START) |
@@ -164,9 +304,11 @@ static void perf_ibs_del(struct perf_event *event, int flags) | |||
164 | if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) | 304 | if (!test_and_clear_bit(IBS_ENABLED, pcpu->state)) |
165 | return; | 305 | return; |
166 | 306 | ||
167 | perf_ibs_stop(event, 0); | 307 | perf_ibs_stop(event, PERF_EF_UPDATE); |
168 | 308 | ||
169 | pcpu->event = NULL; | 309 | pcpu->event = NULL; |
310 | |||
311 | perf_event_update_userpage(event); | ||
170 | } | 312 | } |
171 | 313 | ||
172 | static void perf_ibs_read(struct perf_event *event) { } | 314 | static void perf_ibs_read(struct perf_event *event) { } |
@@ -187,8 +329,11 @@ static struct perf_ibs perf_ibs_fetch = { | |||
187 | .cnt_mask = IBS_FETCH_MAX_CNT, | 329 | .cnt_mask = IBS_FETCH_MAX_CNT, |
188 | .enable_mask = IBS_FETCH_ENABLE, | 330 | .enable_mask = IBS_FETCH_ENABLE, |
189 | .valid_mask = IBS_FETCH_VAL, | 331 | .valid_mask = IBS_FETCH_VAL, |
332 | .max_period = IBS_FETCH_MAX_CNT << 4, | ||
190 | .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, | 333 | .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, |
191 | .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, | 334 | .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, |
335 | |||
336 | .get_count = get_ibs_fetch_count, | ||
192 | }; | 337 | }; |
193 | 338 | ||
194 | static struct perf_ibs perf_ibs_op = { | 339 | static struct perf_ibs perf_ibs_op = { |
@@ -207,8 +352,11 @@ static struct perf_ibs perf_ibs_op = { | |||
207 | .cnt_mask = IBS_OP_MAX_CNT, | 352 | .cnt_mask = IBS_OP_MAX_CNT, |
208 | .enable_mask = IBS_OP_ENABLE, | 353 | .enable_mask = IBS_OP_ENABLE, |
209 | .valid_mask = IBS_OP_VAL, | 354 | .valid_mask = IBS_OP_VAL, |
355 | .max_period = IBS_OP_MAX_CNT << 4, | ||
210 | .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, | 356 | .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, |
211 | .offset_max = MSR_AMD64_IBSOP_REG_COUNT, | 357 | .offset_max = MSR_AMD64_IBSOP_REG_COUNT, |
358 | |||
359 | .get_count = get_ibs_op_count, | ||
212 | }; | 360 | }; |
213 | 361 | ||
214 | static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) | 362 | static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) |
@@ -220,9 +368,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) | |||
220 | struct perf_raw_record raw; | 368 | struct perf_raw_record raw; |
221 | struct pt_regs regs; | 369 | struct pt_regs regs; |
222 | struct perf_ibs_data ibs_data; | 370 | struct perf_ibs_data ibs_data; |
223 | int offset, size; | 371 | int offset, size, overflow, reenable; |
224 | unsigned int msr; | 372 | unsigned int msr; |
225 | u64 *buf; | 373 | u64 *buf, config; |
226 | 374 | ||
227 | if (!test_bit(IBS_STARTED, pcpu->state)) { | 375 | if (!test_bit(IBS_STARTED, pcpu->state)) { |
228 | /* Catch spurious interrupts after stopping IBS: */ | 376 | /* Catch spurious interrupts after stopping IBS: */ |
@@ -257,11 +405,25 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs) | |||
257 | 405 | ||
258 | regs = *iregs; /* XXX: update ip from ibs sample */ | 406 | regs = *iregs; /* XXX: update ip from ibs sample */ |
259 | 407 | ||
260 | if (perf_event_overflow(event, &data, ®s)) | 408 | /* |
261 | ; /* stop */ | 409 | * Emulate IbsOpCurCnt in MSRC001_1033 (IbsOpCtl), not |
262 | else | 410 | * supported in all cpus. As this triggered an interrupt, we |
263 | /* reenable */ | 411 | * set the current count to the max count. |
264 | wrmsrl(hwc->config_base, hwc->config | perf_ibs->enable_mask); | 412 | */ |
413 | config = ibs_data.regs[0]; | ||
414 | if (perf_ibs == &perf_ibs_op && !(ibs_caps & IBS_CAPS_RDWROPCNT)) { | ||
415 | config &= ~IBS_OP_CUR_CNT; | ||
416 | config |= (config & IBS_OP_MAX_CNT) << 36; | ||
417 | } | ||
418 | |||
419 | perf_ibs_event_update(perf_ibs, event, config); | ||
420 | |||
421 | overflow = perf_ibs_set_period(perf_ibs, hwc, &config); | ||
422 | reenable = !(overflow && perf_event_overflow(event, &data, ®s)); | ||
423 | config = (config >> 4) | (reenable ? perf_ibs->enable_mask : 0); | ||
424 | perf_ibs_enable_event(hwc, config); | ||
425 | |||
426 | perf_event_update_userpage(event); | ||
265 | 427 | ||
266 | return 1; | 428 | return 1; |
267 | } | 429 | } |