diff options
author | Ingo Molnar <mingo@elte.hu> | 2008-10-28 11:54:49 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2008-10-28 11:54:49 -0400 |
commit | d1a76187a5be4f89c6cb19d800cb5fb7aac735c5 (patch) | |
tree | 2fac3ffbfffc7560eeef8364b541d0d7a0057920 /arch/x86/kernel/ds.c | |
parent | c7e78cff6b7518212247fb20b1dc6411540dc9af (diff) | |
parent | 0173a3265b228da319ceb9c1ec6a5682fd1b2d92 (diff) |
Merge commit 'v2.6.28-rc2' into core/locking
Conflicts:
arch/um/include/asm/system.h
Diffstat (limited to 'arch/x86/kernel/ds.c')
-rw-r--r-- | arch/x86/kernel/ds.c | 954 |
1 files changed, 677 insertions, 277 deletions
diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c index 11c11b8ec48..2b69994fd3a 100644 --- a/arch/x86/kernel/ds.c +++ b/arch/x86/kernel/ds.c | |||
@@ -2,26 +2,49 @@ | |||
2 | * Debug Store support | 2 | * Debug Store support |
3 | * | 3 | * |
4 | * This provides a low-level interface to the hardware's Debug Store | 4 | * This provides a low-level interface to the hardware's Debug Store |
5 | * feature that is used for last branch recording (LBR) and | 5 | * feature that is used for branch trace store (BTS) and |
6 | * precise-event based sampling (PEBS). | 6 | * precise-event based sampling (PEBS). |
7 | * | 7 | * |
8 | * Different architectures use a different DS layout/pointer size. | 8 | * It manages: |
9 | * The below functions therefore work on a void*. | 9 | * - per-thread and per-cpu allocation of BTS and PEBS |
10 | * - buffer memory allocation (optional) | ||
11 | * - buffer overflow handling | ||
12 | * - buffer access | ||
10 | * | 13 | * |
14 | * It assumes: | ||
15 | * - get_task_struct on all parameter tasks | ||
16 | * - current is allowed to trace parameter tasks | ||
11 | * | 17 | * |
12 | * Since there is no user for PEBS, yet, only LBR (or branch | ||
13 | * trace store, BTS) is supported. | ||
14 | * | 18 | * |
15 | * | 19 | * Copyright (C) 2007-2008 Intel Corporation. |
16 | * Copyright (C) 2007 Intel Corporation. | 20 | * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008 |
17 | * Markus Metzger <markus.t.metzger@intel.com>, Dec 2007 | ||
18 | */ | 21 | */ |
19 | 22 | ||
23 | |||
24 | #ifdef CONFIG_X86_DS | ||
25 | |||
20 | #include <asm/ds.h> | 26 | #include <asm/ds.h> |
21 | 27 | ||
22 | #include <linux/errno.h> | 28 | #include <linux/errno.h> |
23 | #include <linux/string.h> | 29 | #include <linux/string.h> |
24 | #include <linux/slab.h> | 30 | #include <linux/slab.h> |
31 | #include <linux/sched.h> | ||
32 | #include <linux/mm.h> | ||
33 | |||
34 | |||
35 | /* | ||
36 | * The configuration for a particular DS hardware implementation. | ||
37 | */ | ||
38 | struct ds_configuration { | ||
39 | /* the size of the DS structure in bytes */ | ||
40 | unsigned char sizeof_ds; | ||
41 | /* the size of one pointer-typed field in the DS structure in bytes; | ||
42 | this covers the first 8 fields related to buffer management. */ | ||
43 | unsigned char sizeof_field; | ||
44 | /* the size of a BTS/PEBS record in bytes */ | ||
45 | unsigned char sizeof_rec[2]; | ||
46 | }; | ||
47 | static struct ds_configuration ds_cfg; | ||
25 | 48 | ||
26 | 49 | ||
27 | /* | 50 | /* |
@@ -44,378 +67,747 @@ | |||
44 | * (interrupt occurs when write pointer passes interrupt pointer) | 67 | * (interrupt occurs when write pointer passes interrupt pointer) |
45 | * - value to which counter is reset following counter overflow | 68 | * - value to which counter is reset following counter overflow |
46 | * | 69 | * |
47 | * On later architectures, the last branch recording hardware uses | 70 | * Later architectures use 64bit pointers throughout, whereas earlier |
48 | * 64bit pointers even in 32bit mode. | 71 | * architectures use 32bit pointers in 32bit mode. |
49 | * | ||
50 | * | ||
51 | * Branch Trace Store (BTS) records store information about control | ||
52 | * flow changes. They at least provide the following information: | ||
53 | * - source linear address | ||
54 | * - destination linear address | ||
55 | * | 72 | * |
56 | * Netburst supported a predicated bit that had been dropped in later | ||
57 | * architectures. We do not suppor it. | ||
58 | * | 73 | * |
74 | * We compute the base address for the first 8 fields based on: | ||
75 | * - the field size stored in the DS configuration | ||
76 | * - the relative field position | ||
77 | * - an offset giving the start of the respective region | ||
59 | * | 78 | * |
60 | * In order to abstract from the actual DS and BTS layout, we describe | 79 | * This offset is further used to index various arrays holding |
61 | * the access to the relevant fields. | 80 | * information for BTS and PEBS at the respective index. |
62 | * Thanks to Andi Kleen for proposing this design. | ||
63 | * | 81 | * |
64 | * The implementation, however, is not as general as it might seem. In | 82 | * On later 32bit processors, we only access the lower 32bit of the |
65 | * order to stay somewhat simple and efficient, we assume an | 83 | * 64bit pointer fields. The upper halves will be zeroed out. |
66 | * underlying unsigned type (mostly a pointer type) and we expect the | ||
67 | * field to be at least as big as that type. | ||
68 | */ | 84 | */ |
69 | 85 | ||
70 | /* | 86 | enum ds_field { |
71 | * A special from_ip address to indicate that the BTS record is an | 87 | ds_buffer_base = 0, |
72 | * info record that needs to be interpreted or skipped. | 88 | ds_index, |
73 | */ | 89 | ds_absolute_maximum, |
74 | #define BTS_ESCAPE_ADDRESS (-1) | 90 | ds_interrupt_threshold, |
91 | }; | ||
75 | 92 | ||
76 | /* | 93 | enum ds_qualifier { |
77 | * A field access descriptor | 94 | ds_bts = 0, |
78 | */ | 95 | ds_pebs |
79 | struct access_desc { | ||
80 | unsigned char offset; | ||
81 | unsigned char size; | ||
82 | }; | 96 | }; |
83 | 97 | ||
98 | static inline unsigned long ds_get(const unsigned char *base, | ||
99 | enum ds_qualifier qual, enum ds_field field) | ||
100 | { | ||
101 | base += (ds_cfg.sizeof_field * (field + (4 * qual))); | ||
102 | return *(unsigned long *)base; | ||
103 | } | ||
104 | |||
105 | static inline void ds_set(unsigned char *base, enum ds_qualifier qual, | ||
106 | enum ds_field field, unsigned long value) | ||
107 | { | ||
108 | base += (ds_cfg.sizeof_field * (field + (4 * qual))); | ||
109 | (*(unsigned long *)base) = value; | ||
110 | } | ||
111 | |||
112 | |||
84 | /* | 113 | /* |
85 | * The configuration for a particular DS/BTS hardware implementation. | 114 | * Locking is done only for allocating BTS or PEBS resources and for |
115 | * guarding context and buffer memory allocation. | ||
116 | * | ||
117 | * Most functions require the current task to own the ds context part | ||
118 | * they are going to access. All the locking is done when validating | ||
119 | * access to the context. | ||
86 | */ | 120 | */ |
87 | struct ds_configuration { | 121 | static spinlock_t ds_lock = __SPIN_LOCK_UNLOCKED(ds_lock); |
88 | /* the DS configuration */ | ||
89 | unsigned char sizeof_ds; | ||
90 | struct access_desc bts_buffer_base; | ||
91 | struct access_desc bts_index; | ||
92 | struct access_desc bts_absolute_maximum; | ||
93 | struct access_desc bts_interrupt_threshold; | ||
94 | /* the BTS configuration */ | ||
95 | unsigned char sizeof_bts; | ||
96 | struct access_desc from_ip; | ||
97 | struct access_desc to_ip; | ||
98 | /* BTS variants used to store additional information like | ||
99 | timestamps */ | ||
100 | struct access_desc info_type; | ||
101 | struct access_desc info_data; | ||
102 | unsigned long debugctl_mask; | ||
103 | }; | ||
104 | 122 | ||
105 | /* | 123 | /* |
106 | * The global configuration used by the below accessor functions | 124 | * Validate that the current task is allowed to access the BTS/PEBS |
125 | * buffer of the parameter task. | ||
126 | * | ||
127 | * Returns 0, if access is granted; -Eerrno, otherwise. | ||
107 | */ | 128 | */ |
108 | static struct ds_configuration ds_cfg; | 129 | static inline int ds_validate_access(struct ds_context *context, |
130 | enum ds_qualifier qual) | ||
131 | { | ||
132 | if (!context) | ||
133 | return -EPERM; | ||
134 | |||
135 | if (context->owner[qual] == current) | ||
136 | return 0; | ||
137 | |||
138 | return -EPERM; | ||
139 | } | ||
140 | |||
109 | 141 | ||
110 | /* | 142 | /* |
111 | * Accessor functions for some DS and BTS fields using the above | 143 | * We either support (system-wide) per-cpu or per-thread allocation. |
112 | * global ptrace_bts_cfg. | 144 | * We distinguish the two based on the task_struct pointer, where a |
145 | * NULL pointer indicates per-cpu allocation for the current cpu. | ||
146 | * | ||
147 | * Allocations are use-counted. As soon as resources are allocated, | ||
148 | * further allocations must be of the same type (per-cpu or | ||
149 | * per-thread). We model this by counting allocations (i.e. the number | ||
150 | * of tracers of a certain type) for one type negatively: | ||
151 | * =0 no tracers | ||
152 | * >0 number of per-thread tracers | ||
153 | * <0 number of per-cpu tracers | ||
154 | * | ||
155 | * The below functions to get and put tracers and to check the | ||
156 | * allocation type require the ds_lock to be held by the caller. | ||
157 | * | ||
158 | * Tracers essentially gives the number of ds contexts for a certain | ||
159 | * type of allocation. | ||
113 | */ | 160 | */ |
114 | static inline unsigned long get_bts_buffer_base(char *base) | 161 | static long tracers; |
162 | |||
163 | static inline void get_tracer(struct task_struct *task) | ||
115 | { | 164 | { |
116 | return *(unsigned long *)(base + ds_cfg.bts_buffer_base.offset); | 165 | tracers += (task ? 1 : -1); |
117 | } | 166 | } |
118 | static inline void set_bts_buffer_base(char *base, unsigned long value) | 167 | |
168 | static inline void put_tracer(struct task_struct *task) | ||
119 | { | 169 | { |
120 | (*(unsigned long *)(base + ds_cfg.bts_buffer_base.offset)) = value; | 170 | tracers -= (task ? 1 : -1); |
121 | } | 171 | } |
122 | static inline unsigned long get_bts_index(char *base) | 172 | |
173 | static inline int check_tracer(struct task_struct *task) | ||
123 | { | 174 | { |
124 | return *(unsigned long *)(base + ds_cfg.bts_index.offset); | 175 | return (task ? (tracers >= 0) : (tracers <= 0)); |
125 | } | 176 | } |
126 | static inline void set_bts_index(char *base, unsigned long value) | 177 | |
178 | |||
179 | /* | ||
180 | * The DS context is either attached to a thread or to a cpu: | ||
181 | * - in the former case, the thread_struct contains a pointer to the | ||
182 | * attached context. | ||
183 | * - in the latter case, we use a static array of per-cpu context | ||
184 | * pointers. | ||
185 | * | ||
186 | * Contexts are use-counted. They are allocated on first access and | ||
187 | * deallocated when the last user puts the context. | ||
188 | * | ||
189 | * We distinguish between an allocating and a non-allocating get of a | ||
190 | * context: | ||
191 | * - the allocating get is used for requesting BTS/PEBS resources. It | ||
192 | * requires the caller to hold the global ds_lock. | ||
193 | * - the non-allocating get is used for all other cases. A | ||
194 | * non-existing context indicates an error. It acquires and releases | ||
195 | * the ds_lock itself for obtaining the context. | ||
196 | * | ||
197 | * A context and its DS configuration are allocated and deallocated | ||
198 | * together. A context always has a DS configuration of the | ||
199 | * appropriate size. | ||
200 | */ | ||
201 | static DEFINE_PER_CPU(struct ds_context *, system_context); | ||
202 | |||
203 | #define this_system_context per_cpu(system_context, smp_processor_id()) | ||
204 | |||
205 | /* | ||
206 | * Returns the pointer to the parameter task's context or to the | ||
207 | * system-wide context, if task is NULL. | ||
208 | * | ||
209 | * Increases the use count of the returned context, if not NULL. | ||
210 | */ | ||
211 | static inline struct ds_context *ds_get_context(struct task_struct *task) | ||
127 | { | 212 | { |
128 | (*(unsigned long *)(base + ds_cfg.bts_index.offset)) = value; | 213 | struct ds_context *context; |
214 | |||
215 | spin_lock(&ds_lock); | ||
216 | |||
217 | context = (task ? task->thread.ds_ctx : this_system_context); | ||
218 | if (context) | ||
219 | context->count++; | ||
220 | |||
221 | spin_unlock(&ds_lock); | ||
222 | |||
223 | return context; | ||
129 | } | 224 | } |
130 | static inline unsigned long get_bts_absolute_maximum(char *base) | 225 | |
226 | /* | ||
227 | * Same as ds_get_context, but allocates the context and it's DS | ||
228 | * structure, if necessary; returns NULL; if out of memory. | ||
229 | * | ||
230 | * pre: requires ds_lock to be held | ||
231 | */ | ||
232 | static inline struct ds_context *ds_alloc_context(struct task_struct *task) | ||
131 | { | 233 | { |
132 | return *(unsigned long *)(base + ds_cfg.bts_absolute_maximum.offset); | 234 | struct ds_context **p_context = |
235 | (task ? &task->thread.ds_ctx : &this_system_context); | ||
236 | struct ds_context *context = *p_context; | ||
237 | |||
238 | if (!context) { | ||
239 | context = kzalloc(sizeof(*context), GFP_KERNEL); | ||
240 | |||
241 | if (!context) | ||
242 | return NULL; | ||
243 | |||
244 | context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL); | ||
245 | if (!context->ds) { | ||
246 | kfree(context); | ||
247 | return NULL; | ||
248 | } | ||
249 | |||
250 | *p_context = context; | ||
251 | |||
252 | context->this = p_context; | ||
253 | context->task = task; | ||
254 | |||
255 | if (task) | ||
256 | set_tsk_thread_flag(task, TIF_DS_AREA_MSR); | ||
257 | |||
258 | if (!task || (task == current)) | ||
259 | wrmsr(MSR_IA32_DS_AREA, (unsigned long)context->ds, 0); | ||
260 | |||
261 | get_tracer(task); | ||
262 | } | ||
263 | |||
264 | context->count++; | ||
265 | |||
266 | return context; | ||
133 | } | 267 | } |
134 | static inline void set_bts_absolute_maximum(char *base, unsigned long value) | 268 | |
269 | /* | ||
270 | * Decreases the use count of the parameter context, if not NULL. | ||
271 | * Deallocates the context, if the use count reaches zero. | ||
272 | */ | ||
273 | static inline void ds_put_context(struct ds_context *context) | ||
135 | { | 274 | { |
136 | (*(unsigned long *)(base + ds_cfg.bts_absolute_maximum.offset)) = value; | 275 | if (!context) |
276 | return; | ||
277 | |||
278 | spin_lock(&ds_lock); | ||
279 | |||
280 | if (--context->count) | ||
281 | goto out; | ||
282 | |||
283 | *(context->this) = NULL; | ||
284 | |||
285 | if (context->task) | ||
286 | clear_tsk_thread_flag(context->task, TIF_DS_AREA_MSR); | ||
287 | |||
288 | if (!context->task || (context->task == current)) | ||
289 | wrmsrl(MSR_IA32_DS_AREA, 0); | ||
290 | |||
291 | put_tracer(context->task); | ||
292 | |||
293 | /* free any leftover buffers from tracers that did not | ||
294 | * deallocate them properly. */ | ||
295 | kfree(context->buffer[ds_bts]); | ||
296 | kfree(context->buffer[ds_pebs]); | ||
297 | kfree(context->ds); | ||
298 | kfree(context); | ||
299 | out: | ||
300 | spin_unlock(&ds_lock); | ||
137 | } | 301 | } |
138 | static inline unsigned long get_bts_interrupt_threshold(char *base) | 302 | |
303 | |||
304 | /* | ||
305 | * Handle a buffer overflow | ||
306 | * | ||
307 | * task: the task whose buffers are overflowing; | ||
308 | * NULL for a buffer overflow on the current cpu | ||
309 | * context: the ds context | ||
310 | * qual: the buffer type | ||
311 | */ | ||
312 | static void ds_overflow(struct task_struct *task, struct ds_context *context, | ||
313 | enum ds_qualifier qual) | ||
139 | { | 314 | { |
140 | return *(unsigned long *)(base + ds_cfg.bts_interrupt_threshold.offset); | 315 | if (!context) |
316 | return; | ||
317 | |||
318 | if (context->callback[qual]) | ||
319 | (*context->callback[qual])(task); | ||
320 | |||
321 | /* todo: do some more overflow handling */ | ||
141 | } | 322 | } |
142 | static inline void set_bts_interrupt_threshold(char *base, unsigned long value) | 323 | |
324 | |||
325 | /* | ||
326 | * Allocate a non-pageable buffer of the parameter size. | ||
327 | * Checks the memory and the locked memory rlimit. | ||
328 | * | ||
329 | * Returns the buffer, if successful; | ||
330 | * NULL, if out of memory or rlimit exceeded. | ||
331 | * | ||
332 | * size: the requested buffer size in bytes | ||
333 | * pages (out): if not NULL, contains the number of pages reserved | ||
334 | */ | ||
335 | static inline void *ds_allocate_buffer(size_t size, unsigned int *pages) | ||
143 | { | 336 | { |
144 | (*(unsigned long *)(base + ds_cfg.bts_interrupt_threshold.offset)) = value; | 337 | unsigned long rlim, vm, pgsz; |
338 | void *buffer; | ||
339 | |||
340 | pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT; | ||
341 | |||
342 | rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT; | ||
343 | vm = current->mm->total_vm + pgsz; | ||
344 | if (rlim < vm) | ||
345 | return NULL; | ||
346 | |||
347 | rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT; | ||
348 | vm = current->mm->locked_vm + pgsz; | ||
349 | if (rlim < vm) | ||
350 | return NULL; | ||
351 | |||
352 | buffer = kzalloc(size, GFP_KERNEL); | ||
353 | if (!buffer) | ||
354 | return NULL; | ||
355 | |||
356 | current->mm->total_vm += pgsz; | ||
357 | current->mm->locked_vm += pgsz; | ||
358 | |||
359 | if (pages) | ||
360 | *pages = pgsz; | ||
361 | |||
362 | return buffer; | ||
145 | } | 363 | } |
146 | static inline unsigned long get_from_ip(char *base) | 364 | |
365 | static int ds_request(struct task_struct *task, void *base, size_t size, | ||
366 | ds_ovfl_callback_t ovfl, enum ds_qualifier qual) | ||
147 | { | 367 | { |
148 | return *(unsigned long *)(base + ds_cfg.from_ip.offset); | 368 | struct ds_context *context; |
369 | unsigned long buffer, adj; | ||
370 | const unsigned long alignment = (1 << 3); | ||
371 | int error = 0; | ||
372 | |||
373 | if (!ds_cfg.sizeof_ds) | ||
374 | return -EOPNOTSUPP; | ||
375 | |||
376 | /* we require some space to do alignment adjustments below */ | ||
377 | if (size < (alignment + ds_cfg.sizeof_rec[qual])) | ||
378 | return -EINVAL; | ||
379 | |||
380 | /* buffer overflow notification is not yet implemented */ | ||
381 | if (ovfl) | ||
382 | return -EOPNOTSUPP; | ||
383 | |||
384 | |||
385 | spin_lock(&ds_lock); | ||
386 | |||
387 | if (!check_tracer(task)) | ||
388 | return -EPERM; | ||
389 | |||
390 | error = -ENOMEM; | ||
391 | context = ds_alloc_context(task); | ||
392 | if (!context) | ||
393 | goto out_unlock; | ||
394 | |||
395 | error = -EALREADY; | ||
396 | if (context->owner[qual] == current) | ||
397 | goto out_unlock; | ||
398 | error = -EPERM; | ||
399 | if (context->owner[qual] != NULL) | ||
400 | goto out_unlock; | ||
401 | context->owner[qual] = current; | ||
402 | |||
403 | spin_unlock(&ds_lock); | ||
404 | |||
405 | |||
406 | error = -ENOMEM; | ||
407 | if (!base) { | ||
408 | base = ds_allocate_buffer(size, &context->pages[qual]); | ||
409 | if (!base) | ||
410 | goto out_release; | ||
411 | |||
412 | context->buffer[qual] = base; | ||
413 | } | ||
414 | error = 0; | ||
415 | |||
416 | context->callback[qual] = ovfl; | ||
417 | |||
418 | /* adjust the buffer address and size to meet alignment | ||
419 | * constraints: | ||
420 | * - buffer is double-word aligned | ||
421 | * - size is multiple of record size | ||
422 | * | ||
423 | * We checked the size at the very beginning; we have enough | ||
424 | * space to do the adjustment. | ||
425 | */ | ||
426 | buffer = (unsigned long)base; | ||
427 | |||
428 | adj = ALIGN(buffer, alignment) - buffer; | ||
429 | buffer += adj; | ||
430 | size -= adj; | ||
431 | |||
432 | size /= ds_cfg.sizeof_rec[qual]; | ||
433 | size *= ds_cfg.sizeof_rec[qual]; | ||
434 | |||
435 | ds_set(context->ds, qual, ds_buffer_base, buffer); | ||
436 | ds_set(context->ds, qual, ds_index, buffer); | ||
437 | ds_set(context->ds, qual, ds_absolute_maximum, buffer + size); | ||
438 | |||
439 | if (ovfl) { | ||
440 | /* todo: select a suitable interrupt threshold */ | ||
441 | } else | ||
442 | ds_set(context->ds, qual, | ||
443 | ds_interrupt_threshold, buffer + size + 1); | ||
444 | |||
445 | /* we keep the context until ds_release */ | ||
446 | return error; | ||
447 | |||
448 | out_release: | ||
449 | context->owner[qual] = NULL; | ||
450 | ds_put_context(context); | ||
451 | return error; | ||
452 | |||
453 | out_unlock: | ||
454 | spin_unlock(&ds_lock); | ||
455 | ds_put_context(context); | ||
456 | return error; | ||
149 | } | 457 | } |
150 | static inline void set_from_ip(char *base, unsigned long value) | 458 | |
459 | int ds_request_bts(struct task_struct *task, void *base, size_t size, | ||
460 | ds_ovfl_callback_t ovfl) | ||
151 | { | 461 | { |
152 | (*(unsigned long *)(base + ds_cfg.from_ip.offset)) = value; | 462 | return ds_request(task, base, size, ovfl, ds_bts); |
153 | } | 463 | } |
154 | static inline unsigned long get_to_ip(char *base) | 464 | |
465 | int ds_request_pebs(struct task_struct *task, void *base, size_t size, | ||
466 | ds_ovfl_callback_t ovfl) | ||
155 | { | 467 | { |
156 | return *(unsigned long *)(base + ds_cfg.to_ip.offset); | 468 | return ds_request(task, base, size, ovfl, ds_pebs); |
157 | } | 469 | } |
158 | static inline void set_to_ip(char *base, unsigned long value) | 470 | |
471 | static int ds_release(struct task_struct *task, enum ds_qualifier qual) | ||
159 | { | 472 | { |
160 | (*(unsigned long *)(base + ds_cfg.to_ip.offset)) = value; | 473 | struct ds_context *context; |
474 | int error; | ||
475 | |||
476 | context = ds_get_context(task); | ||
477 | error = ds_validate_access(context, qual); | ||
478 | if (error < 0) | ||
479 | goto out; | ||
480 | |||
481 | kfree(context->buffer[qual]); | ||
482 | context->buffer[qual] = NULL; | ||
483 | |||
484 | current->mm->total_vm -= context->pages[qual]; | ||
485 | current->mm->locked_vm -= context->pages[qual]; | ||
486 | context->pages[qual] = 0; | ||
487 | context->owner[qual] = NULL; | ||
488 | |||
489 | /* | ||
490 | * we put the context twice: | ||
491 | * once for the ds_get_context | ||
492 | * once for the corresponding ds_request | ||
493 | */ | ||
494 | ds_put_context(context); | ||
495 | out: | ||
496 | ds_put_context(context); | ||
497 | return error; | ||
161 | } | 498 | } |
162 | static inline unsigned char get_info_type(char *base) | 499 | |
500 | int ds_release_bts(struct task_struct *task) | ||
163 | { | 501 | { |
164 | return *(unsigned char *)(base + ds_cfg.info_type.offset); | 502 | return ds_release(task, ds_bts); |
165 | } | 503 | } |
166 | static inline void set_info_type(char *base, unsigned char value) | 504 | |
505 | int ds_release_pebs(struct task_struct *task) | ||
167 | { | 506 | { |
168 | (*(unsigned char *)(base + ds_cfg.info_type.offset)) = value; | 507 | return ds_release(task, ds_pebs); |
169 | } | 508 | } |
170 | static inline unsigned long get_info_data(char *base) | 509 | |
510 | static int ds_get_index(struct task_struct *task, size_t *pos, | ||
511 | enum ds_qualifier qual) | ||
171 | { | 512 | { |
172 | return *(unsigned long *)(base + ds_cfg.info_data.offset); | 513 | struct ds_context *context; |
514 | unsigned long base, index; | ||
515 | int error; | ||
516 | |||
517 | context = ds_get_context(task); | ||
518 | error = ds_validate_access(context, qual); | ||
519 | if (error < 0) | ||
520 | goto out; | ||
521 | |||
522 | base = ds_get(context->ds, qual, ds_buffer_base); | ||
523 | index = ds_get(context->ds, qual, ds_index); | ||
524 | |||
525 | error = ((index - base) / ds_cfg.sizeof_rec[qual]); | ||
526 | if (pos) | ||
527 | *pos = error; | ||
528 | out: | ||
529 | ds_put_context(context); | ||
530 | return error; | ||
173 | } | 531 | } |
174 | static inline void set_info_data(char *base, unsigned long value) | 532 | |
533 | int ds_get_bts_index(struct task_struct *task, size_t *pos) | ||
175 | { | 534 | { |
176 | (*(unsigned long *)(base + ds_cfg.info_data.offset)) = value; | 535 | return ds_get_index(task, pos, ds_bts); |
177 | } | 536 | } |
178 | 537 | ||
538 | int ds_get_pebs_index(struct task_struct *task, size_t *pos) | ||
539 | { | ||
540 | return ds_get_index(task, pos, ds_pebs); | ||
541 | } | ||
179 | 542 | ||
180 | int ds_allocate(void **dsp, size_t bts_size_in_bytes) | 543 | static int ds_get_end(struct task_struct *task, size_t *pos, |
544 | enum ds_qualifier qual) | ||
181 | { | 545 | { |
182 | size_t bts_size_in_records; | 546 | struct ds_context *context; |
183 | unsigned long bts; | 547 | unsigned long base, end; |
184 | void *ds; | 548 | int error; |
549 | |||
550 | context = ds_get_context(task); | ||
551 | error = ds_validate_access(context, qual); | ||
552 | if (error < 0) | ||
553 | goto out; | ||
554 | |||
555 | base = ds_get(context->ds, qual, ds_buffer_base); | ||
556 | end = ds_get(context->ds, qual, ds_absolute_maximum); | ||
557 | |||
558 | error = ((end - base) / ds_cfg.sizeof_rec[qual]); | ||
559 | if (pos) | ||
560 | *pos = error; | ||
561 | out: | ||
562 | ds_put_context(context); | ||
563 | return error; | ||
564 | } | ||
185 | 565 | ||
186 | if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts) | 566 | int ds_get_bts_end(struct task_struct *task, size_t *pos) |
187 | return -EOPNOTSUPP; | 567 | { |
568 | return ds_get_end(task, pos, ds_bts); | ||
569 | } | ||
188 | 570 | ||
189 | if (bts_size_in_bytes < 0) | 571 | int ds_get_pebs_end(struct task_struct *task, size_t *pos) |
190 | return -EINVAL; | 572 | { |
573 | return ds_get_end(task, pos, ds_pebs); | ||
574 | } | ||
191 | 575 | ||
192 | bts_size_in_records = | 576 | static int ds_access(struct task_struct *task, size_t index, |
193 | bts_size_in_bytes / ds_cfg.sizeof_bts; | 577 | const void **record, enum ds_qualifier qual) |
194 | bts_size_in_bytes = | 578 | { |
195 | bts_size_in_records * ds_cfg.sizeof_bts; | 579 | struct ds_context *context; |
580 | unsigned long base, idx; | ||
581 | int error; | ||
196 | 582 | ||
197 | if (bts_size_in_bytes <= 0) | 583 | if (!record) |
198 | return -EINVAL; | 584 | return -EINVAL; |
199 | 585 | ||
200 | bts = (unsigned long)kzalloc(bts_size_in_bytes, GFP_KERNEL); | 586 | context = ds_get_context(task); |
201 | 587 | error = ds_validate_access(context, qual); | |
202 | if (!bts) | 588 | if (error < 0) |
203 | return -ENOMEM; | 589 | goto out; |
204 | 590 | ||
205 | ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL); | 591 | base = ds_get(context->ds, qual, ds_buffer_base); |
592 | idx = base + (index * ds_cfg.sizeof_rec[qual]); | ||
206 | 593 | ||
207 | if (!ds) { | 594 | error = -EINVAL; |
208 | kfree((void *)bts); | 595 | if (idx > ds_get(context->ds, qual, ds_absolute_maximum)) |
209 | return -ENOMEM; | 596 | goto out; |
210 | } | ||
211 | |||
212 | set_bts_buffer_base(ds, bts); | ||
213 | set_bts_index(ds, bts); | ||
214 | set_bts_absolute_maximum(ds, bts + bts_size_in_bytes); | ||
215 | set_bts_interrupt_threshold(ds, bts + bts_size_in_bytes + 1); | ||
216 | 597 | ||
217 | *dsp = ds; | 598 | *record = (const void *)idx; |
218 | return 0; | 599 | error = ds_cfg.sizeof_rec[qual]; |
600 | out: | ||
601 | ds_put_context(context); | ||
602 | return error; | ||
219 | } | 603 | } |
220 | 604 | ||
221 | int ds_free(void **dsp) | 605 | int ds_access_bts(struct task_struct *task, size_t index, const void **record) |
222 | { | 606 | { |
223 | if (*dsp) { | 607 | return ds_access(task, index, record, ds_bts); |
224 | kfree((void *)get_bts_buffer_base(*dsp)); | ||
225 | kfree(*dsp); | ||
226 | *dsp = NULL; | ||
227 | } | ||
228 | return 0; | ||
229 | } | 608 | } |
230 | 609 | ||
231 | int ds_get_bts_size(void *ds) | 610 | int ds_access_pebs(struct task_struct *task, size_t index, const void **record) |
232 | { | 611 | { |
233 | int size_in_bytes; | 612 | return ds_access(task, index, record, ds_pebs); |
234 | |||
235 | if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts) | ||
236 | return -EOPNOTSUPP; | ||
237 | |||
238 | if (!ds) | ||
239 | return 0; | ||
240 | |||
241 | size_in_bytes = | ||
242 | get_bts_absolute_maximum(ds) - | ||
243 | get_bts_buffer_base(ds); | ||
244 | return size_in_bytes; | ||
245 | } | 613 | } |
246 | 614 | ||
247 | int ds_get_bts_end(void *ds) | 615 | static int ds_write(struct task_struct *task, const void *record, size_t size, |
616 | enum ds_qualifier qual, int force) | ||
248 | { | 617 | { |
249 | int size_in_bytes = ds_get_bts_size(ds); | 618 | struct ds_context *context; |
250 | 619 | int error; | |
251 | if (size_in_bytes <= 0) | ||
252 | return size_in_bytes; | ||
253 | 620 | ||
254 | return size_in_bytes / ds_cfg.sizeof_bts; | 621 | if (!record) |
255 | } | 622 | return -EINVAL; |
256 | 623 | ||
257 | int ds_get_bts_index(void *ds) | 624 | error = -EPERM; |
258 | { | 625 | context = ds_get_context(task); |
259 | int index_offset_in_bytes; | 626 | if (!context) |
627 | goto out; | ||
260 | 628 | ||
261 | if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts) | 629 | if (!force) { |
262 | return -EOPNOTSUPP; | 630 | error = ds_validate_access(context, qual); |
631 | if (error < 0) | ||
632 | goto out; | ||
633 | } | ||
263 | 634 | ||
264 | index_offset_in_bytes = | 635 | error = 0; |
265 | get_bts_index(ds) - | 636 | while (size) { |
266 | get_bts_buffer_base(ds); | 637 | unsigned long base, index, end, write_end, int_th; |
638 | unsigned long write_size, adj_write_size; | ||
639 | |||
640 | /* | ||
641 | * write as much as possible without producing an | ||
642 | * overflow interrupt. | ||
643 | * | ||
644 | * interrupt_threshold must either be | ||
645 | * - bigger than absolute_maximum or | ||
646 | * - point to a record between buffer_base and absolute_maximum | ||
647 | * | ||
648 | * index points to a valid record. | ||
649 | */ | ||
650 | base = ds_get(context->ds, qual, ds_buffer_base); | ||
651 | index = ds_get(context->ds, qual, ds_index); | ||
652 | end = ds_get(context->ds, qual, ds_absolute_maximum); | ||
653 | int_th = ds_get(context->ds, qual, ds_interrupt_threshold); | ||
654 | |||
655 | write_end = min(end, int_th); | ||
656 | |||
657 | /* if we are already beyond the interrupt threshold, | ||
658 | * we fill the entire buffer */ | ||
659 | if (write_end <= index) | ||
660 | write_end = end; | ||
661 | |||
662 | if (write_end <= index) | ||
663 | goto out; | ||
664 | |||
665 | write_size = min((unsigned long) size, write_end - index); | ||
666 | memcpy((void *)index, record, write_size); | ||
667 | |||
668 | record = (const char *)record + write_size; | ||
669 | size -= write_size; | ||
670 | error += write_size; | ||
671 | |||
672 | adj_write_size = write_size / ds_cfg.sizeof_rec[qual]; | ||
673 | adj_write_size *= ds_cfg.sizeof_rec[qual]; | ||
674 | |||
675 | /* zero out trailing bytes */ | ||
676 | memset((char *)index + write_size, 0, | ||
677 | adj_write_size - write_size); | ||
678 | index += adj_write_size; | ||
679 | |||
680 | if (index >= end) | ||
681 | index = base; | ||
682 | ds_set(context->ds, qual, ds_index, index); | ||
683 | |||
684 | if (index >= int_th) | ||
685 | ds_overflow(task, context, qual); | ||
686 | } | ||
267 | 687 | ||
268 | return index_offset_in_bytes / ds_cfg.sizeof_bts; | 688 | out: |
689 | ds_put_context(context); | ||
690 | return error; | ||
269 | } | 691 | } |
270 | 692 | ||
271 | int ds_set_overflow(void *ds, int method) | 693 | int ds_write_bts(struct task_struct *task, const void *record, size_t size) |
272 | { | 694 | { |
273 | switch (method) { | 695 | return ds_write(task, record, size, ds_bts, /* force = */ 0); |
274 | case DS_O_SIGNAL: | ||
275 | return -EOPNOTSUPP; | ||
276 | case DS_O_WRAP: | ||
277 | return 0; | ||
278 | default: | ||
279 | return -EINVAL; | ||
280 | } | ||
281 | } | 696 | } |
282 | 697 | ||
283 | int ds_get_overflow(void *ds) | 698 | int ds_write_pebs(struct task_struct *task, const void *record, size_t size) |
284 | { | 699 | { |
285 | return DS_O_WRAP; | 700 | return ds_write(task, record, size, ds_pebs, /* force = */ 0); |
286 | } | 701 | } |
287 | 702 | ||
288 | int ds_clear(void *ds) | 703 | int ds_unchecked_write_bts(struct task_struct *task, |
704 | const void *record, size_t size) | ||
289 | { | 705 | { |
290 | int bts_size = ds_get_bts_size(ds); | 706 | return ds_write(task, record, size, ds_bts, /* force = */ 1); |
291 | unsigned long bts_base; | ||
292 | |||
293 | if (bts_size <= 0) | ||
294 | return bts_size; | ||
295 | |||
296 | bts_base = get_bts_buffer_base(ds); | ||
297 | memset((void *)bts_base, 0, bts_size); | ||
298 | |||
299 | set_bts_index(ds, bts_base); | ||
300 | return 0; | ||
301 | } | 707 | } |
302 | 708 | ||
303 | int ds_read_bts(void *ds, int index, struct bts_struct *out) | 709 | int ds_unchecked_write_pebs(struct task_struct *task, |
710 | const void *record, size_t size) | ||
304 | { | 711 | { |
305 | void *bts; | 712 | return ds_write(task, record, size, ds_pebs, /* force = */ 1); |
713 | } | ||
306 | 714 | ||
307 | if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts) | 715 | static int ds_reset_or_clear(struct task_struct *task, |
308 | return -EOPNOTSUPP; | 716 | enum ds_qualifier qual, int clear) |
717 | { | ||
718 | struct ds_context *context; | ||
719 | unsigned long base, end; | ||
720 | int error; | ||
309 | 721 | ||
310 | if (index < 0) | 722 | context = ds_get_context(task); |
311 | return -EINVAL; | 723 | error = ds_validate_access(context, qual); |
724 | if (error < 0) | ||
725 | goto out; | ||
312 | 726 | ||
313 | if (index >= ds_get_bts_size(ds)) | 727 | base = ds_get(context->ds, qual, ds_buffer_base); |
314 | return -EINVAL; | 728 | end = ds_get(context->ds, qual, ds_absolute_maximum); |
315 | 729 | ||
316 | bts = (void *)(get_bts_buffer_base(ds) + (index * ds_cfg.sizeof_bts)); | 730 | if (clear) |
731 | memset((void *)base, 0, end - base); | ||
317 | 732 | ||
318 | memset(out, 0, sizeof(*out)); | 733 | ds_set(context->ds, qual, ds_index, base); |
319 | if (get_from_ip(bts) == BTS_ESCAPE_ADDRESS) { | ||
320 | out->qualifier = get_info_type(bts); | ||
321 | out->variant.jiffies = get_info_data(bts); | ||
322 | } else { | ||
323 | out->qualifier = BTS_BRANCH; | ||
324 | out->variant.lbr.from_ip = get_from_ip(bts); | ||
325 | out->variant.lbr.to_ip = get_to_ip(bts); | ||
326 | } | ||
327 | 734 | ||
328 | return sizeof(*out);; | 735 | error = 0; |
736 | out: | ||
737 | ds_put_context(context); | ||
738 | return error; | ||
329 | } | 739 | } |
330 | 740 | ||
331 | int ds_write_bts(void *ds, const struct bts_struct *in) | 741 | int ds_reset_bts(struct task_struct *task) |
332 | { | 742 | { |
333 | unsigned long bts; | 743 | return ds_reset_or_clear(task, ds_bts, /* clear = */ 0); |
334 | 744 | } | |
335 | if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts) | ||
336 | return -EOPNOTSUPP; | ||
337 | |||
338 | if (ds_get_bts_size(ds) <= 0) | ||
339 | return -ENXIO; | ||
340 | 745 | ||
341 | bts = get_bts_index(ds); | 746 | int ds_reset_pebs(struct task_struct *task) |
747 | { | ||
748 | return ds_reset_or_clear(task, ds_pebs, /* clear = */ 0); | ||
749 | } | ||
342 | 750 | ||
343 | memset((void *)bts, 0, ds_cfg.sizeof_bts); | 751 | int ds_clear_bts(struct task_struct *task) |
344 | switch (in->qualifier) { | 752 | { |
345 | case BTS_INVALID: | 753 | return ds_reset_or_clear(task, ds_bts, /* clear = */ 1); |
346 | break; | 754 | } |
347 | 755 | ||
348 | case BTS_BRANCH: | 756 | int ds_clear_pebs(struct task_struct *task) |
349 | set_from_ip((void *)bts, in->variant.lbr.from_ip); | 757 | { |
350 | set_to_ip((void *)bts, in->variant.lbr.to_ip); | 758 | return ds_reset_or_clear(task, ds_pebs, /* clear = */ 1); |
351 | break; | 759 | } |
352 | 760 | ||
353 | case BTS_TASK_ARRIVES: | 761 | int ds_get_pebs_reset(struct task_struct *task, u64 *value) |
354 | case BTS_TASK_DEPARTS: | 762 | { |
355 | set_from_ip((void *)bts, BTS_ESCAPE_ADDRESS); | 763 | struct ds_context *context; |
356 | set_info_type((void *)bts, in->qualifier); | 764 | int error; |
357 | set_info_data((void *)bts, in->variant.jiffies); | ||
358 | break; | ||
359 | 765 | ||
360 | default: | 766 | if (!value) |
361 | return -EINVAL; | 767 | return -EINVAL; |
362 | } | ||
363 | 768 | ||
364 | bts = bts + ds_cfg.sizeof_bts; | 769 | context = ds_get_context(task); |
365 | if (bts >= get_bts_absolute_maximum(ds)) | 770 | error = ds_validate_access(context, ds_pebs); |
366 | bts = get_bts_buffer_base(ds); | 771 | if (error < 0) |
367 | set_bts_index(ds, bts); | 772 | goto out; |
368 | 773 | ||
369 | return ds_cfg.sizeof_bts; | 774 | *value = *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)); |
775 | |||
776 | error = 0; | ||
777 | out: | ||
778 | ds_put_context(context); | ||
779 | return error; | ||
370 | } | 780 | } |
371 | 781 | ||
372 | unsigned long ds_debugctl_mask(void) | 782 | int ds_set_pebs_reset(struct task_struct *task, u64 value) |
373 | { | 783 | { |
374 | return ds_cfg.debugctl_mask; | 784 | struct ds_context *context; |
375 | } | 785 | int error; |
376 | 786 | ||
377 | #ifdef __i386__ | 787 | context = ds_get_context(task); |
378 | static const struct ds_configuration ds_cfg_netburst = { | 788 | error = ds_validate_access(context, ds_pebs); |
379 | .sizeof_ds = 9 * 4, | 789 | if (error < 0) |
380 | .bts_buffer_base = { 0, 4 }, | 790 | goto out; |
381 | .bts_index = { 4, 4 }, | ||
382 | .bts_absolute_maximum = { 8, 4 }, | ||
383 | .bts_interrupt_threshold = { 12, 4 }, | ||
384 | .sizeof_bts = 3 * 4, | ||
385 | .from_ip = { 0, 4 }, | ||
386 | .to_ip = { 4, 4 }, | ||
387 | .info_type = { 4, 1 }, | ||
388 | .info_data = { 8, 4 }, | ||
389 | .debugctl_mask = (1<<2)|(1<<3) | ||
390 | }; | ||
391 | 791 | ||
392 | static const struct ds_configuration ds_cfg_pentium_m = { | 792 | *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)) = value; |
393 | .sizeof_ds = 9 * 4, | 793 | |
394 | .bts_buffer_base = { 0, 4 }, | 794 | error = 0; |
395 | .bts_index = { 4, 4 }, | 795 | out: |
396 | .bts_absolute_maximum = { 8, 4 }, | 796 | ds_put_context(context); |
397 | .bts_interrupt_threshold = { 12, 4 }, | 797 | return error; |
398 | .sizeof_bts = 3 * 4, | 798 | } |
399 | .from_ip = { 0, 4 }, | 799 | |
400 | .to_ip = { 4, 4 }, | 800 | static const struct ds_configuration ds_cfg_var = { |
401 | .info_type = { 4, 1 }, | 801 | .sizeof_ds = sizeof(long) * 12, |
402 | .info_data = { 8, 4 }, | 802 | .sizeof_field = sizeof(long), |
403 | .debugctl_mask = (1<<6)|(1<<7) | 803 | .sizeof_rec[ds_bts] = sizeof(long) * 3, |
804 | .sizeof_rec[ds_pebs] = sizeof(long) * 10 | ||
404 | }; | 805 | }; |
405 | #endif /* _i386_ */ | 806 | static const struct ds_configuration ds_cfg_64 = { |
406 | 807 | .sizeof_ds = 8 * 12, | |
407 | static const struct ds_configuration ds_cfg_core2 = { | 808 | .sizeof_field = 8, |
408 | .sizeof_ds = 9 * 8, | 809 | .sizeof_rec[ds_bts] = 8 * 3, |
409 | .bts_buffer_base = { 0, 8 }, | 810 | .sizeof_rec[ds_pebs] = 8 * 10 |
410 | .bts_index = { 8, 8 }, | ||
411 | .bts_absolute_maximum = { 16, 8 }, | ||
412 | .bts_interrupt_threshold = { 24, 8 }, | ||
413 | .sizeof_bts = 3 * 8, | ||
414 | .from_ip = { 0, 8 }, | ||
415 | .to_ip = { 8, 8 }, | ||
416 | .info_type = { 8, 1 }, | ||
417 | .info_data = { 16, 8 }, | ||
418 | .debugctl_mask = (1<<6)|(1<<7)|(1<<9) | ||
419 | }; | 811 | }; |
420 | 812 | ||
421 | static inline void | 813 | static inline void |
@@ -429,14 +821,13 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c) | |||
429 | switch (c->x86) { | 821 | switch (c->x86) { |
430 | case 0x6: | 822 | case 0x6: |
431 | switch (c->x86_model) { | 823 | switch (c->x86_model) { |
432 | #ifdef __i386__ | ||
433 | case 0xD: | 824 | case 0xD: |
434 | case 0xE: /* Pentium M */ | 825 | case 0xE: /* Pentium M */ |
435 | ds_configure(&ds_cfg_pentium_m); | 826 | ds_configure(&ds_cfg_var); |
436 | break; | 827 | break; |
437 | #endif /* _i386_ */ | ||
438 | case 0xF: /* Core2 */ | 828 | case 0xF: /* Core2 */ |
439 | ds_configure(&ds_cfg_core2); | 829 | case 0x1C: /* Atom */ |
830 | ds_configure(&ds_cfg_64); | ||
440 | break; | 831 | break; |
441 | default: | 832 | default: |
442 | /* sorry, don't know about them */ | 833 | /* sorry, don't know about them */ |
@@ -445,13 +836,11 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c) | |||
445 | break; | 836 | break; |
446 | case 0xF: | 837 | case 0xF: |
447 | switch (c->x86_model) { | 838 | switch (c->x86_model) { |
448 | #ifdef __i386__ | ||
449 | case 0x0: | 839 | case 0x0: |
450 | case 0x1: | 840 | case 0x1: |
451 | case 0x2: /* Netburst */ | 841 | case 0x2: /* Netburst */ |
452 | ds_configure(&ds_cfg_netburst); | 842 | ds_configure(&ds_cfg_var); |
453 | break; | 843 | break; |
454 | #endif /* _i386_ */ | ||
455 | default: | 844 | default: |
456 | /* sorry, don't know about them */ | 845 | /* sorry, don't know about them */ |
457 | break; | 846 | break; |
@@ -462,3 +851,14 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c) | |||
462 | break; | 851 | break; |
463 | } | 852 | } |
464 | } | 853 | } |
854 | |||
855 | void ds_free(struct ds_context *context) | ||
856 | { | ||
857 | /* This is called when the task owning the parameter context | ||
858 | * is dying. There should not be any user of that context left | ||
859 | * to disturb us, anymore. */ | ||
860 | unsigned long leftovers = context->count; | ||
861 | while (leftovers--) | ||
862 | ds_put_context(context); | ||
863 | } | ||
864 | #endif /* CONFIG_X86_DS */ | ||