1 files changed, 676 insertions, 277 deletions

diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c
index 11c11b8ec48d..5b32b6d062b4 100644
--- a/arch/x86/kernel/ds.c
+++ b/arch/x86/kernel/ds.c
@@ -2,26 +2,48 @@
  * Debug Store support
  *
  * This provides a low-level interface to the hardware's Debug Store
- * feature that is used for last branch recording (LBR) and
+ * feature that is used for branch trace store (BTS) and
  * precise-event based sampling (PEBS).
  *
- * Different architectures use a different DS layout/pointer size.
- * The below functions therefore work on a void*.
+ * It manages:
+ * - per-thread and per-cpu allocation of BTS and PEBS
+ * - buffer memory allocation (optional)
+ * - buffer overflow handling
+ * - buffer access
  *
+ * It assumes:
+ * - get_task_struct on all parameter tasks
+ * - current is allowed to trace parameter tasks
  *
- * Since there is no user for PEBS, yet, only LBR (or branch
- * trace store, BTS) is supported.
  *
- *
- * Copyright (C) 2007 Intel Corporation.
- * Markus Metzger <markus.t.metzger@intel.com>, Dec 2007
+ * Copyright (C) 2007-2008 Intel Corporation.
+ * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008
  */
 
+
+#ifdef CONFIG_X86_DS
+
 #include <asm/ds.h>
 
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/slab.h>
+#include <linux/sched.h>
+
+
+/*
+ * The configuration for a particular DS hardware implementation.
+ */
+struct ds_configuration {
+        /* the size of the DS structure in bytes */
+        unsigned char  sizeof_ds;
+        /* the size of one pointer-typed field in the DS structure in bytes;
+           this covers the first 8 fields related to buffer management. */
+        unsigned char  sizeof_field;
+        /* the size of a BTS/PEBS record in bytes */
+        unsigned char  sizeof_rec[2];
+};
+static struct ds_configuration ds_cfg;
 
 
 /*
@@ -44,378 +66,747 @@
  *   (interrupt occurs when write pointer passes interrupt pointer)
  * - value to which counter is reset following counter overflow
  *
- * On later architectures, the last branch recording hardware uses
- * 64bit pointers even in 32bit mode.
- *
- *
- * Branch Trace Store (BTS) records store information about control
- * flow changes. They at least provide the following information:
- * - source linear address
- * - destination linear address
+ * Later architectures use 64bit pointers throughout, whereas earlier
+ * architectures use 32bit pointers in 32bit mode.
  *
- * Netburst supported a predicated bit that had been dropped in later
- * architectures. We do not suppor it.
  *
+ * We compute the base address for the first 8 fields based on:
+ * - the field size stored in the DS configuration
+ * - the relative field position
+ * - an offset giving the start of the respective region
  *
- * In order to abstract from the actual DS and BTS layout, we describe
- * the access to the relevant fields.
- * Thanks to Andi Kleen for proposing this design.
+ * This offset is further used to index various arrays holding
+ * information for BTS and PEBS at the respective index.
  *
- * The implementation, however, is not as general as it might seem. In
- * order to stay somewhat simple and efficient, we assume an
- * underlying unsigned type (mostly a pointer type) and we expect the
- * field to be at least as big as that type.
+ * On later 32bit processors, we only access the lower 32bit of the
+ * 64bit pointer fields. The upper halves will be zeroed out.
  */
 
-/*
- * A special from_ip address to indicate that the BTS record is an
- * info record that needs to be interpreted or skipped.
- */
-#define BTS_ESCAPE_ADDRESS (-1)
+enum ds_field {
+        ds_buffer_base = 0,
+        ds_index,
+        ds_absolute_maximum,
+        ds_interrupt_threshold,
+};
 
-/*
- * A field access descriptor
- */
-struct access_desc {
-        unsigned char offset;
-        unsigned char size;
+enum ds_qualifier {
+        ds_bts  = 0,
+        ds_pebs
 };
 
+static inline unsigned long ds_get(const unsigned char *base,
+                                   enum ds_qualifier qual, enum ds_field field)
+{
+        base += (ds_cfg.sizeof_field * (field + (4 * qual)));
+        return *(unsigned long *)base;
+}
+
+static inline void ds_set(unsigned char *base, enum ds_qualifier qual,
+                          enum ds_field field, unsigned long value)
+{
+        base += (ds_cfg.sizeof_field * (field + (4 * qual)));
+        (*(unsigned long *)base) = value;
+}
+
+
 /*
- * The configuration for a particular DS/BTS hardware implementation.
+ * Locking is done only for allocating BTS or PEBS resources and for
+ * guarding context and buffer memory allocation.
+ *
+ * Most functions require the current task to own the ds context part
+ * they are going to access. All the locking is done when validating
+ * access to the context.
  */
-struct ds_configuration {
-        /* the DS configuration */
-        unsigned char  sizeof_ds;
-        struct access_desc bts_buffer_base;
-        struct access_desc bts_index;
-        struct access_desc bts_absolute_maximum;
-        struct access_desc bts_interrupt_threshold;
-        /* the BTS configuration */
-        unsigned char  sizeof_bts;
-        struct access_desc from_ip;
-        struct access_desc to_ip;
-        /* BTS variants used to store additional information like
-           timestamps */
-        struct access_desc info_type;
-        struct access_desc info_data;
-        unsigned long debugctl_mask;
-};
+static spinlock_t ds_lock = __SPIN_LOCK_UNLOCKED(ds_lock);
 
 /*
- * The global configuration used by the below accessor functions
+ * Validate that the current task is allowed to access the BTS/PEBS
+ * buffer of the parameter task.
+ *
+ * Returns 0, if access is granted; -Eerrno, otherwise.
  */
-static struct ds_configuration ds_cfg;
+static inline int ds_validate_access(struct ds_context *context,
+                                     enum ds_qualifier qual)
+{
+        if (!context)
+                return -EPERM;
+
+        if (context->owner[qual] == current)
+                return 0;
+
+        return -EPERM;
+}
+
 
 /*
- * Accessor functions for some DS and BTS fields using the above
- * global ptrace_bts_cfg.
+ * We either support (system-wide) per-cpu or per-thread allocation.
+ * We distinguish the two based on the task_struct pointer, where a
+ * NULL pointer indicates per-cpu allocation for the current cpu.
+ *
+ * Allocations are use-counted. As soon as resources are allocated,
+ * further allocations must be of the same type (per-cpu or
+ * per-thread). We model this by counting allocations (i.e. the number
+ * of tracers of a certain type) for one type negatively:
+ *   =0  no tracers
+ *   >0  number of per-thread tracers
+ *   <0  number of per-cpu tracers
+ *
+ * The below functions to get and put tracers and to check the
+ * allocation type require the ds_lock to be held by the caller.
+ *
+ * Tracers essentially gives the number of ds contexts for a certain
+ * type of allocation.
  */
-static inline unsigned long get_bts_buffer_base(char *base)
+static long tracers;
+
+static inline void get_tracer(struct task_struct *task)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_buffer_base.offset);
+        tracers += (task ? 1 : -1);
 }
-static inline void set_bts_buffer_base(char *base, unsigned long value)
+
+static inline void put_tracer(struct task_struct *task)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_buffer_base.offset)) = value;
+        tracers -= (task ? 1 : -1);
 }
-static inline unsigned long get_bts_index(char *base)
+
+static inline int check_tracer(struct task_struct *task)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_index.offset);
+        return (task ? (tracers >= 0) : (tracers <= 0));
 }
-static inline void set_bts_index(char *base, unsigned long value)
+
+
+/*
+ * The DS context is either attached to a thread or to a cpu:
+ * - in the former case, the thread_struct contains a pointer to the
+ *   attached context.
+ * - in the latter case, we use a static array of per-cpu context
+ *   pointers.
+ *
+ * Contexts are use-counted. They are allocated on first access and
+ * deallocated when the last user puts the context.
+ *
+ * We distinguish between an allocating and a non-allocating get of a
+ * context:
+ * - the allocating get is used for requesting BTS/PEBS resources. It
+ *   requires the caller to hold the global ds_lock.
+ * - the non-allocating get is used for all other cases. A
+ *   non-existing context indicates an error. It acquires and releases
+ *   the ds_lock itself for obtaining the context.
+ *
+ * A context and its DS configuration are allocated and deallocated
+ * together. A context always has a DS configuration of the
+ * appropriate size.
+ */
+static DEFINE_PER_CPU(struct ds_context *, system_context);
+
+#define this_system_context per_cpu(system_context, smp_processor_id())
+
+/*
+ * Returns the pointer to the parameter task's context or to the
+ * system-wide context, if task is NULL.
+ *
+ * Increases the use count of the returned context, if not NULL.
+ */
+static inline struct ds_context *ds_get_context(struct task_struct *task)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_index.offset)) = value;
+        struct ds_context *context;
+
+        spin_lock(&ds_lock);
+
+        context = (task ? task->thread.ds_ctx : this_system_context);
+        if (context)
+                context->count++;
+
+        spin_unlock(&ds_lock);
+
+        return context;
 }
-static inline unsigned long get_bts_absolute_maximum(char *base)
+
+/*
+ * Same as ds_get_context, but allocates the context and it's DS
+ * structure, if necessary; returns NULL; if out of memory.
+ *
+ * pre: requires ds_lock to be held
+ */
+static inline struct ds_context *ds_alloc_context(struct task_struct *task)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_absolute_maximum.offset);
+        struct ds_context **p_context =
+                (task ? &task->thread.ds_ctx : &this_system_context);
+        struct ds_context *context = *p_context;
+
+        if (!context) {
+                context = kzalloc(sizeof(*context), GFP_KERNEL);
+
+                if (!context)
+                        return 0;
+
+                context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL);
+                if (!context->ds) {
+                        kfree(context);
+                        return 0;
+                }
+
+                *p_context = context;
+
+                context->this = p_context;
+                context->task = task;
+
+                if (task)
+                        set_tsk_thread_flag(task, TIF_DS_AREA_MSR);
+
+                if (!task || (task == current))
+                        wrmsr(MSR_IA32_DS_AREA, (unsigned long)context->ds, 0);
+
+                get_tracer(task);
+        }
+
+        context->count++;
+
+        return context;
 }
-static inline void set_bts_absolute_maximum(char *base, unsigned long value)
+
+/*
+ * Decreases the use count of the parameter context, if not NULL.
+ * Deallocates the context, if the use count reaches zero.
+ */
+static inline void ds_put_context(struct ds_context *context)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_absolute_maximum.offset)) = value;
+        if (!context)
+                return;
+
+        spin_lock(&ds_lock);
+
+        if (--context->count)
+                goto out;
+
+        *(context->this) = 0;
+
+        if (context->task)
+                clear_tsk_thread_flag(context->task, TIF_DS_AREA_MSR);
+
+        if (!context->task || (context->task == current))
+                wrmsrl(MSR_IA32_DS_AREA, 0);
+
+        put_tracer(context->task);
+
+        /* free any leftover buffers from tracers that did not
+         * deallocate them properly. */
+        kfree(context->buffer[ds_bts]);
+        kfree(context->buffer[ds_pebs]);
+        kfree(context->ds);
+        kfree(context);
+ out:
+        spin_unlock(&ds_lock);
 }
-static inline unsigned long get_bts_interrupt_threshold(char *base)
+
+
+/*
+ * Handle a buffer overflow
+ *
+ * task: the task whose buffers are overflowing;
+ *       NULL for a buffer overflow on the current cpu
+ * context: the ds context
+ * qual: the buffer type
+ */
+static void ds_overflow(struct task_struct *task, struct ds_context *context,
+                        enum ds_qualifier qual)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_interrupt_threshold.offset);
+        if (!context)
+                return;
+
+        if (context->callback[qual])
+                (*context->callback[qual])(task);
+
+        /* todo: do some more overflow handling */
 }
-static inline void set_bts_interrupt_threshold(char *base, unsigned long value)
+
+
+/*
+ * Allocate a non-pageable buffer of the parameter size.
+ * Checks the memory and the locked memory rlimit.
+ *
+ * Returns the buffer, if successful;
+ *         NULL, if out of memory or rlimit exceeded.
+ *
+ * size: the requested buffer size in bytes
+ * pages (out): if not NULL, contains the number of pages reserved
+ */
+static inline void *ds_allocate_buffer(size_t size, unsigned int *pages)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_interrupt_threshold.offset)) = value;
+        unsigned long rlim, vm, pgsz;
+        void *buffer;
+
+        pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+        rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
+        vm   = current->mm->total_vm  + pgsz;
+        if (rlim < vm)
+                return 0;
+
+        rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
+        vm   = current->mm->locked_vm  + pgsz;
+        if (rlim < vm)
+                return 0;
+
+        buffer = kzalloc(size, GFP_KERNEL);
+        if (!buffer)
+                return 0;
+
+        current->mm->total_vm  += pgsz;
+        current->mm->locked_vm += pgsz;
+
+        if (pages)
+                *pages = pgsz;
+
+        return buffer;
 }
-static inline unsigned long get_from_ip(char *base)
+
+static int ds_request(struct task_struct *task, void *base, size_t size,
+                      ds_ovfl_callback_t ovfl, enum ds_qualifier qual)
 {
-        return *(unsigned long *)(base + ds_cfg.from_ip.offset);
+        struct ds_context *context;
+        unsigned long buffer, adj;
+        const unsigned long alignment = (1 << 3);
+        int error = 0;
+
+        if (!ds_cfg.sizeof_ds)
+                return -EOPNOTSUPP;
+
+        /* we require some space to do alignment adjustments below */
+        if (size < (alignment + ds_cfg.sizeof_rec[qual]))
+                return -EINVAL;
+
+        /* buffer overflow notification is not yet implemented */
+        if (ovfl)
+                return -EOPNOTSUPP;
+
+
+        spin_lock(&ds_lock);
+
+        if (!check_tracer(task))
+                return -EPERM;
+
+        error = -ENOMEM;
+        context = ds_alloc_context(task);
+        if (!context)
+                goto out_unlock;
+
+        error = -EALREADY;
+        if (context->owner[qual] == current)
+                goto out_unlock;
+        error = -EPERM;
+        if (context->owner[qual] != 0)
+                goto out_unlock;
+        context->owner[qual] = current;
+
+        spin_unlock(&ds_lock);
+
+
+        error = -ENOMEM;
+        if (!base) {
+                base = ds_allocate_buffer(size, &context->pages[qual]);
+                if (!base)
+                        goto out_release;
+
+                context->buffer[qual]   = base;
+        }
+        error = 0;
+
+        context->callback[qual] = ovfl;
+
+        /* adjust the buffer address and size to meet alignment
+         * constraints:
+         * - buffer is double-word aligned
+         * - size is multiple of record size
+         *
+         * We checked the size at the very beginning; we have enough
+         * space to do the adjustment.
+         */
+        buffer = (unsigned long)base;
+
+        adj = ALIGN(buffer, alignment) - buffer;
+        buffer += adj;
+        size   -= adj;
+
+        size /= ds_cfg.sizeof_rec[qual];
+        size *= ds_cfg.sizeof_rec[qual];
+
+        ds_set(context->ds, qual, ds_buffer_base, buffer);
+        ds_set(context->ds, qual, ds_index, buffer);
+        ds_set(context->ds, qual, ds_absolute_maximum, buffer + size);
+
+        if (ovfl) {
+                /* todo: select a suitable interrupt threshold */
+        } else
+                ds_set(context->ds, qual,
+                       ds_interrupt_threshold, buffer + size + 1);
+
+        /* we keep the context until ds_release */
+        return error;
+
+ out_release:
+        context->owner[qual] = 0;
+        ds_put_context(context);
+        return error;
+
+ out_unlock:
+        spin_unlock(&ds_lock);
+        ds_put_context(context);
+        return error;
 }
-static inline void set_from_ip(char *base, unsigned long value)
+
+int ds_request_bts(struct task_struct *task, void *base, size_t size,
+                   ds_ovfl_callback_t ovfl)
 {
-        (*(unsigned long *)(base + ds_cfg.from_ip.offset)) = value;
+        return ds_request(task, base, size, ovfl, ds_bts);
 }
-static inline unsigned long get_to_ip(char *base)
+
+int ds_request_pebs(struct task_struct *task, void *base, size_t size,
+                    ds_ovfl_callback_t ovfl)
 {
-        return *(unsigned long *)(base + ds_cfg.to_ip.offset);
+        return ds_request(task, base, size, ovfl, ds_pebs);
 }
-static inline void set_to_ip(char *base, unsigned long value)
+
+static int ds_release(struct task_struct *task, enum ds_qualifier qual)
 {
-        (*(unsigned long *)(base + ds_cfg.to_ip.offset)) = value;
+        struct ds_context *context;
+        int error;
+
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
+
+        kfree(context->buffer[qual]);
+        context->buffer[qual] = 0;
+
+        current->mm->total_vm  -= context->pages[qual];
+        current->mm->locked_vm -= context->pages[qual];
+        context->pages[qual] = 0;
+        context->owner[qual] = 0;
+
+        /*
+         * we put the context twice:
+         *   once for the ds_get_context
+         *   once for the corresponding ds_request
+         */
+        ds_put_context(context);
+ out:
+        ds_put_context(context);
+        return error;
 }
-static inline unsigned char get_info_type(char *base)
+
+int ds_release_bts(struct task_struct *task)
 {
-        return *(unsigned char *)(base + ds_cfg.info_type.offset);
+        return ds_release(task, ds_bts);
 }
-static inline void set_info_type(char *base, unsigned char value)
+
+int ds_release_pebs(struct task_struct *task)
 {
-        (*(unsigned char *)(base + ds_cfg.info_type.offset)) = value;
+        return ds_release(task, ds_pebs);
 }
-static inline unsigned long get_info_data(char *base)
+
+static int ds_get_index(struct task_struct *task, size_t *pos,
+                        enum ds_qualifier qual)
 {
-        return *(unsigned long *)(base + ds_cfg.info_data.offset);
+        struct ds_context *context;
+        unsigned long base, index;
+        int error;
+
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
+
+        base  = ds_get(context->ds, qual, ds_buffer_base);
+        index = ds_get(context->ds, qual, ds_index);
+
+        error = ((index - base) / ds_cfg.sizeof_rec[qual]);
+        if (pos)
+                *pos = error;
+ out:
+        ds_put_context(context);
+        return error;
 }
-static inline void set_info_data(char *base, unsigned long value)
+
+int ds_get_bts_index(struct task_struct *task, size_t *pos)
 {
-        (*(unsigned long *)(base + ds_cfg.info_data.offset)) = value;
+        return ds_get_index(task, pos, ds_bts);
 }
 
+int ds_get_pebs_index(struct task_struct *task, size_t *pos)
+{
+        return ds_get_index(task, pos, ds_pebs);
+}
 
-int ds_allocate(void **dsp, size_t bts_size_in_bytes)
+static int ds_get_end(struct task_struct *task, size_t *pos,
+                      enum ds_qualifier qual)
 {
-        size_t bts_size_in_records;
-        unsigned long bts;
-        void *ds;
+        struct ds_context *context;
+        unsigned long base, end;
+        int error;
+
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
+
+        base = ds_get(context->ds, qual, ds_buffer_base);
+        end  = ds_get(context->ds, qual, ds_absolute_maximum);
+
+        error = ((end - base) / ds_cfg.sizeof_rec[qual]);
+        if (pos)
+                *pos = error;
+ out:
+        ds_put_context(context);
+        return error;
+}
 
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
+int ds_get_bts_end(struct task_struct *task, size_t *pos)
+{
+        return ds_get_end(task, pos, ds_bts);
+}
 
-        if (bts_size_in_bytes < 0)
-                return -EINVAL;
+int ds_get_pebs_end(struct task_struct *task, size_t *pos)
+{
+        return ds_get_end(task, pos, ds_pebs);
+}
 
-        bts_size_in_records =
-                bts_size_in_bytes / ds_cfg.sizeof_bts;
-        bts_size_in_bytes =
-                bts_size_in_records * ds_cfg.sizeof_bts;
+static int ds_access(struct task_struct *task, size_t index,
+                     const void **record, enum ds_qualifier qual)
+{
+        struct ds_context *context;
+        unsigned long base, idx;
+        int error;
 
-        if (bts_size_in_bytes <= 0)
+        if (!record)
                 return -EINVAL;
 
-        bts = (unsigned long)kzalloc(bts_size_in_bytes, GFP_KERNEL);
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
 
-        if (!bts)
-                return -ENOMEM;
+        base = ds_get(context->ds, qual, ds_buffer_base);
+        idx = base + (index * ds_cfg.sizeof_rec[qual]);
 
-        ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL);
+        error = -EINVAL;
+        if (idx > ds_get(context->ds, qual, ds_absolute_maximum))
+                goto out;
 
-        if (!ds) {
-                kfree((void *)bts);
-                return -ENOMEM;
-        }
-
-        set_bts_buffer_base(ds, bts);
-        set_bts_index(ds, bts);
-        set_bts_absolute_maximum(ds, bts + bts_size_in_bytes);
-        set_bts_interrupt_threshold(ds, bts + bts_size_in_bytes + 1);
-
-        *dsp = ds;
-        return 0;
+        *record = (const void *)idx;
+        error = ds_cfg.sizeof_rec[qual];
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-int ds_free(void **dsp)
+int ds_access_bts(struct task_struct *task, size_t index, const void **record)
 {
-        if (*dsp) {
-                kfree((void *)get_bts_buffer_base(*dsp));
-                kfree(*dsp);
-                *dsp = NULL;
-        }
-        return 0;
+        return ds_access(task, index, record, ds_bts);
 }
 
-int ds_get_bts_size(void *ds)
+int ds_access_pebs(struct task_struct *task, size_t index, const void **record)
 {
-        int size_in_bytes;
-
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
-
-        if (!ds)
-                return 0;
-
-        size_in_bytes =
-                get_bts_absolute_maximum(ds) -
-                get_bts_buffer_base(ds);
-        return size_in_bytes;
+        return ds_access(task, index, record, ds_pebs);
 }
 
-int ds_get_bts_end(void *ds)
+static int ds_write(struct task_struct *task, const void *record, size_t size,
+                    enum ds_qualifier qual, int force)
 {
-        int size_in_bytes = ds_get_bts_size(ds);
-
-        if (size_in_bytes <= 0)
-                return size_in_bytes;
+        struct ds_context *context;
+        int error;
 
-        return size_in_bytes / ds_cfg.sizeof_bts;
-}
+        if (!record)
+                return -EINVAL;
 
-int ds_get_bts_index(void *ds)
-{
-        int index_offset_in_bytes;
+        error = -EPERM;
+        context = ds_get_context(task);
+        if (!context)
+                goto out;
 
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
+        if (!force) {
+                error = ds_validate_access(context, qual);
+                if (error < 0)
+                        goto out;
+        }
 
-        index_offset_in_bytes =
-                get_bts_index(ds) -
-                get_bts_buffer_base(ds);
+        error = 0;
+        while (size) {
+                unsigned long base, index, end, write_end, int_th;
+                unsigned long write_size, adj_write_size;
+
+                /*
+                 * write as much as possible without producing an
+                 * overflow interrupt.
+                 *
+                 * interrupt_threshold must either be
+                 * - bigger than absolute_maximum or
+                 * - point to a record between buffer_base and absolute_maximum
+                 *
+                 * index points to a valid record.
+                 */
+                base   = ds_get(context->ds, qual, ds_buffer_base);
+                index  = ds_get(context->ds, qual, ds_index);
+                end    = ds_get(context->ds, qual, ds_absolute_maximum);
+                int_th = ds_get(context->ds, qual, ds_interrupt_threshold);
+
+                write_end = min(end, int_th);
+
+                /* if we are already beyond the interrupt threshold,
+                 * we fill the entire buffer */
+                if (write_end <= index)
+                        write_end = end;
+
+                if (write_end <= index)
+                        goto out;
+
+                write_size = min((unsigned long) size, write_end - index);
+                memcpy((void *)index, record, write_size);
+
+                record = (const char *)record + write_size;
+                size  -= write_size;
+                error += write_size;
+
+                adj_write_size = write_size / ds_cfg.sizeof_rec[qual];
+                adj_write_size *= ds_cfg.sizeof_rec[qual];
+
+                /* zero out trailing bytes */
+                memset((char *)index + write_size, 0,
+                       adj_write_size - write_size);
+                index += adj_write_size;
+
+                if (index >= end)
+                        index = base;
+                ds_set(context->ds, qual, ds_index, index);
+
+                if (index >= int_th)
+                        ds_overflow(task, context, qual);
+        }
 
-        return index_offset_in_bytes / ds_cfg.sizeof_bts;
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-int ds_set_overflow(void *ds, int method)
+int ds_write_bts(struct task_struct *task, const void *record, size_t size)
 {
-        switch (method) {
-        case DS_O_SIGNAL:
-                return -EOPNOTSUPP;
-        case DS_O_WRAP:
-                return 0;
-        default:
-                return -EINVAL;
-        }
+        return ds_write(task, record, size, ds_bts, /* force = */ 0);
 }
 
-int ds_get_overflow(void *ds)
+int ds_write_pebs(struct task_struct *task, const void *record, size_t size)
 {
-        return DS_O_WRAP;
+        return ds_write(task, record, size, ds_pebs, /* force = */ 0);
 }
 
-int ds_clear(void *ds)
+int ds_unchecked_write_bts(struct task_struct *task,
+                           const void *record, size_t size)
 {
-        int bts_size = ds_get_bts_size(ds);
-        unsigned long bts_base;
-
-        if (bts_size <= 0)
-                return bts_size;
-
-        bts_base = get_bts_buffer_base(ds);
-        memset((void *)bts_base, 0, bts_size);
-
-        set_bts_index(ds, bts_base);
-        return 0;
+        return ds_write(task, record, size, ds_bts, /* force = */ 1);
 }
 
-int ds_read_bts(void *ds, int index, struct bts_struct *out)
+int ds_unchecked_write_pebs(struct task_struct *task,
+                            const void *record, size_t size)
 {
-        void *bts;
+        return ds_write(task, record, size, ds_pebs, /* force = */ 1);
+}
 
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
+static int ds_reset_or_clear(struct task_struct *task,
+                             enum ds_qualifier qual, int clear)
+{
+        struct ds_context *context;
+        unsigned long base, end;
+        int error;
 
-        if (index < 0)
-                return -EINVAL;
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
 
-        if (index >= ds_get_bts_size(ds))
-                return -EINVAL;
+        base = ds_get(context->ds, qual, ds_buffer_base);
+        end  = ds_get(context->ds, qual, ds_absolute_maximum);
 
-        bts = (void *)(get_bts_buffer_base(ds) + (index * ds_cfg.sizeof_bts));
+        if (clear)
+                memset((void *)base, 0, end - base);
 
-        memset(out, 0, sizeof(*out));
-        if (get_from_ip(bts) == BTS_ESCAPE_ADDRESS) {
-                out->qualifier       = get_info_type(bts);
-                out->variant.jiffies = get_info_data(bts);
-        } else {
-                out->qualifier = BTS_BRANCH;
-                out->variant.lbr.from_ip = get_from_ip(bts);
-                out->variant.lbr.to_ip   = get_to_ip(bts);
-        }
+        ds_set(context->ds, qual, ds_index, base);
 
-        return sizeof(*out);;
+        error = 0;
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-int ds_write_bts(void *ds, const struct bts_struct *in)
+int ds_reset_bts(struct task_struct *task)
 {
-        unsigned long bts;
-
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
-
-        if (ds_get_bts_size(ds) <= 0)
-                return -ENXIO;
+        return ds_reset_or_clear(task, ds_bts, /* clear = */ 0);
+}
 
-        bts = get_bts_index(ds);
+int ds_reset_pebs(struct task_struct *task)
+{
+        return ds_reset_or_clear(task, ds_pebs, /* clear = */ 0);
+}
 
-        memset((void *)bts, 0, ds_cfg.sizeof_bts);
-        switch (in->qualifier) {
-        case BTS_INVALID:
-                break;
+int ds_clear_bts(struct task_struct *task)
+{
+        return ds_reset_or_clear(task, ds_bts, /* clear = */ 1);
+}
 
-        case BTS_BRANCH:
-                set_from_ip((void *)bts, in->variant.lbr.from_ip);
-                set_to_ip((void *)bts, in->variant.lbr.to_ip);
-                break;
+int ds_clear_pebs(struct task_struct *task)
+{
+        return ds_reset_or_clear(task, ds_pebs, /* clear = */ 1);
+}
 
-        case BTS_TASK_ARRIVES:
-        case BTS_TASK_DEPARTS:
-                set_from_ip((void *)bts, BTS_ESCAPE_ADDRESS);
-                set_info_type((void *)bts, in->qualifier);
-                set_info_data((void *)bts, in->variant.jiffies);
-                break;
+int ds_get_pebs_reset(struct task_struct *task, u64 *value)
+{
+        struct ds_context *context;
+        int error;
 
-        default:
+        if (!value)
                 return -EINVAL;
-        }
 
-        bts = bts + ds_cfg.sizeof_bts;
-        if (bts >= get_bts_absolute_maximum(ds))
-                bts = get_bts_buffer_base(ds);
-        set_bts_index(ds, bts);
+        context = ds_get_context(task);
+        error = ds_validate_access(context, ds_pebs);
+        if (error < 0)
+                goto out;
 
-        return ds_cfg.sizeof_bts;
+        *value = *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8));
+
+        error = 0;
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-unsigned long ds_debugctl_mask(void)
+int ds_set_pebs_reset(struct task_struct *task, u64 value)
 {
-        return ds_cfg.debugctl_mask;
-}
+        struct ds_context *context;
+        int error;
 
-#ifdef __i386__
-static const struct ds_configuration ds_cfg_netburst = {
-        .sizeof_ds = 9 * 4,
-        .bts_buffer_base = { 0, 4 },
-        .bts_index = { 4, 4 },
-        .bts_absolute_maximum = { 8, 4 },
-        .bts_interrupt_threshold = { 12, 4 },
-        .sizeof_bts = 3 * 4,
-        .from_ip = { 0, 4 },
-        .to_ip = { 4, 4 },
-        .info_type = { 4, 1 },
-        .info_data = { 8, 4 },
-        .debugctl_mask = (1<<2)|(1<<3)
-};
+        context = ds_get_context(task);
+        error = ds_validate_access(context, ds_pebs);
+        if (error < 0)
+                goto out;
 
-static const struct ds_configuration ds_cfg_pentium_m = {
-        .sizeof_ds = 9 * 4,
-        .bts_buffer_base = { 0, 4 },
-        .bts_index = { 4, 4 },
-        .bts_absolute_maximum = { 8, 4 },
-        .bts_interrupt_threshold = { 12, 4 },
-        .sizeof_bts = 3 * 4,
-        .from_ip = { 0, 4 },
-        .to_ip = { 4, 4 },
-        .info_type = { 4, 1 },
-        .info_data = { 8, 4 },
-        .debugctl_mask = (1<<6)|(1<<7)
+        *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)) = value;
+
+        error = 0;
+ out:
+        ds_put_context(context);
+        return error;
+}
+
+static const struct ds_configuration ds_cfg_var = {
+        .sizeof_ds    = sizeof(long) * 12,
+        .sizeof_field = sizeof(long),
+        .sizeof_rec[ds_bts]   = sizeof(long) * 3,
+        .sizeof_rec[ds_pebs]  = sizeof(long) * 10
 };
-#endif /* _i386_ */
-
-static const struct ds_configuration ds_cfg_core2 = {
-        .sizeof_ds = 9 * 8,
-        .bts_buffer_base = { 0, 8 },
-        .bts_index = { 8, 8 },
-        .bts_absolute_maximum = { 16, 8 },
-        .bts_interrupt_threshold = { 24, 8 },
-        .sizeof_bts = 3 * 8,
-        .from_ip = { 0, 8 },
-        .to_ip = { 8, 8 },
-        .info_type = { 8, 1 },
-        .info_data = { 16, 8 },
-        .debugctl_mask = (1<<6)|(1<<7)|(1<<9)
+static const struct ds_configuration ds_cfg_64 = {
+        .sizeof_ds    = 8 * 12,
+        .sizeof_field = 8,
+        .sizeof_rec[ds_bts]   = 8 * 3,
+        .sizeof_rec[ds_pebs]  = 8 * 10
 };
 
 static inline void
@@ -429,14 +820,13 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
         switch (c->x86) {
         case 0x6:
                 switch (c->x86_model) {
-#ifdef __i386__
                 case 0xD:
                 case 0xE: /* Pentium M */
-                        ds_configure(&ds_cfg_pentium_m);
+                        ds_configure(&ds_cfg_var);
                         break;
-#endif /* _i386_ */
                 case 0xF: /* Core2 */
-                        ds_configure(&ds_cfg_core2);
+        case 0x1C: /* Atom */
+                        ds_configure(&ds_cfg_64);
                         break;
                 default:
                         /* sorry, don't know about them */
@@ -445,13 +835,11 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
                 break;
         case 0xF:
                 switch (c->x86_model) {
-#ifdef __i386__
                 case 0x0:
                 case 0x1:
                 case 0x2: /* Netburst */
-                        ds_configure(&ds_cfg_netburst);
+                        ds_configure(&ds_cfg_var);
                         break;
-#endif /* _i386_ */
                 default:
                         /* sorry, don't know about them */
                         break;
@@ -462,3 +850,14 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
                 break;
         }
 }
+
+void ds_free(struct ds_context *context)
+{
+        /* This is called when the task owning the parameter context
+         * is dying. There should not be any user of that context left
+         * to disturb us, anymore. */
+        unsigned long leftovers = context->count;
+        while (leftovers--)
+                ds_put_context(context);
+}
+#endif /* CONFIG_X86_DS */