x86, ptrace: PEBS support

Polish the ds.h interface and add support for PEBS.

Ds.c is meant to be the resource allocator for per-thread and per-cpu
BTS and PEBS recording.
It is used by ptrace/utrace to provide execution tracing of debugged tasks.
It will be used by profilers (e.g. perfmon2).
It may be used by kernel debuggers to provide a kernel execution trace.

Changes in detail:
- guard DS and ptrace by CONFIG macros
- separate DS and BTS more clearly
- simplify field accesses
- add functions to manage PEBS buffers
- add simple protection/allocation mechanism
- added support for Atom

Opens:
- buffer overflow handling
  Currently, only circular buffers are supported. This is all we need
  for debugging. Profilers would want an overflow notification.
  This is planned to be added when perfmon2 is made to use the ds.h
  interface.
- utrace intermediate layer

Signed-off-by: Markus Metzger <markus.t.metzger@intel.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

1 files changed, 676 insertions, 277 deletions

diff --git a/arch/x86/kernel/ds.c b/arch/x86/kernel/ds.c
index 11c11b8ec48d..5b32b6d062b4 100644
--- a/arch/x86/kernel/ds.c
+++ b/arch/x86/kernel/ds.c
@@ -2,26 +2,48 @@
  * Debug Store support
  *
  * This provides a low-level interface to the hardware's Debug Store
- * feature that is used for last branch recording (LBR) and
+ * feature that is used for branch trace store (BTS) and
  * precise-event based sampling (PEBS).
  *
- * Different architectures use a different DS layout/pointer size.
- * The below functions therefore work on a void*.
+ * It manages:
+ * - per-thread and per-cpu allocation of BTS and PEBS
+ * - buffer memory allocation (optional)
+ * - buffer overflow handling
+ * - buffer access
  *
+ * It assumes:
+ * - get_task_struct on all parameter tasks
+ * - current is allowed to trace parameter tasks
  *
- * Since there is no user for PEBS, yet, only LBR (or branch
- * trace store, BTS) is supported.
  *
- *
- * Copyright (C) 2007 Intel Corporation.
- * Markus Metzger <markus.t.metzger@intel.com>, Dec 2007
+ * Copyright (C) 2007-2008 Intel Corporation.
+ * Markus Metzger <markus.t.metzger@intel.com>, 2007-2008
  */
 
+
+#ifdef CONFIG_X86_DS
+
 #include <asm/ds.h>
 
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/slab.h>
+#include <linux/sched.h>
+
+
+/*
+ * The configuration for a particular DS hardware implementation.
+ */
+struct ds_configuration {
+        /* the size of the DS structure in bytes */
+        unsigned char  sizeof_ds;
+        /* the size of one pointer-typed field in the DS structure in bytes;
+           this covers the first 8 fields related to buffer management. */
+        unsigned char  sizeof_field;
+        /* the size of a BTS/PEBS record in bytes */
+        unsigned char  sizeof_rec[2];
+};
+static struct ds_configuration ds_cfg;
 
 
 /*
@@ -44,378 +66,747 @@
  *   (interrupt occurs when write pointer passes interrupt pointer)
  * - value to which counter is reset following counter overflow
  *
- * On later architectures, the last branch recording hardware uses
- * 64bit pointers even in 32bit mode.
- *
- *
- * Branch Trace Store (BTS) records store information about control
- * flow changes. They at least provide the following information:
- * - source linear address
- * - destination linear address
+ * Later architectures use 64bit pointers throughout, whereas earlier
+ * architectures use 32bit pointers in 32bit mode.
  *
- * Netburst supported a predicated bit that had been dropped in later
- * architectures. We do not suppor it.
  *
+ * We compute the base address for the first 8 fields based on:
+ * - the field size stored in the DS configuration
+ * - the relative field position
+ * - an offset giving the start of the respective region
  *
- * In order to abstract from the actual DS and BTS layout, we describe
- * the access to the relevant fields.
- * Thanks to Andi Kleen for proposing this design.
+ * This offset is further used to index various arrays holding
+ * information for BTS and PEBS at the respective index.
  *
- * The implementation, however, is not as general as it might seem. In
- * order to stay somewhat simple and efficient, we assume an
- * underlying unsigned type (mostly a pointer type) and we expect the
- * field to be at least as big as that type.
+ * On later 32bit processors, we only access the lower 32bit of the
+ * 64bit pointer fields. The upper halves will be zeroed out.
  */
 
-/*
- * A special from_ip address to indicate that the BTS record is an
- * info record that needs to be interpreted or skipped.
- */
-#define BTS_ESCAPE_ADDRESS (-1)
+enum ds_field {
+        ds_buffer_base = 0,
+        ds_index,
+        ds_absolute_maximum,
+        ds_interrupt_threshold,
+};
 
-/*
- * A field access descriptor
- */
-struct access_desc {
-        unsigned char offset;
-        unsigned char size;
+enum ds_qualifier {
+        ds_bts  = 0,
+        ds_pebs
 };
 
+static inline unsigned long ds_get(const unsigned char *base,
+                                   enum ds_qualifier qual, enum ds_field field)
+{
+        base += (ds_cfg.sizeof_field * (field + (4 * qual)));
+        return *(unsigned long *)base;
+}
+
+static inline void ds_set(unsigned char *base, enum ds_qualifier qual,
+                          enum ds_field field, unsigned long value)
+{
+        base += (ds_cfg.sizeof_field * (field + (4 * qual)));
+        (*(unsigned long *)base) = value;
+}
+
+
 /*
- * The configuration for a particular DS/BTS hardware implementation.
+ * Locking is done only for allocating BTS or PEBS resources and for
+ * guarding context and buffer memory allocation.
+ *
+ * Most functions require the current task to own the ds context part
+ * they are going to access. All the locking is done when validating
+ * access to the context.
  */
-struct ds_configuration {
-        /* the DS configuration */
-        unsigned char  sizeof_ds;
-        struct access_desc bts_buffer_base;
-        struct access_desc bts_index;
-        struct access_desc bts_absolute_maximum;
-        struct access_desc bts_interrupt_threshold;
-        /* the BTS configuration */
-        unsigned char  sizeof_bts;
-        struct access_desc from_ip;
-        struct access_desc to_ip;
-        /* BTS variants used to store additional information like
-           timestamps */
-        struct access_desc info_type;
-        struct access_desc info_data;
-        unsigned long debugctl_mask;
-};
+static spinlock_t ds_lock = __SPIN_LOCK_UNLOCKED(ds_lock);
 
 /*
- * The global configuration used by the below accessor functions
+ * Validate that the current task is allowed to access the BTS/PEBS
+ * buffer of the parameter task.
+ *
+ * Returns 0, if access is granted; -Eerrno, otherwise.
  */
-static struct ds_configuration ds_cfg;
+static inline int ds_validate_access(struct ds_context *context,
+                                     enum ds_qualifier qual)
+{
+        if (!context)
+                return -EPERM;
+
+        if (context->owner[qual] == current)
+                return 0;
+
+        return -EPERM;
+}
+
 
 /*
- * Accessor functions for some DS and BTS fields using the above
- * global ptrace_bts_cfg.
+ * We either support (system-wide) per-cpu or per-thread allocation.
+ * We distinguish the two based on the task_struct pointer, where a
+ * NULL pointer indicates per-cpu allocation for the current cpu.
+ *
+ * Allocations are use-counted. As soon as resources are allocated,
+ * further allocations must be of the same type (per-cpu or
+ * per-thread). We model this by counting allocations (i.e. the number
+ * of tracers of a certain type) for one type negatively:
+ *   =0  no tracers
+ *   >0  number of per-thread tracers
+ *   <0  number of per-cpu tracers
+ *
+ * The below functions to get and put tracers and to check the
+ * allocation type require the ds_lock to be held by the caller.
+ *
+ * Tracers essentially gives the number of ds contexts for a certain
+ * type of allocation.
  */
-static inline unsigned long get_bts_buffer_base(char *base)
+static long tracers;
+
+static inline void get_tracer(struct task_struct *task)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_buffer_base.offset);
+        tracers += (task ? 1 : -1);
 }
-static inline void set_bts_buffer_base(char *base, unsigned long value)
+
+static inline void put_tracer(struct task_struct *task)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_buffer_base.offset)) = value;
+        tracers -= (task ? 1 : -1);
 }
-static inline unsigned long get_bts_index(char *base)
+
+static inline int check_tracer(struct task_struct *task)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_index.offset);
+        return (task ? (tracers >= 0) : (tracers <= 0));
 }
-static inline void set_bts_index(char *base, unsigned long value)
+
+
+/*
+ * The DS context is either attached to a thread or to a cpu:
+ * - in the former case, the thread_struct contains a pointer to the
+ *   attached context.
+ * - in the latter case, we use a static array of per-cpu context
+ *   pointers.
+ *
+ * Contexts are use-counted. They are allocated on first access and
+ * deallocated when the last user puts the context.
+ *
+ * We distinguish between an allocating and a non-allocating get of a
+ * context:
+ * - the allocating get is used for requesting BTS/PEBS resources. It
+ *   requires the caller to hold the global ds_lock.
+ * - the non-allocating get is used for all other cases. A
+ *   non-existing context indicates an error. It acquires and releases
+ *   the ds_lock itself for obtaining the context.
+ *
+ * A context and its DS configuration are allocated and deallocated
+ * together. A context always has a DS configuration of the
+ * appropriate size.
+ */
+static DEFINE_PER_CPU(struct ds_context *, system_context);
+
+#define this_system_context per_cpu(system_context, smp_processor_id())
+
+/*
+ * Returns the pointer to the parameter task's context or to the
+ * system-wide context, if task is NULL.
+ *
+ * Increases the use count of the returned context, if not NULL.
+ */
+static inline struct ds_context *ds_get_context(struct task_struct *task)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_index.offset)) = value;
+        struct ds_context *context;
+
+        spin_lock(&ds_lock);
+
+        context = (task ? task->thread.ds_ctx : this_system_context);
+        if (context)
+                context->count++;
+
+        spin_unlock(&ds_lock);
+
+        return context;
 }
-static inline unsigned long get_bts_absolute_maximum(char *base)
+
+/*
+ * Same as ds_get_context, but allocates the context and it's DS
+ * structure, if necessary; returns NULL; if out of memory.
+ *
+ * pre: requires ds_lock to be held
+ */
+static inline struct ds_context *ds_alloc_context(struct task_struct *task)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_absolute_maximum.offset);
+        struct ds_context **p_context =
+                (task ? &task->thread.ds_ctx : &this_system_context);
+        struct ds_context *context = *p_context;
+
+        if (!context) {
+                context = kzalloc(sizeof(*context), GFP_KERNEL);
+
+                if (!context)
+                        return 0;
+
+                context->ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL);
+                if (!context->ds) {
+                        kfree(context);
+                        return 0;
+                }
+
+                *p_context = context;
+
+                context->this = p_context;
+                context->task = task;
+
+                if (task)
+                        set_tsk_thread_flag(task, TIF_DS_AREA_MSR);
+
+                if (!task || (task == current))
+                        wrmsr(MSR_IA32_DS_AREA, (unsigned long)context->ds, 0);
+
+                get_tracer(task);
+        }
+
+        context->count++;
+
+        return context;
 }
-static inline void set_bts_absolute_maximum(char *base, unsigned long value)
+
+/*
+ * Decreases the use count of the parameter context, if not NULL.
+ * Deallocates the context, if the use count reaches zero.
+ */
+static inline void ds_put_context(struct ds_context *context)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_absolute_maximum.offset)) = value;
+        if (!context)
+                return;
+
+        spin_lock(&ds_lock);
+
+        if (--context->count)
+                goto out;
+
+        *(context->this) = 0;
+
+        if (context->task)
+                clear_tsk_thread_flag(context->task, TIF_DS_AREA_MSR);
+
+        if (!context->task || (context->task == current))
+                wrmsrl(MSR_IA32_DS_AREA, 0);
+
+        put_tracer(context->task);
+
+        /* free any leftover buffers from tracers that did not
+         * deallocate them properly. */
+        kfree(context->buffer[ds_bts]);
+        kfree(context->buffer[ds_pebs]);
+        kfree(context->ds);
+        kfree(context);
+ out:
+        spin_unlock(&ds_lock);
 }
-static inline unsigned long get_bts_interrupt_threshold(char *base)
+
+
+/*
+ * Handle a buffer overflow
+ *
+ * task: the task whose buffers are overflowing;
+ *       NULL for a buffer overflow on the current cpu
+ * context: the ds context
+ * qual: the buffer type
+ */
+static void ds_overflow(struct task_struct *task, struct ds_context *context,
+                        enum ds_qualifier qual)
 {
-        return *(unsigned long *)(base + ds_cfg.bts_interrupt_threshold.offset);
+        if (!context)
+                return;
+
+        if (context->callback[qual])
+                (*context->callback[qual])(task);
+
+        /* todo: do some more overflow handling */
 }
-static inline void set_bts_interrupt_threshold(char *base, unsigned long value)
+
+
+/*
+ * Allocate a non-pageable buffer of the parameter size.
+ * Checks the memory and the locked memory rlimit.
+ *
+ * Returns the buffer, if successful;
+ *         NULL, if out of memory or rlimit exceeded.
+ *
+ * size: the requested buffer size in bytes
+ * pages (out): if not NULL, contains the number of pages reserved
+ */
+static inline void *ds_allocate_buffer(size_t size, unsigned int *pages)
 {
-        (*(unsigned long *)(base + ds_cfg.bts_interrupt_threshold.offset)) = value;
+        unsigned long rlim, vm, pgsz;
+        void *buffer;
+
+        pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
+
+        rlim = current->signal->rlim[RLIMIT_AS].rlim_cur >> PAGE_SHIFT;
+        vm   = current->mm->total_vm  + pgsz;
+        if (rlim < vm)
+                return 0;
+
+        rlim = current->signal->rlim[RLIMIT_MEMLOCK].rlim_cur >> PAGE_SHIFT;
+        vm   = current->mm->locked_vm  + pgsz;
+        if (rlim < vm)
+                return 0;
+
+        buffer = kzalloc(size, GFP_KERNEL);
+        if (!buffer)
+                return 0;
+
+        current->mm->total_vm  += pgsz;
+        current->mm->locked_vm += pgsz;
+
+        if (pages)
+                *pages = pgsz;
+
+        return buffer;
 }
-static inline unsigned long get_from_ip(char *base)
+
+static int ds_request(struct task_struct *task, void *base, size_t size,
+                      ds_ovfl_callback_t ovfl, enum ds_qualifier qual)
 {
-        return *(unsigned long *)(base + ds_cfg.from_ip.offset);
+        struct ds_context *context;
+        unsigned long buffer, adj;
+        const unsigned long alignment = (1 << 3);
+        int error = 0;
+
+        if (!ds_cfg.sizeof_ds)
+                return -EOPNOTSUPP;
+
+        /* we require some space to do alignment adjustments below */
+        if (size < (alignment + ds_cfg.sizeof_rec[qual]))
+                return -EINVAL;
+
+        /* buffer overflow notification is not yet implemented */
+        if (ovfl)
+                return -EOPNOTSUPP;
+
+
+        spin_lock(&ds_lock);
+
+        if (!check_tracer(task))
+                return -EPERM;
+
+        error = -ENOMEM;
+        context = ds_alloc_context(task);
+        if (!context)
+                goto out_unlock;
+
+        error = -EALREADY;
+        if (context->owner[qual] == current)
+                goto out_unlock;
+        error = -EPERM;
+        if (context->owner[qual] != 0)
+                goto out_unlock;
+        context->owner[qual] = current;
+
+        spin_unlock(&ds_lock);
+
+
+        error = -ENOMEM;
+        if (!base) {
+                base = ds_allocate_buffer(size, &context->pages[qual]);
+                if (!base)
+                        goto out_release;
+
+                context->buffer[qual]   = base;
+        }
+        error = 0;
+
+        context->callback[qual] = ovfl;
+
+        /* adjust the buffer address and size to meet alignment
+         * constraints:
+         * - buffer is double-word aligned
+         * - size is multiple of record size
+         *
+         * We checked the size at the very beginning; we have enough
+         * space to do the adjustment.
+         */
+        buffer = (unsigned long)base;
+
+        adj = ALIGN(buffer, alignment) - buffer;
+        buffer += adj;
+        size   -= adj;
+
+        size /= ds_cfg.sizeof_rec[qual];
+        size *= ds_cfg.sizeof_rec[qual];
+
+        ds_set(context->ds, qual, ds_buffer_base, buffer);
+        ds_set(context->ds, qual, ds_index, buffer);
+        ds_set(context->ds, qual, ds_absolute_maximum, buffer + size);
+
+        if (ovfl) {
+                /* todo: select a suitable interrupt threshold */
+        } else
+                ds_set(context->ds, qual,
+                       ds_interrupt_threshold, buffer + size + 1);
+
+        /* we keep the context until ds_release */
+        return error;
+
+ out_release:
+        context->owner[qual] = 0;
+        ds_put_context(context);
+        return error;
+
+ out_unlock:
+        spin_unlock(&ds_lock);
+        ds_put_context(context);
+        return error;
 }
-static inline void set_from_ip(char *base, unsigned long value)
+
+int ds_request_bts(struct task_struct *task, void *base, size_t size,
+                   ds_ovfl_callback_t ovfl)
 {
-        (*(unsigned long *)(base + ds_cfg.from_ip.offset)) = value;
+        return ds_request(task, base, size, ovfl, ds_bts);
 }
-static inline unsigned long get_to_ip(char *base)
+
+int ds_request_pebs(struct task_struct *task, void *base, size_t size,
+                    ds_ovfl_callback_t ovfl)
 {
-        return *(unsigned long *)(base + ds_cfg.to_ip.offset);
+        return ds_request(task, base, size, ovfl, ds_pebs);
 }
-static inline void set_to_ip(char *base, unsigned long value)
+
+static int ds_release(struct task_struct *task, enum ds_qualifier qual)
 {
-        (*(unsigned long *)(base + ds_cfg.to_ip.offset)) = value;
+        struct ds_context *context;
+        int error;
+
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
+
+        kfree(context->buffer[qual]);
+        context->buffer[qual] = 0;
+
+        current->mm->total_vm  -= context->pages[qual];
+        current->mm->locked_vm -= context->pages[qual];
+        context->pages[qual] = 0;
+        context->owner[qual] = 0;
+
+        /*
+         * we put the context twice:
+         *   once for the ds_get_context
+         *   once for the corresponding ds_request
+         */
+        ds_put_context(context);
+ out:
+        ds_put_context(context);
+        return error;
 }
-static inline unsigned char get_info_type(char *base)
+
+int ds_release_bts(struct task_struct *task)
 {
-        return *(unsigned char *)(base + ds_cfg.info_type.offset);
+        return ds_release(task, ds_bts);
 }
-static inline void set_info_type(char *base, unsigned char value)
+
+int ds_release_pebs(struct task_struct *task)
 {
-        (*(unsigned char *)(base + ds_cfg.info_type.offset)) = value;
+        return ds_release(task, ds_pebs);
 }
-static inline unsigned long get_info_data(char *base)
+
+static int ds_get_index(struct task_struct *task, size_t *pos,
+                        enum ds_qualifier qual)
 {
-        return *(unsigned long *)(base + ds_cfg.info_data.offset);
+        struct ds_context *context;
+        unsigned long base, index;
+        int error;
+
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
+
+        base  = ds_get(context->ds, qual, ds_buffer_base);
+        index = ds_get(context->ds, qual, ds_index);
+
+        error = ((index - base) / ds_cfg.sizeof_rec[qual]);
+        if (pos)
+                *pos = error;
+ out:
+        ds_put_context(context);
+        return error;
 }
-static inline void set_info_data(char *base, unsigned long value)
+
+int ds_get_bts_index(struct task_struct *task, size_t *pos)
 {
-        (*(unsigned long *)(base + ds_cfg.info_data.offset)) = value;
+        return ds_get_index(task, pos, ds_bts);
 }
 
+int ds_get_pebs_index(struct task_struct *task, size_t *pos)
+{
+        return ds_get_index(task, pos, ds_pebs);
+}
 
-int ds_allocate(void **dsp, size_t bts_size_in_bytes)
+static int ds_get_end(struct task_struct *task, size_t *pos,
+                      enum ds_qualifier qual)
 {
-        size_t bts_size_in_records;
-        unsigned long bts;
-        void *ds;
+        struct ds_context *context;
+        unsigned long base, end;
+        int error;
+
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
+
+        base = ds_get(context->ds, qual, ds_buffer_base);
+        end  = ds_get(context->ds, qual, ds_absolute_maximum);
+
+        error = ((end - base) / ds_cfg.sizeof_rec[qual]);
+        if (pos)
+                *pos = error;
+ out:
+        ds_put_context(context);
+        return error;
+}
 
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
+int ds_get_bts_end(struct task_struct *task, size_t *pos)
+{
+        return ds_get_end(task, pos, ds_bts);
+}
 
-        if (bts_size_in_bytes < 0)
-                return -EINVAL;
+int ds_get_pebs_end(struct task_struct *task, size_t *pos)
+{
+        return ds_get_end(task, pos, ds_pebs);
+}
 
-        bts_size_in_records =
-                bts_size_in_bytes / ds_cfg.sizeof_bts;
-        bts_size_in_bytes =
-                bts_size_in_records * ds_cfg.sizeof_bts;
+static int ds_access(struct task_struct *task, size_t index,
+                     const void **record, enum ds_qualifier qual)
+{
+        struct ds_context *context;
+        unsigned long base, idx;
+        int error;
 
-        if (bts_size_in_bytes <= 0)
+        if (!record)
                 return -EINVAL;
 
-        bts = (unsigned long)kzalloc(bts_size_in_bytes, GFP_KERNEL);
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
 
-        if (!bts)
-                return -ENOMEM;
+        base = ds_get(context->ds, qual, ds_buffer_base);
+        idx = base + (index * ds_cfg.sizeof_rec[qual]);
 
-        ds = kzalloc(ds_cfg.sizeof_ds, GFP_KERNEL);
+        error = -EINVAL;
+        if (idx > ds_get(context->ds, qual, ds_absolute_maximum))
+                goto out;
 
-        if (!ds) {
-                kfree((void *)bts);
-                return -ENOMEM;
-        }
-
-        set_bts_buffer_base(ds, bts);
-        set_bts_index(ds, bts);
-        set_bts_absolute_maximum(ds, bts + bts_size_in_bytes);
-        set_bts_interrupt_threshold(ds, bts + bts_size_in_bytes + 1);
-
-        *dsp = ds;
-        return 0;
+        *record = (const void *)idx;
+        error = ds_cfg.sizeof_rec[qual];
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-int ds_free(void **dsp)
+int ds_access_bts(struct task_struct *task, size_t index, const void **record)
 {
-        if (*dsp) {
-                kfree((void *)get_bts_buffer_base(*dsp));
-                kfree(*dsp);
-                *dsp = NULL;
-        }
-        return 0;
+        return ds_access(task, index, record, ds_bts);
 }
 
-int ds_get_bts_size(void *ds)
+int ds_access_pebs(struct task_struct *task, size_t index, const void **record)
 {
-        int size_in_bytes;
-
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
-
-        if (!ds)
-                return 0;
-
-        size_in_bytes =
-                get_bts_absolute_maximum(ds) -
-                get_bts_buffer_base(ds);
-        return size_in_bytes;
+        return ds_access(task, index, record, ds_pebs);
 }
 
-int ds_get_bts_end(void *ds)
+static int ds_write(struct task_struct *task, const void *record, size_t size,
+                    enum ds_qualifier qual, int force)
 {
-        int size_in_bytes = ds_get_bts_size(ds);
-
-        if (size_in_bytes <= 0)
-                return size_in_bytes;
+        struct ds_context *context;
+        int error;
 
-        return size_in_bytes / ds_cfg.sizeof_bts;
-}
+        if (!record)
+                return -EINVAL;
 
-int ds_get_bts_index(void *ds)
-{
-        int index_offset_in_bytes;
+        error = -EPERM;
+        context = ds_get_context(task);
+        if (!context)
+                goto out;
 
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
+        if (!force) {
+                error = ds_validate_access(context, qual);
+                if (error < 0)
+                        goto out;
+        }
 
-        index_offset_in_bytes =
-                get_bts_index(ds) -
-                get_bts_buffer_base(ds);
+        error = 0;
+        while (size) {
+                unsigned long base, index, end, write_end, int_th;
+                unsigned long write_size, adj_write_size;
+
+                /*
+                 * write as much as possible without producing an
+                 * overflow interrupt.
+                 *
+                 * interrupt_threshold must either be
+                 * - bigger than absolute_maximum or
+                 * - point to a record between buffer_base and absolute_maximum
+                 *
+                 * index points to a valid record.
+                 */
+                base   = ds_get(context->ds, qual, ds_buffer_base);
+                index  = ds_get(context->ds, qual, ds_index);
+                end    = ds_get(context->ds, qual, ds_absolute_maximum);
+                int_th = ds_get(context->ds, qual, ds_interrupt_threshold);
+
+                write_end = min(end, int_th);
+
+                /* if we are already beyond the interrupt threshold,
+                 * we fill the entire buffer */
+                if (write_end <= index)
+                        write_end = end;
+
+                if (write_end <= index)
+                        goto out;
+
+                write_size = min((unsigned long) size, write_end - index);
+                memcpy((void *)index, record, write_size);
+
+                record = (const char *)record + write_size;
+                size  -= write_size;
+                error += write_size;
+
+                adj_write_size = write_size / ds_cfg.sizeof_rec[qual];
+                adj_write_size *= ds_cfg.sizeof_rec[qual];
+
+                /* zero out trailing bytes */
+                memset((char *)index + write_size, 0,
+                       adj_write_size - write_size);
+                index += adj_write_size;
+
+                if (index >= end)
+                        index = base;
+                ds_set(context->ds, qual, ds_index, index);
+
+                if (index >= int_th)
+                        ds_overflow(task, context, qual);
+        }
 
-        return index_offset_in_bytes / ds_cfg.sizeof_bts;
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-int ds_set_overflow(void *ds, int method)
+int ds_write_bts(struct task_struct *task, const void *record, size_t size)
 {
-        switch (method) {
-        case DS_O_SIGNAL:
-                return -EOPNOTSUPP;
-        case DS_O_WRAP:
-                return 0;
-        default:
-                return -EINVAL;
-        }
+        return ds_write(task, record, size, ds_bts, /* force = */ 0);
 }
 
-int ds_get_overflow(void *ds)
+int ds_write_pebs(struct task_struct *task, const void *record, size_t size)
 {
-        return DS_O_WRAP;
+        return ds_write(task, record, size, ds_pebs, /* force = */ 0);
 }
 
-int ds_clear(void *ds)
+int ds_unchecked_write_bts(struct task_struct *task,
+                           const void *record, size_t size)
 {
-        int bts_size = ds_get_bts_size(ds);
-        unsigned long bts_base;
-
-        if (bts_size <= 0)
-                return bts_size;
-
-        bts_base = get_bts_buffer_base(ds);
-        memset((void *)bts_base, 0, bts_size);
-
-        set_bts_index(ds, bts_base);
-        return 0;
+        return ds_write(task, record, size, ds_bts, /* force = */ 1);
 }
 
-int ds_read_bts(void *ds, int index, struct bts_struct *out)
+int ds_unchecked_write_pebs(struct task_struct *task,
+                            const void *record, size_t size)
 {
-        void *bts;
+        return ds_write(task, record, size, ds_pebs, /* force = */ 1);
+}
 
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
+static int ds_reset_or_clear(struct task_struct *task,
+                             enum ds_qualifier qual, int clear)
+{
+        struct ds_context *context;
+        unsigned long base, end;
+        int error;
 
-        if (index < 0)
-                return -EINVAL;
+        context = ds_get_context(task);
+        error = ds_validate_access(context, qual);
+        if (error < 0)
+                goto out;
 
-        if (index >= ds_get_bts_size(ds))
-                return -EINVAL;
+        base = ds_get(context->ds, qual, ds_buffer_base);
+        end  = ds_get(context->ds, qual, ds_absolute_maximum);
 
-        bts = (void *)(get_bts_buffer_base(ds) + (index * ds_cfg.sizeof_bts));
+        if (clear)
+                memset((void *)base, 0, end - base);
 
-        memset(out, 0, sizeof(*out));
-        if (get_from_ip(bts) == BTS_ESCAPE_ADDRESS) {
-                out->qualifier       = get_info_type(bts);
-                out->variant.jiffies = get_info_data(bts);
-        } else {
-                out->qualifier = BTS_BRANCH;
-                out->variant.lbr.from_ip = get_from_ip(bts);
-                out->variant.lbr.to_ip   = get_to_ip(bts);
-        }
+        ds_set(context->ds, qual, ds_index, base);
 
-        return sizeof(*out);;
+        error = 0;
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-int ds_write_bts(void *ds, const struct bts_struct *in)
+int ds_reset_bts(struct task_struct *task)
 {
-        unsigned long bts;
-
-        if (!ds_cfg.sizeof_ds || !ds_cfg.sizeof_bts)
-                return -EOPNOTSUPP;
-
-        if (ds_get_bts_size(ds) <= 0)
-                return -ENXIO;
+        return ds_reset_or_clear(task, ds_bts, /* clear = */ 0);
+}
 
-        bts = get_bts_index(ds);
+int ds_reset_pebs(struct task_struct *task)
+{
+        return ds_reset_or_clear(task, ds_pebs, /* clear = */ 0);
+}
 
-        memset((void *)bts, 0, ds_cfg.sizeof_bts);
-        switch (in->qualifier) {
-        case BTS_INVALID:
-                break;
+int ds_clear_bts(struct task_struct *task)
+{
+        return ds_reset_or_clear(task, ds_bts, /* clear = */ 1);
+}
 
-        case BTS_BRANCH:
-                set_from_ip((void *)bts, in->variant.lbr.from_ip);
-                set_to_ip((void *)bts, in->variant.lbr.to_ip);
-                break;
+int ds_clear_pebs(struct task_struct *task)
+{
+        return ds_reset_or_clear(task, ds_pebs, /* clear = */ 1);
+}
 
-        case BTS_TASK_ARRIVES:
-        case BTS_TASK_DEPARTS:
-                set_from_ip((void *)bts, BTS_ESCAPE_ADDRESS);
-                set_info_type((void *)bts, in->qualifier);
-                set_info_data((void *)bts, in->variant.jiffies);
-                break;
+int ds_get_pebs_reset(struct task_struct *task, u64 *value)
+{
+        struct ds_context *context;
+        int error;
 
-        default:
+        if (!value)
                 return -EINVAL;
-        }
 
-        bts = bts + ds_cfg.sizeof_bts;
-        if (bts >= get_bts_absolute_maximum(ds))
-                bts = get_bts_buffer_base(ds);
-        set_bts_index(ds, bts);
+        context = ds_get_context(task);
+        error = ds_validate_access(context, ds_pebs);
+        if (error < 0)
+                goto out;
 
-        return ds_cfg.sizeof_bts;
+        *value = *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8));
+
+        error = 0;
+ out:
+        ds_put_context(context);
+        return error;
 }
 
-unsigned long ds_debugctl_mask(void)
+int ds_set_pebs_reset(struct task_struct *task, u64 value)
 {
-        return ds_cfg.debugctl_mask;
-}
+        struct ds_context *context;
+        int error;
 
-#ifdef __i386__
-static const struct ds_configuration ds_cfg_netburst = {
-        .sizeof_ds = 9 * 4,
-        .bts_buffer_base = { 0, 4 },
-        .bts_index = { 4, 4 },
-        .bts_absolute_maximum = { 8, 4 },
-        .bts_interrupt_threshold = { 12, 4 },
-        .sizeof_bts = 3 * 4,
-        .from_ip = { 0, 4 },
-        .to_ip = { 4, 4 },
-        .info_type = { 4, 1 },
-        .info_data = { 8, 4 },
-        .debugctl_mask = (1<<2)|(1<<3)
-};
+        context = ds_get_context(task);
+        error = ds_validate_access(context, ds_pebs);
+        if (error < 0)
+                goto out;
 
-static const struct ds_configuration ds_cfg_pentium_m = {
-        .sizeof_ds = 9 * 4,
-        .bts_buffer_base = { 0, 4 },
-        .bts_index = { 4, 4 },
-        .bts_absolute_maximum = { 8, 4 },
-        .bts_interrupt_threshold = { 12, 4 },
-        .sizeof_bts = 3 * 4,
-        .from_ip = { 0, 4 },
-        .to_ip = { 4, 4 },
-        .info_type = { 4, 1 },
-        .info_data = { 8, 4 },
-        .debugctl_mask = (1<<6)|(1<<7)
+        *(u64 *)(context->ds + (ds_cfg.sizeof_field * 8)) = value;
+
+        error = 0;
+ out:
+        ds_put_context(context);
+        return error;
+}
+
+static const struct ds_configuration ds_cfg_var = {
+        .sizeof_ds    = sizeof(long) * 12,
+        .sizeof_field = sizeof(long),
+        .sizeof_rec[ds_bts]   = sizeof(long) * 3,
+        .sizeof_rec[ds_pebs]  = sizeof(long) * 10
 };
-#endif /* _i386_ */
-
-static const struct ds_configuration ds_cfg_core2 = {
-        .sizeof_ds = 9 * 8,
-        .bts_buffer_base = { 0, 8 },
-        .bts_index = { 8, 8 },
-        .bts_absolute_maximum = { 16, 8 },
-        .bts_interrupt_threshold = { 24, 8 },
-        .sizeof_bts = 3 * 8,
-        .from_ip = { 0, 8 },
-        .to_ip = { 8, 8 },
-        .info_type = { 8, 1 },
-        .info_data = { 16, 8 },
-        .debugctl_mask = (1<<6)|(1<<7)|(1<<9)
+static const struct ds_configuration ds_cfg_64 = {
+        .sizeof_ds    = 8 * 12,
+        .sizeof_field = 8,
+        .sizeof_rec[ds_bts]   = 8 * 3,
+        .sizeof_rec[ds_pebs]  = 8 * 10
 };
 
 static inline void
@@ -429,14 +820,13 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
         switch (c->x86) {
         case 0x6:
                 switch (c->x86_model) {
-#ifdef __i386__
                 case 0xD:
                 case 0xE: /* Pentium M */
-                        ds_configure(&ds_cfg_pentium_m);
+                        ds_configure(&ds_cfg_var);
                         break;
-#endif /* _i386_ */
                 case 0xF: /* Core2 */
-                        ds_configure(&ds_cfg_core2);
+        case 0x1C: /* Atom */
+                        ds_configure(&ds_cfg_64);
                         break;
                 default:
                         /* sorry, don't know about them */
@@ -445,13 +835,11 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
                 break;
         case 0xF:
                 switch (c->x86_model) {
-#ifdef __i386__
                 case 0x0:
                 case 0x1:
                 case 0x2: /* Netburst */
-                        ds_configure(&ds_cfg_netburst);
+                        ds_configure(&ds_cfg_var);
                         break;
-#endif /* _i386_ */
                 default:
                         /* sorry, don't know about them */
                         break;
@@ -462,3 +850,14 @@ void __cpuinit ds_init_intel(struct cpuinfo_x86 *c)
                 break;
         }
 }
+
+void ds_free(struct ds_context *context)
+{
+        /* This is called when the task owning the parameter context
+         * is dying. There should not be any user of that context left
+         * to disturb us, anymore. */
+        unsigned long leftovers = context->count;
+        while (leftovers--)
+                ds_put_context(context);
+}
+#endif /* CONFIG_X86_DS */