Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

7 files changed, 381 insertions, 160 deletions

diff --git a/arch/x86/oprofile/backtrace.c b/arch/x86/oprofile/backtrace.c
index 3855096c59b8..a5b64ab4cd6e 100644
--- a/arch/x86/oprofile/backtrace.c
+++ b/arch/x86/oprofile/backtrace.c
@@ -14,17 +14,7 @@
 #include <asm/ptrace.h>
 #include <asm/uaccess.h>
 #include <asm/stacktrace.h>
-
-static void backtrace_warning_symbol(void *data, char *msg,
-                                     unsigned long symbol)
-{
-        /* Ignore warnings */
-}
-
-static void backtrace_warning(void *data, char *msg)
-{
-        /* Ignore warnings */
-}
+#include <linux/compat.h>
 
 static int backtrace_stack(void *data, char *name)
 {
@@ -41,21 +31,17 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
 }
 
 static struct stacktrace_ops backtrace_ops = {
-        .warning        = backtrace_warning,
-        .warning_symbol = backtrace_warning_symbol,
         .stack          = backtrace_stack,
         .address        = backtrace_address,
         .walk_stack     = print_context_stack,
 };
 
-struct frame_head {
-        struct frame_head *bp;
-        unsigned long ret;
-} __attribute__((packed));
-
-static struct frame_head *dump_user_backtrace(struct frame_head *head)
+#ifdef CONFIG_COMPAT
+static struct stack_frame_ia32 *
+dump_user_backtrace_32(struct stack_frame_ia32 *head)
 {
-        struct frame_head bufhead[2];
+        struct stack_frame_ia32 bufhead[2];
+        struct stack_frame_ia32 *fp;
 
         /* Also check accessibility of one struct frame_head beyond */
         if (!access_ok(VERIFY_READ, head, sizeof(bufhead)))
@@ -63,20 +49,66 @@ static struct frame_head *dump_user_backtrace(struct frame_head *head)
         if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead)))
                 return NULL;
 
-        oprofile_add_trace(bufhead[0].ret);
+        fp = (struct stack_frame_ia32 *) compat_ptr(bufhead[0].next_frame);
+
+        oprofile_add_trace(bufhead[0].return_address);
+
+        /* frame pointers should strictly progress back up the stack
+        * (towards higher addresses) */
+        if (head >= fp)
+                return NULL;
+
+        return fp;
+}
+
+static inline int
+x86_backtrace_32(struct pt_regs * const regs, unsigned int depth)
+{
+        struct stack_frame_ia32 *head;
+
+        /* User process is 32-bit */
+        if (!current || !test_thread_flag(TIF_IA32))
+                return 0;
+
+        head = (struct stack_frame_ia32 *) regs->bp;
+        while (depth-- && head)
+                head = dump_user_backtrace_32(head);
+
+        return 1;
+}
+
+#else
+static inline int
+x86_backtrace_32(struct pt_regs * const regs, unsigned int depth)
+{
+        return 0;
+}
+#endif /* CONFIG_COMPAT */
+
+static struct stack_frame *dump_user_backtrace(struct stack_frame *head)
+{
+        struct stack_frame bufhead[2];
+
+        /* Also check accessibility of one struct stack_frame beyond */
+        if (!access_ok(VERIFY_READ, head, sizeof(bufhead)))
+                return NULL;
+        if (__copy_from_user_inatomic(bufhead, head, sizeof(bufhead)))
+                return NULL;
+
+        oprofile_add_trace(bufhead[0].return_address);
 
         /* frame pointers should strictly progress back up the stack
          * (towards higher addresses) */
-        if (head >= bufhead[0].bp)
+        if (head >= bufhead[0].next_frame)
                 return NULL;
 
-        return bufhead[0].bp;
+        return bufhead[0].next_frame;
 }
 
 void
 x86_backtrace(struct pt_regs * const regs, unsigned int depth)
 {
-        struct frame_head *head = (struct frame_head *)frame_pointer(regs);
+        struct stack_frame *head = (struct stack_frame *)frame_pointer(regs);
 
         if (!user_mode_vm(regs)) {
                 unsigned long stack = kernel_stack_pointer(regs);
@@ -86,6 +118,9 @@ x86_backtrace(struct pt_regs * const regs, unsigned int depth)
                 return;
         }
 
+        if (x86_backtrace_32(regs, depth))
+                return;
+
         while (depth-- && head)
                 head = dump_user_backtrace(head);
 }
diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c
index f1575c9a2572..68894fdc034b 100644
--- a/arch/x86/oprofile/nmi_int.c
+++ b/arch/x86/oprofile/nmi_int.c
@@ -15,7 +15,7 @@
 #include <linux/notifier.h>
 #include <linux/smp.h>
 #include <linux/oprofile.h>
-#include <linux/sysdev.h>
+#include <linux/syscore_ops.h>
 #include <linux/slab.h>
 #include <linux/moduleparam.h>
 #include <linux/kdebug.h>
@@ -49,6 +49,10 @@ u64 op_x86_get_ctrl(struct op_x86_model_spec const *model,
         val |= counter_config->user ? ARCH_PERFMON_EVENTSEL_USR : 0;
         val |= counter_config->kernel ? ARCH_PERFMON_EVENTSEL_OS : 0;
         val |= (counter_config->unit_mask & 0xFF) << 8;
+        counter_config->extra &= (ARCH_PERFMON_EVENTSEL_INV |
+                                  ARCH_PERFMON_EVENTSEL_EDGE |
+                                  ARCH_PERFMON_EVENTSEL_CMASK);
+        val |= counter_config->extra;
         event &= model->event_mask ? model->event_mask : 0xFF;
         val |= event & 0xFF;
         val |= (event & 0x0F00) << 24;
@@ -65,7 +69,6 @@ static int profile_exceptions_notify(struct notifier_block *self,
 
         switch (val) {
         case DIE_NMI:
-        case DIE_NMI_IPI:
                 if (ctr_running)
                         model->check_ctrs(args->regs, &__get_cpu_var(cpu_msrs));
                 else if (!nmi_enabled)
@@ -109,8 +112,10 @@ static void nmi_cpu_start(void *dummy)
 static int nmi_start(void)
 {
         get_online_cpus();
-        on_each_cpu(nmi_cpu_start, NULL, 1);
         ctr_running = 1;
+        /* make ctr_running visible to the nmi handler: */
+        smp_mb();
+        on_each_cpu(nmi_cpu_start, NULL, 1);
         put_online_cpus();
         return 0;
 }
@@ -143,7 +148,7 @@ static inline int has_mux(void)
 
 inline int op_x86_phys_to_virt(int phys)
 {
-        return __get_cpu_var(switch_index) + phys;
+        return __this_cpu_read(switch_index) + phys;
 }
 
 inline int op_x86_virt_to_phys(int virt)
@@ -361,7 +366,7 @@ static void nmi_cpu_setup(void *dummy)
 static struct notifier_block profile_exceptions_nb = {
         .notifier_call = profile_exceptions_notify,
         .next = NULL,
-        .priority = 2
+        .priority = NMI_LOCAL_LOW_PRIOR,
 };
 
 static void nmi_cpu_restore_registers(struct op_msrs *msrs)
@@ -441,6 +446,7 @@ static int nmi_create_files(struct super_block *sb, struct dentry *root)
                 oprofilefs_create_ulong(sb, dir, "unit_mask", &counter_config[i].unit_mask);
                 oprofilefs_create_ulong(sb, dir, "kernel", &counter_config[i].kernel);
                 oprofilefs_create_ulong(sb, dir, "user", &counter_config[i].user);
+                oprofilefs_create_ulong(sb, dir, "extra", &counter_config[i].extra);
         }
 
         return 0;
@@ -500,15 +506,18 @@ static int nmi_setup(void)
 
         nmi_enabled = 0;
         ctr_running = 0;
-        barrier();
+        /* make variables visible to the nmi handler: */
+        smp_mb();
         err = register_die_notifier(&profile_exceptions_nb);
         if (err)
                 goto fail;
 
         get_online_cpus();
         register_cpu_notifier(&oprofile_cpu_nb);
-        on_each_cpu(nmi_cpu_setup, NULL, 1);
         nmi_enabled = 1;
+        /* make nmi_enabled visible to the nmi handler: */
+        smp_mb();
+        on_each_cpu(nmi_cpu_setup, NULL, 1);
         put_online_cpus();
 
         return 0;
@@ -527,7 +536,8 @@ static void nmi_shutdown(void)
         nmi_enabled = 0;
         ctr_running = 0;
         put_online_cpus();
-        barrier();
+        /* make variables visible to the nmi handler: */
+        smp_mb();
         unregister_die_notifier(&profile_exceptions_nb);
         msrs = &get_cpu_var(cpu_msrs);
         model->shutdown(msrs);
@@ -537,7 +547,7 @@ static void nmi_shutdown(void)
 
 #ifdef CONFIG_PM
 
-static int nmi_suspend(struct sys_device *dev, pm_message_t state)
+static int nmi_suspend(void)
 {
         /* Only one CPU left, just stop that one */
         if (nmi_enabled == 1)
@@ -545,49 +555,31 @@ static int nmi_suspend(struct sys_device *dev, pm_message_t state)
         return 0;
 }
 
-static int nmi_resume(struct sys_device *dev)
+static void nmi_resume(void)
 {
         if (nmi_enabled == 1)
                 nmi_cpu_start(NULL);
-        return 0;
 }
 
-static struct sysdev_class oprofile_sysclass = {
-        .name           = "oprofile",
+static struct syscore_ops oprofile_syscore_ops = {
         .resume         = nmi_resume,
         .suspend        = nmi_suspend,
 };
 
-static struct sys_device device_oprofile = {
-        .id     = 0,
-        .cls    = &oprofile_sysclass,
-};
-
-static int __init init_sysfs(void)
+static void __init init_suspend_resume(void)
 {
-        int error;
-
-        error = sysdev_class_register(&oprofile_sysclass);
-        if (error)
-                return error;
-
-        error = sysdev_register(&device_oprofile);
-        if (error)
-                sysdev_class_unregister(&oprofile_sysclass);
-
-        return error;
+        register_syscore_ops(&oprofile_syscore_ops);
 }
 
-static void exit_sysfs(void)
+static void exit_suspend_resume(void)
 {
-        sysdev_unregister(&device_oprofile);
-        sysdev_class_unregister(&oprofile_sysclass);
+        unregister_syscore_ops(&oprofile_syscore_ops);
 }
 
 #else
 
-static inline int  init_sysfs(void) { return 0; }
-static inline void exit_sysfs(void) { }
+static inline void init_suspend_resume(void) { }
+static inline void exit_suspend_resume(void) { }
 
 #endif /* CONFIG_PM */
 
@@ -695,9 +687,6 @@ static int __init ppro_init(char **cpu_type)
         return 1;
 }
 
-/* in order to get sysfs right */
-static int using_nmi;
-
 int __init op_nmi_init(struct oprofile_operations *ops)
 {
         __u8 vendor = boot_cpu_data.x86_vendor;
@@ -705,8 +694,6 @@ int __init op_nmi_init(struct oprofile_operations *ops)
         char *cpu_type = NULL;
         int ret = 0;
 
-        using_nmi = 0;
-
         if (!cpu_has_apic)
                 return -ENODEV;
 
@@ -731,6 +718,15 @@ int __init op_nmi_init(struct oprofile_operations *ops)
                 case 0x11:
                         cpu_type = "x86-64/family11h";
                         break;
+                case 0x12:
+                        cpu_type = "x86-64/family12h";
+                        break;
+                case 0x14:
+                        cpu_type = "x86-64/family14h";
+                        break;
+                case 0x15:
+                        cpu_type = "x86-64/family15h";
+                        break;
                 default:
                         return -ENODEV;
                 }
@@ -786,17 +782,13 @@ int __init op_nmi_init(struct oprofile_operations *ops)
 
         mux_init(ops);
 
-        ret = init_sysfs();
-        if (ret)
-                return ret;
+        init_suspend_resume();
 
-        using_nmi = 1;
         printk(KERN_INFO "oprofile: using NMI interrupt.\n");
         return 0;
 }
 
 void op_nmi_exit(void)
 {
-        if (using_nmi)
-                exit_sysfs();
+        exit_suspend_resume();
 }
diff --git a/arch/x86/oprofile/nmi_timer_int.c b/arch/x86/oprofile/nmi_timer_int.c
index e3ecb71b5790..720bf5a53c51 100644
--- a/arch/x86/oprofile/nmi_timer_int.c
+++ b/arch/x86/oprofile/nmi_timer_int.c
@@ -38,7 +38,7 @@ static int profile_timer_exceptions_notify(struct notifier_block *self,
 static struct notifier_block profile_timer_exceptions_nb = {
         .notifier_call = profile_timer_exceptions_notify,
         .next = NULL,
-        .priority = 0
+        .priority = NMI_LOW_PRIOR,
 };
 
 static int timer_start(void)
@@ -58,9 +58,6 @@ static void timer_stop(void)
 
 int __init op_nmi_timer_init(struct oprofile_operations *ops)
 {
-        if ((nmi_watchdog != NMI_IO_APIC) || (atomic_read(&nmi_active) <= 0))
-                return -ENODEV;
-
         ops->start = timer_start;
         ops->stop = timer_stop;
         ops->cpu_type = "timer";
diff --git a/arch/x86/oprofile/op_counter.h b/arch/x86/oprofile/op_counter.h
index e28398df0df2..0b7b7b179cbe 100644
--- a/arch/x86/oprofile/op_counter.h
+++ b/arch/x86/oprofile/op_counter.h
@@ -22,6 +22,7 @@ struct op_counter_config {
         unsigned long kernel;
         unsigned long user;
         unsigned long unit_mask;
+        unsigned long extra;
 };
 
 extern struct op_counter_config counter_config[];
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c
index b67a6b5aa8d4..9cbb710dc94b 100644
--- a/arch/x86/oprofile/op_model_amd.c
+++ b/arch/x86/oprofile/op_model_amd.c
@@ -29,11 +29,12 @@
 #include "op_x86_model.h"
 #include "op_counter.h"
 
-#define NUM_COUNTERS 4
+#define NUM_COUNTERS            4
+#define NUM_COUNTERS_F15H       6
 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
-#define NUM_VIRT_COUNTERS 32
+#define NUM_VIRT_COUNTERS       32
 #else
-#define NUM_VIRT_COUNTERS NUM_COUNTERS
+#define NUM_VIRT_COUNTERS       0
 #endif
 
 #define OP_EVENT_MASK                   0x0FFF
@@ -41,38 +42,61 @@
 
 #define MSR_AMD_EVENTSEL_RESERVED       ((0xFFFFFCF0ULL<<32)|(1ULL<<21))
 
-static unsigned long reset_value[NUM_VIRT_COUNTERS];
+static int num_counters;
+static unsigned long reset_value[OP_MAX_COUNTER];
 
 #define IBS_FETCH_SIZE                  6
 #define IBS_OP_SIZE                     12
 
 static u32 ibs_caps;
 
-struct op_ibs_config {
+struct ibs_config {
         unsigned long op_enabled;
         unsigned long fetch_enabled;
         unsigned long max_cnt_fetch;
         unsigned long max_cnt_op;
         unsigned long rand_en;
         unsigned long dispatched_ops;
+        unsigned long branch_target;
 };
 
-static struct op_ibs_config ibs_config;
-static u64 ibs_op_ctl;
+struct ibs_state {
+        u64             ibs_op_ctl;
+        int             branch_target;
+        unsigned long   sample_size;
+};
+
+static struct ibs_config ibs_config;
+static struct ibs_state ibs_state;
 
 /*
  * IBS cpuid feature detection
  */
 
-#define IBS_CPUID_FEATURES      0x8000001b
+#define IBS_CPUID_FEATURES              0x8000001b
 
 /*
  * Same bit mask as for IBS cpuid feature flags (Fn8000_001B_EAX), but
  * bit 0 is used to indicate the existence of IBS.
  */
-#define IBS_CAPS_AVAIL                  (1LL<<0)
-#define IBS_CAPS_RDWROPCNT              (1LL<<3)
-#define IBS_CAPS_OPCNT                  (1LL<<4)
+#define IBS_CAPS_AVAIL                  (1U<<0)
+#define IBS_CAPS_FETCHSAM               (1U<<1)
+#define IBS_CAPS_OPSAM                  (1U<<2)
+#define IBS_CAPS_RDWROPCNT              (1U<<3)
+#define IBS_CAPS_OPCNT                  (1U<<4)
+#define IBS_CAPS_BRNTRGT                (1U<<5)
+#define IBS_CAPS_OPCNTEXT               (1U<<6)
+
+#define IBS_CAPS_DEFAULT                (IBS_CAPS_AVAIL         \
+                                         | IBS_CAPS_FETCHSAM    \
+                                         | IBS_CAPS_OPSAM)
+
+/*
+ * IBS APIC setup
+ */
+#define IBSCTL                          0x1cc
+#define IBSCTL_LVT_OFFSET_VALID         (1ULL<<8)
+#define IBSCTL_LVT_OFFSET_MASK          0x0F
 
 /*
  * IBS randomization macros
@@ -92,12 +116,12 @@ static u32 get_ibs_caps(void)
         /* check IBS cpuid feature flags */
         max_level = cpuid_eax(0x80000000);
         if (max_level < IBS_CPUID_FEATURES)
-                return IBS_CAPS_AVAIL;
+                return IBS_CAPS_DEFAULT;
 
         ibs_caps = cpuid_eax(IBS_CPUID_FEATURES);
         if (!(ibs_caps & IBS_CAPS_AVAIL))
                 /* cpuid flags not valid */
-                return IBS_CAPS_AVAIL;
+                return IBS_CAPS_DEFAULT;
 
         return ibs_caps;
 }
@@ -190,8 +214,8 @@ op_amd_handle_ibs(struct pt_regs * const regs,
                 rdmsrl(MSR_AMD64_IBSOPCTL, ctl);
                 if (ctl & IBS_OP_VAL) {
                         rdmsrl(MSR_AMD64_IBSOPRIP, val);
-                        oprofile_write_reserve(&entry, regs, val,
-                                               IBS_OP_CODE, IBS_OP_SIZE);
+                        oprofile_write_reserve(&entry, regs, val, IBS_OP_CODE,
+                                               ibs_state.sample_size);
                         oprofile_add_data64(&entry, val);
                         rdmsrl(MSR_AMD64_IBSOPDATA, val);
                         oprofile_add_data64(&entry, val);
@@ -203,10 +227,14 @@ op_amd_handle_ibs(struct pt_regs * const regs,
                         oprofile_add_data64(&entry, val);
                         rdmsrl(MSR_AMD64_IBSDCPHYSAD, val);
                         oprofile_add_data64(&entry, val);
+                        if (ibs_state.branch_target) {
+                                rdmsrl(MSR_AMD64_IBSBRTARGET, val);
+                                oprofile_add_data(&entry, (unsigned long)val);
+                        }
                         oprofile_write_commit(&entry);
 
                         /* reenable the IRQ */
-                        ctl = op_amd_randomize_ibs_op(ibs_op_ctl);
+                        ctl = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
                         wrmsrl(MSR_AMD64_IBSOPCTL, ctl);
                 }
         }
@@ -219,21 +247,32 @@ static inline void op_amd_start_ibs(void)
         if (!ibs_caps)
                 return;
 
+        memset(&ibs_state, 0, sizeof(ibs_state));
+
+        /*
+         * Note: Since the max count settings may out of range we
+         * write back the actual used values so that userland can read
+         * it.
+         */
+
         if (ibs_config.fetch_enabled) {
-                val = (ibs_config.max_cnt_fetch >> 4) & IBS_FETCH_MAX_CNT;
+                val = ibs_config.max_cnt_fetch >> 4;
+                val = min(val, IBS_FETCH_MAX_CNT);
+                ibs_config.max_cnt_fetch = val << 4;
                 val |= ibs_config.rand_en ? IBS_FETCH_RAND_EN : 0;
                 val |= IBS_FETCH_ENABLE;
                 wrmsrl(MSR_AMD64_IBSFETCHCTL, val);
         }
 
         if (ibs_config.op_enabled) {
-                ibs_op_ctl = ibs_config.max_cnt_op >> 4;
+                val = ibs_config.max_cnt_op >> 4;
                 if (!(ibs_caps & IBS_CAPS_RDWROPCNT)) {
                         /*
                          * IbsOpCurCnt not supported.  See
                          * op_amd_randomize_ibs_op() for details.
                          */
-                        ibs_op_ctl = clamp(ibs_op_ctl, 0x0081ULL, 0xFF80ULL);
+                        val = clamp(val, 0x0081ULL, 0xFF80ULL);
+                        ibs_config.max_cnt_op = val << 4;
                 } else {
                         /*
                          * The start value is randomized with a
@@ -241,13 +280,24 @@ static inline void op_amd_start_ibs(void)
                          * with the half of the randomized range. Also
                          * avoid underflows.
                          */
-                        ibs_op_ctl = min(ibs_op_ctl + IBS_RANDOM_MAXCNT_OFFSET,
-                                         IBS_OP_MAX_CNT);
+                        val += IBS_RANDOM_MAXCNT_OFFSET;
+                        if (ibs_caps & IBS_CAPS_OPCNTEXT)
+                                val = min(val, IBS_OP_MAX_CNT_EXT);
+                        else
+                                val = min(val, IBS_OP_MAX_CNT);
+                        ibs_config.max_cnt_op =
+                                (val - IBS_RANDOM_MAXCNT_OFFSET) << 4;
                 }
-                if (ibs_caps & IBS_CAPS_OPCNT && ibs_config.dispatched_ops)
-                        ibs_op_ctl |= IBS_OP_CNT_CTL;
-                ibs_op_ctl |= IBS_OP_ENABLE;
-                val = op_amd_randomize_ibs_op(ibs_op_ctl);
+                val = ((val & ~IBS_OP_MAX_CNT) << 4) | (val & IBS_OP_MAX_CNT);
+                val |= ibs_config.dispatched_ops ? IBS_OP_CNT_CTL : 0;
+                val |= IBS_OP_ENABLE;
+                ibs_state.ibs_op_ctl = val;
+                ibs_state.sample_size = IBS_OP_SIZE;
+                if (ibs_config.branch_target) {
+                        ibs_state.branch_target = 1;
+                        ibs_state.sample_size++;
+                }
+                val = op_amd_randomize_ibs_op(ibs_state.ibs_op_ctl);
                 wrmsrl(MSR_AMD64_IBSOPCTL, val);
         }
 }
@@ -266,6 +316,81 @@ static void op_amd_stop_ibs(void)
                 wrmsrl(MSR_AMD64_IBSOPCTL, 0);
 }
 
+static inline int get_eilvt(int offset)
+{
+        return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1);
+}
+
+static inline int put_eilvt(int offset)
+{
+        return !setup_APIC_eilvt(offset, 0, 0, 1);
+}
+
+static inline int ibs_eilvt_valid(void)
+{
+        int offset;
+        u64 val;
+        int valid = 0;
+
+        preempt_disable();
+
+        rdmsrl(MSR_AMD64_IBSCTL, val);
+        offset = val & IBSCTL_LVT_OFFSET_MASK;
+
+        if (!(val & IBSCTL_LVT_OFFSET_VALID)) {
+                pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n",
+                       smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
+                goto out;
+        }
+
+        if (!get_eilvt(offset)) {
+                pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n",
+                       smp_processor_id(), offset, MSR_AMD64_IBSCTL, val);
+                goto out;
+        }
+
+        valid = 1;
+out:
+        preempt_enable();
+
+        return valid;
+}
+
+static inline int get_ibs_offset(void)
+{
+        u64 val;
+
+        rdmsrl(MSR_AMD64_IBSCTL, val);
+        if (!(val & IBSCTL_LVT_OFFSET_VALID))
+                return -EINVAL;
+
+        return val & IBSCTL_LVT_OFFSET_MASK;
+}
+
+static void setup_APIC_ibs(void)
+{
+        int offset;
+
+        offset = get_ibs_offset();
+        if (offset < 0)
+                goto failed;
+
+        if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0))
+                return;
+failed:
+        pr_warn("oprofile: IBS APIC setup failed on cpu #%d\n",
+                smp_processor_id());
+}
+
+static void clear_APIC_ibs(void)
+{
+        int offset;
+
+        offset = get_ibs_offset();
+        if (offset >= 0)
+                setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1);
+}
+
 #ifdef CONFIG_OPROFILE_EVENT_MULTIPLEX
 
 static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
@@ -275,7 +400,7 @@ static void op_mux_switch_ctrl(struct op_x86_model_spec const *model,
         int i;
 
         /* enable active counters */
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 int virt = op_x86_phys_to_virt(i);
                 if (!reset_value[virt])
                         continue;
@@ -294,7 +419,7 @@ static void op_amd_shutdown(struct op_msrs const * const msrs)
 {
         int i;
 
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 if (!msrs->counters[i].addr)
                         continue;
                 release_perfctr_nmi(MSR_K7_PERFCTR0 + i);
@@ -306,7 +431,7 @@ static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
 {
         int i;
 
-        for (i = 0; i < NUM_COUNTERS; i++) {
+        for (i = 0; i < num_counters; i++) {
                 if (!reserve_perfctr_nmi(MSR_K7_PERFCTR0 + i))
                         goto fail;
                 if (!reserve_evntsel_nmi(MSR_K7_EVNTSEL0 + i)) {
@@ -314,8 +439,13 @@ static int op_amd_fill_in_addresses(struct op_msrs * const msrs)
                         goto fail;
                 }
                 /* both registers must be reserved */
-                msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
-                msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
+                if (num_counters == NUM_COUNTERS_F15H) {
+                        msrs->counters[i].addr = MSR_F15H_PERF_CTR + (i << 1);
+                        msrs->controls[i].addr = MSR_F15H_PERF_CTL + (i << 1);
+                } else {
+                        msrs->controls[i].addr = MSR_K7_EVNTSEL0 + i;
+                        msrs->counters[i].addr = MSR_K7_PERFCTR0 + i;
+                }
                 continue;
         fail:
                 if (!counter_config[i].enabled)
@@ -335,7 +465,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
         int i;
 
         /* setup reset_value */
-        for (i = 0; i < NUM_VIRT_COUNTERS; ++i) {
+        for (i = 0; i < OP_MAX_COUNTER; ++i) {
                 if (counter_config[i].enabled
                     && msrs->counters[op_x86_virt_to_phys(i)].addr)
                         reset_value[i] = counter_config[i].count;
@@ -344,7 +474,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
         }
 
         /* clear all counters */
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 if (!msrs->controls[i].addr)
                         continue;
                 rdmsrl(msrs->controls[i].addr, val);
@@ -360,7 +490,7 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
         }
 
         /* enable active counters */
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 int virt = op_x86_phys_to_virt(i);
                 if (!reset_value[virt])
                         continue;
@@ -376,13 +506,13 @@ static void op_amd_setup_ctrs(struct op_x86_model_spec const *model,
         }
 
         if (ibs_caps)
-                setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_NMI, 0);
+                setup_APIC_ibs();
 }
 
 static void op_amd_cpu_shutdown(void)
 {
         if (ibs_caps)
-                setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
+                clear_APIC_ibs();
 }
 
 static int op_amd_check_ctrs(struct pt_regs * const regs,
@@ -391,7 +521,7 @@ static int op_amd_check_ctrs(struct pt_regs * const regs,
         u64 val;
         int i;
 
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 int virt = op_x86_phys_to_virt(i);
                 if (!reset_value[virt])
                         continue;
@@ -414,7 +544,7 @@ static void op_amd_start(struct op_msrs const * const msrs)
         u64 val;
         int i;
 
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 if (!reset_value[op_x86_phys_to_virt(i)])
                         continue;
                 rdmsrl(msrs->controls[i].addr, val);
@@ -434,7 +564,7 @@ static void op_amd_stop(struct op_msrs const * const msrs)
          * Subtle: stop on all counters to avoid race with setting our
          * pm callback
          */
-        for (i = 0; i < NUM_COUNTERS; ++i) {
+        for (i = 0; i < num_counters; ++i) {
                 if (!reset_value[op_x86_phys_to_virt(i)])
                         continue;
                 rdmsrl(msrs->controls[i].addr, val);
@@ -445,16 +575,11 @@ static void op_amd_stop(struct op_msrs const * const msrs)
         op_amd_stop_ibs();
 }
 
-static int __init_ibs_nmi(void)
+static int setup_ibs_ctl(int ibs_eilvt_off)
 {
-#define IBSCTL_LVTOFFSETVAL             (1 << 8)
-#define IBSCTL                          0x1cc
         struct pci_dev *cpu_cfg;
         int nodes;
         u32 value = 0;
-        u8 ibs_eilvt_off;
-
-        ibs_eilvt_off = setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
 
         nodes = 0;
         cpu_cfg = NULL;
@@ -466,25 +591,75 @@ static int __init_ibs_nmi(void)
                         break;
                 ++nodes;
                 pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off
-                                       | IBSCTL_LVTOFFSETVAL);
+                                       | IBSCTL_LVT_OFFSET_VALID);
                 pci_read_config_dword(cpu_cfg, IBSCTL, &value);
-                if (value != (ibs_eilvt_off | IBSCTL_LVTOFFSETVAL)) {
+                if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) {
                         pci_dev_put(cpu_cfg);
                         printk(KERN_DEBUG "Failed to setup IBS LVT offset, "
-                                "IBSCTL = 0x%08x", value);
-                        return 1;
+                               "IBSCTL = 0x%08x\n", value);
+                        return -EINVAL;
                 }
         } while (1);
 
         if (!nodes) {
-                printk(KERN_DEBUG "No CPU node configured for IBS");
-                return 1;
+                printk(KERN_DEBUG "No CPU node configured for IBS\n");
+                return -ENODEV;
+        }
+
+        return 0;
+}
+
+/*
+ * This runs only on the current cpu. We try to find an LVT offset and
+ * setup the local APIC. For this we must disable preemption. On
+ * success we initialize all nodes with this offset. This updates then
+ * the offset in the IBS_CTL per-node msr. The per-core APIC setup of
+ * the IBS interrupt vector is called from op_amd_setup_ctrs()/op_-
+ * amd_cpu_shutdown() using the new offset.
+ */
+static int force_ibs_eilvt_setup(void)
+{
+        int offset;
+        int ret;
+
+        preempt_disable();
+        /* find the next free available EILVT entry, skip offset 0 */
+        for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) {
+                if (get_eilvt(offset))
+                        break;
+        }
+        preempt_enable();
+
+        if (offset == APIC_EILVT_NR_MAX) {
+                printk(KERN_DEBUG "No EILVT entry available\n");
+                return -EBUSY;
         }
 
+        ret = setup_ibs_ctl(offset);
+        if (ret)
+                goto out;
+
+        if (!ibs_eilvt_valid()) {
+                ret = -EFAULT;
+                goto out;
+        }
+
+        pr_err(FW_BUG "using offset %d for IBS interrupts\n", offset);
+        pr_err(FW_BUG "workaround enabled for IBS LVT offset\n");
+
         return 0;
+out:
+        preempt_disable();
+        put_eilvt(offset);
+        preempt_enable();
+        return ret;
 }
 
-/* initialize the APIC for the IBS interrupts if available */
+/*
+ * check and reserve APIC extended interrupt LVT offset for IBS if
+ * available
+ */
+
 static void init_ibs(void)
 {
         ibs_caps = get_ibs_caps();
@@ -492,13 +667,18 @@ static void init_ibs(void)
         if (!ibs_caps)
                 return;
 
-        if (__init_ibs_nmi()) {
-                ibs_caps = 0;
-                return;
-        }
+        if (ibs_eilvt_valid())
+                goto out;
+
+        if (!force_ibs_eilvt_setup())
+                goto out;
 
-        printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n",
-               (unsigned)ibs_caps);
+        /* Failed to setup ibs */
+        ibs_caps = 0;
+        return;
+
+out:
+        printk(KERN_INFO "oprofile: AMD IBS detected (0x%08x)\n", ibs_caps);
 }
 
 static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
@@ -521,44 +701,60 @@ static int setup_ibs_files(struct super_block *sb, struct dentry *root)
         /* model specific files */
 
         /* setup some reasonable defaults */
+        memset(&ibs_config, 0, sizeof(ibs_config));
         ibs_config.max_cnt_fetch = 250000;
-        ibs_config.fetch_enabled = 0;
         ibs_config.max_cnt_op = 250000;
-        ibs_config.op_enabled = 0;
-        ibs_config.dispatched_ops = 0;
-
-        dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
-        oprofilefs_create_ulong(sb, dir, "enable",
-                                &ibs_config.fetch_enabled);
-        oprofilefs_create_ulong(sb, dir, "max_count",
-                                &ibs_config.max_cnt_fetch);
-        oprofilefs_create_ulong(sb, dir, "rand_enable",
-                                &ibs_config.rand_en);
-
-        dir = oprofilefs_mkdir(sb, root, "ibs_op");
-        oprofilefs_create_ulong(sb, dir, "enable",
-                                &ibs_config.op_enabled);
-        oprofilefs_create_ulong(sb, dir, "max_count",
-                                &ibs_config.max_cnt_op);
-        if (ibs_caps & IBS_CAPS_OPCNT)
-                oprofilefs_create_ulong(sb, dir, "dispatched_ops",
-                                        &ibs_config.dispatched_ops);
+
+        if (ibs_caps & IBS_CAPS_FETCHSAM) {
+                dir = oprofilefs_mkdir(sb, root, "ibs_fetch");
+                oprofilefs_create_ulong(sb, dir, "enable",
+                                        &ibs_config.fetch_enabled);
+                oprofilefs_create_ulong(sb, dir, "max_count",
+                                        &ibs_config.max_cnt_fetch);
+                oprofilefs_create_ulong(sb, dir, "rand_enable",
+                                        &ibs_config.rand_en);
+        }
+
+        if (ibs_caps & IBS_CAPS_OPSAM) {
+                dir = oprofilefs_mkdir(sb, root, "ibs_op");
+                oprofilefs_create_ulong(sb, dir, "enable",
+                                        &ibs_config.op_enabled);
+                oprofilefs_create_ulong(sb, dir, "max_count",
+                                        &ibs_config.max_cnt_op);
+                if (ibs_caps & IBS_CAPS_OPCNT)
+                        oprofilefs_create_ulong(sb, dir, "dispatched_ops",
+                                                &ibs_config.dispatched_ops);
+                if (ibs_caps & IBS_CAPS_BRNTRGT)
+                        oprofilefs_create_ulong(sb, dir, "branch_target",
+                                                &ibs_config.branch_target);
+        }
 
         return 0;
 }
 
+struct op_x86_model_spec op_amd_spec;
+
 static int op_amd_init(struct oprofile_operations *ops)
 {
         init_ibs();
         create_arch_files = ops->create_files;
         ops->create_files = setup_ibs_files;
+
+        if (boot_cpu_data.x86 == 0x15) {
+                num_counters = NUM_COUNTERS_F15H;
+        } else {
+                num_counters = NUM_COUNTERS;
+        }
+
+        op_amd_spec.num_counters = num_counters;
+        op_amd_spec.num_controls = num_counters;
+        op_amd_spec.num_virt_counters = max(num_counters, NUM_VIRT_COUNTERS);
+
         return 0;
 }
 
 struct op_x86_model_spec op_amd_spec = {
-        .num_counters           = NUM_COUNTERS,
-        .num_controls           = NUM_COUNTERS,
-        .num_virt_counters      = NUM_VIRT_COUNTERS,
+        /* num_counters/num_controls filled in at runtime */
         .reserved               = MSR_AMD_EVENTSEL_RESERVED,
         .event_mask             = OP_EVENT_MASK,
         .init                   = op_amd_init,
diff --git a/arch/x86/oprofile/op_model_p4.c b/arch/x86/oprofile/op_model_p4.c
index 182558dd5515..98ab13058f89 100644
--- a/arch/x86/oprofile/op_model_p4.c
+++ b/arch/x86/oprofile/op_model_p4.c
@@ -11,7 +11,7 @@
 #include <linux/oprofile.h>
 #include <linux/smp.h>
 #include <linux/ptrace.h>
-#include <linux/nmi.h>
+#include <asm/nmi.h>
 #include <asm/msr.h>
 #include <asm/fixmap.h>
 #include <asm/apic.h>
@@ -50,7 +50,7 @@ static inline void setup_num_counters(void)
 #endif
 }
 
-static int inline addr_increment(void)
+static inline int addr_increment(void)
 {
 #ifdef CONFIG_SMP
         return smp_num_siblings == 2 ? 2 : 1;
diff --git a/arch/x86/oprofile/op_model_ppro.c b/arch/x86/oprofile/op_model_ppro.c
index d769cda54082..94b745045e45 100644
--- a/arch/x86/oprofile/op_model_ppro.c
+++ b/arch/x86/oprofile/op_model_ppro.c
@@ -95,8 +95,8 @@ static void ppro_setup_ctrs(struct op_x86_model_spec const *model,
                  * counter width:
                  */
                 if (!(eax.split.version_id == 0 &&
-                        current_cpu_data.x86 == 6 &&
-                                current_cpu_data.x86_model == 15)) {
+                        __this_cpu_read(cpu_info.x86) == 6 &&
+                                __this_cpu_read(cpu_info.x86_model) == 15)) {
 
                         if (counter_width < eax.split.bit_width)
                                 counter_width = eax.split.bit_width;
@@ -235,8 +235,8 @@ static void arch_perfmon_setup_counters(void)
         eax.full = cpuid_eax(0xa);
 
         /* Workaround for BIOS bugs in 6/15. Taken from perfmon2 */
-        if (eax.split.version_id == 0 && current_cpu_data.x86 == 6 &&
-                current_cpu_data.x86_model == 15) {
+        if (eax.split.version_id == 0 && __this_cpu_read(cpu_info.x86) == 6 &&
+                __this_cpu_read(cpu_info.x86_model) == 15) {
                 eax.split.version_id = 2;
                 eax.split.num_counters = 2;
                 eax.split.bit_width = 40;