Merge branch 'mpi-master' into wip-k-fmlpwip-k-fmlp

Conflicts:
	litmus/sched_cedf.c

56 files changed, 7403 insertions, 4686 deletions

diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile
index 980b78e31328..a5b31af5c2b8 100644
--- a/arch/arm/kernel/Makefile
+++ b/arch/arm/kernel/Makefile
@@ -5,7 +5,7 @@
 CPPFLAGS_vmlinux.lds := -DTEXT_OFFSET=$(TEXT_OFFSET)
 AFLAGS_head.o        := -DTEXT_OFFSET=$(TEXT_OFFSET)
 
-ifdef CONFIG_DYNAMIC_FTRACE
+ifdef CONFIG_FUNCTION_TRACER
 CFLAGS_REMOVE_ftrace.o = -pg
 endif
 
@@ -24,15 +24,18 @@ obj-$(CONFIG_OC_ETM)		+= etm.o
 
 obj-$(CONFIG_ISA_DMA_API)       += dma.o
 obj-$(CONFIG_ARCH_ACORN)        += ecard.o 
-obj-$(CONFIG_FIQ)               += fiq.o
+obj-$(CONFIG_FIQ)               += fiq.o fiqasm.o
 obj-$(CONFIG_MODULES)           += armksyms.o module.o
 obj-$(CONFIG_ARTHUR)            += arthur.o
 obj-$(CONFIG_ISA_DMA)           += dma-isa.o
 obj-$(CONFIG_PCI)               += bios32.o isa.o
-obj-$(CONFIG_SMP)               += smp.o
+obj-$(CONFIG_PM_SLEEP)          += sleep.o
+obj-$(CONFIG_HAVE_SCHED_CLOCK)  += sched_clock.o
+obj-$(CONFIG_SMP)               += smp.o smp_tlb.o
 obj-$(CONFIG_HAVE_ARM_SCU)      += smp_scu.o
 obj-$(CONFIG_HAVE_ARM_TWD)      += smp_twd.o
 obj-$(CONFIG_DYNAMIC_FTRACE)    += ftrace.o
+obj-$(CONFIG_FUNCTION_GRAPH_TRACER)     += ftrace.o
 obj-$(CONFIG_KEXEC)             += machine_kexec.o relocate_kernel.o
 obj-$(CONFIG_KPROBES)           += kprobes.o kprobes-decode.o
 obj-$(CONFIG_ATAGS_PROC)        += atags.o
@@ -41,7 +44,11 @@ obj-$(CONFIG_ARM_THUMBEE)	+= thumbee.o
 obj-$(CONFIG_KGDB)              += kgdb.o
 obj-$(CONFIG_ARM_UNWIND)        += unwind.o
 obj-$(CONFIG_HAVE_TCM)          += tcm.o
+obj-$(CONFIG_OF)                += devtree.o
 obj-$(CONFIG_CRASH_DUMP)        += crash_dump.o
+obj-$(CONFIG_SWP_EMULATE)       += swp_emulate.o
+CFLAGS_swp_emulate.o            := -Wa,-march=armv7-a
+obj-$(CONFIG_HAVE_HW_BREAKPOINT)        += hw_breakpoint.o
 
 obj-$(CONFIG_CRUNCH)            += crunch.o crunch-bits.o
 AFLAGS_crunch-bits.o            := -Wa,-mcpu=ep9312
@@ -49,6 +56,7 @@ AFLAGS_crunch-bits.o		:= -Wa,-mcpu=ep9312
 obj-$(CONFIG_CPU_XSCALE)        += xscale-cp0.o
 obj-$(CONFIG_CPU_XSC3)          += xscale-cp0.o
 obj-$(CONFIG_CPU_MOHAWK)        += xscale-cp0.o
+obj-$(CONFIG_CPU_PJ4)           += pj4-cp0.o
 obj-$(CONFIG_IWMMXT)            += iwmmxt.o
 obj-$(CONFIG_CPU_HAS_PMU)       += pmu.o
 obj-$(CONFIG_HW_PERF_EVENTS)    += perf_event.o
diff --git a/arch/arm/kernel/armksyms.c b/arch/arm/kernel/armksyms.c
index 8214bfebfaca..acca35aebe28 100644
--- a/arch/arm/kernel/armksyms.c
+++ b/arch/arm/kernel/armksyms.c
@@ -140,24 +140,18 @@ EXPORT_SYMBOL(__aeabi_ulcmp);
 #endif
 
         /* bitops */
-EXPORT_SYMBOL(_set_bit_le);
-EXPORT_SYMBOL(_test_and_set_bit_le);
-EXPORT_SYMBOL(_clear_bit_le);
-EXPORT_SYMBOL(_test_and_clear_bit_le);
-EXPORT_SYMBOL(_change_bit_le);
-EXPORT_SYMBOL(_test_and_change_bit_le);
+EXPORT_SYMBOL(_set_bit);
+EXPORT_SYMBOL(_test_and_set_bit);
+EXPORT_SYMBOL(_clear_bit);
+EXPORT_SYMBOL(_test_and_clear_bit);
+EXPORT_SYMBOL(_change_bit);
+EXPORT_SYMBOL(_test_and_change_bit);
 EXPORT_SYMBOL(_find_first_zero_bit_le);
 EXPORT_SYMBOL(_find_next_zero_bit_le);
 EXPORT_SYMBOL(_find_first_bit_le);
 EXPORT_SYMBOL(_find_next_bit_le);
 
 #ifdef __ARMEB__
-EXPORT_SYMBOL(_set_bit_be);
-EXPORT_SYMBOL(_test_and_set_bit_be);
-EXPORT_SYMBOL(_clear_bit_be);
-EXPORT_SYMBOL(_test_and_clear_bit_be);
-EXPORT_SYMBOL(_change_bit_be);
-EXPORT_SYMBOL(_test_and_change_bit_be);
 EXPORT_SYMBOL(_find_first_zero_bit_be);
 EXPORT_SYMBOL(_find_next_zero_bit_be);
 EXPORT_SYMBOL(_find_first_bit_be);
@@ -165,6 +159,12 @@ EXPORT_SYMBOL(_find_next_bit_be);
 #endif
 
 #ifdef CONFIG_FUNCTION_TRACER
+#ifdef CONFIG_OLD_MCOUNT
 EXPORT_SYMBOL(mcount);
+#endif
 EXPORT_SYMBOL(__gnu_mcount_nc);
 #endif
+
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
+EXPORT_SYMBOL(__pv_phys_offset);
+#endif
diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c
index 85f2a019f77b..927522cfc12e 100644
--- a/arch/arm/kernel/asm-offsets.c
+++ b/arch/arm/kernel/asm-offsets.c
@@ -13,6 +13,9 @@
 #include <linux/sched.h>
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
+#include <asm/cacheflush.h>
+#include <asm/glue-df.h>
+#include <asm/glue-pf.h>
 #include <asm/mach/arch.h>
 #include <asm/thread_info.h>
 #include <asm/memory.h>
@@ -102,8 +105,6 @@ int main(void)
   DEFINE(SIZEOF_MACHINE_DESC,   sizeof(struct machine_desc));
   DEFINE(MACHINFO_TYPE,         offsetof(struct machine_desc, nr));
   DEFINE(MACHINFO_NAME,         offsetof(struct machine_desc, name));
-  DEFINE(MACHINFO_PHYSIO,       offsetof(struct machine_desc, phys_io));
-  DEFINE(MACHINFO_PGOFFIO,      offsetof(struct machine_desc, io_pg_offst));
   BLANK();
   DEFINE(PROC_INFO_SZ,          sizeof(struct proc_info_list));
   DEFINE(PROCINFO_INITFUNC,     offsetof(struct proc_info_list, __cpu_flush));
@@ -116,6 +117,14 @@ int main(void)
 #ifdef MULTI_PABORT
   DEFINE(PROCESSOR_PABT_FUNC,   offsetof(struct processor, _prefetch_abort));
 #endif
+#ifdef MULTI_CPU
+  DEFINE(CPU_SLEEP_SIZE,        offsetof(struct processor, suspend_size));
+  DEFINE(CPU_DO_SUSPEND,        offsetof(struct processor, do_suspend));
+  DEFINE(CPU_DO_RESUME,         offsetof(struct processor, do_resume));
+#endif
+#ifdef MULTI_CACHE
+  DEFINE(CACHE_FLUSH_KERN_ALL,  offsetof(struct cpu_cache_fns, flush_kern_all));
+#endif
   BLANK();
   DEFINE(DMA_BIDIRECTIONAL,     DMA_BIDIRECTIONAL);
   DEFINE(DMA_TO_DEVICE,         DMA_TO_DEVICE);
diff --git a/arch/arm/kernel/bios32.c b/arch/arm/kernel/bios32.c
index c6273a3bfc25..e4ee050aad7d 100644
--- a/arch/arm/kernel/bios32.c
+++ b/arch/arm/kernel/bios32.c
@@ -159,31 +159,6 @@ static void __devinit pci_fixup_dec21285(struct pci_dev *dev)
 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_DEC_21285, pci_fixup_dec21285);
 
 /*
- * Same as above. The PrPMC800 carrier board for the PrPMC1100 
- * card maps the host-bridge @ 00:01:00 for some reason and it
- * ends up getting scanned. Note that we only want to do this
- * fixup when we find the IXP4xx on a PrPMC system, which is why
- * we check the machine type. We could be running on a board
- * with an IXP4xx target device and we don't want to kill the
- * resources in that case.
- */
-static void __devinit pci_fixup_prpmc1100(struct pci_dev *dev)
-{
-        int i;
-
-        if (machine_is_prpmc1100()) {
-                dev->class &= 0xff;
-                dev->class |= PCI_CLASS_BRIDGE_HOST << 8;
-                for (i = 0; i < PCI_NUM_RESOURCES; i++) {
-                        dev->resource[i].start = 0;
-                        dev->resource[i].end   = 0;
-                        dev->resource[i].flags = 0;
-                }
-        }
-}
-DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IXP4XX, pci_fixup_prpmc1100);
-
-/*
  * PCI IDE controllers use non-standard I/O port decoding, respect it.
  */
 static void __devinit pci_fixup_ide_bases(struct pci_dev *dev)
@@ -583,6 +558,11 @@ void __init pci_common_init(struct hw_pci *hw)
                          * Assign resources.
                          */
                         pci_bus_assign_resources(bus);
+
+                        /*
+                         * Enable bridges
+                         */
+                        pci_enable_bridges(bus);
                 }
 
                 /*
diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S
index b99087ac85b9..ed2ae934127e 100644
--- a/arch/arm/kernel/calls.S
+++ b/arch/arm/kernel/calls.S
@@ -379,17 +379,23 @@
                 CALL(sys_fanotify_init)
                 CALL(sys_fanotify_mark)
                 CALL(sys_prlimit64)
-/* 370 */       CALL(sys_set_rt_task_param)
+/* 370 */       CALL(sys_name_to_handle_at)
+                CALL(sys_open_by_handle_at)
+                CALL(sys_clock_adjtime)
+                CALL(sys_syncfs)
+                CALL(sys_sendmmsg)
+/* 375 */       CALL(sys_setns)
+                CALL(sys_set_rt_task_param)
                 CALL(sys_get_rt_task_param)
                 CALL(sys_complete_job)
                 CALL(sys_od_open)
-                CALL(sys_od_close)
-/* 375 */       CALL(sys_litmus_lock)
+/* 380 */       CALL(sys_od_close)
+                CALL(sys_litmus_lock)
                 CALL(sys_litmus_unlock)
                 CALL(sys_query_job_no)
                 CALL(sys_wait_for_job_release)
-                CALL(sys_wait_for_ts_release)
-/* 380 */       CALL(sys_release_ts)
+/* 385 */       CALL(sys_wait_for_ts_release)
+                CALL(sys_release_ts)
                 CALL(sys_null_call)
 #ifndef syscalls_counted
 .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
diff --git a/arch/arm/kernel/crash_dump.c b/arch/arm/kernel/crash_dump.c
index cd3b853a8a6d..90c50d4b43f7 100644
--- a/arch/arm/kernel/crash_dump.c
+++ b/arch/arm/kernel/crash_dump.c
@@ -18,9 +18,6 @@
 #include <linux/uaccess.h>
 #include <linux/io.h>
 
-/* stores the physical address of elf header of crash image */
-unsigned long long elfcorehdr_addr = ELFCORE_ADDR_MAX;
-
 /**
  * copy_oldmem_page() - copy one page from old kernel memory
  * @pfn: page frame number to be copied
diff --git a/arch/arm/kernel/debug.S b/arch/arm/kernel/debug.S
index a38b4879441d..bcd66e00bdbe 100644
--- a/arch/arm/kernel/debug.S
+++ b/arch/arm/kernel/debug.S
@@ -22,11 +22,11 @@
 #if defined(CONFIG_DEBUG_ICEDCC)
                 @@ debug using ARM EmbeddedICE DCC channel
 
-#if defined(CONFIG_CPU_V6)
-
-                .macro  addruart, rx, tmp
+                .macro  addruart, rp, rv
                 .endm
 
+#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K) || defined(CONFIG_CPU_V7)
+
                 .macro  senduart, rd, rx
                 mcr     p14, 0, \rd, c0, c5, 0
                 .endm
@@ -49,31 +49,8 @@
 1002:
                 .endm
 
-#elif defined(CONFIG_CPU_V7)
-
-                .macro  addruart, rx, tmp
-                .endm
-
-                .macro  senduart, rd, rx
-                mcr     p14, 0, \rd, c0, c5, 0
-                .endm
-
-                .macro  busyuart, rd, rx
-busy:           mrc     p14, 0, pc, c0, c1, 0
-                bcs     busy
-                .endm
-
-                .macro  waituart, rd, rx
-wait:           mrc     p14, 0, pc, c0, c1, 0
-                bcs     wait
-
-                .endm
-
 #elif defined(CONFIG_CPU_XSCALE)
 
-                .macro  addruart, rx, tmp
-                .endm
-
                 .macro  senduart, rd, rx
                 mcr     p14, 0, \rd, c8, c0, 0
                 .endm
@@ -98,9 +75,6 @@ wait:		mrc	p14, 0, pc, c0, c1, 0
 
 #else
 
-                .macro  addruart, rx, tmp
-                .endm
-
                 .macro  senduart, rd, rx
                 mcr     p14, 0, \rd, c1, c0, 0
                 .endm
@@ -130,6 +104,22 @@ wait:		mrc	p14, 0, pc, c0, c1, 0
 #include <mach/debug-macro.S>
 #endif  /* CONFIG_DEBUG_ICEDCC */
 
+#ifdef CONFIG_MMU
+                .macro  addruart_current, rx, tmp1, tmp2
+                addruart        \tmp1, \tmp2
+                mrc             p15, 0, \rx, c1, c0
+                tst             \rx, #1
+                moveq           \rx, \tmp1
+                movne           \rx, \tmp2
+                .endm
+
+#else /* !CONFIG_MMU */
+                .macro  addruart_current, rx, tmp1, tmp2
+                addruart        \rx, \tmp1
+                .endm
+
+#endif /* CONFIG_MMU */
+
 /*
  * Useful debugging routines
  */
@@ -164,7 +154,7 @@ ENDPROC(printhex2)
                 .ltorg
 
 ENTRY(printascii)
-                addruart r3, r1
+                addruart_current r3, r1, r2
                 b       2f
 1:              waituart r2, r3
                 senduart r1, r3
@@ -180,7 +170,7 @@ ENTRY(printascii)
 ENDPROC(printascii)
 
 ENTRY(printch)
-                addruart r3, r1
+                addruart_current r3, r1, r2
                 mov     r1, r0
                 mov     r0, #0
                 b       1b
diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c
new file mode 100644
index 000000000000..0cdd7b456cb2
--- /dev/null
+++ b/arch/arm/kernel/devtree.c
@@ -0,0 +1,148 @@
+/*
+ *  linux/arch/arm/kernel/devtree.c
+ *
+ *  Copyright (C) 2009 Canonical Ltd. <jeremy.kerr@canonical.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/bootmem.h>
+#include <linux/memblock.h>
+#include <linux/of.h>
+#include <linux/of_fdt.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+
+#include <asm/setup.h>
+#include <asm/page.h>
+#include <asm/mach/arch.h>
+#include <asm/mach-types.h>
+
+void __init early_init_dt_add_memory_arch(u64 base, u64 size)
+{
+        arm_add_memory(base, size);
+}
+
+void * __init early_init_dt_alloc_memory_arch(u64 size, u64 align)
+{
+        return alloc_bootmem_align(size, align);
+}
+
+void __init arm_dt_memblock_reserve(void)
+{
+        u64 *reserve_map, base, size;
+
+        if (!initial_boot_params)
+                return;
+
+        /* Reserve the dtb region */
+        memblock_reserve(virt_to_phys(initial_boot_params),
+                         be32_to_cpu(initial_boot_params->totalsize));
+
+        /*
+         * Process the reserve map.  This will probably overlap the initrd
+         * and dtb locations which are already reserved, but overlaping
+         * doesn't hurt anything
+         */
+        reserve_map = ((void*)initial_boot_params) +
+                        be32_to_cpu(initial_boot_params->off_mem_rsvmap);
+        while (1) {
+                base = be64_to_cpup(reserve_map++);
+                size = be64_to_cpup(reserve_map++);
+                if (!size)
+                        break;
+                memblock_reserve(base, size);
+        }
+}
+
+/**
+ * setup_machine_fdt - Machine setup when an dtb was passed to the kernel
+ * @dt_phys: physical address of dt blob
+ *
+ * If a dtb was passed to the kernel in r2, then use it to choose the
+ * correct machine_desc and to setup the system.
+ */
+struct machine_desc * __init setup_machine_fdt(unsigned int dt_phys)
+{
+        struct boot_param_header *devtree;
+        struct machine_desc *mdesc, *mdesc_best = NULL;
+        unsigned int score, mdesc_score = ~1;
+        unsigned long dt_root;
+        const char *model;
+
+        if (!dt_phys)
+                return NULL;
+
+        devtree = phys_to_virt(dt_phys);
+
+        /* check device tree validity */
+        if (be32_to_cpu(devtree->magic) != OF_DT_HEADER)
+                return NULL;
+
+        /* Search the mdescs for the 'best' compatible value match */
+        initial_boot_params = devtree;
+        dt_root = of_get_flat_dt_root();
+        for_each_machine_desc(mdesc) {
+                score = of_flat_dt_match(dt_root, mdesc->dt_compat);
+                if (score > 0 && score < mdesc_score) {
+                        mdesc_best = mdesc;
+                        mdesc_score = score;
+                }
+        }
+        if (!mdesc_best) {
+                const char *prop;
+                long size;
+
+                early_print("\nError: unrecognized/unsupported "
+                            "device tree compatible list:\n[ ");
+
+                prop = of_get_flat_dt_prop(dt_root, "compatible", &size);
+                while (size > 0) {
+                        early_print("'%s' ", prop);
+                        size -= strlen(prop) + 1;
+                        prop += strlen(prop) + 1;
+                }
+                early_print("]\n\n");
+
+                dump_machine_table(); /* does not return */
+        }
+
+        model = of_get_flat_dt_prop(dt_root, "model", NULL);
+        if (!model)
+                model = of_get_flat_dt_prop(dt_root, "compatible", NULL);
+        if (!model)
+                model = "<unknown>";
+        pr_info("Machine: %s, model: %s\n", mdesc_best->name, model);
+
+        /* Retrieve various information from the /chosen node */
+        of_scan_flat_dt(early_init_dt_scan_chosen, boot_command_line);
+        /* Initialize {size,address}-cells info */
+        of_scan_flat_dt(early_init_dt_scan_root, NULL);
+        /* Setup memory, calling early_init_dt_add_memory_arch */
+        of_scan_flat_dt(early_init_dt_scan_memory, NULL);
+
+        /* Change machine number to match the mdesc we're using */
+        __machine_arch_type = mdesc_best->nr;
+
+        return mdesc_best;
+}
+
+/**
+ * irq_create_of_mapping - Hook to resolve OF irq specifier into a Linux irq#
+ *
+ * Currently the mapping mechanism is trivial; simple flat hwirq numbers are
+ * mapped 1:1 onto Linux irq numbers.  Cascaded irq controllers are not
+ * supported.
+ */
+unsigned int irq_create_of_mapping(struct device_node *controller,
+                                   const u32 *intspec, unsigned int intsize)
+{
+        return intspec[0];
+}
+EXPORT_SYMBOL_GPL(irq_create_of_mapping);
diff --git a/arch/arm/kernel/ecard.c b/arch/arm/kernel/ecard.c
index eed2f795e1b3..d16500110ee9 100644
--- a/arch/arm/kernel/ecard.c
+++ b/arch/arm/kernel/ecard.c
@@ -443,40 +443,40 @@ static expansioncard_ops_t ecard_default_ops = {
  *
  * They are not meant to be called directly, but via enable/disable_irq.
  */
-static void ecard_irq_unmask(unsigned int irqnr)
+static void ecard_irq_unmask(struct irq_data *d)
 {
-        ecard_t *ec = slot_to_ecard(irqnr - 32);
+        ecard_t *ec = slot_to_ecard(d->irq - 32);
 
         if (ec) {
                 if (!ec->ops)
                         ec->ops = &ecard_default_ops;
 
                 if (ec->claimed && ec->ops->irqenable)
-                        ec->ops->irqenable(ec, irqnr);
+                        ec->ops->irqenable(ec, d->irq);
                 else
                         printk(KERN_ERR "ecard: rejecting request to "
-                                "enable IRQs for %d\n", irqnr);
+                                "enable IRQs for %d\n", d->irq);
         }
 }
 
-static void ecard_irq_mask(unsigned int irqnr)
+static void ecard_irq_mask(struct irq_data *d)
 {
-        ecard_t *ec = slot_to_ecard(irqnr - 32);
+        ecard_t *ec = slot_to_ecard(d->irq - 32);
 
         if (ec) {
                 if (!ec->ops)
                         ec->ops = &ecard_default_ops;
 
                 if (ec->ops && ec->ops->irqdisable)
-                        ec->ops->irqdisable(ec, irqnr);
+                        ec->ops->irqdisable(ec, d->irq);
         }
 }
 
 static struct irq_chip ecard_chip = {
-        .name   = "ECARD",
-        .ack    = ecard_irq_mask,
-        .mask   = ecard_irq_mask,
-        .unmask = ecard_irq_unmask,
+        .name           = "ECARD",
+        .irq_ack        = ecard_irq_mask,
+        .irq_mask       = ecard_irq_mask,
+        .irq_unmask     = ecard_irq_unmask,
 };
 
 void ecard_enablefiq(unsigned int fiqnr)
@@ -551,7 +551,7 @@ static void ecard_check_lockup(struct irq_desc *desc)
                         printk(KERN_ERR "\nInterrupt lockup detected - "
                                "disabling all expansion card interrupts\n");
 
-                        desc->chip->mask(IRQ_EXPANSIONCARD);
+                        desc->irq_data.chip->irq_mask(&desc->irq_data);
                         ecard_dump_irq_state();
                 }
         } else
@@ -574,7 +574,7 @@ ecard_irq_handler(unsigned int irq, struct irq_desc *desc)
         ecard_t *ec;
         int called = 0;
 
-        desc->chip->mask(irq);
+        desc->irq_data.chip->irq_mask(&desc->irq_data);
         for (ec = cards; ec; ec = ec->next) {
                 int pending;
 
@@ -591,7 +591,7 @@ ecard_irq_handler(unsigned int irq, struct irq_desc *desc)
                         called ++;
                 }
         }
-        desc->chip->unmask(irq);
+        desc->irq_data.chip->irq_unmask(&desc->irq_data);
 
         if (called == 0)
                 ecard_check_lockup(desc);
@@ -1043,8 +1043,8 @@ ecard_probe(int slot, card_type_t type)
          */
         if (slot < 8) {
                 ec->irq = 32 + slot;
-                set_irq_chip(ec->irq, &ecard_chip);
-                set_irq_handler(ec->irq, handle_level_irq);
+                irq_set_chip_and_handler(ec->irq, &ecard_chip,
+                                         handle_level_irq);
                 set_irq_flags(ec->irq, IRQF_VALID);
         }
 
@@ -1103,7 +1103,7 @@ static int __init ecard_init(void)
 
         irqhw = ecard_probeirqhw();
 
-        set_irq_chained_handler(IRQ_EXPANSIONCARD,
+        irq_set_chained_handler(IRQ_EXPANSIONCARD,
                                 irqhw ? ecard_irqexp_handler : ecard_irq_handler);
 
         ecard_proc_init();
diff --git a/arch/arm/kernel/elf.c b/arch/arm/kernel/elf.c
index d4a0da1e48f4..9b05c6a0dcea 100644
--- a/arch/arm/kernel/elf.c
+++ b/arch/arm/kernel/elf.c
@@ -40,15 +40,22 @@ EXPORT_SYMBOL(elf_check_arch);
 void elf_set_personality(const struct elf32_hdr *x)
 {
         unsigned int eflags = x->e_flags;
-        unsigned int personality = PER_LINUX_32BIT;
+        unsigned int personality = current->personality & ~PER_MASK;
+
+        /*
+         * We only support Linux ELF executables, so always set the
+         * personality to LINUX.
+         */
+        personality |= PER_LINUX;
 
         /*
          * APCS-26 is only valid for OABI executables
          */
-        if ((eflags & EF_ARM_EABI_MASK) == EF_ARM_EABI_UNKNOWN) {
-                if (eflags & EF_ARM_APCS_26)
-                        personality = PER_LINUX;
-        }
+        if ((eflags & EF_ARM_EABI_MASK) == EF_ARM_EABI_UNKNOWN &&
+            (eflags & EF_ARM_APCS_26))
+                personality &= ~ADDR_LIMIT_32BIT;
+        else
+                personality |= ADDR_LIMIT_32BIT;
 
         set_personality(personality);
 
diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S
index bb8e93a76407..90c62cd51ca9 100644
--- a/arch/arm/kernel/entry-armv.S
+++ b/arch/arm/kernel/entry-armv.S
@@ -16,7 +16,8 @@
  */
 
 #include <asm/memory.h>
-#include <asm/glue.h>
+#include <asm/glue-df.h>
+#include <asm/glue-pf.h>
 #include <asm/vfpmacros.h>
 #include <mach/entry-macro.S>
 #include <asm/thread_notify.h>
@@ -25,40 +26,22 @@
 #include <asm/tls.h>
 
 #include "entry-header.S"
+#include <asm/entry-macro-multi.S>
 
 /*
  * Interrupt handling.  Preserves r7, r8, r9
  */
         .macro  irq_handler
-        get_irqnr_preamble r5, lr
-1:      get_irqnr_and_base r0, r6, r5, lr
-        movne   r1, sp
-        @
-        @ routine called with r0 = irq number, r1 = struct pt_regs *
-        @
-        adrne   lr, BSYM(1b)
-        bne     asm_do_IRQ
-
-#ifdef CONFIG_SMP
-        /*
-         * XXX
-         *
-         * this macro assumes that irqstat (r6) and base (r5) are
-         * preserved from get_irqnr_and_base above
-         */
-        test_for_ipi r0, r6, r5, lr
-        movne   r0, sp
-        adrne   lr, BSYM(1b)
-        bne     do_IPI
-
-#ifdef CONFIG_LOCAL_TIMERS
-        test_for_ltirq r0, r6, r5, lr
-        movne   r0, sp
-        adrne   lr, BSYM(1b)
-        bne     do_local_timer
-#endif
+#ifdef CONFIG_MULTI_IRQ_HANDLER
+        ldr     r5, =handle_arch_irq
+        mov     r0, sp
+        ldr     r5, [r5]
+        adr     lr, BSYM(9997f)
+        teq     r5, #0
+        movne   pc, r5
 #endif
-
+        arch_irq_handler_default
+9997:
         .endm
 
 #ifdef CONFIG_KPROBES
@@ -196,6 +179,7 @@ __dabt_svc:
         @
         @ set desired IRQ state, then call main handler
         @
+        debug_entry r1
         msr     cpsr_c, r9
         mov     r2, sp
         bl      do_DataAbort
@@ -322,6 +306,7 @@ __pabt_svc:
 #else
         bl      CPU_PABORT_HANDLER
 #endif
+        debug_entry r1
         msr     cpsr_c, r9                      @ Maybe enable interrupts
         mov     r2, sp                          @ regs
         bl      do_PrefetchAbort                @ call abort handler
@@ -437,6 +422,7 @@ __dabt_usr:
         @
         @ IRQs on, then call the main handler
         @
+        debug_entry r1
         enable_irq
         mov     r2, sp
         adr     lr, BSYM(ret_from_exception)
@@ -449,6 +435,10 @@ __irq_usr:
         usr_entry
         kuser_cmpxchg_check
 
+#ifdef CONFIG_IRQSOFF_TRACER
+        bl      trace_hardirqs_off
+#endif
+
         get_thread_info tsk
 #ifdef CONFIG_PREEMPT
         ldr     r8, [tsk, #TI_PREEMPT]          @ get preempt count
@@ -467,7 +457,7 @@ __irq_usr:
 #endif
 
         mov     why, #0
-        b       ret_to_user
+        b       ret_to_user_from_irq
  UNWIND(.fnend          )
 ENDPROC(__irq_usr)
 
@@ -701,6 +691,7 @@ __pabt_usr:
 #else
         bl      CPU_PABORT_HANDLER
 #endif
+        debug_entry r1
         enable_irq                              @ Enable interrupts
         mov     r2, sp                          @ regs
         bl      do_PrefetchAbort                @ call abort handler
@@ -733,7 +724,7 @@ ENTRY(__switch_to)
  THUMB( stmia   ip!, {r4 - sl, fp}         )    @ Store most regs on stack
  THUMB( str     sp, [ip], #4               )
  THUMB( str     lr, [ip], #4               )
-#ifdef CONFIG_MMU
+#ifdef CONFIG_CPU_USE_DOMAINS
         ldr     r6, [r2, #TI_CPU_DOMAIN]
 #endif
         set_tls r3, r4, r5
@@ -742,7 +733,7 @@ ENTRY(__switch_to)
         ldr     r8, =__stack_chk_guard
         ldr     r7, [r7, #TSK_STACK_CANARY]
 #endif
-#ifdef CONFIG_MMU
+#ifdef CONFIG_CPU_USE_DOMAINS
         mcr     p15, 0, r6, c3, c0, 0           @ Set domain register
 #endif
         mov     r5, r0
@@ -840,7 +831,7 @@ __kuser_helper_start:
  */
 
 __kuser_memory_barrier:                         @ 0xffff0fa0
-        smp_dmb
+        smp_dmb arm
         usr_ret lr
 
         .align  5
@@ -909,7 +900,7 @@ __kuser_cmpxchg:				@ 0xffff0fc0
          * A special ghost syscall is used for that (see traps.c).
          */
         stmfd   sp!, {r7, lr}
-        ldr     r7, =1f                 @ it's 20 bits
+        ldr     r7, 1f                  @ it's 20 bits
         swi     __ARM_NR_cmpxchg
         ldmfd   sp!, {r7, pc}
 1:      .word   __ARM_NR_cmpxchg
@@ -957,7 +948,7 @@ kuser_cmpxchg_fixup:
 
 #else
 
-        smp_dmb
+        smp_dmb arm
 1:      ldrex   r3, [r2]
         subs    r3, r3, r0
         strexeq r3, r1, [r2]
@@ -965,11 +956,8 @@ kuser_cmpxchg_fixup:
         beq     1b
         rsbs    r0, r3, #0
         /* beware -- each __kuser slot must be 8 instructions max */
-#ifdef CONFIG_SMP
-        b       __kuser_memory_barrier
-#else
-        usr_ret lr
-#endif
+        ALT_SMP(b       __kuser_memory_barrier)
+        ALT_UP(usr_ret  lr)
 
 #endif
 
@@ -1246,3 +1234,9 @@ cr_alignment:
         .space  4
 cr_no_alignment:
         .space  4
+
+#ifdef CONFIG_MULTI_IRQ_HANDLER
+        .globl  handle_arch_irq
+handle_arch_irq:
+        .space  4
+#endif
diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S
index 7885722bdf4e..b2a27b6b0046 100644
--- a/arch/arm/kernel/entry-common.S
+++ b/arch/arm/kernel/entry-common.S
@@ -29,6 +29,9 @@ ret_fast_syscall:
         ldr     r1, [tsk, #TI_FLAGS]
         tst     r1, #_TIF_WORK_MASK
         bne     fast_work_pending
+#if defined(CONFIG_IRQSOFF_TRACER)
+        asm_trace_hardirqs_on
+#endif
 
         /* perform architecture specific actions before user return */
         arch_ret_to_user r1, lr
@@ -61,14 +64,19 @@ work_resched:
 ENTRY(ret_to_user)
 ret_slow_syscall:
         disable_irq                             @ disable interrupts
+ENTRY(ret_to_user_from_irq)
         ldr     r1, [tsk, #TI_FLAGS]
         tst     r1, #_TIF_WORK_MASK
         bne     work_pending
 no_work_pending:
+#if defined(CONFIG_IRQSOFF_TRACER)
+        asm_trace_hardirqs_on
+#endif
         /* perform architecture specific actions before user return */
         arch_ret_to_user r1, lr
 
         restore_user_regs fast = 0, offset = 0
+ENDPROC(ret_to_user_from_irq)
 ENDPROC(ret_to_user)
 
 /*
@@ -129,76 +137,187 @@ ENDPROC(ret_from_fork)
  * clobber the ip register.  This is OK because the ARM calling convention
  * allows it to be clobbered in subroutines and doesn't use it to hold
  * parameters.)
+ *
+ * When using dynamic ftrace, we patch out the mcount call by a "mov r0, r0"
+ * for the mcount case, and a "pop {lr}" for the __gnu_mcount_nc case (see
+ * arch/arm/kernel/ftrace.c).
  */
-#ifdef CONFIG_DYNAMIC_FTRACE
-ENTRY(mcount)
-        stmdb   sp!, {r0-r3, lr}
-        mov     r0, lr
+
+#ifndef CONFIG_OLD_MCOUNT
+#if (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 4))
+#error Ftrace requires CONFIG_FRAME_POINTER=y with GCC older than 4.4.0.
+#endif
+#endif
+
+.macro __mcount suffix
+        mcount_enter
+        ldr     r0, =ftrace_trace_function
+        ldr     r2, [r0]
+        adr     r0, .Lftrace_stub
+        cmp     r0, r2
+        bne     1f
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+        ldr     r1, =ftrace_graph_return
+        ldr     r2, [r1]
+        cmp     r0, r2
+        bne     ftrace_graph_caller\suffix
+
+        ldr     r1, =ftrace_graph_entry
+        ldr     r2, [r1]
+        ldr     r0, =ftrace_graph_entry_stub
+        cmp     r0, r2
+        bne     ftrace_graph_caller\suffix
+#endif
+
+        mcount_exit
+
+1:      mcount_get_lr   r1                      @ lr of instrumented func
+        mov     r0, lr                          @ instrumented function
         sub     r0, r0, #MCOUNT_INSN_SIZE
+        adr     lr, BSYM(2f)
+        mov     pc, r2
+2:      mcount_exit
+.endm
 
-        .globl mcount_call
-mcount_call:
-        bl      ftrace_stub
-        ldr     lr, [fp, #-4]                   @ restore lr
-        ldmia   sp!, {r0-r3, pc}
+.macro __ftrace_caller suffix
+        mcount_enter
 
-ENTRY(ftrace_caller)
-        stmdb   sp!, {r0-r3, lr}
-        ldr     r1, [fp, #-4]
-        mov     r0, lr
+        mcount_get_lr   r1                      @ lr of instrumented func
+        mov     r0, lr                          @ instrumented function
         sub     r0, r0, #MCOUNT_INSN_SIZE
 
-        .globl ftrace_call
-ftrace_call:
+        .globl ftrace_call\suffix
+ftrace_call\suffix:
         bl      ftrace_stub
-        ldr     lr, [fp, #-4]                   @ restore lr
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+        .globl ftrace_graph_call\suffix
+ftrace_graph_call\suffix:
+        mov     r0, r0
+#endif
+
+        mcount_exit
+.endm
+
+.macro __ftrace_graph_caller
+        sub     r0, fp, #4              @ &lr of instrumented routine (&parent)
+#ifdef CONFIG_DYNAMIC_FTRACE
+        @ called from __ftrace_caller, saved in mcount_enter
+        ldr     r1, [sp, #16]           @ instrumented routine (func)
+#else
+        @ called from __mcount, untouched in lr
+        mov     r1, lr                  @ instrumented routine (func)
+#endif
+        sub     r1, r1, #MCOUNT_INSN_SIZE
+        mov     r2, fp                  @ frame pointer
+        bl      prepare_ftrace_return
+        mcount_exit
+.endm
+
+#ifdef CONFIG_OLD_MCOUNT
+/*
+ * mcount
+ */
+
+.macro mcount_enter
+        stmdb   sp!, {r0-r3, lr}
+.endm
+
+.macro mcount_get_lr reg
+        ldr     \reg, [fp, #-4]
+.endm
+
+.macro mcount_exit
+        ldr     lr, [fp, #-4]
         ldmia   sp!, {r0-r3, pc}
+.endm
 
+ENTRY(mcount)
+#ifdef CONFIG_DYNAMIC_FTRACE
+        stmdb   sp!, {lr}
+        ldr     lr, [fp, #-4]
+        ldmia   sp!, {pc}
 #else
+        __mcount _old
+#endif
+ENDPROC(mcount)
 
-ENTRY(__gnu_mcount_nc)
+#ifdef CONFIG_DYNAMIC_FTRACE
+ENTRY(ftrace_caller_old)
+        __ftrace_caller _old
+ENDPROC(ftrace_caller_old)
+#endif
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(ftrace_graph_caller_old)
+        __ftrace_graph_caller
+ENDPROC(ftrace_graph_caller_old)
+#endif
+
+.purgem mcount_enter
+.purgem mcount_get_lr
+.purgem mcount_exit
+#endif
+
+/*
+ * __gnu_mcount_nc
+ */
+
+.macro mcount_enter
         stmdb   sp!, {r0-r3, lr}
-        ldr     r0, =ftrace_trace_function
-        ldr     r2, [r0]
-        adr     r0, ftrace_stub
-        cmp     r0, r2
-        bne     gnu_trace
+.endm
+
+.macro mcount_get_lr reg
+        ldr     \reg, [sp, #20]
+.endm
+
+.macro mcount_exit
         ldmia   sp!, {r0-r3, ip, lr}
         mov     pc, ip
+.endm
 
-gnu_trace:
-        ldr     r1, [sp, #20]                   @ lr of instrumented routine
-        mov     r0, lr
-        sub     r0, r0, #MCOUNT_INSN_SIZE
-        mov     lr, pc
-        mov     pc, r2
-        ldmia   sp!, {r0-r3, ip, lr}
+ENTRY(__gnu_mcount_nc)
+#ifdef CONFIG_DYNAMIC_FTRACE
+        mov     ip, lr
+        ldmia   sp!, {lr}
         mov     pc, ip
+#else
+        __mcount
+#endif
+ENDPROC(__gnu_mcount_nc)
 
-ENTRY(mcount)
-        stmdb   sp!, {r0-r3, lr}
-        ldr     r0, =ftrace_trace_function
-        ldr     r2, [r0]
-        adr     r0, ftrace_stub
-        cmp     r0, r2
-        bne     trace
-        ldr     lr, [fp, #-4]                   @ restore lr
-        ldmia   sp!, {r0-r3, pc}
+#ifdef CONFIG_DYNAMIC_FTRACE
+ENTRY(ftrace_caller)
+        __ftrace_caller
+ENDPROC(ftrace_caller)
+#endif
 
-trace:
-        ldr     r1, [fp, #-4]                   @ lr of instrumented routine
-        mov     r0, lr
-        sub     r0, r0, #MCOUNT_INSN_SIZE
-        mov     lr, pc
-        mov     pc, r2
-        ldr     lr, [fp, #-4]                   @ restore lr
-        ldmia   sp!, {r0-r3, pc}
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+ENTRY(ftrace_graph_caller)
+        __ftrace_graph_caller
+ENDPROC(ftrace_graph_caller)
+#endif
 
-#endif /* CONFIG_DYNAMIC_FTRACE */
+.purgem mcount_enter
+.purgem mcount_get_lr
+.purgem mcount_exit
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+        .globl return_to_handler
+return_to_handler:
+        stmdb   sp!, {r0-r3}
+        mov     r0, fp                  @ frame pointer
+        bl      ftrace_return_to_handler
+        mov     lr, r0                  @ r0 has real ret addr
+        ldmia   sp!, {r0-r3}
+        mov     pc, lr
+#endif
 
-        .globl ftrace_stub
-ftrace_stub:
+ENTRY(ftrace_stub)
+.Lftrace_stub:
         mov     pc, lr
+ENDPROC(ftrace_stub)
 
 #endif /* CONFIG_FUNCTION_TRACER */
 
@@ -295,7 +414,6 @@ ENTRY(vector_swi)
 
         get_thread_info tsk
         adr     tbl, sys_call_table             @ load syscall table pointer
-        ldr     ip, [tsk, #TI_FLAGS]            @ check for syscall tracing
 
 #if defined(CONFIG_OABI_COMPAT)
         /*
@@ -312,8 +430,20 @@ ENTRY(vector_swi)
         eor     scno, scno, #__NR_SYSCALL_BASE  @ check OS number
 #endif
 
+        ldr     r10, [tsk, #TI_FLAGS]           @ check for syscall tracing
         stmdb   sp!, {r4, r5}                   @ push fifth and sixth args
-        tst     ip, #_TIF_SYSCALL_TRACE         @ are we tracing syscalls?
+
+#ifdef CONFIG_SECCOMP
+        tst     r10, #_TIF_SECCOMP
+        beq     1f
+        mov     r0, scno
+        bl      __secure_computing      
+        add     r0, sp, #S_R0 + S_OFF           @ pointer to regs
+        ldmia   r0, {r0 - r3}                   @ have to reload r0 - r3
+1:
+#endif
+
+        tst     r10, #_TIF_SYSCALL_TRACE                @ are we tracing syscalls?
         bne     __sys_trace
 
         cmp     scno, #NR_syscalls              @ check upper syscall limit
diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S
index d93f976fb389..051166c2a932 100644
--- a/arch/arm/kernel/entry-header.S
+++ b/arch/arm/kernel/entry-header.S
@@ -76,13 +76,13 @@
 #ifndef CONFIG_THUMB2_KERNEL
         .macro  svc_exit, rpsr
         msr     spsr_cxsf, \rpsr
-#if defined(CONFIG_CPU_32v6K)
-        clrex                                   @ clear the exclusive monitor
-        ldmia   sp, {r0 - pc}^                  @ load r0 - pc, cpsr
-#elif defined (CONFIG_CPU_V6)
+#if defined(CONFIG_CPU_V6)
         ldr     r0, [sp]
         strex   r1, r2, [sp]                    @ clear the exclusive monitor
         ldmib   sp, {r1 - pc}^                  @ load r1 - pc, cpsr
+#elif defined(CONFIG_CPU_32v6K)
+        clrex                                   @ clear the exclusive monitor
+        ldmia   sp, {r0 - pc}^                  @ load r0 - pc, cpsr
 #else
         ldmia   sp, {r0 - pc}^                  @ load r0 - pc, cpsr
 #endif
@@ -92,10 +92,10 @@
         ldr     r1, [sp, #\offset + S_PSR]      @ get calling cpsr
         ldr     lr, [sp, #\offset + S_PC]!      @ get pc
         msr     spsr_cxsf, r1                   @ save in spsr_svc
-#if defined(CONFIG_CPU_32v6K)
-        clrex                                   @ clear the exclusive monitor
-#elif defined (CONFIG_CPU_V6)
+#if defined(CONFIG_CPU_V6)
         strex   r1, r2, [sp]                    @ clear the exclusive monitor
+#elif defined(CONFIG_CPU_32v6K)
+        clrex                                   @ clear the exclusive monitor
 #endif
         .if     \fast
         ldmdb   sp, {r1 - lr}^                  @ get calling r1 - lr
@@ -165,6 +165,25 @@
         .endm
 #endif  /* !CONFIG_THUMB2_KERNEL */
 
+        @
+        @ Debug exceptions are taken as prefetch or data aborts.
+        @ We must disable preemption during the handler so that
+        @ we can access the debug registers safely.
+        @
+        .macro  debug_entry, fsr
+#if defined(CONFIG_HAVE_HW_BREAKPOINT) && defined(CONFIG_PREEMPT)
+        ldr     r4, =0x40f              @ mask out fsr.fs
+        and     r5, r4, \fsr
+        cmp     r5, #2                  @ debug exception
+        bne     1f
+        get_thread_info r10
+        ldr     r6, [r10, #TI_PREEMPT]  @ get preempt count
+        add     r11, r6, #1             @ increment it
+        str     r11, [r10, #TI_PREEMPT]
+1:
+#endif
+        .endm
+
 /*
  * These are the registers used in the syscall handler, and allow us to
  * have in theory up to 7 arguments to a function - r0 to r6.
diff --git a/arch/arm/kernel/etm.c b/arch/arm/kernel/etm.c
index 33c7077174db..1bec8b5f22f0 100644
--- a/arch/arm/kernel/etm.c
+++ b/arch/arm/kernel/etm.c
@@ -30,6 +30,21 @@
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Alexander Shishkin");
 
+/*
+ * ETM tracer state
+ */
+struct tracectx {
+        unsigned int    etb_bufsz;
+        void __iomem    *etb_regs;
+        void __iomem    *etm_regs;
+        unsigned long   flags;
+        int             ncmppairs;
+        int             etm_portsz;
+        struct device   *dev;
+        struct clk      *emu_clk;
+        struct mutex    mutex;
+};
+
 static struct tracectx tracer;
 
 static inline bool trace_isrunning(struct tracectx *t)
@@ -314,6 +329,7 @@ static const struct file_operations etb_fops = {
         .read = etb_read,
         .open = etb_open,
         .release = etb_release,
+        .llseek = no_llseek,
 };
 
 static struct miscdevice etb_miscdev = {
@@ -322,7 +338,7 @@ static struct miscdevice etb_miscdev = {
         .fops = &etb_fops,
 };
 
-static int __init etb_probe(struct amba_device *dev, struct amba_id *id)
+static int __devinit etb_probe(struct amba_device *dev, const struct amba_id *id)
 {
         struct tracectx *t = &tracer;
         int ret = 0;
@@ -514,7 +530,7 @@ static ssize_t trace_mode_store(struct kobject *kobj,
 static struct kobj_attribute trace_mode_attr =
         __ATTR(trace_mode, 0644, trace_mode_show, trace_mode_store);
 
-static int __init etm_probe(struct amba_device *dev, struct amba_id *id)
+static int __devinit etm_probe(struct amba_device *dev, const struct amba_id *id)
 {
         struct tracectx *t = &tracer;
         int ret = 0;
diff --git a/arch/arm/kernel/fiq.c b/arch/arm/kernel/fiq.c
index 6ff7919613d7..4c164ece5891 100644
--- a/arch/arm/kernel/fiq.c
+++ b/arch/arm/kernel/fiq.c
@@ -45,6 +45,7 @@
 #include <asm/fiq.h>
 #include <asm/irq.h>
 #include <asm/system.h>
+#include <asm/traps.h>
 
 static unsigned long no_fiq_insn;
 
@@ -67,63 +68,27 @@ static struct fiq_handler default_owner = {
 
 static struct fiq_handler *current_fiq = &default_owner;
 
-int show_fiq_list(struct seq_file *p, void *v)
+int show_fiq_list(struct seq_file *p, int prec)
 {
         if (current_fiq != &default_owner)
-                seq_printf(p, "FIQ:              %s\n", current_fiq->name);
+                seq_printf(p, "%*s:              %s\n", prec, "FIQ",
+                        current_fiq->name);
 
         return 0;
 }
 
 void set_fiq_handler(void *start, unsigned int length)
 {
+#if defined(CONFIG_CPU_USE_DOMAINS)
         memcpy((void *)0xffff001c, start, length);
+#else
+        memcpy(vectors_page + 0x1c, start, length);
+#endif
         flush_icache_range(0xffff001c, 0xffff001c + length);
         if (!vectors_high())
                 flush_icache_range(0x1c, 0x1c + length);
 }
 
-/*
- * Taking an interrupt in FIQ mode is death, so both these functions
- * disable irqs for the duration.  Note - these functions are almost
- * entirely coded in assembly.
- */
-void __naked set_fiq_regs(struct pt_regs *regs)
-{
-        register unsigned long tmp;
-        asm volatile (
-        "mov    ip, sp\n\
-        stmfd   sp!, {fp, ip, lr, pc}\n\
-        sub     fp, ip, #4\n\
-        mrs     %0, cpsr\n\
-        msr     cpsr_c, %2      @ select FIQ mode\n\
-        mov     r0, r0\n\
-        ldmia   %1, {r8 - r14}\n\
-        msr     cpsr_c, %0      @ return to SVC mode\n\
-        mov     r0, r0\n\
-        ldmfd   sp, {fp, sp, pc}"
-        : "=&r" (tmp)
-        : "r" (&regs->ARM_r8), "I" (PSR_I_BIT | PSR_F_BIT | FIQ_MODE));
-}
-
-void __naked get_fiq_regs(struct pt_regs *regs)
-{
-        register unsigned long tmp;
-        asm volatile (
-        "mov    ip, sp\n\
-        stmfd   sp!, {fp, ip, lr, pc}\n\
-        sub     fp, ip, #4\n\
-        mrs     %0, cpsr\n\
-        msr     cpsr_c, %2      @ select FIQ mode\n\
-        mov     r0, r0\n\
-        stmia   %1, {r8 - r14}\n\
-        msr     cpsr_c, %0      @ return to SVC mode\n\
-        mov     r0, r0\n\
-        ldmfd   sp, {fp, sp, pc}"
-        : "=&r" (tmp)
-        : "r" (&regs->ARM_r8), "I" (PSR_I_BIT | PSR_F_BIT | FIQ_MODE));
-}
-
 int claim_fiq(struct fiq_handler *f)
 {
         int ret = 0;
@@ -168,8 +133,8 @@ void disable_fiq(int fiq)
 }
 
 EXPORT_SYMBOL(set_fiq_handler);
-EXPORT_SYMBOL(set_fiq_regs);
-EXPORT_SYMBOL(get_fiq_regs);
+EXPORT_SYMBOL(__set_fiq_regs);  /* defined in fiqasm.S */
+EXPORT_SYMBOL(__get_fiq_regs);  /* defined in fiqasm.S */
 EXPORT_SYMBOL(claim_fiq);
 EXPORT_SYMBOL(release_fiq);
 EXPORT_SYMBOL(enable_fiq);
diff --git a/arch/arm/kernel/fiqasm.S b/arch/arm/kernel/fiqasm.S
new file mode 100644
index 000000000000..207f9d652010
--- /dev/null
+++ b/arch/arm/kernel/fiqasm.S
@@ -0,0 +1,49 @@
+/*
+ *  linux/arch/arm/kernel/fiqasm.S
+ *
+ *  Derived from code originally in linux/arch/arm/kernel/fiq.c:
+ *
+ *  Copyright (C) 1998 Russell King
+ *  Copyright (C) 1998, 1999 Phil Blundell
+ *  Copyright (C) 2011, Linaro Limited
+ *
+ *  FIQ support written by Philip Blundell <philb@gnu.org>, 1998.
+ *
+ *  FIQ support re-written by Russell King to be more generic
+ *
+ *  v7/Thumb-2 compatibility modifications by Linaro Limited, 2011.
+ */
+
+#include <linux/linkage.h>
+#include <asm/assembler.h>
+
+/*
+ * Taking an interrupt in FIQ mode is death, so both these functions
+ * disable irqs for the duration.
+ */
+
+ENTRY(__set_fiq_regs)
+        mov     r2, #PSR_I_BIT | PSR_F_BIT | FIQ_MODE
+        mrs     r1, cpsr
+        msr     cpsr_c, r2      @ select FIQ mode
+        mov     r0, r0          @ avoid hazard prior to ARMv4
+        ldmia   r0!, {r8 - r12}
+        ldr     sp, [r0], #4
+        ldr     lr, [r0]
+        msr     cpsr_c, r1      @ return to SVC mode
+        mov     r0, r0          @ avoid hazard prior to ARMv4
+        mov     pc, lr
+ENDPROC(__set_fiq_regs)
+
+ENTRY(__get_fiq_regs)
+        mov     r2, #PSR_I_BIT | PSR_F_BIT | FIQ_MODE
+        mrs     r1, cpsr
+        msr     cpsr_c, r2      @ select FIQ mode
+        mov     r0, r0          @ avoid hazard prior to ARMv4
+        stmia   r0!, {r8 - r12}
+        str     sp, [r0], #4
+        str     lr, [r0]
+        msr     cpsr_c, r1      @ return to SVC mode
+        mov     r0, r0          @ avoid hazard prior to ARMv4
+        mov     pc, lr
+ENDPROC(__get_fiq_regs)
diff --git a/arch/arm/kernel/ftrace.c b/arch/arm/kernel/ftrace.c
index 0298286ad4ad..c0062ad1e847 100644
--- a/arch/arm/kernel/ftrace.c
+++ b/arch/arm/kernel/ftrace.c
@@ -2,102 +2,287 @@
  * Dynamic function tracing support.
  *
  * Copyright (C) 2008 Abhishek Sagar <sagar.abhishek@gmail.com>
+ * Copyright (C) 2010 Rabin Vincent <rabin@rab.in>
  *
  * For licencing details, see COPYING.
  *
  * Defines low-level handling of mcount calls when the kernel
  * is compiled with the -pg flag. When using dynamic ftrace, the
- * mcount call-sites get patched lazily with NOP till they are
- * enabled. All code mutation routines here take effect atomically.
+ * mcount call-sites get patched with NOP till they are enabled.
+ * All code mutation routines here are called under stop_machine().
  */
 
 #include <linux/ftrace.h>
+#include <linux/uaccess.h>
 
 #include <asm/cacheflush.h>
 #include <asm/ftrace.h>
 
-#define PC_OFFSET      8
-#define BL_OPCODE      0xeb000000
-#define BL_OFFSET_MASK 0x00ffffff
+#ifdef CONFIG_THUMB2_KERNEL
+#define NOP             0xeb04f85d      /* pop.w {lr} */
+#else
+#define NOP             0xe8bd4000      /* pop {lr} */
+#endif
 
-static unsigned long bl_insn;
-static const unsigned long NOP = 0xe1a00000; /* mov r0, r0 */
+#ifdef CONFIG_DYNAMIC_FTRACE
+#ifdef CONFIG_OLD_MCOUNT
+#define OLD_MCOUNT_ADDR ((unsigned long) mcount)
+#define OLD_FTRACE_ADDR ((unsigned long) ftrace_caller_old)
 
-unsigned char *ftrace_nop_replace(void)
+#define OLD_NOP         0xe1a00000      /* mov r0, r0 */
+
+static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
+{
+        return rec->arch.old_mcount ? OLD_NOP : NOP;
+}
+
+static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
+{
+        if (!rec->arch.old_mcount)
+                return addr;
+
+        if (addr == MCOUNT_ADDR)
+                addr = OLD_MCOUNT_ADDR;
+        else if (addr == FTRACE_ADDR)
+                addr = OLD_FTRACE_ADDR;
+
+        return addr;
+}
+#else
+static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
+{
+        return NOP;
+}
+
+static unsigned long adjust_address(struct dyn_ftrace *rec, unsigned long addr)
 {
-        return (char *)&NOP;
+        return addr;
 }
+#endif
+
+#ifdef CONFIG_THUMB2_KERNEL
+static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr,
+                                       bool link)
+{
+        unsigned long s, j1, j2, i1, i2, imm10, imm11;
+        unsigned long first, second;
+        long offset;
+
+        offset = (long)addr - (long)(pc + 4);
+        if (offset < -16777216 || offset > 16777214) {
+                WARN_ON_ONCE(1);
+                return 0;
+        }
+
+        s       = (offset >> 24) & 0x1;
+        i1      = (offset >> 23) & 0x1;
+        i2      = (offset >> 22) & 0x1;
+        imm10   = (offset >> 12) & 0x3ff;
+        imm11   = (offset >>  1) & 0x7ff;
+
+        j1 = (!i1) ^ s;
+        j2 = (!i2) ^ s;
+
+        first = 0xf000 | (s << 10) | imm10;
+        second = 0x9000 | (j1 << 13) | (j2 << 11) | imm11;
+        if (link)
+                second |= 1 << 14;
 
-/* construct a branch (BL) instruction to addr */
-unsigned char *ftrace_call_replace(unsigned long pc, unsigned long addr)
+        return (second << 16) | first;
+}
+#else
+static unsigned long ftrace_gen_branch(unsigned long pc, unsigned long addr,
+                                       bool link)
 {
+        unsigned long opcode = 0xea000000;
         long offset;
 
-        offset = (long)addr - (long)(pc + PC_OFFSET);
+        if (link)
+                opcode |= 1 << 24;
+
+        offset = (long)addr - (long)(pc + 8);
         if (unlikely(offset < -33554432 || offset > 33554428)) {
                 /* Can't generate branches that far (from ARM ARM). Ftrace
                  * doesn't generate branches outside of kernel text.
                  */
                 WARN_ON_ONCE(1);
-                return NULL;
+                return 0;
         }
-        offset = (offset >> 2) & BL_OFFSET_MASK;
-        bl_insn = BL_OPCODE | offset;
-        return (unsigned char *)&bl_insn;
+
+        offset = (offset >> 2) & 0x00ffffff;
+
+        return opcode | offset;
 }
+#endif
 
-int ftrace_modify_code(unsigned long pc, unsigned char *old_code,
-                       unsigned char *new_code)
+static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr)
 {
-        unsigned long err = 0, replaced = 0, old, new;
-
-        old = *(unsigned long *)old_code;
-        new = *(unsigned long *)new_code;
+        return ftrace_gen_branch(pc, addr, true);
+}
 
-        __asm__ __volatile__ (
-                "1:  ldr    %1, [%2]  \n"
-                "    cmp    %1, %4    \n"
-                "2:  streq  %3, [%2]  \n"
-                "    cmpne  %1, %3    \n"
-                "    movne  %0, #2    \n"
-                "3:\n"
+static int ftrace_modify_code(unsigned long pc, unsigned long old,
+                              unsigned long new)
+{
+        unsigned long replaced;
 
-                ".pushsection .fixup, \"ax\"\n"
-                "4:  mov  %0, #1  \n"
-                "    b    3b      \n"
-                ".popsection\n"
+        if (probe_kernel_read(&replaced, (void *)pc, MCOUNT_INSN_SIZE))
+                return -EFAULT;
 
-                ".pushsection __ex_table, \"a\"\n"
-                "    .long 1b, 4b \n"
-                "    .long 2b, 4b \n"
-                ".popsection\n"
+        if (replaced != old)
+                return -EINVAL;
 
-                : "=r"(err), "=r"(replaced)
-                : "r"(pc), "r"(new), "r"(old), "0"(err), "1"(replaced)
-                : "memory");
+        if (probe_kernel_write((void *)pc, &new, MCOUNT_INSN_SIZE))
+                return -EPERM;
 
-        if (!err && (replaced == old))
-                flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
+        flush_icache_range(pc, pc + MCOUNT_INSN_SIZE);
 
-        return err;
+        return 0;
 }
 
 int ftrace_update_ftrace_func(ftrace_func_t func)
 {
-        int ret;
         unsigned long pc, old;
-        unsigned char *new;
+        unsigned long new;
+        int ret;
 
         pc = (unsigned long)&ftrace_call;
         memcpy(&old, &ftrace_call, MCOUNT_INSN_SIZE);
         new = ftrace_call_replace(pc, (unsigned long)func);
-        ret = ftrace_modify_code(pc, (unsigned char *)&old, new);
+
+        ret = ftrace_modify_code(pc, old, new);
+
+#ifdef CONFIG_OLD_MCOUNT
+        if (!ret) {
+                pc = (unsigned long)&ftrace_call_old;
+                memcpy(&old, &ftrace_call_old, MCOUNT_INSN_SIZE);
+                new = ftrace_call_replace(pc, (unsigned long)func);
+
+                ret = ftrace_modify_code(pc, old, new);
+        }
+#endif
+
+        return ret;
+}
+
+int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
+{
+        unsigned long new, old;
+        unsigned long ip = rec->ip;
+
+        old = ftrace_nop_replace(rec);
+        new = ftrace_call_replace(ip, adjust_address(rec, addr));
+
+        return ftrace_modify_code(rec->ip, old, new);
+}
+
+int ftrace_make_nop(struct module *mod,
+                    struct dyn_ftrace *rec, unsigned long addr)
+{
+        unsigned long ip = rec->ip;
+        unsigned long old;
+        unsigned long new;
+        int ret;
+
+        old = ftrace_call_replace(ip, adjust_address(rec, addr));
+        new = ftrace_nop_replace(rec);
+        ret = ftrace_modify_code(ip, old, new);
+
+#ifdef CONFIG_OLD_MCOUNT
+        if (ret == -EINVAL && addr == MCOUNT_ADDR) {
+                rec->arch.old_mcount = true;
+
+                old = ftrace_call_replace(ip, adjust_address(rec, addr));
+                new = ftrace_nop_replace(rec);
+                ret = ftrace_modify_code(ip, old, new);
+        }
+#endif
+
         return ret;
 }
 
-/* run from ftrace_init with irqs disabled */
 int __init ftrace_dyn_arch_init(void *data)
 {
-        ftrace_mcount_set(data);
+        *(unsigned long *)data = 0;
+
         return 0;
 }
+#endif /* CONFIG_DYNAMIC_FTRACE */
+
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
+                           unsigned long frame_pointer)
+{
+        unsigned long return_hooker = (unsigned long) &return_to_handler;
+        struct ftrace_graph_ent trace;
+        unsigned long old;
+        int err;
+
+        if (unlikely(atomic_read(&current->tracing_graph_pause)))
+                return;
+
+        old = *parent;
+        *parent = return_hooker;
+
+        err = ftrace_push_return_trace(old, self_addr, &trace.depth,
+                                       frame_pointer);
+        if (err == -EBUSY) {
+                *parent = old;
+                return;
+        }
+
+        trace.func = self_addr;
+
+        /* Only trace if the calling function expects to */
+        if (!ftrace_graph_entry(&trace)) {
+                current->curr_ret_stack--;
+                *parent = old;
+        }
+}
+
+#ifdef CONFIG_DYNAMIC_FTRACE
+extern unsigned long ftrace_graph_call;
+extern unsigned long ftrace_graph_call_old;
+extern void ftrace_graph_caller_old(void);
+
+static int __ftrace_modify_caller(unsigned long *callsite,
+                                  void (*func) (void), bool enable)
+{
+        unsigned long caller_fn = (unsigned long) func;
+        unsigned long pc = (unsigned long) callsite;
+        unsigned long branch = ftrace_gen_branch(pc, caller_fn, false);
+        unsigned long nop = 0xe1a00000; /* mov r0, r0 */
+        unsigned long old = enable ? nop : branch;
+        unsigned long new = enable ? branch : nop;
+
+        return ftrace_modify_code(pc, old, new);
+}
+
+static int ftrace_modify_graph_caller(bool enable)
+{
+        int ret;
+
+        ret = __ftrace_modify_caller(&ftrace_graph_call,
+                                     ftrace_graph_caller,
+                                     enable);
+
+#ifdef CONFIG_OLD_MCOUNT
+        if (!ret)
+                ret = __ftrace_modify_caller(&ftrace_graph_call_old,
+                                             ftrace_graph_caller_old,
+                                             enable);
+#endif
+
+        return ret;
+}
+
+int ftrace_enable_ftrace_graph_caller(void)
+{
+        return ftrace_modify_graph_caller(true);
+}
+
+int ftrace_disable_ftrace_graph_caller(void)
+{
+        return ftrace_modify_graph_caller(false);
+}
+#endif /* CONFIG_DYNAMIC_FTRACE */
+#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
diff --git a/arch/arm/kernel/head-common.S b/arch/arm/kernel/head-common.S
index b9505aa267c0..854bd22380d3 100644
--- a/arch/arm/kernel/head-common.S
+++ b/arch/arm/kernel/head-common.S
@@ -15,19 +15,57 @@
 #define ATAG_CORE_SIZE ((2*4 + 3*4) >> 2)
 #define ATAG_CORE_SIZE_EMPTY ((2*4) >> 2)
 
-        .align  2
-        .type   __switch_data, %object
-__switch_data:
-        .long   __mmap_switched
-        .long   __data_loc                      @ r4
-        .long   _data                           @ r5
-        .long   __bss_start                     @ r6
-        .long   _end                            @ r7
-        .long   processor_id                    @ r4
-        .long   __machine_arch_type             @ r5
-        .long   __atags_pointer                 @ r6
-        .long   cr_alignment                    @ r7
-        .long   init_thread_union + THREAD_START_SP @ sp
+#ifdef CONFIG_CPU_BIG_ENDIAN
+#define OF_DT_MAGIC 0xd00dfeed
+#else
+#define OF_DT_MAGIC 0xedfe0dd0 /* 0xd00dfeed in big-endian */
+#endif
+
+/*
+ * Exception handling.  Something went wrong and we can't proceed.  We
+ * ought to tell the user, but since we don't have any guarantee that
+ * we're even running on the right architecture, we do virtually nothing.
+ *
+ * If CONFIG_DEBUG_LL is set we try to print out something about the error
+ * and hope for the best (useful if bootloader fails to pass a proper
+ * machine ID for example).
+ */
+        __HEAD
+
+/* Determine validity of the r2 atags pointer.  The heuristic requires
+ * that the pointer be aligned, in the first 16k of physical RAM and
+ * that the ATAG_CORE marker is first and present.  If CONFIG_OF_FLATTREE
+ * is selected, then it will also accept a dtb pointer.  Future revisions
+ * of this function may be more lenient with the physical address and
+ * may also be able to move the ATAGS block if necessary.
+ *
+ * Returns:
+ *  r2 either valid atags pointer, valid dtb pointer, or zero
+ *  r5, r6 corrupted
+ */
+__vet_atags:
+        tst     r2, #0x3                        @ aligned?
+        bne     1f
+
+        ldr     r5, [r2, #0]
+#ifdef CONFIG_OF_FLATTREE
+        ldr     r6, =OF_DT_MAGIC                @ is it a DTB?
+        cmp     r5, r6
+        beq     2f
+#endif
+        cmp     r5, #ATAG_CORE_SIZE             @ is first tag ATAG_CORE?
+        cmpne   r5, #ATAG_CORE_SIZE_EMPTY
+        bne     1f
+        ldr     r5, [r2, #4]
+        ldr     r6, =ATAG_CORE
+        cmp     r5, r6
+        bne     1f
+
+2:      mov     pc, lr                          @ atag/dtb pointer is ok
+
+1:      mov     r2, #0
+        mov     pc, lr
+ENDPROC(__vet_atags)
 
 /*
  * The following fragment of code is executed with the MMU on in MMU mode,
@@ -35,11 +73,12 @@ __switch_data:
  *
  *  r0  = cp#15 control register
  *  r1  = machine ID
- *  r2  = atags pointer
+ *  r2  = atags/dtb pointer
  *  r9  = processor ID
  */
+        __INIT
 __mmap_switched:
-        adr     r3, __switch_data + 4
+        adr     r3, __mmap_switched_data
 
         ldmia   r3!, {r4, r5, r6, r7}
         cmp     r4, r5                          @ Copy data segment if needed
@@ -64,85 +103,30 @@ __mmap_switched:
         b       start_kernel
 ENDPROC(__mmap_switched)
 
-/*
- * Exception handling.  Something went wrong and we can't proceed.  We
- * ought to tell the user, but since we don't have any guarantee that
- * we're even running on the right architecture, we do virtually nothing.
- *
- * If CONFIG_DEBUG_LL is set we try to print out something about the error
- * and hope for the best (useful if bootloader fails to pass a proper
- * machine ID for example).
- */
-__error_p:
-#ifdef CONFIG_DEBUG_LL
-        adr     r0, str_p1
-        bl      printascii
-        mov     r0, r9
-        bl      printhex8
-        adr     r0, str_p2
-        bl      printascii
-        b       __error
-str_p1: .asciz  "\nError: unrecognized/unsupported processor variant (0x"
-str_p2: .asciz  ").\n"
-        .align
-#endif
-ENDPROC(__error_p)
-
-__error_a:
-#ifdef CONFIG_DEBUG_LL
-        mov     r4, r1                          @ preserve machine ID
-        adr     r0, str_a1
-        bl      printascii
-        mov     r0, r4
-        bl      printhex8
-        adr     r0, str_a2
-        bl      printascii
-        adr     r3, 4f
-        ldmia   r3, {r4, r5, r6}                @ get machine desc list
-        sub     r4, r3, r4                      @ get offset between virt&phys
-        add     r5, r5, r4                      @ convert virt addresses to
-        add     r6, r6, r4                      @ physical address space
-1:      ldr     r0, [r5, #MACHINFO_TYPE]        @ get machine type
-        bl      printhex8
-        mov     r0, #'\t'
-        bl      printch
-        ldr     r0, [r5, #MACHINFO_NAME]        @ get machine name
-        add     r0, r0, r4
-        bl      printascii
-        mov     r0, #'\n'
-        bl      printch
-        add     r5, r5, #SIZEOF_MACHINE_DESC    @ next machine_desc
-        cmp     r5, r6
-        blo     1b
-        adr     r0, str_a3
-        bl      printascii
-        b       __error
-ENDPROC(__error_a)
-
-str_a1: .asciz  "\nError: unrecognized/unsupported machine ID (r1 = 0x"
-str_a2: .asciz  ").\n\nAvailable machine support:\n\nID (hex)\tNAME\n"
-str_a3: .asciz  "\nPlease check your kernel config and/or bootloader.\n"
-        .align
-#endif
+        .align  2
+        .type   __mmap_switched_data, %object
+__mmap_switched_data:
+        .long   __data_loc                      @ r4
+        .long   _sdata                          @ r5
+        .long   __bss_start                     @ r6
+        .long   _end                            @ r7
+        .long   processor_id                    @ r4
+        .long   __machine_arch_type             @ r5
+        .long   __atags_pointer                 @ r6
+        .long   cr_alignment                    @ r7
+        .long   init_thread_union + THREAD_START_SP @ sp
+        .size   __mmap_switched_data, . - __mmap_switched_data
 
-__error:
-#ifdef CONFIG_ARCH_RPC
 /*
- * Turn the screen red on a error - RiscPC only.
+ * This provides a C-API version of __lookup_processor_type
  */
-        mov     r0, #0x02000000
-        mov     r3, #0x11
-        orr     r3, r3, r3, lsl #8
-        orr     r3, r3, r3, lsl #16
-        str     r3, [r0], #4
-        str     r3, [r0], #4
-        str     r3, [r0], #4
-        str     r3, [r0], #4
-#endif
-1:      mov     r0, r0
-        b       1b
-ENDPROC(__error)
-
+ENTRY(lookup_processor_type)
+        stmfd   sp!, {r4 - r6, r9, lr}
+        mov     r9, r0
+        bl      __lookup_processor_type
+        mov     r0, r5
+        ldmfd   sp!, {r4 - r6, r9, pc}
+ENDPROC(lookup_processor_type)
 
 /*
  * Read processor ID register (CP#15, CR0), and look up in the linker-built
@@ -157,11 +141,11 @@ ENDPROC(__error)
  *      r5 = proc_info pointer in physical address space
  *      r9 = cpuid (preserved)
  */
+        __CPUINIT
 __lookup_processor_type:
-        adr     r3, 3f
-        ldmia   r3, {r5 - r7}
-        add     r3, r3, #8
-        sub     r3, r3, r7                      @ get offset between virt&phys
+        adr     r3, __lookup_processor_type_data
+        ldmia   r3, {r4 - r6}
+        sub     r3, r3, r4                      @ get offset between virt&phys
         add     r5, r5, r3                      @ convert virt addresses to
         add     r6, r6, r3                      @ physical address space
 1:      ldmia   r5, {r3, r4}                    @ value, mask
@@ -176,92 +160,45 @@ __lookup_processor_type:
 ENDPROC(__lookup_processor_type)
 
 /*
- * This provides a C-API version of the above function.
- */
-ENTRY(lookup_processor_type)
-        stmfd   sp!, {r4 - r7, r9, lr}
-        mov     r9, r0
-        bl      __lookup_processor_type
-        mov     r0, r5
-        ldmfd   sp!, {r4 - r7, r9, pc}
-ENDPROC(lookup_processor_type)
-
-/*
- * Look in <asm/procinfo.h> and arch/arm/kernel/arch.[ch] for
- * more information about the __proc_info and __arch_info structures.
+ * Look in <asm/procinfo.h> for information about the __proc_info structure.
  */
         .align  2
-3:      .long   __proc_info_begin
+        .type   __lookup_processor_type_data, %object
+__lookup_processor_type_data:
+        .long   .
+        .long   __proc_info_begin
         .long   __proc_info_end
-4:      .long   .
-        .long   __arch_info_begin
-        .long   __arch_info_end
+        .size   __lookup_processor_type_data, . - __lookup_processor_type_data
 
-/*
- * Lookup machine architecture in the linker-build list of architectures.
- * Note that we can't use the absolute addresses for the __arch_info
- * lists since we aren't running with the MMU on (and therefore, we are
- * not in the correct address space).  We have to calculate the offset.
- *
- *  r1 = machine architecture number
- * Returns:
- *  r3, r4, r6 corrupted
- *  r5 = mach_info pointer in physical address space
- */
-__lookup_machine_type:
-        adr     r3, 4b
-        ldmia   r3, {r4, r5, r6}
-        sub     r3, r3, r4                      @ get offset between virt&phys
-        add     r5, r5, r3                      @ convert virt addresses to
-        add     r6, r6, r3                      @ physical address space
-1:      ldr     r3, [r5, #MACHINFO_TYPE]        @ get machine type
-        teq     r3, r1                          @ matches loader number?
-        beq     2f                              @ found
-        add     r5, r5, #SIZEOF_MACHINE_DESC    @ next machine_desc
-        cmp     r5, r6
-        blo     1b
-        mov     r5, #0                          @ unknown machine
-2:      mov     pc, lr
-ENDPROC(__lookup_machine_type)
+__error_p:
+#ifdef CONFIG_DEBUG_LL
+        adr     r0, str_p1
+        bl      printascii
+        mov     r0, r9
+        bl      printhex8
+        adr     r0, str_p2
+        bl      printascii
+        b       __error
+str_p1: .asciz  "\nError: unrecognized/unsupported processor variant (0x"
+str_p2: .asciz  ").\n"
+        .align
+#endif
+ENDPROC(__error_p)
 
+__error:
+#ifdef CONFIG_ARCH_RPC
 /*
- * This provides a C-API version of the above function.
- */
-ENTRY(lookup_machine_type)
-        stmfd   sp!, {r4 - r6, lr}
-        mov     r1, r0
-        bl      __lookup_machine_type
-        mov     r0, r5
-        ldmfd   sp!, {r4 - r6, pc}
-ENDPROC(lookup_machine_type)
-
-/* Determine validity of the r2 atags pointer.  The heuristic requires
- * that the pointer be aligned, in the first 16k of physical RAM and
- * that the ATAG_CORE marker is first and present.  Future revisions
- * of this function may be more lenient with the physical address and
- * may also be able to move the ATAGS block if necessary.
- *
- * r8  = machinfo
- *
- * Returns:
- *  r2 either valid atags pointer, or zero
- *  r5, r6 corrupted
+ * Turn the screen red on a error - RiscPC only.
  */
-__vet_atags:
-        tst     r2, #0x3                        @ aligned?
-        bne     1f
-
-        ldr     r5, [r2, #0]                    @ is first tag ATAG_CORE?
-        cmp     r5, #ATAG_CORE_SIZE
-        cmpne   r5, #ATAG_CORE_SIZE_EMPTY
-        bne     1f
-        ldr     r5, [r2, #4]
-        ldr     r6, =ATAG_CORE
-        cmp     r5, r6
-        bne     1f
-
-        mov     pc, lr                          @ atag pointer is ok
-
-1:      mov     r2, #0
-        mov     pc, lr
-ENDPROC(__vet_atags)
+        mov     r0, #0x02000000
+        mov     r3, #0x11
+        orr     r3, r3, r3, lsl #8
+        orr     r3, r3, r3, lsl #16
+        str     r3, [r0], #4
+        str     r3, [r0], #4
+        str     r3, [r0], #4
+        str     r3, [r0], #4
+#endif
+1:      mov     r0, r0
+        b       1b
+ENDPROC(__error)
diff --git a/arch/arm/kernel/head-nommu.S b/arch/arm/kernel/head-nommu.S
index 573b803dc6bf..6b1e0ad9ec3b 100644
--- a/arch/arm/kernel/head-nommu.S
+++ b/arch/arm/kernel/head-nommu.S
@@ -44,12 +44,7 @@ ENTRY(stext)
         bl      __lookup_processor_type         @ r5=procinfo r9=cpuid
         movs    r10, r5                         @ invalid processor (r5=0)?
         beq     __error_p                               @ yes, error 'p'
-        bl      __lookup_machine_type           @ r5=machinfo
-        movs    r8, r5                          @ invalid machine (r5=0)?
-        beq     __error_a                       @ yes, error 'a'
 
-        ldr     r13, __switch_data              @ address to jump to after
-                                                @ the initialization is done
         adr     lr, BSYM(__after_proc_init)     @ return (PIC) address
  ARM(   add     pc, r10, #PROCINFO_INITFUNC     )
  THUMB( add     r12, r10, #PROCINFO_INITFUNC    )
@@ -87,8 +82,7 @@ __after_proc_init:
         mcr     p15, 0, r0, c1, c0, 0           @ write control reg
 #endif /* CONFIG_CPU_CP15 */
 
-        mov     r3, r13
-        mov     pc, r3                          @ clear the BSS and jump
+        b       __mmap_switched                 @ clear the BSS and jump
                                                 @ to start_kernel
 ENDPROC(__after_proc_init)
         .ltorg
diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index eb62bf947212..278c1b0ebb2e 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -22,14 +22,10 @@
 #include <asm/thread_info.h>
 #include <asm/system.h>
 
-#if (PHYS_OFFSET & 0x001fffff)
-#error "PHYS_OFFSET must be at an even 2MiB boundary!"
+#ifdef CONFIG_DEBUG_LL
+#include <mach/debug-macro.S>
 #endif
 
-#define KERNEL_RAM_VADDR        (PAGE_OFFSET + TEXT_OFFSET)
-#define KERNEL_RAM_PADDR        (PHYS_OFFSET + TEXT_OFFSET)
-
-
 /*
  * swapper_pg_dir is the virtual address of the initial page table.
  * We place the page tables 16K below KERNEL_RAM_VADDR.  Therefore, we must
@@ -37,6 +33,7 @@
  * the least significant 16 bits to be 0x8000, but we could probably
  * relax this restriction to KERNEL_RAM_VADDR >= PAGE_OFFSET + 0x4000.
  */
+#define KERNEL_RAM_VADDR        (PAGE_OFFSET + TEXT_OFFSET)
 #if (KERNEL_RAM_VADDR & 0xffff) != 0x8000
 #error KERNEL_RAM_VADDR must start at 0xXXXX8000
 #endif
@@ -44,8 +41,8 @@
         .globl  swapper_pg_dir
         .equ    swapper_pg_dir, KERNEL_RAM_VADDR - 0x4000
 
-        .macro  pgtbl, rd
-        ldr     \rd, =(KERNEL_RAM_PADDR - 0x4000)
+        .macro  pgtbl, rd, phys
+        add     \rd, \phys, #TEXT_OFFSET - 0x4000
         .endm
 
 #ifdef CONFIG_XIP_KERNEL
@@ -62,7 +59,7 @@
  *
  * This is normally called from the decompressor code.  The requirements
  * are: MMU = off, D-cache = off, I-cache = dont care, r0 = 0,
- * r1 = machine nr, r2 = atags pointer.
+ * r1 = machine nr, r2 = atags or dtb pointer.
  *
  * This code is mostly position independent, so if you link the kernel at
  * 0xc0008000, you call this at __pa(0xc0008000).
@@ -81,29 +78,210 @@ ENTRY(stext)
         mrc     p15, 0, r9, c0, c0              @ get processor id
         bl      __lookup_processor_type         @ r5=procinfo r9=cpuid
         movs    r10, r5                         @ invalid processor (r5=0)?
+ THUMB( it      eq )            @ force fixup-able long branch encoding
         beq     __error_p                       @ yes, error 'p'
-        bl      __lookup_machine_type           @ r5=machinfo
-        movs    r8, r5                          @ invalid machine (r5=0)?
-        beq     __error_a                       @ yes, error 'a'
+
+#ifndef CONFIG_XIP_KERNEL
+        adr     r3, 2f
+        ldmia   r3, {r4, r8}
+        sub     r4, r3, r4                      @ (PHYS_OFFSET - PAGE_OFFSET)
+        add     r8, r8, r4                      @ PHYS_OFFSET
+#else
+        ldr     r8, =PLAT_PHYS_OFFSET
+#endif
+
+        /*
+         * r1 = machine no, r2 = atags or dtb,
+         * r8 = phys_offset, r9 = cpuid, r10 = procinfo
+         */
         bl      __vet_atags
+#ifdef CONFIG_SMP_ON_UP
+        bl      __fixup_smp
+#endif
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
+        bl      __fixup_pv_table
+#endif
         bl      __create_page_tables
 
         /*
          * The following calls CPU specific code in a position independent
          * manner.  See arch/arm/mm/proc-*.S for details.  r10 = base of
-         * xxx_proc_info structure selected by __lookup_machine_type
+         * xxx_proc_info structure selected by __lookup_processor_type
          * above.  On return, the CPU will be ready for the MMU to be
          * turned on, and r0 will hold the CPU control register value.
          */
-        ldr     r13, __switch_data              @ address to jump to after
+        ldr     r13, =__mmap_switched           @ address to jump to after
                                                 @ mmu has been enabled
-        adr     lr, BSYM(__enable_mmu)          @ return (PIC) address
+        adr     lr, BSYM(1f)                    @ return (PIC) address
+        mov     r8, r4                          @ set TTBR1 to swapper_pg_dir
  ARM(   add     pc, r10, #PROCINFO_INITFUNC     )
  THUMB( add     r12, r10, #PROCINFO_INITFUNC    )
  THUMB( mov     pc, r12                         )
+1:      b       __enable_mmu
 ENDPROC(stext)
+        .ltorg
+#ifndef CONFIG_XIP_KERNEL
+2:      .long   .
+        .long   PAGE_OFFSET
+#endif
+
+/*
+ * Setup the initial page tables.  We only setup the barest
+ * amount which are required to get the kernel running, which
+ * generally means mapping in the kernel code.
+ *
+ * r8 = phys_offset, r9 = cpuid, r10 = procinfo
+ *
+ * Returns:
+ *  r0, r3, r5-r7 corrupted
+ *  r4 = physical page table address
+ */
+__create_page_tables:
+        pgtbl   r4, r8                          @ page table address
+
+        /*
+         * Clear the 16K level 1 swapper page table
+         */
+        mov     r0, r4
+        mov     r3, #0
+        add     r6, r0, #0x4000
+1:      str     r3, [r0], #4
+        str     r3, [r0], #4
+        str     r3, [r0], #4
+        str     r3, [r0], #4
+        teq     r0, r6
+        bne     1b
+
+        ldr     r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags
+
+        /*
+         * Create identity mapping to cater for __enable_mmu.
+         * This identity mapping will be removed by paging_init().
+         */
+        adr     r0, __enable_mmu_loc
+        ldmia   r0, {r3, r5, r6}
+        sub     r0, r0, r3                      @ virt->phys offset
+        add     r5, r5, r0                      @ phys __enable_mmu
+        add     r6, r6, r0                      @ phys __enable_mmu_end
+        mov     r5, r5, lsr #20
+        mov     r6, r6, lsr #20
+
+1:      orr     r3, r7, r5, lsl #20             @ flags + kernel base
+        str     r3, [r4, r5, lsl #2]            @ identity mapping
+        teq     r5, r6
+        addne   r5, r5, #1                      @ next section
+        bne     1b
+
+        /*
+         * Now setup the pagetables for our kernel direct
+         * mapped region.
+         */
+        mov     r3, pc
+        mov     r3, r3, lsr #20
+        orr     r3, r7, r3, lsl #20
+        add     r0, r4,  #(KERNEL_START & 0xff000000) >> 18
+        str     r3, [r0, #(KERNEL_START & 0x00f00000) >> 18]!
+        ldr     r6, =(KERNEL_END - 1)
+        add     r0, r0, #4
+        add     r6, r4, r6, lsr #18
+1:      cmp     r0, r6
+        add     r3, r3, #1 << 20
+        strls   r3, [r0], #4
+        bls     1b
+
+#ifdef CONFIG_XIP_KERNEL
+        /*
+         * Map some ram to cover our .data and .bss areas.
+         */
+        add     r3, r8, #TEXT_OFFSET
+        orr     r3, r3, r7
+        add     r0, r4,  #(KERNEL_RAM_VADDR & 0xff000000) >> 18
+        str     r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> 18]!
+        ldr     r6, =(_end - 1)
+        add     r0, r0, #4
+        add     r6, r4, r6, lsr #18
+1:      cmp     r0, r6
+        add     r3, r3, #1 << 20
+        strls   r3, [r0], #4
+        bls     1b
+#endif
+
+        /*
+         * Then map boot params address in r2 or
+         * the first 1MB of ram if boot params address is not specified.
+         */
+        mov     r0, r2, lsr #20
+        movs    r0, r0, lsl #20
+        moveq   r0, r8
+        sub     r3, r0, r8
+        add     r3, r3, #PAGE_OFFSET
+        add     r3, r4, r3, lsr #18
+        orr     r6, r7, r0
+        str     r6, [r3]
+
+#ifdef CONFIG_DEBUG_LL
+#ifndef CONFIG_DEBUG_ICEDCC
+        /*
+         * Map in IO space for serial debugging.
+         * This allows debug messages to be output
+         * via a serial console before paging_init.
+         */
+        addruart r7, r3
+
+        mov     r3, r3, lsr #20
+        mov     r3, r3, lsl #2
+
+        add     r0, r4, r3
+        rsb     r3, r3, #0x4000                 @ PTRS_PER_PGD*sizeof(long)
+        cmp     r3, #0x0800                     @ limit to 512MB
+        movhi   r3, #0x0800
+        add     r6, r0, r3
+        mov     r3, r7, lsr #20
+        ldr     r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags
+        orr     r3, r7, r3, lsl #20
+1:      str     r3, [r0], #4
+        add     r3, r3, #1 << 20
+        teq     r0, r6
+        bne     1b
+
+#else /* CONFIG_DEBUG_ICEDCC */
+        /* we don't need any serial debugging mappings for ICEDCC */
+        ldr     r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags
+#endif /* !CONFIG_DEBUG_ICEDCC */
+
+#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS)
+        /*
+         * If we're using the NetWinder or CATS, we also need to map
+         * in the 16550-type serial port for the debug messages
+         */
+        add     r0, r4, #0xff000000 >> 18
+        orr     r3, r7, #0x7c000000
+        str     r3, [r0]
+#endif
+#ifdef CONFIG_ARCH_RPC
+        /*
+         * Map in screen at 0x02000000 & SCREEN2_BASE
+         * Similar reasons here - for debug.  This is
+         * only for Acorn RiscPC architectures.
+         */
+        add     r0, r4, #0x02000000 >> 18
+        orr     r3, r7, #0x02000000
+        str     r3, [r0]
+        add     r0, r4, #0xd8000000 >> 18
+        str     r3, [r0]
+#endif
+#endif
+        mov     pc, lr
+ENDPROC(__create_page_tables)
+        .ltorg
+        .align
+__enable_mmu_loc:
+        .long   .
+        .long   __enable_mmu
+        .long   __enable_mmu_end
 
 #if defined(CONFIG_SMP)
+        __CPUINIT
 ENTRY(secondary_startup)
         /*
          * Common entry point for secondary CPUs.
@@ -117,15 +295,18 @@ ENTRY(secondary_startup)
         bl      __lookup_processor_type
         movs    r10, r5                         @ invalid processor?
         moveq   r0, #'p'                        @ yes, error 'p'
-        beq     __error
+ THUMB( it      eq )            @ force fixup-able long branch encoding
+        beq     __error_p
 
         /*
          * Use the page tables supplied from  __cpu_up.
          */
         adr     r4, __secondary_data
         ldmia   r4, {r5, r7, r12}               @ address to jump to after
-        sub     r4, r4, r5                      @ mmu has been enabled
-        ldr     r4, [r7, r4]                    @ get secondary_data.pgdir
+        sub     lr, r4, r5                      @ mmu has been enabled
+        ldr     r4, [r7, lr]                    @ get secondary_data.pgdir
+        add     r7, r7, #4
+        ldr     r8, [r7, lr]                    @ get secondary_data.swapper_pg_dir
         adr     lr, BSYM(__enable_mmu)          @ return address
         mov     r13, r12                        @ __secondary_switched address
  ARM(   add     pc, r10, #PROCINFO_INITFUNC     ) @ initialise processor
@@ -143,6 +324,8 @@ ENTRY(__secondary_switched)
         b       secondary_start_kernel
 ENDPROC(__secondary_switched)
 
+        .align
+
         .type   __secondary_data, %object
 __secondary_data:
         .long   .
@@ -156,6 +339,13 @@ __secondary_data:
  * Setup common bits before finally enabling the MMU.  Essentially
  * this is just loading the page table pointer and domain access
  * registers.
+ *
+ *  r0  = cp#15 control register
+ *  r1  = machine ID
+ *  r2  = atags or dtb pointer
+ *  r4  = page table pointer
+ *  r9  = processor ID
+ *  r13 = *virtual* address to jump to upon completion
  */
 __enable_mmu:
 #ifdef CONFIG_ALIGNMENT_TRAP
@@ -188,6 +378,9 @@ ENDPROC(__enable_mmu)
  * mailing list archives BEFORE sending another post to the list.
  *
  *  r0  = cp#15 control register
+ *  r1  = machine ID
+ *  r2  = atags or dtb pointer
+ *  r9  = processor ID
  *  r13 = *virtual* address to jump to upon completion
  *
  * other registers depend on the function called upon completion
@@ -200,137 +393,200 @@ __turn_mmu_on:
         mov     r3, r3
         mov     r3, r13
         mov     pc, r3
+__enable_mmu_end:
 ENDPROC(__turn_mmu_on)
 
 
-/*
- * Setup the initial page tables.  We only setup the barest
- * amount which are required to get the kernel running, which
- * generally means mapping in the kernel code.
- *
- * r8  = machinfo
- * r9  = cpuid
- * r10 = procinfo
- *
- * Returns:
- *  r0, r3, r6, r7 corrupted
- *  r4 = physical page table address
- */
-__create_page_tables:
-        pgtbl   r4                              @ page table address
+#ifdef CONFIG_SMP_ON_UP
+        __INIT
+__fixup_smp:
+        and     r3, r9, #0x000f0000     @ architecture version
+        teq     r3, #0x000f0000         @ CPU ID supported?
+        bne     __fixup_smp_on_up       @ no, assume UP
 
-        /*
-         * Clear the 16K level 1 swapper page table
-         */
-        mov     r0, r4
-        mov     r3, #0
-        add     r6, r0, #0x4000
-1:      str     r3, [r0], #4
-        str     r3, [r0], #4
-        str     r3, [r0], #4
-        str     r3, [r0], #4
-        teq     r0, r6
-        bne     1b
+        bic     r3, r9, #0x00ff0000
+        bic     r3, r3, #0x0000000f     @ mask 0xff00fff0
+        mov     r4, #0x41000000
+        orr     r4, r4, #0x0000b000
+        orr     r4, r4, #0x00000020     @ val 0x4100b020
+        teq     r3, r4                  @ ARM 11MPCore?
+        moveq   pc, lr                  @ yes, assume SMP
 
-        ldr     r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags
+        mrc     p15, 0, r0, c0, c0, 5   @ read MPIDR
+        and     r0, r0, #0xc0000000     @ multiprocessing extensions and
+        teq     r0, #0x80000000         @ not part of a uniprocessor system?
+        moveq   pc, lr                  @ yes, assume SMP
 
-        /*
-         * Create identity mapping for first MB of kernel to
-         * cater for the MMU enable.  This identity mapping
-         * will be removed by paging_init().  We use our current program
-         * counter to determine corresponding section base address.
-         */
-        mov     r6, pc
-        mov     r6, r6, lsr #20                 @ start of kernel section
-        orr     r3, r7, r6, lsl #20             @ flags + kernel base
-        str     r3, [r4, r6, lsl #2]            @ identity mapping
+__fixup_smp_on_up:
+        adr     r0, 1f
+        ldmia   r0, {r3 - r5}
+        sub     r3, r0, r3
+        add     r4, r4, r3
+        add     r5, r5, r3
+        b       __do_fixup_smp_on_up
+ENDPROC(__fixup_smp)
 
-        /*
-         * Now setup the pagetables for our kernel direct
-         * mapped region.
-         */
-        add     r0, r4,  #(KERNEL_START & 0xff000000) >> 18
-        str     r3, [r0, #(KERNEL_START & 0x00f00000) >> 18]!
-        ldr     r6, =(KERNEL_END - 1)
-        add     r0, r0, #4
-        add     r6, r4, r6, lsr #18
-1:      cmp     r0, r6
-        add     r3, r3, #1 << 20
-        strls   r3, [r0], #4
-        bls     1b
+        .align
+1:      .word   .
+        .word   __smpalt_begin
+        .word   __smpalt_end
 
-#ifdef CONFIG_XIP_KERNEL
-        /*
-         * Map some ram to cover our .data and .bss areas.
-         */
-        orr     r3, r7, #(KERNEL_RAM_PADDR & 0xff000000)
-        .if     (KERNEL_RAM_PADDR & 0x00f00000)
-        orr     r3, r3, #(KERNEL_RAM_PADDR & 0x00f00000)
-        .endif
-        add     r0, r4,  #(KERNEL_RAM_VADDR & 0xff000000) >> 18
-        str     r3, [r0, #(KERNEL_RAM_VADDR & 0x00f00000) >> 18]!
-        ldr     r6, =(_end - 1)
-        add     r0, r0, #4
-        add     r6, r4, r6, lsr #18
-1:      cmp     r0, r6
-        add     r3, r3, #1 << 20
-        strls   r3, [r0], #4
-        bls     1b
+        .pushsection .data
+        .globl  smp_on_up
+smp_on_up:
+        ALT_SMP(.long   1)
+        ALT_UP(.long    0)
+        .popsection
 #endif
 
-        /*
-         * Then map first 1MB of ram in case it contains our boot params.
-         */
-        add     r0, r4, #PAGE_OFFSET >> 18
-        orr     r6, r7, #(PHYS_OFFSET & 0xff000000)
-        .if     (PHYS_OFFSET & 0x00f00000)
-        orr     r6, r6, #(PHYS_OFFSET & 0x00f00000)
-        .endif
-        str     r6, [r0]
+        .text
+__do_fixup_smp_on_up:
+        cmp     r4, r5
+        movhs   pc, lr
+        ldmia   r4!, {r0, r6}
+ ARM(   str     r6, [r0, r3]    )
+ THUMB( add     r0, r0, r3      )
+#ifdef __ARMEB__
+ THUMB( mov     r6, r6, ror #16 )       @ Convert word order for big-endian.
+#endif
+ THUMB( strh    r6, [r0], #2    )       @ For Thumb-2, store as two halfwords
+ THUMB( mov     r6, r6, lsr #16 )       @ to be robust against misaligned r3.
+ THUMB( strh    r6, [r0]        )
+        b       __do_fixup_smp_on_up
+ENDPROC(__do_fixup_smp_on_up)
 
-#ifdef CONFIG_DEBUG_LL
-        ldr     r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags
-        /*
-         * Map in IO space for serial debugging.
-         * This allows debug messages to be output
-         * via a serial console before paging_init.
-         */
-        ldr     r3, [r8, #MACHINFO_PGOFFIO]
-        add     r0, r4, r3
-        rsb     r3, r3, #0x4000                 @ PTRS_PER_PGD*sizeof(long)
-        cmp     r3, #0x0800                     @ limit to 512MB
-        movhi   r3, #0x0800
-        add     r6, r0, r3
-        ldr     r3, [r8, #MACHINFO_PHYSIO]
-        orr     r3, r3, r7
-1:      str     r3, [r0], #4
-        add     r3, r3, #1 << 20
-        teq     r0, r6
-        bne     1b
-#if defined(CONFIG_ARCH_NETWINDER) || defined(CONFIG_ARCH_CATS)
-        /*
-         * If we're using the NetWinder or CATS, we also need to map
-         * in the 16550-type serial port for the debug messages
-         */
-        add     r0, r4, #0xff000000 >> 18
-        orr     r3, r7, #0x7c000000
-        str     r3, [r0]
+ENTRY(fixup_smp)
+        stmfd   sp!, {r4 - r6, lr}
+        mov     r4, r0
+        add     r5, r0, r1
+        mov     r3, #0
+        bl      __do_fixup_smp_on_up
+        ldmfd   sp!, {r4 - r6, pc}
+ENDPROC(fixup_smp)
+
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
+
+/* __fixup_pv_table - patch the stub instructions with the delta between
+ * PHYS_OFFSET and PAGE_OFFSET, which is assumed to be 16MiB aligned and
+ * can be expressed by an immediate shifter operand. The stub instruction
+ * has a form of '(add|sub) rd, rn, #imm'.
+ */
+        __HEAD
+__fixup_pv_table:
+        adr     r0, 1f
+        ldmia   r0, {r3-r5, r7}
+        sub     r3, r0, r3      @ PHYS_OFFSET - PAGE_OFFSET
+        add     r4, r4, r3      @ adjust table start address
+        add     r5, r5, r3      @ adjust table end address
+        add     r7, r7, r3      @ adjust __pv_phys_offset address
+        str     r8, [r7]        @ save computed PHYS_OFFSET to __pv_phys_offset
+#ifndef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT
+        mov     r6, r3, lsr #24 @ constant for add/sub instructions
+        teq     r3, r6, lsl #24 @ must be 16MiB aligned
+#else
+        mov     r6, r3, lsr #16 @ constant for add/sub instructions
+        teq     r3, r6, lsl #16 @ must be 64kiB aligned
 #endif
-#ifdef CONFIG_ARCH_RPC
-        /*
-         * Map in screen at 0x02000000 & SCREEN2_BASE
-         * Similar reasons here - for debug.  This is
-         * only for Acorn RiscPC architectures.
-         */
-        add     r0, r4, #0x02000000 >> 18
-        orr     r3, r7, #0x02000000
-        str     r3, [r0]
-        add     r0, r4, #0xd8000000 >> 18
-        str     r3, [r0]
+THUMB(  it      ne              @ cross section branch )
+        bne     __error
+        str     r6, [r7, #4]    @ save to __pv_offset
+        b       __fixup_a_pv_table
+ENDPROC(__fixup_pv_table)
+
+        .align
+1:      .long   .
+        .long   __pv_table_begin
+        .long   __pv_table_end
+2:      .long   __pv_phys_offset
+
+        .text
+__fixup_a_pv_table:
+#ifdef CONFIG_THUMB2_KERNEL
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT
+        lsls    r0, r6, #24
+        lsr     r6, #8
+        beq     1f
+        clz     r7, r0
+        lsr     r0, #24
+        lsl     r0, r7
+        bic     r0, 0x0080
+        lsrs    r7, #1
+        orrcs   r0, #0x0080
+        orr     r0, r0, r7, lsl #12
 #endif
+1:      lsls    r6, #24
+        beq     4f
+        clz     r7, r6
+        lsr     r6, #24
+        lsl     r6, r7
+        bic     r6, #0x0080
+        lsrs    r7, #1
+        orrcs   r6, #0x0080
+        orr     r6, r6, r7, lsl #12
+        orr     r6, #0x4000
+        b       4f
+2:      @ at this point the C flag is always clear
+        add     r7, r3
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT
+        ldrh    ip, [r7]
+        tst     ip, 0x0400      @ the i bit tells us LS or MS byte
+        beq     3f
+        cmp     r0, #0          @ set C flag, and ...
+        biceq   ip, 0x0400      @ immediate zero value has a special encoding
+        streqh  ip, [r7]        @ that requires the i bit cleared
+#endif
+3:      ldrh    ip, [r7, #2]
+        and     ip, 0x8f00
+        orrcc   ip, r6  @ mask in offset bits 31-24
+        orrcs   ip, r0  @ mask in offset bits 23-16
+        strh    ip, [r7, #2]
+4:      cmp     r4, r5
+        ldrcc   r7, [r4], #4    @ use branch for delay slot
+        bcc     2b
+        bx      lr
+#else
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT_16BIT
+        and     r0, r6, #255    @ offset bits 23-16
+        mov     r6, r6, lsr #8  @ offset bits 31-24
+#else
+        mov     r0, #0          @ just in case...
 #endif
+        b       3f
+2:      ldr     ip, [r7, r3]
+        bic     ip, ip, #0x000000ff
+        tst     ip, #0x400      @ rotate shift tells us LS or MS byte
+        orrne   ip, ip, r6      @ mask in offset bits 31-24
+        orreq   ip, ip, r0      @ mask in offset bits 23-16
+        str     ip, [r7, r3]
+3:      cmp     r4, r5
+        ldrcc   r7, [r4], #4    @ use branch for delay slot
+        bcc     2b
         mov     pc, lr
-ENDPROC(__create_page_tables)
-        .ltorg
+#endif
+ENDPROC(__fixup_a_pv_table)
+
+ENTRY(fixup_pv_table)
+        stmfd   sp!, {r4 - r7, lr}
+        ldr     r2, 2f                  @ get address of __pv_phys_offset
+        mov     r3, #0                  @ no offset
+        mov     r4, r0                  @ r0 = table start
+        add     r5, r0, r1              @ r1 = table size
+        ldr     r6, [r2, #4]            @ get __pv_offset
+        bl      __fixup_a_pv_table
+        ldmfd   sp!, {r4 - r7, pc}
+ENDPROC(fixup_pv_table)
+
+        .align
+2:      .long   __pv_phys_offset
+
+        .data
+        .globl  __pv_phys_offset
+        .type   __pv_phys_offset, %object
+__pv_phys_offset:
+        .long   0
+        .size   __pv_phys_offset, . - __pv_phys_offset
+__pv_offset:
+        .long   0
+#endif
 
 #include "head-common.S"
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
new file mode 100644
index 000000000000..87acc25d7a3e
--- /dev/null
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -0,0 +1,978 @@
+/*
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) 2009, 2010 ARM Limited
+ *
+ * Author: Will Deacon <will.deacon@arm.com>
+ */
+
+/*
+ * HW_breakpoint: a unified kernel/user-space hardware breakpoint facility,
+ * using the CPU's debug registers.
+ */
+#define pr_fmt(fmt) "hw-breakpoint: " fmt
+
+#include <linux/errno.h>
+#include <linux/hardirq.h>
+#include <linux/perf_event.h>
+#include <linux/hw_breakpoint.h>
+#include <linux/smp.h>
+
+#include <asm/cacheflush.h>
+#include <asm/cputype.h>
+#include <asm/current.h>
+#include <asm/hw_breakpoint.h>
+#include <asm/kdebug.h>
+#include <asm/system.h>
+#include <asm/traps.h>
+
+/* Breakpoint currently in use for each BRP. */
+static DEFINE_PER_CPU(struct perf_event *, bp_on_reg[ARM_MAX_BRP]);
+
+/* Watchpoint currently in use for each WRP. */
+static DEFINE_PER_CPU(struct perf_event *, wp_on_reg[ARM_MAX_WRP]);
+
+/* Number of BRP/WRP registers on this CPU. */
+static int core_num_brps;
+static int core_num_reserved_brps;
+static int core_num_wrps;
+
+/* Debug architecture version. */
+static u8 debug_arch;
+
+/* Maximum supported watchpoint length. */
+static u8 max_watchpoint_len;
+
+#define READ_WB_REG_CASE(OP2, M, VAL)           \
+        case ((OP2 << 4) + M):                  \
+                ARM_DBG_READ(c ## M, OP2, VAL); \
+                break
+
+#define WRITE_WB_REG_CASE(OP2, M, VAL)          \
+        case ((OP2 << 4) + M):                  \
+                ARM_DBG_WRITE(c ## M, OP2, VAL);\
+                break
+
+#define GEN_READ_WB_REG_CASES(OP2, VAL)         \
+        READ_WB_REG_CASE(OP2, 0, VAL);          \
+        READ_WB_REG_CASE(OP2, 1, VAL);          \
+        READ_WB_REG_CASE(OP2, 2, VAL);          \
+        READ_WB_REG_CASE(OP2, 3, VAL);          \
+        READ_WB_REG_CASE(OP2, 4, VAL);          \
+        READ_WB_REG_CASE(OP2, 5, VAL);          \
+        READ_WB_REG_CASE(OP2, 6, VAL);          \
+        READ_WB_REG_CASE(OP2, 7, VAL);          \
+        READ_WB_REG_CASE(OP2, 8, VAL);          \
+        READ_WB_REG_CASE(OP2, 9, VAL);          \
+        READ_WB_REG_CASE(OP2, 10, VAL);         \
+        READ_WB_REG_CASE(OP2, 11, VAL);         \
+        READ_WB_REG_CASE(OP2, 12, VAL);         \
+        READ_WB_REG_CASE(OP2, 13, VAL);         \
+        READ_WB_REG_CASE(OP2, 14, VAL);         \
+        READ_WB_REG_CASE(OP2, 15, VAL)
+
+#define GEN_WRITE_WB_REG_CASES(OP2, VAL)        \
+        WRITE_WB_REG_CASE(OP2, 0, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 1, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 2, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 3, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 4, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 5, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 6, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 7, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 8, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 9, VAL);         \
+        WRITE_WB_REG_CASE(OP2, 10, VAL);        \
+        WRITE_WB_REG_CASE(OP2, 11, VAL);        \
+        WRITE_WB_REG_CASE(OP2, 12, VAL);        \
+        WRITE_WB_REG_CASE(OP2, 13, VAL);        \
+        WRITE_WB_REG_CASE(OP2, 14, VAL);        \
+        WRITE_WB_REG_CASE(OP2, 15, VAL)
+
+static u32 read_wb_reg(int n)
+{
+        u32 val = 0;
+
+        switch (n) {
+        GEN_READ_WB_REG_CASES(ARM_OP2_BVR, val);
+        GEN_READ_WB_REG_CASES(ARM_OP2_BCR, val);
+        GEN_READ_WB_REG_CASES(ARM_OP2_WVR, val);
+        GEN_READ_WB_REG_CASES(ARM_OP2_WCR, val);
+        default:
+                pr_warning("attempt to read from unknown breakpoint "
+                                "register %d\n", n);
+        }
+
+        return val;
+}
+
+static void write_wb_reg(int n, u32 val)
+{
+        switch (n) {
+        GEN_WRITE_WB_REG_CASES(ARM_OP2_BVR, val);
+        GEN_WRITE_WB_REG_CASES(ARM_OP2_BCR, val);
+        GEN_WRITE_WB_REG_CASES(ARM_OP2_WVR, val);
+        GEN_WRITE_WB_REG_CASES(ARM_OP2_WCR, val);
+        default:
+                pr_warning("attempt to write to unknown breakpoint "
+                                "register %d\n", n);
+        }
+        isb();
+}
+
+/* Determine debug architecture. */
+static u8 get_debug_arch(void)
+{
+        u32 didr;
+
+        /* Do we implement the extended CPUID interface? */
+        if (WARN_ONCE((((read_cpuid_id() >> 16) & 0xf) != 0xf),
+            "CPUID feature registers not supported. "
+            "Assuming v6 debug is present.\n"))
+                return ARM_DEBUG_ARCH_V6;
+
+        ARM_DBG_READ(c0, 0, didr);
+        return (didr >> 16) & 0xf;
+}
+
+u8 arch_get_debug_arch(void)
+{
+        return debug_arch;
+}
+
+static int debug_arch_supported(void)
+{
+        u8 arch = get_debug_arch();
+        return arch >= ARM_DEBUG_ARCH_V6 && arch <= ARM_DEBUG_ARCH_V7_ECP14;
+}
+
+/* Determine number of BRP register available. */
+static int get_num_brp_resources(void)
+{
+        u32 didr;
+        ARM_DBG_READ(c0, 0, didr);
+        return ((didr >> 24) & 0xf) + 1;
+}
+
+/* Does this core support mismatch breakpoints? */
+static int core_has_mismatch_brps(void)
+{
+        return (get_debug_arch() >= ARM_DEBUG_ARCH_V7_ECP14 &&
+                get_num_brp_resources() > 1);
+}
+
+/* Determine number of usable WRPs available. */
+static int get_num_wrps(void)
+{
+        /*
+         * FIXME: When a watchpoint fires, the only way to work out which
+         * watchpoint it was is by disassembling the faulting instruction
+         * and working out the address of the memory access.
+         *
+         * Furthermore, we can only do this if the watchpoint was precise
+         * since imprecise watchpoints prevent us from calculating register
+         * based addresses.
+         *
+         * Providing we have more than 1 breakpoint register, we only report
+         * a single watchpoint register for the time being. This way, we always
+         * know which watchpoint fired. In the future we can either add a
+         * disassembler and address generation emulator, or we can insert a
+         * check to see if the DFAR is set on watchpoint exception entry
+         * [the ARM ARM states that the DFAR is UNKNOWN, but experience shows
+         * that it is set on some implementations].
+         */
+
+#if 0
+        int wrps;
+        u32 didr;
+        ARM_DBG_READ(c0, 0, didr);
+        wrps = ((didr >> 28) & 0xf) + 1;
+#endif
+        int wrps = 1;
+
+        if (core_has_mismatch_brps() && wrps >= get_num_brp_resources())
+                wrps = get_num_brp_resources() - 1;
+
+        return wrps;
+}
+
+/* We reserve one breakpoint for each watchpoint. */
+static int get_num_reserved_brps(void)
+{
+        if (core_has_mismatch_brps())
+                return get_num_wrps();
+        return 0;
+}
+
+/* Determine number of usable BRPs available. */
+static int get_num_brps(void)
+{
+        int brps = get_num_brp_resources();
+        if (core_has_mismatch_brps())
+                brps -= get_num_reserved_brps();
+        return brps;
+}
+
+/*
+ * In order to access the breakpoint/watchpoint control registers,
+ * we must be running in debug monitor mode. Unfortunately, we can
+ * be put into halting debug mode at any time by an external debugger
+ * but there is nothing we can do to prevent that.
+ */
+static int enable_monitor_mode(void)
+{
+        u32 dscr;
+        int ret = 0;
+
+        ARM_DBG_READ(c1, 0, dscr);
+
+        /* Ensure that halting mode is disabled. */
+        if (WARN_ONCE(dscr & ARM_DSCR_HDBGEN,
+                        "halting debug mode enabled. Unable to access hardware resources.\n")) {
+                ret = -EPERM;
+                goto out;
+        }
+
+        /* If monitor mode is already enabled, just return. */
+        if (dscr & ARM_DSCR_MDBGEN)
+                goto out;
+
+        /* Write to the corresponding DSCR. */
+        switch (get_debug_arch()) {
+        case ARM_DEBUG_ARCH_V6:
+        case ARM_DEBUG_ARCH_V6_1:
+                ARM_DBG_WRITE(c1, 0, (dscr | ARM_DSCR_MDBGEN));
+                break;
+        case ARM_DEBUG_ARCH_V7_ECP14:
+                ARM_DBG_WRITE(c2, 2, (dscr | ARM_DSCR_MDBGEN));
+                break;
+        default:
+                ret = -ENODEV;
+                goto out;
+        }
+
+        /* Check that the write made it through. */
+        ARM_DBG_READ(c1, 0, dscr);
+        if (!(dscr & ARM_DSCR_MDBGEN))
+                ret = -EPERM;
+
+out:
+        return ret;
+}
+
+int hw_breakpoint_slots(int type)
+{
+        if (!debug_arch_supported())
+                return 0;
+
+        /*
+         * We can be called early, so don't rely on
+         * our static variables being initialised.
+         */
+        switch (type) {
+        case TYPE_INST:
+                return get_num_brps();
+        case TYPE_DATA:
+                return get_num_wrps();
+        default:
+                pr_warning("unknown slot type: %d\n", type);
+                return 0;
+        }
+}
+
+/*
+ * Check if 8-bit byte-address select is available.
+ * This clobbers WRP 0.
+ */
+static u8 get_max_wp_len(void)
+{
+        u32 ctrl_reg;
+        struct arch_hw_breakpoint_ctrl ctrl;
+        u8 size = 4;
+
+        if (debug_arch < ARM_DEBUG_ARCH_V7_ECP14)
+                goto out;
+
+        memset(&ctrl, 0, sizeof(ctrl));
+        ctrl.len = ARM_BREAKPOINT_LEN_8;
+        ctrl_reg = encode_ctrl_reg(ctrl);
+
+        write_wb_reg(ARM_BASE_WVR, 0);
+        write_wb_reg(ARM_BASE_WCR, ctrl_reg);
+        if ((read_wb_reg(ARM_BASE_WCR) & ctrl_reg) == ctrl_reg)
+                size = 8;
+
+out:
+        return size;
+}
+
+u8 arch_get_max_wp_len(void)
+{
+        return max_watchpoint_len;
+}
+
+/*
+ * Install a perf counter breakpoint.
+ */
+int arch_install_hw_breakpoint(struct perf_event *bp)
+{
+        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+        struct perf_event **slot, **slots;
+        int i, max_slots, ctrl_base, val_base, ret = 0;
+        u32 addr, ctrl;
+
+        /* Ensure that we are in monitor mode and halting mode is disabled. */
+        ret = enable_monitor_mode();
+        if (ret)
+                goto out;
+
+        addr = info->address;
+        ctrl = encode_ctrl_reg(info->ctrl) | 0x1;
+
+        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
+                /* Breakpoint */
+                ctrl_base = ARM_BASE_BCR;
+                val_base = ARM_BASE_BVR;
+                slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
+                max_slots = core_num_brps;
+                if (info->step_ctrl.enabled) {
+                        /* Override the breakpoint data with the step data. */
+                        addr = info->trigger & ~0x3;
+                        ctrl = encode_ctrl_reg(info->step_ctrl);
+                }
+        } else {
+                /* Watchpoint */
+                if (info->step_ctrl.enabled) {
+                        /* Install into the reserved breakpoint region. */
+                        ctrl_base = ARM_BASE_BCR + core_num_brps;
+                        val_base = ARM_BASE_BVR + core_num_brps;
+                        /* Override the watchpoint data with the step data. */
+                        addr = info->trigger & ~0x3;
+                        ctrl = encode_ctrl_reg(info->step_ctrl);
+                } else {
+                        ctrl_base = ARM_BASE_WCR;
+                        val_base = ARM_BASE_WVR;
+                }
+                slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
+                max_slots = core_num_wrps;
+        }
+
+        for (i = 0; i < max_slots; ++i) {
+                slot = &slots[i];
+
+                if (!*slot) {
+                        *slot = bp;
+                        break;
+                }
+        }
+
+        if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot\n")) {
+                ret = -EBUSY;
+                goto out;
+        }
+
+        /* Setup the address register. */
+        write_wb_reg(val_base + i, addr);
+
+        /* Setup the control register. */
+        write_wb_reg(ctrl_base + i, ctrl);
+
+out:
+        return ret;
+}
+
+void arch_uninstall_hw_breakpoint(struct perf_event *bp)
+{
+        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+        struct perf_event **slot, **slots;
+        int i, max_slots, base;
+
+        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE) {
+                /* Breakpoint */
+                base = ARM_BASE_BCR;
+                slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
+                max_slots = core_num_brps;
+        } else {
+                /* Watchpoint */
+                if (info->step_ctrl.enabled)
+                        base = ARM_BASE_BCR + core_num_brps;
+                else
+                        base = ARM_BASE_WCR;
+                slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
+                max_slots = core_num_wrps;
+        }
+
+        /* Remove the breakpoint. */
+        for (i = 0; i < max_slots; ++i) {
+                slot = &slots[i];
+
+                if (*slot == bp) {
+                        *slot = NULL;
+                        break;
+                }
+        }
+
+        if (WARN_ONCE(i == max_slots, "Can't find any breakpoint slot\n"))
+                return;
+
+        /* Reset the control register. */
+        write_wb_reg(base + i, 0);
+}
+
+static int get_hbp_len(u8 hbp_len)
+{
+        unsigned int len_in_bytes = 0;
+
+        switch (hbp_len) {
+        case ARM_BREAKPOINT_LEN_1:
+                len_in_bytes = 1;
+                break;
+        case ARM_BREAKPOINT_LEN_2:
+                len_in_bytes = 2;
+                break;
+        case ARM_BREAKPOINT_LEN_4:
+                len_in_bytes = 4;
+                break;
+        case ARM_BREAKPOINT_LEN_8:
+                len_in_bytes = 8;
+                break;
+        }
+
+        return len_in_bytes;
+}
+
+/*
+ * Check whether bp virtual address is in kernel space.
+ */
+int arch_check_bp_in_kernelspace(struct perf_event *bp)
+{
+        unsigned int len;
+        unsigned long va;
+        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+
+        va = info->address;
+        len = get_hbp_len(info->ctrl.len);
+
+        return (va >= TASK_SIZE) && ((va + len - 1) >= TASK_SIZE);
+}
+
+/*
+ * Extract generic type and length encodings from an arch_hw_breakpoint_ctrl.
+ * Hopefully this will disappear when ptrace can bypass the conversion
+ * to generic breakpoint descriptions.
+ */
+int arch_bp_generic_fields(struct arch_hw_breakpoint_ctrl ctrl,
+                           int *gen_len, int *gen_type)
+{
+        /* Type */
+        switch (ctrl.type) {
+        case ARM_BREAKPOINT_EXECUTE:
+                *gen_type = HW_BREAKPOINT_X;
+                break;
+        case ARM_BREAKPOINT_LOAD:
+                *gen_type = HW_BREAKPOINT_R;
+                break;
+        case ARM_BREAKPOINT_STORE:
+                *gen_type = HW_BREAKPOINT_W;
+                break;
+        case ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE:
+                *gen_type = HW_BREAKPOINT_RW;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        /* Len */
+        switch (ctrl.len) {
+        case ARM_BREAKPOINT_LEN_1:
+                *gen_len = HW_BREAKPOINT_LEN_1;
+                break;
+        case ARM_BREAKPOINT_LEN_2:
+                *gen_len = HW_BREAKPOINT_LEN_2;
+                break;
+        case ARM_BREAKPOINT_LEN_4:
+                *gen_len = HW_BREAKPOINT_LEN_4;
+                break;
+        case ARM_BREAKPOINT_LEN_8:
+                *gen_len = HW_BREAKPOINT_LEN_8;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+/*
+ * Construct an arch_hw_breakpoint from a perf_event.
+ */
+static int arch_build_bp_info(struct perf_event *bp)
+{
+        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+
+        /* Type */
+        switch (bp->attr.bp_type) {
+        case HW_BREAKPOINT_X:
+                info->ctrl.type = ARM_BREAKPOINT_EXECUTE;
+                break;
+        case HW_BREAKPOINT_R:
+                info->ctrl.type = ARM_BREAKPOINT_LOAD;
+                break;
+        case HW_BREAKPOINT_W:
+                info->ctrl.type = ARM_BREAKPOINT_STORE;
+                break;
+        case HW_BREAKPOINT_RW:
+                info->ctrl.type = ARM_BREAKPOINT_LOAD | ARM_BREAKPOINT_STORE;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        /* Len */
+        switch (bp->attr.bp_len) {
+        case HW_BREAKPOINT_LEN_1:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_1;
+                break;
+        case HW_BREAKPOINT_LEN_2:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_2;
+                break;
+        case HW_BREAKPOINT_LEN_4:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_4;
+                break;
+        case HW_BREAKPOINT_LEN_8:
+                info->ctrl.len = ARM_BREAKPOINT_LEN_8;
+                if ((info->ctrl.type != ARM_BREAKPOINT_EXECUTE)
+                        && max_watchpoint_len >= 8)
+                        break;
+        default:
+                return -EINVAL;
+        }
+
+        /*
+         * Breakpoints must be of length 2 (thumb) or 4 (ARM) bytes.
+         * Watchpoints can be of length 1, 2, 4 or 8 bytes if supported
+         * by the hardware and must be aligned to the appropriate number of
+         * bytes.
+         */
+        if (info->ctrl.type == ARM_BREAKPOINT_EXECUTE &&
+            info->ctrl.len != ARM_BREAKPOINT_LEN_2 &&
+            info->ctrl.len != ARM_BREAKPOINT_LEN_4)
+                return -EINVAL;
+
+        /* Address */
+        info->address = bp->attr.bp_addr;
+
+        /* Privilege */
+        info->ctrl.privilege = ARM_BREAKPOINT_USER;
+        if (arch_check_bp_in_kernelspace(bp))
+                info->ctrl.privilege |= ARM_BREAKPOINT_PRIV;
+
+        /* Enabled? */
+        info->ctrl.enabled = !bp->attr.disabled;
+
+        /* Mismatch */
+        info->ctrl.mismatch = 0;
+
+        return 0;
+}
+
+/*
+ * Validate the arch-specific HW Breakpoint register settings.
+ */
+int arch_validate_hwbkpt_settings(struct perf_event *bp)
+{
+        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+        int ret = 0;
+        u32 offset, alignment_mask = 0x3;
+
+        /* Build the arch_hw_breakpoint. */
+        ret = arch_build_bp_info(bp);
+        if (ret)
+                goto out;
+
+        /* Check address alignment. */
+        if (info->ctrl.len == ARM_BREAKPOINT_LEN_8)
+                alignment_mask = 0x7;
+        offset = info->address & alignment_mask;
+        switch (offset) {
+        case 0:
+                /* Aligned */
+                break;
+        case 1:
+                /* Allow single byte watchpoint. */
+                if (info->ctrl.len == ARM_BREAKPOINT_LEN_1)
+                        break;
+        case 2:
+                /* Allow halfword watchpoints and breakpoints. */
+                if (info->ctrl.len == ARM_BREAKPOINT_LEN_2)
+                        break;
+        default:
+                ret = -EINVAL;
+                goto out;
+        }
+
+        info->address &= ~alignment_mask;
+        info->ctrl.len <<= offset;
+
+        /*
+         * Currently we rely on an overflow handler to take
+         * care of single-stepping the breakpoint when it fires.
+         * In the case of userspace breakpoints on a core with V7 debug,
+         * we can use the mismatch feature as a poor-man's hardware
+         * single-step, but this only works for per-task breakpoints.
+         */
+        if (WARN_ONCE(!bp->overflow_handler &&
+                (arch_check_bp_in_kernelspace(bp) || !core_has_mismatch_brps()
+                 || !bp->hw.bp_target),
+                        "overflow handler required but none found\n")) {
+                ret = -EINVAL;
+        }
+out:
+        return ret;
+}
+
+/*
+ * Enable/disable single-stepping over the breakpoint bp at address addr.
+ */
+static void enable_single_step(struct perf_event *bp, u32 addr)
+{
+        struct arch_hw_breakpoint *info = counter_arch_bp(bp);
+
+        arch_uninstall_hw_breakpoint(bp);
+        info->step_ctrl.mismatch  = 1;
+        info->step_ctrl.len       = ARM_BREAKPOINT_LEN_4;
+        info->step_ctrl.type      = ARM_BREAKPOINT_EXECUTE;
+        info->step_ctrl.privilege = info->ctrl.privilege;
+        info->step_ctrl.enabled   = 1;
+        info->trigger             = addr;
+        arch_install_hw_breakpoint(bp);
+}
+
+static void disable_single_step(struct perf_event *bp)
+{
+        arch_uninstall_hw_breakpoint(bp);
+        counter_arch_bp(bp)->step_ctrl.enabled = 0;
+        arch_install_hw_breakpoint(bp);
+}
+
+static void watchpoint_handler(unsigned long unknown, struct pt_regs *regs)
+{
+        int i;
+        struct perf_event *wp, **slots;
+        struct arch_hw_breakpoint *info;
+
+        slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
+
+        /* Without a disassembler, we can only handle 1 watchpoint. */
+        BUG_ON(core_num_wrps > 1);
+
+        for (i = 0; i < core_num_wrps; ++i) {
+                rcu_read_lock();
+
+                wp = slots[i];
+
+                if (wp == NULL) {
+                        rcu_read_unlock();
+                        continue;
+                }
+
+                /*
+                 * The DFAR is an unknown value. Since we only allow a
+                 * single watchpoint, we can set the trigger to the lowest
+                 * possible faulting address.
+                 */
+                info = counter_arch_bp(wp);
+                info->trigger = wp->attr.bp_addr;
+                pr_debug("watchpoint fired: address = 0x%x\n", info->trigger);
+                perf_bp_event(wp, regs);
+
+                /*
+                 * If no overflow handler is present, insert a temporary
+                 * mismatch breakpoint so we can single-step over the
+                 * watchpoint trigger.
+                 */
+                if (!wp->overflow_handler)
+                        enable_single_step(wp, instruction_pointer(regs));
+
+                rcu_read_unlock();
+        }
+}
+
+static void watchpoint_single_step_handler(unsigned long pc)
+{
+        int i;
+        struct perf_event *wp, **slots;
+        struct arch_hw_breakpoint *info;
+
+        slots = (struct perf_event **)__get_cpu_var(wp_on_reg);
+
+        for (i = 0; i < core_num_reserved_brps; ++i) {
+                rcu_read_lock();
+
+                wp = slots[i];
+
+                if (wp == NULL)
+                        goto unlock;
+
+                info = counter_arch_bp(wp);
+                if (!info->step_ctrl.enabled)
+                        goto unlock;
+
+                /*
+                 * Restore the original watchpoint if we've completed the
+                 * single-step.
+                 */
+                if (info->trigger != pc)
+                        disable_single_step(wp);
+
+unlock:
+                rcu_read_unlock();
+        }
+}
+
+static void breakpoint_handler(unsigned long unknown, struct pt_regs *regs)
+{
+        int i;
+        u32 ctrl_reg, val, addr;
+        struct perf_event *bp, **slots;
+        struct arch_hw_breakpoint *info;
+        struct arch_hw_breakpoint_ctrl ctrl;
+
+        slots = (struct perf_event **)__get_cpu_var(bp_on_reg);
+
+        /* The exception entry code places the amended lr in the PC. */
+        addr = regs->ARM_pc;
+
+        /* Check the currently installed breakpoints first. */
+        for (i = 0; i < core_num_brps; ++i) {
+                rcu_read_lock();
+
+                bp = slots[i];
+
+                if (bp == NULL)
+                        goto unlock;
+
+                info = counter_arch_bp(bp);
+
+                /* Check if the breakpoint value matches. */
+                val = read_wb_reg(ARM_BASE_BVR + i);
+                if (val != (addr & ~0x3))
+                        goto mismatch;
+
+                /* Possible match, check the byte address select to confirm. */
+                ctrl_reg = read_wb_reg(ARM_BASE_BCR + i);
+                decode_ctrl_reg(ctrl_reg, &ctrl);
+                if ((1 << (addr & 0x3)) & ctrl.len) {
+                        info->trigger = addr;
+                        pr_debug("breakpoint fired: address = 0x%x\n", addr);
+                        perf_bp_event(bp, regs);
+                        if (!bp->overflow_handler)
+                                enable_single_step(bp, addr);
+                        goto unlock;
+                }
+
+mismatch:
+                /* If we're stepping a breakpoint, it can now be restored. */
+                if (info->step_ctrl.enabled)
+                        disable_single_step(bp);
+unlock:
+                rcu_read_unlock();
+        }
+
+        /* Handle any pending watchpoint single-step breakpoints. */
+        watchpoint_single_step_handler(addr);
+}
+
+/*
+ * Called from either the Data Abort Handler [watchpoint] or the
+ * Prefetch Abort Handler [breakpoint] with preemption disabled.
+ */
+static int hw_breakpoint_pending(unsigned long addr, unsigned int fsr,
+                                 struct pt_regs *regs)
+{
+        int ret = 0;
+        u32 dscr;
+
+        /* We must be called with preemption disabled. */
+        WARN_ON(preemptible());
+
+        /* We only handle watchpoints and hardware breakpoints. */
+        ARM_DBG_READ(c1, 0, dscr);
+
+        /* Perform perf callbacks. */
+        switch (ARM_DSCR_MOE(dscr)) {
+        case ARM_ENTRY_BREAKPOINT:
+                breakpoint_handler(addr, regs);
+                break;
+        case ARM_ENTRY_ASYNC_WATCHPOINT:
+                WARN(1, "Asynchronous watchpoint exception taken. Debugging results may be unreliable\n");
+        case ARM_ENTRY_SYNC_WATCHPOINT:
+                watchpoint_handler(addr, regs);
+                break;
+        default:
+                ret = 1; /* Unhandled fault. */
+        }
+
+        /*
+         * Re-enable preemption after it was disabled in the
+         * low-level exception handling code.
+         */
+        preempt_enable();
+
+        return ret;
+}
+
+/*
+ * One-time initialisation.
+ */
+static void reset_ctrl_regs(void *info)
+{
+        int i, cpu = smp_processor_id();
+        u32 dbg_power;
+        cpumask_t *cpumask = info;
+
+        /*
+         * v7 debug contains save and restore registers so that debug state
+         * can be maintained across low-power modes without leaving the debug
+         * logic powered up. It is IMPLEMENTATION DEFINED whether we can access
+         * the debug registers out of reset, so we must unlock the OS Lock
+         * Access Register to avoid taking undefined instruction exceptions
+         * later on.
+         */
+        if (debug_arch >= ARM_DEBUG_ARCH_V7_ECP14) {
+                /*
+                 * Ensure sticky power-down is clear (i.e. debug logic is
+                 * powered up).
+                 */
+                asm volatile("mrc p14, 0, %0, c1, c5, 4" : "=r" (dbg_power));
+                if ((dbg_power & 0x1) == 0) {
+                        pr_warning("CPU %d debug is powered down!\n", cpu);
+                        cpumask_or(cpumask, cpumask, cpumask_of(cpu));
+                        return;
+                }
+
+                /*
+                 * Unconditionally clear the lock by writing a value
+                 * other than 0xC5ACCE55 to the access register.
+                 */
+                asm volatile("mcr p14, 0, %0, c1, c0, 4" : : "r" (0));
+                isb();
+
+                /*
+                 * Clear any configured vector-catch events before
+                 * enabling monitor mode.
+                 */
+                asm volatile("mcr p14, 0, %0, c0, c7, 0" : : "r" (0));
+                isb();
+        }
+
+        if (enable_monitor_mode())
+                return;
+
+        /* We must also reset any reserved registers. */
+        for (i = 0; i < core_num_brps + core_num_reserved_brps; ++i) {
+                write_wb_reg(ARM_BASE_BCR + i, 0UL);
+                write_wb_reg(ARM_BASE_BVR + i, 0UL);
+        }
+
+        for (i = 0; i < core_num_wrps; ++i) {
+                write_wb_reg(ARM_BASE_WCR + i, 0UL);
+                write_wb_reg(ARM_BASE_WVR + i, 0UL);
+        }
+}
+
+static int __cpuinit dbg_reset_notify(struct notifier_block *self,
+                                      unsigned long action, void *cpu)
+{
+        if (action == CPU_ONLINE)
+                smp_call_function_single((int)cpu, reset_ctrl_regs, NULL, 1);
+        return NOTIFY_OK;
+}
+
+static struct notifier_block __cpuinitdata dbg_reset_nb = {
+        .notifier_call = dbg_reset_notify,
+};
+
+static int __init arch_hw_breakpoint_init(void)
+{
+        u32 dscr;
+        cpumask_t cpumask = { CPU_BITS_NONE };
+
+        debug_arch = get_debug_arch();
+
+        if (!debug_arch_supported()) {
+                pr_info("debug architecture 0x%x unsupported.\n", debug_arch);
+                return 0;
+        }
+
+        /* Determine how many BRPs/WRPs are available. */
+        core_num_brps = get_num_brps();
+        core_num_reserved_brps = get_num_reserved_brps();
+        core_num_wrps = get_num_wrps();
+
+        pr_info("found %d breakpoint and %d watchpoint registers.\n",
+                core_num_brps + core_num_reserved_brps, core_num_wrps);
+
+        if (core_num_reserved_brps)
+                pr_info("%d breakpoint(s) reserved for watchpoint "
+                                "single-step.\n", core_num_reserved_brps);
+
+        /*
+         * Reset the breakpoint resources. We assume that a halting
+         * debugger will leave the world in a nice state for us.
+         */
+        on_each_cpu(reset_ctrl_regs, &cpumask, 1);
+        if (!cpumask_empty(&cpumask)) {
+                core_num_brps = 0;
+                core_num_reserved_brps = 0;
+                core_num_wrps = 0;
+                return 0;
+        }
+
+        ARM_DBG_READ(c1, 0, dscr);
+        if (dscr & ARM_DSCR_HDBGEN) {
+                max_watchpoint_len = 4;
+                pr_warning("halting debug mode enabled. Assuming maximum watchpoint size of %u bytes.\n",
+                           max_watchpoint_len);
+        } else {
+                /* Work out the maximum supported watchpoint length. */
+                max_watchpoint_len = get_max_wp_len();
+                pr_info("maximum watchpoint size is %u bytes.\n",
+                                max_watchpoint_len);
+        }
+
+        /* Register debug fault handler. */
+        hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT,
+                        "watchpoint debug exception");
+        hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT,
+                        "breakpoint debug exception");
+
+        /* Register hotplug notifier. */
+        register_cpu_notifier(&dbg_reset_nb);
+        return 0;
+}
+arch_initcall(arch_hw_breakpoint_init);
+
+void hw_breakpoint_pmu_read(struct perf_event *bp)
+{
+}
+
+/*
+ * Dummy function to register with die_notifier.
+ */
+int hw_breakpoint_exceptions_notify(struct notifier_block *unused,
+                                        unsigned long val, void *data)
+{
+        return NOTIFY_DONE;
+}
diff --git a/arch/arm/kernel/irq.c b/arch/arm/kernel/irq.c
index c0d5c3b3a760..83bbad03fcc6 100644
--- a/arch/arm/kernel/irq.c
+++ b/arch/arm/kernel/irq.c
@@ -35,8 +35,10 @@
 #include <linux/list.h>
 #include <linux/kallsyms.h>
 #include <linux/proc_fs.h>
+#include <linux/ftrace.h>
 
 #include <asm/system.h>
+#include <asm/mach/arch.h>
 #include <asm/mach/irq.h>
 #include <asm/mach/time.h>
 
@@ -47,56 +49,20 @@
 #define irq_finish(irq) do { } while (0)
 #endif
 
-unsigned int arch_nr_irqs;
-void (*init_arch_irq)(void) __initdata = NULL;
 unsigned long irq_err_count;
 
-int show_interrupts(struct seq_file *p, void *v)
+int arch_show_interrupts(struct seq_file *p, int prec)
 {
-        int i = *(loff_t *) v, cpu;
-        struct irq_desc *desc;
-        struct irqaction * action;
-        unsigned long flags;
-
-        if (i == 0) {
-                char cpuname[12];
-
-                seq_printf(p, "    ");
-                for_each_present_cpu(cpu) {
-                        sprintf(cpuname, "CPU%d", cpu);
-                        seq_printf(p, " %10s", cpuname);
-                }
-                seq_putc(p, '\n');
-        }
-
-        if (i < nr_irqs) {
-                desc = irq_to_desc(i);
-                raw_spin_lock_irqsave(&desc->lock, flags);
-                action = desc->action;
-                if (!action)
-                        goto unlock;
-
-                seq_printf(p, "%3d: ", i);
-                for_each_present_cpu(cpu)
-                        seq_printf(p, "%10u ", kstat_irqs_cpu(i, cpu));
-                seq_printf(p, " %10s", desc->chip->name ? : "-");
-                seq_printf(p, "  %s", action->name);
-                for (action = action->next; action; action = action->next)
-                        seq_printf(p, ", %s", action->name);
-
-                seq_putc(p, '\n');
-unlock:
-                raw_spin_unlock_irqrestore(&desc->lock, flags);
-        } else if (i == nr_irqs) {
 #ifdef CONFIG_FIQ
-                show_fiq_list(p, v);
+        show_fiq_list(p, prec);
 #endif
 #ifdef CONFIG_SMP
-                show_ipi_list(p);
-                show_local_irqs(p);
+        show_ipi_list(p, prec);
 #endif
-                seq_printf(p, "Err: %10lu\n", irq_err_count);
-        }
+#ifdef CONFIG_LOCAL_TIMERS
+        show_local_irqs(p, prec);
+#endif
+        seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
         return 0;
 }
 
@@ -105,7 +71,8 @@ unlock:
  * come via this function.  Instead, they should provide their
  * own 'handler'
  */
-asmlinkage void __exception asm_do_IRQ(unsigned int irq, struct pt_regs *regs)
+asmlinkage void __exception_irq_entry
+asm_do_IRQ(unsigned int irq, struct pt_regs *regs)
 {
         struct pt_regs *old_regs = set_irq_regs(regs);
 
@@ -132,56 +99,53 @@ asmlinkage void __exception asm_do_IRQ(unsigned int irq, struct pt_regs *regs)
 
 void set_irq_flags(unsigned int irq, unsigned int iflags)
 {
-        struct irq_desc *desc;
-        unsigned long flags;
+        unsigned long clr = 0, set = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN;
 
         if (irq >= nr_irqs) {
                 printk(KERN_ERR "Trying to set irq flags for IRQ%d\n", irq);
                 return;
         }
 
-        desc = irq_to_desc(irq);
-        raw_spin_lock_irqsave(&desc->lock, flags);
-        desc->status |= IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN;
         if (iflags & IRQF_VALID)
-                desc->status &= ~IRQ_NOREQUEST;
+                clr |= IRQ_NOREQUEST;
         if (iflags & IRQF_PROBE)
-                desc->status &= ~IRQ_NOPROBE;
+                clr |= IRQ_NOPROBE;
         if (!(iflags & IRQF_NOAUTOEN))
-                desc->status &= ~IRQ_NOAUTOEN;
-        raw_spin_unlock_irqrestore(&desc->lock, flags);
+                clr |= IRQ_NOAUTOEN;
+        /* Order is clear bits in "clr" then set bits in "set" */
+        irq_modify_status(irq, clr, set & ~clr);
 }
 
 void __init init_IRQ(void)
 {
-        struct irq_desc *desc;
-        int irq;
-
-        for (irq = 0; irq < nr_irqs; irq++) {
-                desc = irq_to_desc_alloc_node(irq, 0);
-                desc->status |= IRQ_NOREQUEST | IRQ_NOPROBE;
-        }
-
-        init_arch_irq();
+        machine_desc->init_irq();
 }
 
 #ifdef CONFIG_SPARSE_IRQ
 int __init arch_probe_nr_irqs(void)
 {
-        nr_irqs = arch_nr_irqs ? arch_nr_irqs : NR_IRQS;
-        return 0;
+        nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS;
+        return nr_irqs;
 }
 #endif
 
 #ifdef CONFIG_HOTPLUG_CPU
 
-static void route_irq(struct irq_desc *desc, unsigned int irq, unsigned int cpu)
+static bool migrate_one_irq(struct irq_data *d)
 {
-        pr_debug("IRQ%u: moving from cpu%u to cpu%u\n", irq, desc->node, cpu);
+        unsigned int cpu = cpumask_any_and(d->affinity, cpu_online_mask);
+        bool ret = false;
+
+        if (cpu >= nr_cpu_ids) {
+                cpu = cpumask_any(cpu_online_mask);
+                ret = true;
+        }
 
-        raw_spin_lock_irq(&desc->lock);
-        desc->chip->set_affinity(irq, cpumask_of(cpu));
-        raw_spin_unlock_irq(&desc->lock);
+        pr_debug("IRQ%u: moving from cpu%u to cpu%u\n", d->irq, d->node, cpu);
+
+        d->chip->irq_set_affinity(d, cpumask_of(cpu), true);
+
+        return ret;
 }
 
 /*
@@ -193,23 +157,30 @@ void migrate_irqs(void)
 {
         unsigned int i, cpu = smp_processor_id();
         struct irq_desc *desc;
+        unsigned long flags;
+
+        local_irq_save(flags);
 
         for_each_irq_desc(i, desc) {
-                if (desc->node == cpu) {
-                        unsigned int newcpu = cpumask_any_and(desc->affinity,
-                                                              cpu_online_mask);
-                        if (newcpu >= nr_cpu_ids) {
-                                if (printk_ratelimit())
-                                        printk(KERN_INFO "IRQ%u no longer affine to CPU%u\n",
-                                               i, cpu);
-
-                                cpumask_setall(desc->affinity);
-                                newcpu = cpumask_any_and(desc->affinity,
-                                                         cpu_online_mask);
-                        }
-
-                        route_irq(desc, i, newcpu);
-                }
+                struct irq_data *d = &desc->irq_data;
+                bool affinity_broken = false;
+
+                raw_spin_lock(&desc->lock);
+                do {
+                        if (desc->action == NULL)
+                                break;
+
+                        if (d->node != cpu)
+                                break;
+
+                        affinity_broken = migrate_one_irq(d);
+                } while (0);
+                raw_spin_unlock(&desc->lock);
+
+                if (affinity_broken && printk_ratelimit())
+                        pr_warning("IRQ%u no longer affine to CPU%u\n", i, cpu);
         }
+
+        local_irq_restore(flags);
 }
 #endif /* CONFIG_HOTPLUG_CPU */
diff --git a/arch/arm/kernel/iwmmxt.S b/arch/arm/kernel/iwmmxt.S
index b63b528f22a6..7fa3bb0d2397 100644
--- a/arch/arm/kernel/iwmmxt.S
+++ b/arch/arm/kernel/iwmmxt.S
@@ -19,6 +19,14 @@
 #include <asm/thread_info.h>
 #include <asm/asm-offsets.h>
 
+#if defined(CONFIG_CPU_PJ4)
+#define PJ4(code...)            code
+#define XSC(code...)
+#else
+#define PJ4(code...)
+#define XSC(code...)            code
+#endif
+
 #define MMX_WR0                 (0x00)
 #define MMX_WR1                 (0x08)
 #define MMX_WR2                 (0x10)
@@ -58,11 +66,17 @@
 
 ENTRY(iwmmxt_task_enable)
 
-        mrc     p15, 0, r2, c15, c1, 0
-        tst     r2, #0x3                        @ CP0 and CP1 accessible?
+        XSC(mrc p15, 0, r2, c15, c1, 0)
+        PJ4(mrc p15, 0, r2, c1, c0, 2)
+        @ CP0 and CP1 accessible?
+        XSC(tst r2, #0x3)
+        PJ4(tst r2, #0xf)
         movne   pc, lr                          @ if so no business here
-        orr     r2, r2, #0x3                    @ enable access to CP0 and CP1
-        mcr     p15, 0, r2, c15, c1, 0
+        @ enable access to CP0 and CP1
+        XSC(orr r2, r2, #0x3)
+        XSC(mcr p15, 0, r2, c15, c1, 0)
+        PJ4(orr r2, r2, #0xf)
+        PJ4(mcr p15, 0, r2, c1, c0, 2)
 
         ldr     r3, =concan_owner
         add     r0, r10, #TI_IWMMXT_STATE       @ get task Concan save area
@@ -179,17 +193,26 @@ ENTRY(iwmmxt_task_disable)
         teqne   r1, r2                          @ or specified one?
         bne     1f                              @ no: quit
 
-        mrc     p15, 0, r4, c15, c1, 0
-        orr     r4, r4, #0x3                    @ enable access to CP0 and CP1
-        mcr     p15, 0, r4, c15, c1, 0
+        @ enable access to CP0 and CP1
+        XSC(mrc p15, 0, r4, c15, c1, 0)
+        XSC(orr r4, r4, #0xf)
+        XSC(mcr p15, 0, r4, c15, c1, 0)
+        PJ4(mrc p15, 0, r4, c1, c0, 2)
+        PJ4(orr r4, r4, #0x3)
+        PJ4(mcr p15, 0, r4, c1, c0, 2)
+
         mov     r0, #0                          @ nothing to load
         str     r0, [r3]                        @ no more current owner
         mrc     p15, 0, r2, c2, c0, 0
         mov     r2, r2                          @ cpwait
         bl      concan_save
 
-        bic     r4, r4, #0x3                    @ disable access to CP0 and CP1
-        mcr     p15, 0, r4, c15, c1, 0
+        @ disable access to CP0 and CP1
+        XSC(bic r4, r4, #0x3)
+        XSC(mcr p15, 0, r4, c15, c1, 0)
+        PJ4(bic r4, r4, #0xf)
+        PJ4(mcr p15, 0, r4, c1, c0, 2)
+
         mrc     p15, 0, r2, c2, c0, 0
         mov     r2, r2                          @ cpwait
 
@@ -277,8 +300,11 @@ ENTRY(iwmmxt_task_restore)
  */
 ENTRY(iwmmxt_task_switch)
 
-        mrc     p15, 0, r1, c15, c1, 0
-        tst     r1, #0x3                        @ CP0 and CP1 accessible?
+        XSC(mrc p15, 0, r1, c15, c1, 0)
+        PJ4(mrc p15, 0, r1, c1, c0, 2)
+        @ CP0 and CP1 accessible?
+        XSC(tst r1, #0x3)
+        PJ4(tst r1, #0xf)
         bne     1f                              @ yes: block them for next task
 
         ldr     r2, =concan_owner
@@ -287,8 +313,11 @@ ENTRY(iwmmxt_task_switch)
         teq     r2, r3                          @ next task owns it?
         movne   pc, lr                          @ no: leave Concan disabled
 
-1:      eor     r1, r1, #3                      @ flip Concan access
-        mcr     p15, 0, r1, c15, c1, 0
+1:      @ flip Conan access
+        XSC(eor r1, r1, #0x3)
+        XSC(mcr p15, 0, r1, c15, c1, 0)
+        PJ4(eor r1, r1, #0xf)
+        PJ4(mcr p15, 0, r1, c1, c0, 2)
 
         mrc     p15, 0, r1, c2, c0, 0
         sub     pc, lr, r1, lsr #32             @ cpwait and return
diff --git a/arch/arm/kernel/kgdb.c b/arch/arm/kernel/kgdb.c
index d6e8b4d2e60d..778c2f7024ff 100644
--- a/arch/arm/kernel/kgdb.c
+++ b/arch/arm/kernel/kgdb.c
@@ -79,7 +79,7 @@ sleeping_thread_to_gdb_regs(unsigned long *gdb_regs, struct task_struct *task)
                 return;
 
         /* Initialize to zero */
-        for (regno = 0; regno < DBG_MAX_REG_NUM; regno++)
+        for (regno = 0; regno < GDB_MAX_REGS; regno++)
                 gdb_regs[regno] = 0;
 
         /* Otherwise, we have only some registers from switch_to() */
diff --git a/arch/arm/kernel/kprobes-decode.c b/arch/arm/kernel/kprobes-decode.c
index 2c1f0050c9c4..15eeff6aea0e 100644
--- a/arch/arm/kernel/kprobes-decode.c
+++ b/arch/arm/kernel/kprobes-decode.c
@@ -34,9 +34,6 @@
  *
  *   *) If the PC is written to by the instruction, the
  *      instruction must be fully simulated in software.
- *      If it is a conditional instruction, the handler
- *      will use insn[0] to copy its condition code to
- *      set r0 to 1 and insn[1] to "mov pc, lr" to return.
  *
  *   *) Otherwise, a modified form of the instruction is
  *      directly executed.  Its handler calls the
@@ -68,13 +65,17 @@
 
 #define branch_displacement(insn) sign_extend(((insn) & 0xffffff) << 2, 25)
 
+#define is_r15(insn, bitpos) (((insn) & (0xf << bitpos)) == (0xf << bitpos))
+
+/*
+ * Test if load/store instructions writeback the address register.
+ * if P (bit 24) == 0 or W (bit 21) == 1
+ */
+#define is_writeback(insn) ((insn ^ 0x01000000) & 0x01200000)
+
 #define PSR_fs  (PSR_f|PSR_s)
 
 #define KPROBE_RETURN_INSTRUCTION       0xe1a0f00e      /* mov pc, lr */
-#define SET_R0_TRUE_INSTRUCTION         0xe3a00001      /* mov  r0, #1 */
-
-#define truecc_insn(insn)       (((insn) & 0xf0000000) | \
-                                 (SET_R0_TRUE_INSTRUCTION & 0x0fffffff))
 
 typedef long (insn_0arg_fn_t)(void);
 typedef long (insn_1arg_fn_t)(long);
@@ -419,14 +420,10 @@ insnslot_llret_4arg_rwflags(long r0, long r1, long r2, long r3, long *cpsr,
 
 static void __kprobes simulate_bbl(struct kprobe *p, struct pt_regs *regs)
 {
-        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
         long iaddr = (long)p->addr;
         int disp  = branch_displacement(insn);
 
-        if (!insnslot_1arg_rflags(0, regs->ARM_cpsr, i_fn))
-                return;
-
         if (insn & (1 << 24))
                 regs->ARM_lr = iaddr + 4;
 
@@ -446,14 +443,10 @@ static void __kprobes simulate_blx1(struct kprobe *p, struct pt_regs *regs)
 
 static void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs)
 {
-        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
         int rm = insn & 0xf;
         long rmv = regs->uregs[rm];
 
-        if (!insnslot_1arg_rflags(0, regs->ARM_cpsr, i_fn))
-                return;
-
         if (insn & (1 << 5))
                 regs->ARM_lr = (long)p->addr + 4;
 
@@ -463,9 +456,16 @@ static void __kprobes simulate_blx2bx(struct kprobe *p, struct pt_regs *regs)
                 regs->ARM_cpsr |= PSR_T_BIT;
 }
 
+static void __kprobes simulate_mrs(struct kprobe *p, struct pt_regs *regs)
+{
+        kprobe_opcode_t insn = p->opcode;
+        int rd = (insn >> 12) & 0xf;
+        unsigned long mask = 0xf8ff03df; /* Mask out execution state */
+        regs->uregs[rd] = regs->ARM_cpsr & mask;
+}
+
 static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs)
 {
-        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
         int rn = (insn >> 16) & 0xf;
         int lbit = insn & (1 << 20);
@@ -476,9 +476,6 @@ static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs)
         int reg_bit_vector;
         int reg_count;
 
-        if (!insnslot_1arg_rflags(0, regs->ARM_cpsr, i_fn))
-                return;
-
         reg_count = 0;
         reg_bit_vector = insn & 0xffff;
         while (reg_bit_vector) {
@@ -510,11 +507,6 @@ static void __kprobes simulate_ldm1stm1(struct kprobe *p, struct pt_regs *regs)
 
 static void __kprobes simulate_stm1_pc(struct kprobe *p, struct pt_regs *regs)
 {
-        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
-
-        if (!insnslot_1arg_rflags(0, regs->ARM_cpsr, i_fn))
-                return;
-
         regs->ARM_pc = (long)p->addr + str_pc_offset;
         simulate_ldm1stm1(p, regs);
         regs->ARM_pc = (long)p->addr + 4;
@@ -525,24 +517,16 @@ static void __kprobes simulate_mov_ipsp(struct kprobe *p, struct pt_regs *regs)
         regs->uregs[12] = regs->uregs[13];
 }
 
-static void __kprobes emulate_ldcstc(struct kprobe *p, struct pt_regs *regs)
-{
-        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
-        kprobe_opcode_t insn = p->opcode;
-        int rn = (insn >> 16) & 0xf;
-        long rnv = regs->uregs[rn];
-
-        /* Save Rn in case of writeback. */
-        regs->uregs[rn] = insnslot_1arg_rflags(rnv, regs->ARM_cpsr, i_fn);
-}
-
 static void __kprobes emulate_ldrd(struct kprobe *p, struct pt_regs *regs)
 {
         insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
+        long ppc = (long)p->addr + 8;
         int rd = (insn >> 12) & 0xf;
         int rn = (insn >> 16) & 0xf;
         int rm = insn & 0xf;  /* rm may be invalid, don't care. */
+        long rmv = (rm == 15) ? ppc : regs->uregs[rm];
+        long rnv = (rn == 15) ? ppc : regs->uregs[rn];
 
         /* Not following the C calling convention here, so need asm(). */
         __asm__ __volatile__ (
@@ -554,29 +538,36 @@ static void __kprobes emulate_ldrd(struct kprobe *p, struct pt_regs *regs)
                 "str    r0, %[rn]       \n\t"   /* in case of writeback */
                 "str    r2, %[rd0]      \n\t"
                 "str    r3, %[rd1]      \n\t"
-                : [rn]  "+m" (regs->uregs[rn]),
+                : [rn]  "+m" (rnv),
                   [rd0] "=m" (regs->uregs[rd]),
                   [rd1] "=m" (regs->uregs[rd+1])
-                : [rm]   "m" (regs->uregs[rm]),
+                : [rm]   "m" (rmv),
                   [cpsr] "r" (regs->ARM_cpsr),
                   [i_fn] "r" (i_fn)
                 : "r0", "r1", "r2", "r3", "lr", "cc"
         );
+        if (is_writeback(insn))
+                regs->uregs[rn] = rnv;
 }
 
 static void __kprobes emulate_strd(struct kprobe *p, struct pt_regs *regs)
 {
         insn_4arg_fn_t *i_fn = (insn_4arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
+        long ppc = (long)p->addr + 8;
         int rd = (insn >> 12) & 0xf;
         int rn = (insn >> 16) & 0xf;
         int rm  = insn & 0xf;
-        long rnv = regs->uregs[rn];
-        long rmv = regs->uregs[rm];  /* rm/rmv may be invalid, don't care. */
+        long rnv = (rn == 15) ? ppc : regs->uregs[rn];
+        /* rm/rmv may be invalid, don't care. */
+        long rmv = (rm == 15) ? ppc : regs->uregs[rm];
+        long rnv_wb;
 
-        regs->uregs[rn] = insnslot_4arg_rflags(rnv, rmv, regs->uregs[rd],
+        rnv_wb = insnslot_4arg_rflags(rnv, rmv, regs->uregs[rd],
                                                regs->uregs[rd+1],
                                                regs->ARM_cpsr, i_fn);
+        if (is_writeback(insn))
+                regs->uregs[rn] = rnv_wb;
 }
 
 static void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs)
@@ -594,7 +585,8 @@ static void __kprobes emulate_ldr(struct kprobe *p, struct pt_regs *regs)
         long cpsr = regs->ARM_cpsr;
 
         fnr.dr = insnslot_llret_3arg_rflags(rnv, 0, rmv, cpsr, i_fn);
-        regs->uregs[rn] = fnr.r0;  /* Save Rn in case of writeback. */
+        if (rn != 15)
+                regs->uregs[rn] = fnr.r0;  /* Save Rn in case of writeback. */
         rdv = fnr.r1;
 
         if (rd == 15) {
@@ -622,35 +614,11 @@ static void __kprobes emulate_str(struct kprobe *p, struct pt_regs *regs)
         long rdv = (rd == 15) ? iaddr + str_pc_offset : regs->uregs[rd];
         long rnv = (rn == 15) ? iaddr +  8 : regs->uregs[rn];
         long rmv = regs->uregs[rm];  /* rm/rmv may be invalid, don't care. */
+        long rnv_wb;
 
-        /* Save Rn in case of writeback. */
-        regs->uregs[rn] =
-                insnslot_3arg_rflags(rnv, rdv, rmv, regs->ARM_cpsr, i_fn);
-}
-
-static void __kprobes emulate_mrrc(struct kprobe *p, struct pt_regs *regs)
-{
-        insn_llret_0arg_fn_t *i_fn = (insn_llret_0arg_fn_t *)&p->ainsn.insn[0];
-        kprobe_opcode_t insn = p->opcode;
-        union reg_pair fnr;
-        int rd = (insn >> 12) & 0xf;
-        int rn = (insn >> 16) & 0xf;
-
-        fnr.dr = insnslot_llret_0arg_rflags(regs->ARM_cpsr, i_fn);
-        regs->uregs[rn] = fnr.r0;
-        regs->uregs[rd] = fnr.r1;
-}
-
-static void __kprobes emulate_mcrr(struct kprobe *p, struct pt_regs *regs)
-{
-        insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0];
-        kprobe_opcode_t insn = p->opcode;
-        int rd = (insn >> 12) & 0xf;
-        int rn = (insn >> 16) & 0xf;
-        long rnv = regs->uregs[rn];
-        long rdv = regs->uregs[rd];
-
-        insnslot_2arg_rflags(rnv, rdv, regs->ARM_cpsr, i_fn);
+        rnv_wb = insnslot_3arg_rflags(rnv, rdv, rmv, regs->ARM_cpsr, i_fn);
+        if (rn != 15)
+                regs->uregs[rn] = rnv_wb;  /* Save Rn in case of writeback. */
 }
 
 static void __kprobes emulate_sat(struct kprobe *p, struct pt_regs *regs)
@@ -686,32 +654,32 @@ static void __kprobes emulate_none(struct kprobe *p, struct pt_regs *regs)
         insnslot_0arg_rflags(regs->ARM_cpsr, i_fn);
 }
 
-static void __kprobes emulate_rd12(struct kprobe *p, struct pt_regs *regs)
+static void __kprobes emulate_nop(struct kprobe *p, struct pt_regs *regs)
 {
-        insn_0arg_fn_t *i_fn = (insn_0arg_fn_t *)&p->ainsn.insn[0];
-        kprobe_opcode_t insn = p->opcode;
-        int rd = (insn >> 12) & 0xf;
-
-        regs->uregs[rd] = insnslot_0arg_rflags(regs->ARM_cpsr, i_fn);
 }
 
-static void __kprobes emulate_ird12(struct kprobe *p, struct pt_regs *regs)
+static void __kprobes
+emulate_rd12_modify(struct kprobe *p, struct pt_regs *regs)
 {
         insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
-        int ird = (insn >> 12) & 0xf;
+        int rd = (insn >> 12) & 0xf;
+        long rdv = regs->uregs[rd];
 
-        insnslot_1arg_rflags(regs->uregs[ird], regs->ARM_cpsr, i_fn);
+        regs->uregs[rd] = insnslot_1arg_rflags(rdv, regs->ARM_cpsr, i_fn);
 }
 
-static void __kprobes emulate_rn16(struct kprobe *p, struct pt_regs *regs)
+static void __kprobes
+emulate_rd12rn0_modify(struct kprobe *p, struct pt_regs *regs)
 {
-        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
+        insn_2arg_fn_t *i_fn = (insn_2arg_fn_t *)&p->ainsn.insn[0];
         kprobe_opcode_t insn = p->opcode;
-        int rn = (insn >> 16) & 0xf;
+        int rd = (insn >> 12) & 0xf;
+        int rn = insn & 0xf;
+        long rdv = regs->uregs[rd];
         long rnv = regs->uregs[rn];
 
-        insnslot_1arg_rflags(rnv, regs->ARM_cpsr, i_fn);
+        regs->uregs[rd] = insnslot_2arg_rflags(rdv, rnv, regs->ARM_cpsr, i_fn);
 }
 
 static void __kprobes emulate_rd12rm0(struct kprobe *p, struct pt_regs *regs)
@@ -817,6 +785,17 @@ emulate_alu_imm_rwflags(struct kprobe *p, struct pt_regs *regs)
 }
 
 static void __kprobes
+emulate_alu_tests_imm(struct kprobe *p, struct pt_regs *regs)
+{
+        insn_1arg_fn_t *i_fn = (insn_1arg_fn_t *)&p->ainsn.insn[0];
+        kprobe_opcode_t insn = p->opcode;
+        int rn = (insn >> 16) & 0xf;
+        long rnv = (rn == 15) ? (long)p->addr + 8 : regs->uregs[rn];
+
+        insnslot_1arg_rwflags(rnv, &regs->ARM_cpsr, i_fn);
+}
+
+static void __kprobes
 emulate_alu_rflags(struct kprobe *p, struct pt_regs *regs)
 {
         insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0];
@@ -852,14 +831,34 @@ emulate_alu_rwflags(struct kprobe *p, struct pt_regs *regs)
                 insnslot_3arg_rwflags(rnv, rmv, rsv, &regs->ARM_cpsr, i_fn);
 }
 
+static void __kprobes
+emulate_alu_tests(struct kprobe *p, struct pt_regs *regs)
+{
+        insn_3arg_fn_t *i_fn = (insn_3arg_fn_t *)&p->ainsn.insn[0];
+        kprobe_opcode_t insn = p->opcode;
+        long ppc = (long)p->addr + 8;
+        int rn = (insn >> 16) & 0xf;
+        int rs = (insn >> 8) & 0xf;     /* rs/rsv may be invalid, don't care. */
+        int rm = insn & 0xf;
+        long rnv = (rn == 15) ? ppc : regs->uregs[rn];
+        long rmv = (rm == 15) ? ppc : regs->uregs[rm];
+        long rsv = regs->uregs[rs];
+
+        insnslot_3arg_rwflags(rnv, rmv, rsv, &regs->ARM_cpsr, i_fn);
+}
+
 static enum kprobe_insn __kprobes
 prep_emulate_ldr_str(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
-        int ibit = (insn & (1 << 26)) ? 25 : 22;
+        int not_imm = (insn & (1 << 26)) ? (insn & (1 << 25))
+                                         : (~insn & (1 << 22));
+
+        if (is_writeback(insn) && is_r15(insn, 16))
+                return INSN_REJECTED;   /* Writeback to PC */
 
         insn &= 0xfff00fff;
         insn |= 0x00001000;     /* Rn = r0, Rd = r1 */
-        if (insn & (1 << ibit)) {
+        if (not_imm) {
                 insn &= ~0xf;
                 insn |= 2;      /* Rm = r2 */
         }
@@ -869,20 +868,40 @@ prep_emulate_ldr_str(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 }
 
 static enum kprobe_insn __kprobes
-prep_emulate_rd12rm0(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+prep_emulate_rd12_modify(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
-        insn &= 0xffff0ff0;     /* Rd = r0, Rm = r0 */
+        if (is_r15(insn, 12))
+                return INSN_REJECTED;   /* Rd is PC */
+
+        insn &= 0xffff0fff;     /* Rd = r0 */
         asi->insn[0] = insn;
-        asi->insn_handler = emulate_rd12rm0;
+        asi->insn_handler = emulate_rd12_modify;
         return INSN_GOOD;
 }
 
 static enum kprobe_insn __kprobes
-prep_emulate_rd12(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+prep_emulate_rd12rn0_modify(kprobe_opcode_t insn,
+                            struct arch_specific_insn *asi)
 {
-        insn &= 0xffff0fff;     /* Rd = r0 */
+        if (is_r15(insn, 12))
+                return INSN_REJECTED;   /* Rd is PC */
+
+        insn &= 0xffff0ff0;     /* Rd = r0 */
+        insn |= 0x00000001;     /* Rn = r1 */
+        asi->insn[0] = insn;
+        asi->insn_handler = emulate_rd12rn0_modify;
+        return INSN_GOOD;
+}
+
+static enum kprobe_insn __kprobes
+prep_emulate_rd12rm0(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+{
+        if (is_r15(insn, 12))
+                return INSN_REJECTED;   /* Rd is PC */
+
+        insn &= 0xffff0ff0;     /* Rd = r0, Rm = r0 */
         asi->insn[0] = insn;
-        asi->insn_handler = emulate_rd12;
+        asi->insn_handler = emulate_rd12rm0;
         return INSN_GOOD;
 }
 
@@ -890,6 +909,9 @@ static enum kprobe_insn __kprobes
 prep_emulate_rd12rn16rm0_wflags(kprobe_opcode_t insn,
                                 struct arch_specific_insn *asi)
 {
+        if (is_r15(insn, 12))
+                return INSN_REJECTED;   /* Rd is PC */
+
         insn &= 0xfff00ff0;     /* Rd = r0, Rn = r0 */
         insn |= 0x00000001;     /* Rm = r1 */
         asi->insn[0] = insn;
@@ -901,6 +923,9 @@ static enum kprobe_insn __kprobes
 prep_emulate_rd16rs8rm0_wflags(kprobe_opcode_t insn,
                                struct arch_specific_insn *asi)
 {
+        if (is_r15(insn, 16))
+                return INSN_REJECTED;   /* Rd is PC */
+
         insn &= 0xfff0f0f0;     /* Rd = r0, Rs = r0 */
         insn |= 0x00000001;     /* Rm = r1          */
         asi->insn[0] = insn;
@@ -912,6 +937,9 @@ static enum kprobe_insn __kprobes
 prep_emulate_rd16rn12rs8rm0_wflags(kprobe_opcode_t insn,
                                    struct arch_specific_insn *asi)
 {
+        if (is_r15(insn, 16))
+                return INSN_REJECTED;   /* Rd is PC */
+
         insn &= 0xfff000f0;     /* Rd = r0, Rn = r0 */
         insn |= 0x00000102;     /* Rs = r1, Rm = r2 */
         asi->insn[0] = insn;
@@ -923,6 +951,9 @@ static enum kprobe_insn __kprobes
 prep_emulate_rdhi16rdlo12rs8rm0_wflags(kprobe_opcode_t insn,
                                        struct arch_specific_insn *asi)
 {
+        if (is_r15(insn, 16) || is_r15(insn, 12))
+                return INSN_REJECTED;   /* RdHi or RdLo is PC */
+
         insn &= 0xfff000f0;     /* RdHi = r0, RdLo = r1 */
         insn |= 0x00001203;     /* Rs = r2, Rm = r3 */
         asi->insn[0] = insn;
@@ -943,20 +974,13 @@ prep_emulate_rdhi16rdlo12rs8rm0_wflags(kprobe_opcode_t insn,
 static enum kprobe_insn __kprobes
 space_1111(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
-        /* CPS mmod == 1 : 1111 0001 0000 xx10 xxxx xxxx xx0x xxxx */
-        /* RFE           : 1111 100x x0x1 xxxx xxxx 1010 xxxx xxxx */
-        /* SRS           : 1111 100x x1x0 1101 xxxx 0101 xxxx xxxx */
-        if ((insn & 0xfff30020) == 0xf1020000 ||
-            (insn & 0xfe500f00) == 0xf8100a00 ||
-            (insn & 0xfe5f0f00) == 0xf84d0500)
-                return INSN_REJECTED;
-
-        /* PLD : 1111 01x1 x101 xxxx xxxx xxxx xxxx xxxx : */
-        if ((insn & 0xfd700000) == 0xf4500000) {
-                insn &= 0xfff0ffff;     /* Rn = r0 */
-                asi->insn[0] = insn;
-                asi->insn_handler = emulate_rn16;
-                return INSN_GOOD;
+        /* memory hint : 1111 0100 x001 xxxx xxxx xxxx xxxx xxxx : */
+        /* PLDI        : 1111 0100 x101 xxxx xxxx xxxx xxxx xxxx : */
+        /* PLDW        : 1111 0101 x001 xxxx xxxx xxxx xxxx xxxx : */
+        /* PLD         : 1111 0101 x101 xxxx xxxx xxxx xxxx xxxx : */
+        if ((insn & 0xfe300000) == 0xf4100000) {
+                asi->insn_handler = emulate_nop;
+                return INSN_GOOD_NO_SLOT;
         }
 
         /* BLX(1) : 1111 101x xxxx xxxx xxxx xxxx xxxx xxxx : */
@@ -965,41 +989,22 @@ space_1111(kprobe_opcode_t insn, struct arch_specific_insn *asi)
                 return INSN_GOOD_NO_SLOT;
         }
 
-        /* SETEND : 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */
-        /* CDP2   : 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */
-        if ((insn & 0xffff00f0) == 0xf1010000 ||
-            (insn & 0xff000010) == 0xfe000000) {
-                asi->insn[0] = insn;
-                asi->insn_handler = emulate_none;
-                return INSN_GOOD;
-        }
+        /* CPS   : 1111 0001 0000 xxx0 xxxx xxxx xx0x xxxx */
+        /* SETEND: 1111 0001 0000 0001 xxxx xxxx 0000 xxxx */
+
+        /* SRS   : 1111 100x x1x0 xxxx xxxx xxxx xxxx xxxx */
+        /* RFE   : 1111 100x x0x1 xxxx xxxx xxxx xxxx xxxx */
 
+        /* Coprocessor instructions... */
         /* MCRR2 : 1111 1100 0100 xxxx xxxx xxxx xxxx xxxx : (Rd != Rn) */
         /* MRRC2 : 1111 1100 0101 xxxx xxxx xxxx xxxx xxxx : (Rd != Rn) */
-        if ((insn & 0xffe00000) == 0xfc400000) {
-                insn &= 0xfff00fff;     /* Rn = r0 */
-                insn |= 0x00001000;     /* Rd = r1 */
-                asi->insn[0] = insn;
-                asi->insn_handler =
-                        (insn & (1 << 20)) ? emulate_mrrc : emulate_mcrr;
-                return INSN_GOOD;
-        }
+        /* LDC2  : 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */
+        /* STC2  : 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */
+        /* CDP2  : 1111 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */
+        /* MCR2  : 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */
+        /* MRC2  : 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */
 
-        /* LDC2 : 1111 110x xxx1 xxxx xxxx xxxx xxxx xxxx */
-        /* STC2 : 1111 110x xxx0 xxxx xxxx xxxx xxxx xxxx */
-        if ((insn & 0xfe000000) == 0xfc000000) {
-                insn &= 0xfff0ffff;      /* Rn = r0 */
-                asi->insn[0] = insn;
-                asi->insn_handler = emulate_ldcstc;
-                return INSN_GOOD;
-        }
-
-        /* MCR2 : 1111 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */
-        /* MRC2 : 1111 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */
-        insn &= 0xffff0fff;     /* Rd = r0 */
-        asi->insn[0]      = insn;
-        asi->insn_handler = (insn & (1 << 20)) ? emulate_rd12 : emulate_ird12;
-        return INSN_GOOD;
+        return INSN_REJECTED;
 }
 
 static enum kprobe_insn __kprobes
@@ -1008,19 +1013,18 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
         /* cccc 0001 0xx0 xxxx xxxx xxxx xxxx xxx0 xxxx */
         if ((insn & 0x0f900010) == 0x01000000) {
 
-                /* BXJ  : cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */
-                /* MSR  : cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */
-                if ((insn & 0x0ff000f0) == 0x01200020 ||
-                    (insn & 0x0fb000f0) == 0x01200000)
-                        return INSN_REJECTED;
-
-                /* MRS : cccc 0001 0x00 xxxx xxxx xxxx 0000 xxxx */
-                if ((insn & 0x0fb00010) == 0x01000000)
-                        return prep_emulate_rd12(insn, asi);
+                /* MRS cpsr : cccc 0001 0000 xxxx xxxx xxxx 0000 xxxx */
+                if ((insn & 0x0ff000f0) == 0x01000000) {
+                        if (is_r15(insn, 12))
+                                return INSN_REJECTED;   /* Rd is PC */
+                        asi->insn_handler = simulate_mrs;
+                        return INSN_GOOD_NO_SLOT;
+                }
 
                 /* SMLALxy : cccc 0001 0100 xxxx xxxx xxxx 1xx0 xxxx */
                 if ((insn & 0x0ff00090) == 0x01400080)
-                        return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn, asi);
+                        return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn,
+                                                                        asi);
 
                 /* SMULWy : cccc 0001 0010 xxxx xxxx xxxx 1x10 xxxx */
                 /* SMULxy : cccc 0001 0110 xxxx xxxx xxxx 1xx0 xxxx */
@@ -1029,24 +1033,29 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
                         return prep_emulate_rd16rs8rm0_wflags(insn, asi);
 
                 /* SMLAxy : cccc 0001 0000 xxxx xxxx xxxx 1xx0 xxxx : Q */
-                /* SMLAWy : cccc 0001 0010 xxxx xxxx xxxx 0x00 xxxx : Q */
-                return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
+                /* SMLAWy : cccc 0001 0010 xxxx xxxx xxxx 1x00 xxxx : Q */
+                if ((insn & 0x0ff00090) == 0x01000080 ||
+                    (insn & 0x0ff000b0) == 0x01200080)
+                        return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
 
+                /* BXJ      : cccc 0001 0010 xxxx xxxx xxxx 0010 xxxx */
+                /* MSR      : cccc 0001 0x10 xxxx xxxx xxxx 0000 xxxx */
+                /* MRS spsr : cccc 0001 0100 xxxx xxxx xxxx 0000 xxxx */
+
+                /* Other instruction encodings aren't yet defined */
+                return INSN_REJECTED;
         }
 
         /* cccc 0001 0xx0 xxxx xxxx xxxx xxxx 0xx1 xxxx */
         else if ((insn & 0x0f900090) == 0x01000010) {
 
-                /* BKPT : 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */
-                if ((insn & 0xfff000f0) == 0xe1200070)
-                        return INSN_REJECTED;
-
                 /* BLX(2) : cccc 0001 0010 xxxx xxxx xxxx 0011 xxxx */
                 /* BX     : cccc 0001 0010 xxxx xxxx xxxx 0001 xxxx */
                 if ((insn & 0x0ff000d0) == 0x01200010) {
-                        asi->insn[0] = truecc_insn(insn);
+                        if ((insn & 0x0ff000ff) == 0x0120003f)
+                                return INSN_REJECTED; /* BLX pc */
                         asi->insn_handler = simulate_blx2bx;
-                        return INSN_GOOD;
+                        return INSN_GOOD_NO_SLOT;
                 }
 
                 /* CLZ : cccc 0001 0110 xxxx xxxx xxxx 0001 xxxx */
@@ -1057,17 +1066,27 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
                 /* QSUB    : cccc 0001 0010 xxxx xxxx xxxx 0101 xxxx :Q */
                 /* QDADD   : cccc 0001 0100 xxxx xxxx xxxx 0101 xxxx :Q */
                 /* QDSUB   : cccc 0001 0110 xxxx xxxx xxxx 0101 xxxx :Q */
-                return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+                if ((insn & 0x0f9000f0) == 0x01000050)
+                        return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+
+                /* BKPT : 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */
+                /* SMC  : cccc 0001 0110 xxxx xxxx xxxx 0111 xxxx */
+
+                /* Other instruction encodings aren't yet defined */
+                return INSN_REJECTED;
         }
 
         /* cccc 0000 xxxx xxxx xxxx xxxx xxxx 1001 xxxx */
-        else if ((insn & 0x0f000090) == 0x00000090) {
+        else if ((insn & 0x0f0000f0) == 0x00000090) {
 
                 /* MUL    : cccc 0000 0000 xxxx xxxx xxxx 1001 xxxx :   */
                 /* MULS   : cccc 0000 0001 xxxx xxxx xxxx 1001 xxxx :cc */
                 /* MLA    : cccc 0000 0010 xxxx xxxx xxxx 1001 xxxx :   */
                 /* MLAS   : cccc 0000 0011 xxxx xxxx xxxx 1001 xxxx :cc */
                 /* UMAAL  : cccc 0000 0100 xxxx xxxx xxxx 1001 xxxx :   */
+                /* undef  : cccc 0000 0101 xxxx xxxx xxxx 1001 xxxx :   */
+                /* MLS    : cccc 0000 0110 xxxx xxxx xxxx 1001 xxxx :   */
+                /* undef  : cccc 0000 0111 xxxx xxxx xxxx 1001 xxxx :   */
                 /* UMULL  : cccc 0000 1000 xxxx xxxx xxxx 1001 xxxx :   */
                 /* UMULLS : cccc 0000 1001 xxxx xxxx xxxx 1001 xxxx :cc */
                 /* UMLAL  : cccc 0000 1010 xxxx xxxx xxxx 1001 xxxx :   */
@@ -1076,13 +1095,15 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
                 /* SMULLS : cccc 0000 1101 xxxx xxxx xxxx 1001 xxxx :cc */
                 /* SMLAL  : cccc 0000 1110 xxxx xxxx xxxx 1001 xxxx :   */
                 /* SMLALS : cccc 0000 1111 xxxx xxxx xxxx 1001 xxxx :cc */
-                if ((insn & 0x0fe000f0) == 0x00000090) {
-                       return prep_emulate_rd16rs8rm0_wflags(insn, asi);
-                } else if  ((insn & 0x0fe000f0) == 0x00200090) {
-                       return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
-                } else {
-                       return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn, asi);
-                }
+                if ((insn & 0x00d00000) == 0x00500000)
+                        return INSN_REJECTED;
+                else if ((insn & 0x00e00000) == 0x00000000)
+                        return prep_emulate_rd16rs8rm0_wflags(insn, asi);
+                else if ((insn & 0x00a00000) == 0x00200000)
+                        return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
+                else
+                        return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn,
+                                                                        asi);
         }
 
         /* cccc 000x xxxx xxxx xxxx xxxx xxxx 1xx1 xxxx */
@@ -1090,23 +1111,45 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 
                 /* SWP   : cccc 0001 0000 xxxx xxxx xxxx 1001 xxxx */
                 /* SWPB  : cccc 0001 0100 xxxx xxxx xxxx 1001 xxxx */
-                /* LDRD  : cccc 000x xxx0 xxxx xxxx xxxx 1101 xxxx */
-                /* STRD  : cccc 000x xxx0 xxxx xxxx xxxx 1111 xxxx */
+                /* ???   : cccc 0001 0x01 xxxx xxxx xxxx 1001 xxxx */
+                /* ???   : cccc 0001 0x10 xxxx xxxx xxxx 1001 xxxx */
+                /* ???   : cccc 0001 0x11 xxxx xxxx xxxx 1001 xxxx */
                 /* STREX : cccc 0001 1000 xxxx xxxx xxxx 1001 xxxx */
                 /* LDREX : cccc 0001 1001 xxxx xxxx xxxx 1001 xxxx */
+                /* STREXD: cccc 0001 1010 xxxx xxxx xxxx 1001 xxxx */
+                /* LDREXD: cccc 0001 1011 xxxx xxxx xxxx 1001 xxxx */
+                /* STREXB: cccc 0001 1100 xxxx xxxx xxxx 1001 xxxx */
+                /* LDREXB: cccc 0001 1101 xxxx xxxx xxxx 1001 xxxx */
+                /* STREXH: cccc 0001 1110 xxxx xxxx xxxx 1001 xxxx */
+                /* LDREXH: cccc 0001 1111 xxxx xxxx xxxx 1001 xxxx */
+
+                /* LDRD  : cccc 000x xxx0 xxxx xxxx xxxx 1101 xxxx */
+                /* STRD  : cccc 000x xxx0 xxxx xxxx xxxx 1111 xxxx */
                 /* LDRH  : cccc 000x xxx1 xxxx xxxx xxxx 1011 xxxx */
                 /* STRH  : cccc 000x xxx0 xxxx xxxx xxxx 1011 xxxx */
                 /* LDRSB : cccc 000x xxx1 xxxx xxxx xxxx 1101 xxxx */
                 /* LDRSH : cccc 000x xxx1 xxxx xxxx xxxx 1111 xxxx */
-                if ((insn & 0x0fb000f0) == 0x01000090) {
-                        /* SWP/SWPB */
-                        return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+                if ((insn & 0x0f0000f0) == 0x01000090) {
+                        if ((insn & 0x0fb000f0) == 0x01000090) {
+                                /* SWP/SWPB */
+                                return prep_emulate_rd12rn16rm0_wflags(insn,
+                                                                        asi);
+                        } else {
+                                /* STREX/LDREX variants and unallocaed space */
+                                return INSN_REJECTED;
+                        }
+
                 } else if ((insn & 0x0e1000d0) == 0x00000d0) {
                         /* STRD/LDRD */
+                        if ((insn & 0x0000e000) == 0x0000e000)
+                                return INSN_REJECTED;   /* Rd is LR or PC */
+                        if (is_writeback(insn) && is_r15(insn, 16))
+                                return INSN_REJECTED;   /* Writeback to PC */
+
                         insn &= 0xfff00fff;
                         insn |= 0x00002000;     /* Rn = r0, Rd = r2 */
-                        if (insn & (1 << 22)) {
-                                /* I bit */
+                        if (!(insn & (1 << 22))) {
+                                /* Register index */
                                 insn &= ~0xf;
                                 insn |= 1;      /* Rm = r1 */
                         }
@@ -1116,6 +1159,9 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
                         return INSN_GOOD;
                 }
 
+                /* LDRH/STRH/LDRSB/LDRSH */
+                if (is_r15(insn, 12))
+                        return INSN_REJECTED;   /* Rd is PC */
                 return prep_emulate_ldr_str(insn, asi);
         }
 
@@ -1123,7 +1169,7 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 
         /*
          * ALU op with S bit and Rd == 15 :
-         *      cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx
+         *      cccc 000x xxx1 xxxx 1111 xxxx xxxx xxxx
          */
         if ((insn & 0x0e10f000) == 0x0010f000)
                 return INSN_REJECTED;
@@ -1152,22 +1198,61 @@ space_cccc_000x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
                 insn |= 0x00000200;     /* Rs = r2 */
         }
         asi->insn[0] = insn;
-        asi->insn_handler = (insn & (1 << 20)) ?  /* S-bit */
+
+        if ((insn & 0x0f900000) == 0x01100000) {
+                /*
+                 * TST : cccc 0001 0001 xxxx xxxx xxxx xxxx xxxx
+                 * TEQ : cccc 0001 0011 xxxx xxxx xxxx xxxx xxxx
+                 * CMP : cccc 0001 0101 xxxx xxxx xxxx xxxx xxxx
+                 * CMN : cccc 0001 0111 xxxx xxxx xxxx xxxx xxxx
+                 */
+                asi->insn_handler = emulate_alu_tests;
+        } else {
+                /* ALU ops which write to Rd */
+                asi->insn_handler = (insn & (1 << 20)) ?  /* S-bit */
                                 emulate_alu_rwflags : emulate_alu_rflags;
+        }
         return INSN_GOOD;
 }
 
 static enum kprobe_insn __kprobes
 space_cccc_001x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
+        /* MOVW  : cccc 0011 0000 xxxx xxxx xxxx xxxx xxxx */
+        /* MOVT  : cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx */
+        if ((insn & 0x0fb00000) == 0x03000000)
+                return prep_emulate_rd12_modify(insn, asi);
+
+        /* hints : cccc 0011 0010 0000 xxxx xxxx xxxx xxxx */
+        if ((insn & 0x0fff0000) == 0x03200000) {
+                unsigned op2 = insn & 0x000000ff;
+                if (op2 == 0x01 || op2 == 0x04) {
+                        /* YIELD : cccc 0011 0010 0000 xxxx xxxx 0000 0001 */
+                        /* SEV   : cccc 0011 0010 0000 xxxx xxxx 0000 0100 */
+                        asi->insn[0] = insn;
+                        asi->insn_handler = emulate_none;
+                        return INSN_GOOD;
+                } else if (op2 <= 0x03) {
+                        /* NOP   : cccc 0011 0010 0000 xxxx xxxx 0000 0000 */
+                        /* WFE   : cccc 0011 0010 0000 xxxx xxxx 0000 0010 */
+                        /* WFI   : cccc 0011 0010 0000 xxxx xxxx 0000 0011 */
+                        /*
+                         * We make WFE and WFI true NOPs to avoid stalls due
+                         * to missing events whilst processing the probe.
+                         */
+                        asi->insn_handler = emulate_nop;
+                        return INSN_GOOD_NO_SLOT;
+                }
+                /* For DBG and unallocated hints it's safest to reject them */
+                return INSN_REJECTED;
+        }
+
         /*
          * MSR   : cccc 0011 0x10 xxxx xxxx xxxx xxxx xxxx
-         * Undef : cccc 0011 0100 xxxx xxxx xxxx xxxx xxxx
          * ALU op with S bit and Rd == 15 :
          *         cccc 001x xxx1 xxxx 1111 xxxx xxxx xxxx
          */
         if ((insn & 0x0fb00000) == 0x03200000 ||        /* MSR */
-            (insn & 0x0ff00000) == 0x03400000 ||        /* Undef */
             (insn & 0x0e10f000) == 0x0210f000)          /* ALU s-bit, R15  */
                 return INSN_REJECTED;
 
@@ -1178,10 +1263,22 @@ space_cccc_001x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
          * *S (bit 20) updates condition codes
          * ADC/SBC/RSC reads the C flag
          */
-        insn &= 0xffff0fff;     /* Rd = r0 */
+        insn &= 0xfff00fff;     /* Rn = r0 and Rd = r0 */
         asi->insn[0] = insn;
-        asi->insn_handler = (insn & (1 << 20)) ?  /* S-bit */
+
+        if ((insn & 0x0f900000) == 0x03100000) {
+                /*
+                 * TST : cccc 0011 0001 xxxx xxxx xxxx xxxx xxxx
+                 * TEQ : cccc 0011 0011 xxxx xxxx xxxx xxxx xxxx
+                 * CMP : cccc 0011 0101 xxxx xxxx xxxx xxxx xxxx
+                 * CMN : cccc 0011 0111 xxxx xxxx xxxx xxxx xxxx
+                 */
+                asi->insn_handler = emulate_alu_tests_imm;
+        } else {
+                /* ALU ops which write to Rd */
+                asi->insn_handler = (insn & (1 << 20)) ?  /* S-bit */
                         emulate_alu_imm_rwflags : emulate_alu_imm_rflags;
+        }
         return INSN_GOOD;
 }
 
@@ -1190,6 +1287,8 @@ space_cccc_0110__1(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
         /* SEL : cccc 0110 1000 xxxx xxxx xxxx 1011 xxxx GE: !!! */
         if ((insn & 0x0ff000f0) == 0x068000b0) {
+                if (is_r15(insn, 12))
+                        return INSN_REJECTED;   /* Rd is PC */
                 insn &= 0xfff00ff0;     /* Rd = r0, Rn = r0 */
                 insn |= 0x00000001;     /* Rm = r1 */
                 asi->insn[0] = insn;
@@ -1203,6 +1302,8 @@ space_cccc_0110__1(kprobe_opcode_t insn, struct arch_specific_insn *asi)
         /* USAT16 : cccc 0110 1110 xxxx xxxx xxxx 0011 xxxx :Q */
         if ((insn & 0x0fa00030) == 0x06a00010 ||
             (insn & 0x0fb000f0) == 0x06a00030) {
+                if (is_r15(insn, 12))
+                        return INSN_REJECTED;   /* Rd is PC */
                 insn &= 0xffff0ff0;     /* Rd = r0, Rm = r0 */
                 asi->insn[0] = insn;
                 asi->insn_handler = emulate_sat;
@@ -1211,57 +1312,101 @@ space_cccc_0110__1(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 
         /* REV    : cccc 0110 1011 xxxx xxxx xxxx 0011 xxxx */
         /* REV16  : cccc 0110 1011 xxxx xxxx xxxx 1011 xxxx */
+        /* RBIT   : cccc 0110 1111 xxxx xxxx xxxx 0011 xxxx */
         /* REVSH  : cccc 0110 1111 xxxx xxxx xxxx 1011 xxxx */
         if ((insn & 0x0ff00070) == 0x06b00030 ||
-            (insn & 0x0ff000f0) == 0x06f000b0)
+            (insn & 0x0ff00070) == 0x06f00030)
                 return prep_emulate_rd12rm0(insn, asi);
 
+        /* ???       : cccc 0110 0000 xxxx xxxx xxxx xxx1 xxxx :   */
         /* SADD16    : cccc 0110 0001 xxxx xxxx xxxx 0001 xxxx :GE */
         /* SADDSUBX  : cccc 0110 0001 xxxx xxxx xxxx 0011 xxxx :GE */
         /* SSUBADDX  : cccc 0110 0001 xxxx xxxx xxxx 0101 xxxx :GE */
         /* SSUB16    : cccc 0110 0001 xxxx xxxx xxxx 0111 xxxx :GE */
         /* SADD8     : cccc 0110 0001 xxxx xxxx xxxx 1001 xxxx :GE */
+        /* ???       : cccc 0110 0001 xxxx xxxx xxxx 1011 xxxx :   */
+        /* ???       : cccc 0110 0001 xxxx xxxx xxxx 1101 xxxx :   */
         /* SSUB8     : cccc 0110 0001 xxxx xxxx xxxx 1111 xxxx :GE */
         /* QADD16    : cccc 0110 0010 xxxx xxxx xxxx 0001 xxxx :   */
         /* QADDSUBX  : cccc 0110 0010 xxxx xxxx xxxx 0011 xxxx :   */
         /* QSUBADDX  : cccc 0110 0010 xxxx xxxx xxxx 0101 xxxx :   */
         /* QSUB16    : cccc 0110 0010 xxxx xxxx xxxx 0111 xxxx :   */
         /* QADD8     : cccc 0110 0010 xxxx xxxx xxxx 1001 xxxx :   */
+        /* ???       : cccc 0110 0010 xxxx xxxx xxxx 1011 xxxx :   */
+        /* ???       : cccc 0110 0010 xxxx xxxx xxxx 1101 xxxx :   */
         /* QSUB8     : cccc 0110 0010 xxxx xxxx xxxx 1111 xxxx :   */
         /* SHADD16   : cccc 0110 0011 xxxx xxxx xxxx 0001 xxxx :   */
         /* SHADDSUBX : cccc 0110 0011 xxxx xxxx xxxx 0011 xxxx :   */
         /* SHSUBADDX : cccc 0110 0011 xxxx xxxx xxxx 0101 xxxx :   */
         /* SHSUB16   : cccc 0110 0011 xxxx xxxx xxxx 0111 xxxx :   */
         /* SHADD8    : cccc 0110 0011 xxxx xxxx xxxx 1001 xxxx :   */
+        /* ???       : cccc 0110 0011 xxxx xxxx xxxx 1011 xxxx :   */
+        /* ???       : cccc 0110 0011 xxxx xxxx xxxx 1101 xxxx :   */
         /* SHSUB8    : cccc 0110 0011 xxxx xxxx xxxx 1111 xxxx :   */
+        /* ???       : cccc 0110 0100 xxxx xxxx xxxx xxx1 xxxx :   */
         /* UADD16    : cccc 0110 0101 xxxx xxxx xxxx 0001 xxxx :GE */
         /* UADDSUBX  : cccc 0110 0101 xxxx xxxx xxxx 0011 xxxx :GE */
         /* USUBADDX  : cccc 0110 0101 xxxx xxxx xxxx 0101 xxxx :GE */
         /* USUB16    : cccc 0110 0101 xxxx xxxx xxxx 0111 xxxx :GE */
         /* UADD8     : cccc 0110 0101 xxxx xxxx xxxx 1001 xxxx :GE */
+        /* ???       : cccc 0110 0101 xxxx xxxx xxxx 1011 xxxx :   */
+        /* ???       : cccc 0110 0101 xxxx xxxx xxxx 1101 xxxx :   */
         /* USUB8     : cccc 0110 0101 xxxx xxxx xxxx 1111 xxxx :GE */
         /* UQADD16   : cccc 0110 0110 xxxx xxxx xxxx 0001 xxxx :   */
         /* UQADDSUBX : cccc 0110 0110 xxxx xxxx xxxx 0011 xxxx :   */
         /* UQSUBADDX : cccc 0110 0110 xxxx xxxx xxxx 0101 xxxx :   */
         /* UQSUB16   : cccc 0110 0110 xxxx xxxx xxxx 0111 xxxx :   */
         /* UQADD8    : cccc 0110 0110 xxxx xxxx xxxx 1001 xxxx :   */
+        /* ???       : cccc 0110 0110 xxxx xxxx xxxx 1011 xxxx :   */
+        /* ???       : cccc 0110 0110 xxxx xxxx xxxx 1101 xxxx :   */
         /* UQSUB8    : cccc 0110 0110 xxxx xxxx xxxx 1111 xxxx :   */
         /* UHADD16   : cccc 0110 0111 xxxx xxxx xxxx 0001 xxxx :   */
         /* UHADDSUBX : cccc 0110 0111 xxxx xxxx xxxx 0011 xxxx :   */
         /* UHSUBADDX : cccc 0110 0111 xxxx xxxx xxxx 0101 xxxx :   */
         /* UHSUB16   : cccc 0110 0111 xxxx xxxx xxxx 0111 xxxx :   */
         /* UHADD8    : cccc 0110 0111 xxxx xxxx xxxx 1001 xxxx :   */
+        /* ???       : cccc 0110 0111 xxxx xxxx xxxx 1011 xxxx :   */
+        /* ???       : cccc 0110 0111 xxxx xxxx xxxx 1101 xxxx :   */
         /* UHSUB8    : cccc 0110 0111 xxxx xxxx xxxx 1111 xxxx :   */
+        if ((insn & 0x0f800010) == 0x06000010) {
+                if ((insn & 0x00300000) == 0x00000000 ||
+                    (insn & 0x000000e0) == 0x000000a0 ||
+                    (insn & 0x000000e0) == 0x000000c0)
+                        return INSN_REJECTED;   /* Unallocated space */
+                return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+        }
+
         /* PKHBT     : cccc 0110 1000 xxxx xxxx xxxx x001 xxxx :   */
         /* PKHTB     : cccc 0110 1000 xxxx xxxx xxxx x101 xxxx :   */
+        if ((insn & 0x0ff00030) == 0x06800010)
+                return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+
         /* SXTAB16   : cccc 0110 1000 xxxx xxxx xxxx 0111 xxxx :   */
-        /* SXTB      : cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx :   */
+        /* SXTB16    : cccc 0110 1000 1111 xxxx xxxx 0111 xxxx :   */
+        /* ???       : cccc 0110 1001 xxxx xxxx xxxx 0111 xxxx :   */
         /* SXTAB     : cccc 0110 1010 xxxx xxxx xxxx 0111 xxxx :   */
+        /* SXTB      : cccc 0110 1010 1111 xxxx xxxx 0111 xxxx :   */
         /* SXTAH     : cccc 0110 1011 xxxx xxxx xxxx 0111 xxxx :   */
+        /* SXTH      : cccc 0110 1011 1111 xxxx xxxx 0111 xxxx :   */
         /* UXTAB16   : cccc 0110 1100 xxxx xxxx xxxx 0111 xxxx :   */
+        /* UXTB16    : cccc 0110 1100 1111 xxxx xxxx 0111 xxxx :   */
+        /* ???       : cccc 0110 1101 xxxx xxxx xxxx 0111 xxxx :   */
         /* UXTAB     : cccc 0110 1110 xxxx xxxx xxxx 0111 xxxx :   */
+        /* UXTB      : cccc 0110 1110 1111 xxxx xxxx 0111 xxxx :   */
         /* UXTAH     : cccc 0110 1111 xxxx xxxx xxxx 0111 xxxx :   */
-        return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+        /* UXTH      : cccc 0110 1111 1111 xxxx xxxx 0111 xxxx :   */
+        if ((insn & 0x0f8000f0) == 0x06800070) {
+                if ((insn & 0x00300000) == 0x00100000)
+                        return INSN_REJECTED;   /* Unallocated space */
+
+                if ((insn & 0x000f0000) == 0x000f0000)
+                        return prep_emulate_rd12rm0(insn, asi);
+                else
+                        return prep_emulate_rd12rn16rm0_wflags(insn, asi);
+        }
+
+        /* Other instruction encodings aren't yet defined */
+        return INSN_REJECTED;
 }
 
 static enum kprobe_insn __kprobes
@@ -1271,29 +1416,49 @@ space_cccc_0111__1(kprobe_opcode_t insn, struct arch_specific_insn *asi)
         if ((insn & 0x0ff000f0) == 0x03f000f0)
                 return INSN_REJECTED;
 
-        /* USADA8 : cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx */
-        /* USAD8  : cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx */
-        if ((insn & 0x0ff000f0) == 0x07800010)
-                 return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
-
         /* SMLALD : cccc 0111 0100 xxxx xxxx xxxx 00x1 xxxx */
         /* SMLSLD : cccc 0111 0100 xxxx xxxx xxxx 01x1 xxxx */
         if ((insn & 0x0ff00090) == 0x07400010)
                 return prep_emulate_rdhi16rdlo12rs8rm0_wflags(insn, asi);
 
         /* SMLAD  : cccc 0111 0000 xxxx xxxx xxxx 00x1 xxxx :Q */
+        /* SMUAD  : cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx :Q */
         /* SMLSD  : cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx :Q */
+        /* SMUSD  : cccc 0111 0000 xxxx 1111 xxxx 01x1 xxxx :  */
         /* SMMLA  : cccc 0111 0101 xxxx xxxx xxxx 00x1 xxxx :  */
-        /* SMMLS  : cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx :  */
+        /* SMMUL  : cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx :  */
+        /* USADA8 : cccc 0111 1000 xxxx xxxx xxxx 0001 xxxx :  */
+        /* USAD8  : cccc 0111 1000 xxxx 1111 xxxx 0001 xxxx :  */
         if ((insn & 0x0ff00090) == 0x07000010 ||
             (insn & 0x0ff000d0) == 0x07500010 ||
-            (insn & 0x0ff000d0) == 0x075000d0)
+            (insn & 0x0ff000f0) == 0x07800010) {
+
+                if ((insn & 0x0000f000) == 0x0000f000)
+                        return prep_emulate_rd16rs8rm0_wflags(insn, asi);
+                else
+                        return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
+        }
+
+        /* SMMLS  : cccc 0111 0101 xxxx xxxx xxxx 11x1 xxxx :  */
+        if ((insn & 0x0ff000d0) == 0x075000d0)
                 return prep_emulate_rd16rn12rs8rm0_wflags(insn, asi);
 
-        /* SMUSD  : cccc 0111 0000 xxxx xxxx xxxx 01x1 xxxx :  */
-        /* SMUAD  : cccc 0111 0000 xxxx 1111 xxxx 00x1 xxxx :Q */
-        /* SMMUL  : cccc 0111 0101 xxxx 1111 xxxx 00x1 xxxx :  */
-        return prep_emulate_rd16rs8rm0_wflags(insn, asi);
+        /* SBFX   : cccc 0111 101x xxxx xxxx xxxx x101 xxxx :  */
+        /* UBFX   : cccc 0111 111x xxxx xxxx xxxx x101 xxxx :  */
+        if ((insn & 0x0fa00070) == 0x07a00050)
+                return prep_emulate_rd12rm0(insn, asi);
+
+        /* BFI    : cccc 0111 110x xxxx xxxx xxxx x001 xxxx :  */
+        /* BFC    : cccc 0111 110x xxxx xxxx xxxx x001 1111 :  */
+        if ((insn & 0x0fe00070) == 0x07c00010) {
+
+                if ((insn & 0x0000000f) == 0x0000000f)
+                        return prep_emulate_rd12_modify(insn, asi);
+                else
+                        return prep_emulate_rd12rn0_modify(insn, asi);
+        }
+
+        return INSN_REJECTED;
 }
 
 static enum kprobe_insn __kprobes
@@ -1307,6 +1472,10 @@ space_cccc_01xx(kprobe_opcode_t insn, struct arch_specific_insn *asi)
         /* STRB  : cccc 01xx x1x0 xxxx xxxx xxxx xxxx xxxx */
         /* STRBT : cccc 01x0 x110 xxxx xxxx xxxx xxxx xxxx */
         /* STRT  : cccc 01x0 x010 xxxx xxxx xxxx xxxx xxxx */
+
+        if ((insn & 0x00500000) == 0x00500000 && is_r15(insn, 12))
+                return INSN_REJECTED;   /* LDRB into PC */
+
         return prep_emulate_ldr_str(insn, asi);
 }
 
@@ -1321,10 +1490,9 @@ space_cccc_100x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 
         /* LDM(1) : cccc 100x x0x1 xxxx xxxx xxxx xxxx xxxx */
         /* STM(1) : cccc 100x x0x0 xxxx xxxx xxxx xxxx xxxx */
-        asi->insn[0] = truecc_insn(insn);
         asi->insn_handler = ((insn & 0x108000) == 0x008000) ? /* STM & R15 */
                                 simulate_stm1_pc : simulate_ldm1stm1;
-        return INSN_GOOD;
+        return INSN_GOOD_NO_SLOT;
 }
 
 static enum kprobe_insn __kprobes
@@ -1332,58 +1500,117 @@ space_cccc_101x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
         /* B  : cccc 1010 xxxx xxxx xxxx xxxx xxxx xxxx */
         /* BL : cccc 1011 xxxx xxxx xxxx xxxx xxxx xxxx */
-        asi->insn[0] = truecc_insn(insn);
         asi->insn_handler = simulate_bbl;
-        return INSN_GOOD;
+        return INSN_GOOD_NO_SLOT;
 }
 
 static enum kprobe_insn __kprobes
-space_cccc_1100_010x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+space_cccc_11xx(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
+        /* Coprocessor instructions... */
         /* MCRR : cccc 1100 0100 xxxx xxxx xxxx xxxx xxxx : (Rd!=Rn) */
         /* MRRC : cccc 1100 0101 xxxx xxxx xxxx xxxx xxxx : (Rd!=Rn) */
-        insn &= 0xfff00fff;
-        insn |= 0x00001000;     /* Rn = r0, Rd = r1 */
-        asi->insn[0] = insn;
-        asi->insn_handler = (insn & (1 << 20)) ? emulate_mrrc : emulate_mcrr;
-        return INSN_GOOD;
+        /* LDC  : cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */
+        /* STC  : cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */
+        /* CDP  : cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */
+        /* MCR  : cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */
+        /* MRC  : cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */
+
+        /* SVC  : cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */
+
+        return INSN_REJECTED;
 }
 
-static enum kprobe_insn __kprobes
-space_cccc_110x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+static unsigned long __kprobes __check_eq(unsigned long cpsr)
 {
-        /* LDC : cccc 110x xxx1 xxxx xxxx xxxx xxxx xxxx */
-        /* STC : cccc 110x xxx0 xxxx xxxx xxxx xxxx xxxx */
-        insn &= 0xfff0ffff;     /* Rn = r0 */
-        asi->insn[0] = insn;
-        asi->insn_handler = emulate_ldcstc;
-        return INSN_GOOD;
+        return cpsr & PSR_Z_BIT;
 }
 
-static enum kprobe_insn __kprobes
-space_cccc_111x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
+static unsigned long __kprobes __check_ne(unsigned long cpsr)
 {
-        /* BKPT : 1110 0001 0010 xxxx xxxx xxxx 0111 xxxx */
-        /* SWI  : cccc 1111 xxxx xxxx xxxx xxxx xxxx xxxx */
-        if ((insn & 0xfff000f0) == 0xe1200070 ||
-            (insn & 0x0f000000) == 0x0f000000)
-                return INSN_REJECTED;
+        return (~cpsr) & PSR_Z_BIT;
+}
 
-        /* CDP : cccc 1110 xxxx xxxx xxxx xxxx xxx0 xxxx */
-        if ((insn & 0x0f000010) == 0x0e000000) {
-                asi->insn[0] = insn;
-                asi->insn_handler = emulate_none;
-                return INSN_GOOD;
-        }
+static unsigned long __kprobes __check_cs(unsigned long cpsr)
+{
+        return cpsr & PSR_C_BIT;
+}
 
-        /* MCR : cccc 1110 xxx0 xxxx xxxx xxxx xxx1 xxxx */
-        /* MRC : cccc 1110 xxx1 xxxx xxxx xxxx xxx1 xxxx */
-        insn &= 0xffff0fff;     /* Rd = r0 */
-        asi->insn[0] = insn;
-        asi->insn_handler = (insn & (1 << 20)) ? emulate_rd12 : emulate_ird12;
-        return INSN_GOOD;
+static unsigned long __kprobes __check_cc(unsigned long cpsr)
+{
+        return (~cpsr) & PSR_C_BIT;
 }
 
+static unsigned long __kprobes __check_mi(unsigned long cpsr)
+{
+        return cpsr & PSR_N_BIT;
+}
+
+static unsigned long __kprobes __check_pl(unsigned long cpsr)
+{
+        return (~cpsr) & PSR_N_BIT;
+}
+
+static unsigned long __kprobes __check_vs(unsigned long cpsr)
+{
+        return cpsr & PSR_V_BIT;
+}
+
+static unsigned long __kprobes __check_vc(unsigned long cpsr)
+{
+        return (~cpsr) & PSR_V_BIT;
+}
+
+static unsigned long __kprobes __check_hi(unsigned long cpsr)
+{
+        cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
+        return cpsr & PSR_C_BIT;
+}
+
+static unsigned long __kprobes __check_ls(unsigned long cpsr)
+{
+        cpsr &= ~(cpsr >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
+        return (~cpsr) & PSR_C_BIT;
+}
+
+static unsigned long __kprobes __check_ge(unsigned long cpsr)
+{
+        cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */
+        return (~cpsr) & PSR_N_BIT;
+}
+
+static unsigned long __kprobes __check_lt(unsigned long cpsr)
+{
+        cpsr ^= (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */
+        return cpsr & PSR_N_BIT;
+}
+
+static unsigned long __kprobes __check_gt(unsigned long cpsr)
+{
+        unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */
+        temp |= (cpsr << 1);                     /* PSR_N_BIT |= PSR_Z_BIT */
+        return (~temp) & PSR_N_BIT;
+}
+
+static unsigned long __kprobes __check_le(unsigned long cpsr)
+{
+        unsigned long temp = cpsr ^ (cpsr << 3); /* PSR_N_BIT ^= PSR_V_BIT */
+        temp |= (cpsr << 1);                     /* PSR_N_BIT |= PSR_Z_BIT */
+        return temp & PSR_N_BIT;
+}
+
+static unsigned long __kprobes __check_al(unsigned long cpsr)
+{
+        return true;
+}
+
+static kprobe_check_cc * const condition_checks[16] = {
+        &__check_eq, &__check_ne, &__check_cs, &__check_cc,
+        &__check_mi, &__check_pl, &__check_vs, &__check_vc,
+        &__check_hi, &__check_ls, &__check_ge, &__check_lt,
+        &__check_gt, &__check_le, &__check_al, &__check_al
+};
+
 /* Return:
  *   INSN_REJECTED     If instruction is one not allowed to kprobe,
  *   INSN_GOOD         If instruction is supported and uses instruction slot,
@@ -1399,133 +1626,45 @@ space_cccc_111x(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 enum kprobe_insn __kprobes
 arm_kprobe_decode_insn(kprobe_opcode_t insn, struct arch_specific_insn *asi)
 {
+        asi->insn_check_cc = condition_checks[insn>>28];
         asi->insn[1] = KPROBE_RETURN_INSTRUCTION;
 
-        if ((insn & 0xf0000000) == 0xf0000000) {
+        if ((insn & 0xf0000000) == 0xf0000000)
 
                 return space_1111(insn, asi);
 
-        } else if ((insn & 0x0e000000) == 0x00000000) {
+        else if ((insn & 0x0e000000) == 0x00000000)
 
                 return space_cccc_000x(insn, asi);
 
-        } else if ((insn & 0x0e000000) == 0x02000000) {
+        else if ((insn & 0x0e000000) == 0x02000000)
 
                 return space_cccc_001x(insn, asi);
 
-        } else if ((insn & 0x0f000010) == 0x06000010) {
+        else if ((insn & 0x0f000010) == 0x06000010)
 
                 return space_cccc_0110__1(insn, asi);
 
-        } else if ((insn & 0x0f000010) == 0x07000010) {
+        else if ((insn & 0x0f000010) == 0x07000010)
 
                 return space_cccc_0111__1(insn, asi);
 
-        } else if ((insn & 0x0c000000) == 0x04000000) {
+        else if ((insn & 0x0c000000) == 0x04000000)
 
                 return space_cccc_01xx(insn, asi);
 
-        } else if ((insn & 0x0e000000) == 0x08000000) {
+        else if ((insn & 0x0e000000) == 0x08000000)
 
                 return space_cccc_100x(insn, asi);
 
-        } else if ((insn & 0x0e000000) == 0x0a000000) {
+        else if ((insn & 0x0e000000) == 0x0a000000)
 
                 return space_cccc_101x(insn, asi);
 
-        } else if ((insn & 0x0fe00000) == 0x0c400000) {
-
-                return space_cccc_1100_010x(insn, asi);
-
-        } else if ((insn & 0x0e000000) == 0x0c400000) {
-
-                return space_cccc_110x(insn, asi);
-
-        }
-
-        return space_cccc_111x(insn, asi);
+        return space_cccc_11xx(insn, asi);
 }
 
 void __init arm_kprobe_decode_init(void)
 {
         find_str_pc_offset();
 }
-
-
-/*
- * All ARM instructions listed below.
- *
- * Instructions and their general purpose registers are given.
- * If a particular register may not use R15, it is prefixed with a "!".
- * If marked with a "*" means the value returned by reading R15
- * is implementation defined.
- *
- * ADC/ADD/AND/BIC/CMN/CMP/EOR/MOV/MVN/ORR/RSB/RSC/SBC/SUB/TEQ
- *     TST: Rd, Rn, Rm, !Rs
- * BX: Rm
- * BLX(2): !Rm
- * BX: Rm (R15 legal, but discouraged)
- * BXJ: !Rm,
- * CLZ: !Rd, !Rm
- * CPY: Rd, Rm
- * LDC/2,STC/2 immediate offset & unindex: Rn
- * LDC/2,STC/2 immediate pre/post-indexed: !Rn
- * LDM(1/3): !Rn, register_list
- * LDM(2): !Rn, !register_list
- * LDR,STR,PLD immediate offset: Rd, Rn
- * LDR,STR,PLD register offset: Rd, Rn, !Rm
- * LDR,STR,PLD scaled register offset: Rd, !Rn, !Rm
- * LDR,STR immediate pre/post-indexed: Rd, !Rn
- * LDR,STR register pre/post-indexed: Rd, !Rn, !Rm
- * LDR,STR scaled register pre/post-indexed: Rd, !Rn, !Rm
- * LDRB,STRB immediate offset: !Rd, Rn
- * LDRB,STRB register offset: !Rd, Rn, !Rm
- * LDRB,STRB scaled register offset: !Rd, !Rn, !Rm
- * LDRB,STRB immediate pre/post-indexed: !Rd, !Rn
- * LDRB,STRB register pre/post-indexed: !Rd, !Rn, !Rm
- * LDRB,STRB scaled register pre/post-indexed: !Rd, !Rn, !Rm
- * LDRT,LDRBT,STRBT immediate pre/post-indexed: !Rd, !Rn
- * LDRT,LDRBT,STRBT register pre/post-indexed: !Rd, !Rn, !Rm
- * LDRT,LDRBT,STRBT scaled register pre/post-indexed: !Rd, !Rn, !Rm
- * LDRH/SH/SB/D,STRH/SH/SB/D immediate offset: !Rd, Rn
- * LDRH/SH/SB/D,STRH/SH/SB/D register offset: !Rd, Rn, !Rm
- * LDRH/SH/SB/D,STRH/SH/SB/D immediate pre/post-indexed: !Rd, !Rn
- * LDRH/SH/SB/D,STRH/SH/SB/D register pre/post-indexed: !Rd, !Rn, !Rm
- * LDREX: !Rd, !Rn
- * MCR/2: !Rd
- * MCRR/2,MRRC/2: !Rd, !Rn
- * MLA: !Rd, !Rn, !Rm, !Rs
- * MOV: Rd
- * MRC/2: !Rd (if Rd==15, only changes cond codes, not the register)
- * MRS,MSR: !Rd
- * MUL: !Rd, !Rm, !Rs
- * PKH{BT,TB}: !Rd, !Rn, !Rm
- * QDADD,[U]QADD/16/8/SUBX: !Rd, !Rm, !Rn
- * QDSUB,[U]QSUB/16/8/ADDX: !Rd, !Rm, !Rn
- * REV/16/SH: !Rd, !Rm
- * RFE: !Rn
- * {S,U}[H]ADD{16,8,SUBX},{S,U}[H]SUB{16,8,ADDX}: !Rd, !Rn, !Rm
- * SEL: !Rd, !Rn, !Rm
- * SMLA<x><y>,SMLA{D,W<y>},SMLSD,SMML{A,S}: !Rd, !Rn, !Rm, !Rs
- * SMLAL<x><y>,SMLA{D,LD},SMLSLD,SMMULL,SMULW<y>: !RdHi, !RdLo, !Rm, !Rs
- * SMMUL,SMUAD,SMUL<x><y>,SMUSD: !Rd, !Rm, !Rs
- * SSAT/16: !Rd, !Rm
- * STM(1/2): !Rn, register_list* (R15 in reg list not recommended)
- * STRT immediate pre/post-indexed: Rd*, !Rn
- * STRT register pre/post-indexed: Rd*, !Rn, !Rm
- * STRT scaled register pre/post-indexed: Rd*, !Rn, !Rm
- * STREX: !Rd, !Rn, !Rm
- * SWP/B: !Rd, !Rn, !Rm
- * {S,U}XTA{B,B16,H}: !Rd, !Rn, !Rm
- * {S,U}XT{B,B16,H}: !Rd, !Rm
- * UM{AA,LA,UL}L: !RdHi, !RdLo, !Rm, !Rs
- * USA{D8,A8,T,T16}: !Rd, !Rm, !Rs
- *
- * May transfer control by writing R15 (possible mode changes or alternate
- * mode accesses marked by "*"):
- * ALU op (* with s-bit), B, BL, BKPT, BLX(1/2), BX, BXJ, CPS*, CPY,
- * LDM(1), LDM(2/3)*, LDR, MOV, RFE*, SWI*
- *
- * Instructions that do not take general registers, nor transfer control:
- * CDP/2, SETEND, SRS*
- */
diff --git a/arch/arm/kernel/kprobes.c b/arch/arm/kernel/kprobes.c
index 2ba7deb3072e..1656c87501c0 100644
--- a/arch/arm/kernel/kprobes.c
+++ b/arch/arm/kernel/kprobes.c
@@ -134,7 +134,8 @@ static void __kprobes singlestep(struct kprobe *p, struct pt_regs *regs,
                                  struct kprobe_ctlblk *kcb)
 {
         regs->ARM_pc += 4;
-        p->ainsn.insn_handler(p, regs);
+        if (p->ainsn.insn_check_cc(regs->ARM_cpsr))
+                p->ainsn.insn_handler(p, regs);
 }
 
 /*
diff --git a/arch/arm/kernel/leds.c b/arch/arm/kernel/leds.c
index 31a316c1777b..0f107dcb0347 100644
--- a/arch/arm/kernel/leds.c
+++ b/arch/arm/kernel/leds.c
@@ -10,6 +10,7 @@
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/sysdev.h>
+#include <linux/syscore_ops.h>
 
 #include <asm/leds.h>
 
@@ -69,36 +70,37 @@ static ssize_t leds_store(struct sys_device *dev,
 
 static SYSDEV_ATTR(event, 0200, NULL, leds_store);
 
-static int leds_suspend(struct sys_device *dev, pm_message_t state)
+static struct sysdev_class leds_sysclass = {
+        .name           = "leds",
+};
+
+static struct sys_device leds_device = {
+        .id             = 0,
+        .cls            = &leds_sysclass,
+};
+
+static int leds_suspend(void)
 {
         leds_event(led_stop);
         return 0;
 }
 
-static int leds_resume(struct sys_device *dev)
+static void leds_resume(void)
 {
         leds_event(led_start);
-        return 0;
 }
 
-static int leds_shutdown(struct sys_device *dev)
+static void leds_shutdown(void)
 {
         leds_event(led_halted);
-        return 0;
 }
 
-static struct sysdev_class leds_sysclass = {
-        .name           = "leds",
+static struct syscore_ops leds_syscore_ops = {
         .shutdown       = leds_shutdown,
         .suspend        = leds_suspend,
         .resume         = leds_resume,
 };
 
-static struct sys_device leds_device = {
-        .id             = 0,
-        .cls            = &leds_sysclass,
-};
-
 static int __init leds_init(void)
 {
         int ret;
@@ -107,6 +109,8 @@ static int __init leds_init(void)
                 ret = sysdev_register(&leds_device);
         if (ret == 0)
                 ret = sysdev_create_file(&leds_device, &attr_event);
+        if (ret == 0)
+                register_syscore_ops(&leds_syscore_ops);
         return ret;
 }
 
diff --git a/arch/arm/kernel/machine_kexec.c b/arch/arm/kernel/machine_kexec.c
index 1fc74cbd1a19..e59bbd496c39 100644
--- a/arch/arm/kernel/machine_kexec.c
+++ b/arch/arm/kernel/machine_kexec.c
@@ -23,6 +23,8 @@ extern unsigned long kexec_indirection_page;
 extern unsigned long kexec_mach_type;
 extern unsigned long kexec_boot_atags;
 
+static atomic_t waiting_for_crash_ipi;
+
 /*
  * Provide a dummy crash_notes definition while crash dump arrives to arm.
  * This prevents breakage of crash_notes attribute in kernel/ksysfs.c.
@@ -37,14 +39,47 @@ void machine_kexec_cleanup(struct kimage *image)
 {
 }
 
+void machine_crash_nonpanic_core(void *unused)
+{
+        struct pt_regs regs;
+
+        crash_setup_regs(&regs, NULL);
+        printk(KERN_DEBUG "CPU %u will stop doing anything useful since another CPU has crashed\n",
+               smp_processor_id());
+        crash_save_cpu(&regs, smp_processor_id());
+        flush_cache_all();
+
+        atomic_dec(&waiting_for_crash_ipi);
+        while (1)
+                cpu_relax();
+}
+
 void machine_crash_shutdown(struct pt_regs *regs)
 {
+        unsigned long msecs;
+
         local_irq_disable();
+
+        atomic_set(&waiting_for_crash_ipi, num_online_cpus() - 1);
+        smp_call_function(machine_crash_nonpanic_core, NULL, false);
+        msecs = 1000; /* Wait at most a second for the other cpus to stop */
+        while ((atomic_read(&waiting_for_crash_ipi) > 0) && msecs) {
+                mdelay(1);
+                msecs--;
+        }
+        if (atomic_read(&waiting_for_crash_ipi) > 0)
+                printk(KERN_WARNING "Non-crashing CPUs did not react to IPI\n");
+
         crash_save_cpu(regs, smp_processor_id());
 
         printk(KERN_INFO "Loading crashdump kernel...\n");
 }
 
+/*
+ * Function pointer to optional machine-specific reinitialization
+ */
+void (*kexec_reinit)(void);
+
 void machine_kexec(struct kimage *image)
 {
         unsigned long page_list;
@@ -74,11 +109,16 @@ void machine_kexec(struct kimage *image)
                            (unsigned long) reboot_code_buffer + KEXEC_CONTROL_PAGE_SIZE);
         printk(KERN_INFO "Bye!\n");
 
+        if (kexec_reinit)
+                kexec_reinit();
         local_irq_disable();
         local_fiq_disable();
         setup_mm_for_reboot(0); /* mode is not used, so just pass 0*/
         flush_cache_all();
+        outer_flush_all();
+        outer_disable();
         cpu_proc_fin();
+        outer_inv_all();
         flush_cache_all();
         cpu_reset(reboot_code_buffer_phys);
 }
diff --git a/arch/arm/kernel/module.c b/arch/arm/kernel/module.c
index 6b4605893f1e..016d6a0830a3 100644
--- a/arch/arm/kernel/module.c
+++ b/arch/arm/kernel/module.c
@@ -22,6 +22,7 @@
 
 #include <asm/pgtable.h>
 #include <asm/sections.h>
+#include <asm/smp_plat.h>
 #include <asm/unwind.h>
 
 #ifdef CONFIG_XIP_KERNEL
@@ -38,17 +39,9 @@
 #ifdef CONFIG_MMU
 void *module_alloc(unsigned long size)
 {
-        struct vm_struct *area;
-
-        size = PAGE_ALIGN(size);
-        if (!size)
-                return NULL;
-
-        area = __get_vm_area(size, VM_ALLOC, MODULES_VADDR, MODULES_END);
-        if (!area)
-                return NULL;
-
-        return __vmalloc_area(area, GFP_KERNEL, PAGE_KERNEL_EXEC);
+        return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
+                                GFP_KERNEL, PAGE_KERNEL_EXEC, -1,
+                                __builtin_return_address(0));
 }
 #else /* CONFIG_MMU */
 void *module_alloc(unsigned long size)
@@ -67,24 +60,6 @@ int module_frob_arch_sections(Elf_Ehdr *hdr,
                               char *secstrings,
                               struct module *mod)
 {
-#ifdef CONFIG_ARM_UNWIND
-        Elf_Shdr *s, *sechdrs_end = sechdrs + hdr->e_shnum;
-
-        for (s = sechdrs; s < sechdrs_end; s++) {
-                if (strcmp(".ARM.exidx.init.text", secstrings + s->sh_name) == 0)
-                        mod->arch.unw_sec_init = s;
-                else if (strcmp(".ARM.exidx.devinit.text", secstrings + s->sh_name) == 0)
-                        mod->arch.unw_sec_devinit = s;
-                else if (strcmp(".ARM.exidx", secstrings + s->sh_name) == 0)
-                        mod->arch.unw_sec_core = s;
-                else if (strcmp(".init.text", secstrings + s->sh_name) == 0)
-                        mod->arch.sec_init_text = s;
-                else if (strcmp(".devinit.text", secstrings + s->sh_name) == 0)
-                        mod->arch.sec_devinit_text = s;
-                else if (strcmp(".text", secstrings + s->sh_name) == 0)
-                        mod->arch.sec_core_text = s;
-        }
-#endif
         return 0;
 }
 
@@ -101,6 +76,7 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
         for (i = 0; i < relsec->sh_size / sizeof(Elf32_Rel); i++, rel++) {
                 unsigned long loc;
                 Elf32_Sym *sym;
+                const char *symname;
                 s32 offset;
 #ifdef CONFIG_THUMB2_KERNEL
                 u32 upper, lower, sign, j1, j2;
@@ -108,18 +84,18 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
 
                 offset = ELF32_R_SYM(rel->r_info);
                 if (offset < 0 || offset > (symsec->sh_size / sizeof(Elf32_Sym))) {
-                        printk(KERN_ERR "%s: bad relocation, section %d reloc %d\n",
+                        pr_err("%s: section %u reloc %u: bad relocation sym offset\n",
                                 module->name, relindex, i);
                         return -ENOEXEC;
                 }
 
                 sym = ((Elf32_Sym *)symsec->sh_addr) + offset;
+                symname = strtab + sym->st_name;
 
                 if (rel->r_offset < 0 || rel->r_offset > dstsec->sh_size - sizeof(u32)) {
-                        printk(KERN_ERR "%s: out of bounds relocation, "
-                                "section %d reloc %d offset %d size %d\n",
-                                module->name, relindex, i, rel->r_offset,
-                                dstsec->sh_size);
+                        pr_err("%s: section %u reloc %u sym '%s': out of bounds relocation, offset %d size %u\n",
+                               module->name, relindex, i, symname,
+                               rel->r_offset, dstsec->sh_size);
                         return -ENOEXEC;
                 }
 
@@ -145,10 +121,10 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
                         if (offset & 3 ||
                             offset <= (s32)0xfe000000 ||
                             offset >= (s32)0x02000000) {
-                                printk(KERN_ERR
-                                       "%s: relocation out of range, section "
-                                       "%d reloc %d sym '%s'\n", module->name,
-                                       relindex, i, strtab + sym->st_name);
+                                pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
+                                       module->name, relindex, i, symname,
+                                       ELF32_R_TYPE(rel->r_info), loc,
+                                       sym->st_value);
                                 return -ENOEXEC;
                         }
 
@@ -217,14 +193,23 @@ apply_relocate(Elf32_Shdr *sechdrs, const char *strtab, unsigned int symindex,
                                 offset -= 0x02000000;
                         offset += sym->st_value - loc;
 
-                        /* only Thumb addresses allowed (no interworking) */
-                        if (!(offset & 1) ||
+                        /*
+                         * For function symbols, only Thumb addresses are
+                         * allowed (no interworking).
+                         *
+                         * For non-function symbols, the destination
+                         * has no specific ARM/Thumb disposition, so
+                         * the branch is resolved under the assumption
+                         * that interworking is not required.
+                         */
+                        if ((ELF32_ST_TYPE(sym->st_info) == STT_FUNC &&
+                                !(offset & 1)) ||
                             offset <= (s32)0xff000000 ||
                             offset >= (s32)0x01000000) {
-                                printk(KERN_ERR
-                                       "%s: relocation out of range, section "
-                                       "%d reloc %d sym '%s'\n", module->name,
-                                       relindex, i, strtab + sym->st_name);
+                                pr_err("%s: section %u reloc %u sym '%s': relocation %u out of range (%#lx -> %#x)\n",
+                                       module->name, relindex, i, symname,
+                                       ELF32_R_TYPE(rel->r_info), loc,
+                                       sym->st_value);
                                 return -ENOEXEC;
                         }
 
@@ -289,50 +274,94 @@ apply_relocate_add(Elf32_Shdr *sechdrs, const char *strtab,
         return -ENOEXEC;
 }
 
-#ifdef CONFIG_ARM_UNWIND
-static void register_unwind_tables(struct module *mod)
-{
-        if (mod->arch.unw_sec_init && mod->arch.sec_init_text)
-                mod->arch.unwind_init =
-                        unwind_table_add(mod->arch.unw_sec_init->sh_addr,
-                                         mod->arch.unw_sec_init->sh_size,
-                                         mod->arch.sec_init_text->sh_addr,
-                                         mod->arch.sec_init_text->sh_size);
-        if (mod->arch.unw_sec_devinit && mod->arch.sec_devinit_text)
-                mod->arch.unwind_devinit =
-                        unwind_table_add(mod->arch.unw_sec_devinit->sh_addr,
-                                         mod->arch.unw_sec_devinit->sh_size,
-                                         mod->arch.sec_devinit_text->sh_addr,
-                                         mod->arch.sec_devinit_text->sh_size);
-        if (mod->arch.unw_sec_core && mod->arch.sec_core_text)
-                mod->arch.unwind_core =
-                        unwind_table_add(mod->arch.unw_sec_core->sh_addr,
-                                         mod->arch.unw_sec_core->sh_size,
-                                         mod->arch.sec_core_text->sh_addr,
-                                         mod->arch.sec_core_text->sh_size);
-}
+struct mod_unwind_map {
+        const Elf_Shdr *unw_sec;
+        const Elf_Shdr *txt_sec;
+};
 
-static void unregister_unwind_tables(struct module *mod)
+static const Elf_Shdr *find_mod_section(const Elf32_Ehdr *hdr,
+        const Elf_Shdr *sechdrs, const char *name)
 {
-        unwind_table_del(mod->arch.unwind_init);
-        unwind_table_del(mod->arch.unwind_devinit);
-        unwind_table_del(mod->arch.unwind_core);
+        const Elf_Shdr *s, *se;
+        const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
+
+        for (s = sechdrs, se = sechdrs + hdr->e_shnum; s < se; s++)
+                if (strcmp(name, secstrs + s->sh_name) == 0)
+                        return s;
+
+        return NULL;
 }
-#else
-static inline void register_unwind_tables(struct module *mod) { }
-static inline void unregister_unwind_tables(struct module *mod) { }
-#endif
 
-int
-module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs,
-                struct module *module)
+extern void fixup_pv_table(const void *, unsigned long);
+extern void fixup_smp(const void *, unsigned long);
+
+int module_finalize(const Elf32_Ehdr *hdr, const Elf_Shdr *sechdrs,
+                    struct module *mod)
 {
-        register_unwind_tables(module);
+        const Elf_Shdr *s = NULL;
+#ifdef CONFIG_ARM_UNWIND
+        const char *secstrs = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset;
+        const Elf_Shdr *sechdrs_end = sechdrs + hdr->e_shnum;
+        struct mod_unwind_map maps[ARM_SEC_MAX];
+        int i;
+
+        memset(maps, 0, sizeof(maps));
+
+        for (s = sechdrs; s < sechdrs_end; s++) {
+                const char *secname = secstrs + s->sh_name;
+
+                if (!(s->sh_flags & SHF_ALLOC))
+                        continue;
+
+                if (strcmp(".ARM.exidx.init.text", secname) == 0)
+                        maps[ARM_SEC_INIT].unw_sec = s;
+                else if (strcmp(".ARM.exidx.devinit.text", secname) == 0)
+                        maps[ARM_SEC_DEVINIT].unw_sec = s;
+                else if (strcmp(".ARM.exidx", secname) == 0)
+                        maps[ARM_SEC_CORE].unw_sec = s;
+                else if (strcmp(".ARM.exidx.exit.text", secname) == 0)
+                        maps[ARM_SEC_EXIT].unw_sec = s;
+                else if (strcmp(".ARM.exidx.devexit.text", secname) == 0)
+                        maps[ARM_SEC_DEVEXIT].unw_sec = s;
+                else if (strcmp(".init.text", secname) == 0)
+                        maps[ARM_SEC_INIT].txt_sec = s;
+                else if (strcmp(".devinit.text", secname) == 0)
+                        maps[ARM_SEC_DEVINIT].txt_sec = s;
+                else if (strcmp(".text", secname) == 0)
+                        maps[ARM_SEC_CORE].txt_sec = s;
+                else if (strcmp(".exit.text", secname) == 0)
+                        maps[ARM_SEC_EXIT].txt_sec = s;
+                else if (strcmp(".devexit.text", secname) == 0)
+                        maps[ARM_SEC_DEVEXIT].txt_sec = s;
+        }
+
+        for (i = 0; i < ARM_SEC_MAX; i++)
+                if (maps[i].unw_sec && maps[i].txt_sec)
+                        mod->arch.unwind[i] =
+                                unwind_table_add(maps[i].unw_sec->sh_addr,
+                                                 maps[i].unw_sec->sh_size,
+                                                 maps[i].txt_sec->sh_addr,
+                                                 maps[i].txt_sec->sh_size);
+#endif
+#ifdef CONFIG_ARM_PATCH_PHYS_VIRT
+        s = find_mod_section(hdr, sechdrs, ".pv_table");
+        if (s)
+                fixup_pv_table((void *)s->sh_addr, s->sh_size);
+#endif
+        s = find_mod_section(hdr, sechdrs, ".alt.smp.init");
+        if (s && !is_smp())
+                fixup_smp((void *)s->sh_addr, s->sh_size);
         return 0;
 }
 
 void
 module_arch_cleanup(struct module *mod)
 {
-        unregister_unwind_tables(mod);
+#ifdef CONFIG_ARM_UNWIND
+        int i;
+
+        for (i = 0; i < ARM_SEC_MAX; i++)
+                if (mod->arch.unwind[i])
+                        unwind_table_del(mod->arch.unwind[i]);
+#endif
 }
diff --git a/arch/arm/kernel/perf_event.c b/arch/arm/kernel/perf_event.c
index ecbb0288e5dd..2b5b1421596c 100644
--- a/arch/arm/kernel/perf_event.c
+++ b/arch/arm/kernel/perf_event.c
@@ -4,9 +4,7 @@
  * ARM performance counter support.
  *
  * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
- *
- * ARMv7 support: Jean Pihet <jpihet@mvista.com>
- * 2010 (c) MontaVista Software, LLC.
+ * Copyright (C) 2010 ARM Ltd., Will Deacon <will.deacon@arm.com>
  *
  * This code is based on the sparc64 perf event code, which is in turn based
  * on the x86 code. Callchain code is based on the ARM OProfile backtrace
@@ -34,7 +32,7 @@ static struct platform_device *pmu_device;
  * Hardware lock to serialize accesses to PMU registers. Needed for the
  * read/modify/write sequences.
  */
-DEFINE_SPINLOCK(pmu_lock);
+static DEFINE_RAW_SPINLOCK(pmu_lock);
 
 /*
  * ARMv6 supports a maximum of 3 events, starting from index 1. If we add
@@ -67,31 +65,26 @@ struct cpu_hw_events {
          */
         unsigned long           active_mask[BITS_TO_LONGS(ARMPMU_MAX_HWEVENTS)];
 };
-DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
-
-/* PMU names. */
-static const char *arm_pmu_names[] = {
-        [ARM_PERF_PMU_ID_XSCALE1] = "xscale1",
-        [ARM_PERF_PMU_ID_XSCALE2] = "xscale2",
-        [ARM_PERF_PMU_ID_V6]      = "v6",
-        [ARM_PERF_PMU_ID_V6MP]    = "v6mpcore",
-        [ARM_PERF_PMU_ID_CA8]     = "ARMv7 Cortex-A8",
-        [ARM_PERF_PMU_ID_CA9]     = "ARMv7 Cortex-A9",
-};
+static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
 
 struct arm_pmu {
         enum arm_perf_pmu_ids id;
+        const char      *name;
         irqreturn_t     (*handle_irq)(int irq_num, void *dev);
         void            (*enable)(struct hw_perf_event *evt, int idx);
         void            (*disable)(struct hw_perf_event *evt, int idx);
-        int             (*event_map)(int evt);
-        u64             (*raw_event)(u64);
         int             (*get_event_idx)(struct cpu_hw_events *cpuc,
                                          struct hw_perf_event *hwc);
         u32             (*read_counter)(int idx);
         void            (*write_counter)(int idx, u32 val);
         void            (*start)(void);
         void            (*stop)(void);
+        void            (*reset)(void *);
+        const unsigned  (*cache_map)[PERF_COUNT_HW_CACHE_MAX]
+                                    [PERF_COUNT_HW_CACHE_OP_MAX]
+                                    [PERF_COUNT_HW_CACHE_RESULT_MAX];
+        const unsigned  (*event_map)[PERF_COUNT_HW_MAX];
+        u32             raw_event_mask;
         int             num_events;
         u64             max_period;
 };
@@ -123,6 +116,12 @@ armpmu_get_max_events(void)
 }
 EXPORT_SYMBOL_GPL(armpmu_get_max_events);
 
+int perf_num_counters(void)
+{
+        return armpmu_get_max_events();
+}
+EXPORT_SYMBOL_GPL(perf_num_counters);
+
 #define HW_OP_UNSUPPORTED               0xFFFF
 
 #define C(_x) \
@@ -130,10 +129,6 @@ EXPORT_SYMBOL_GPL(armpmu_get_max_events);
 
 #define CACHE_OP_UNSUPPORTED            0xFFFF
 
-static unsigned armpmu_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
-                                     [PERF_COUNT_HW_CACHE_OP_MAX]
-                                     [PERF_COUNT_HW_CACHE_RESULT_MAX];
-
 static int
 armpmu_map_cache_event(u64 config)
 {
@@ -151,7 +146,7 @@ armpmu_map_cache_event(u64 config)
         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
                 return -EINVAL;
 
-        ret = (int)armpmu_perf_cache_map[cache_type][cache_op][cache_result];
+        ret = (int)(*armpmu->cache_map)[cache_type][cache_op][cache_result];
 
         if (ret == CACHE_OP_UNSUPPORTED)
                 return -ENOENT;
@@ -160,6 +155,19 @@ armpmu_map_cache_event(u64 config)
 }
 
 static int
+armpmu_map_event(u64 config)
+{
+        int mapping = (*armpmu->event_map)[config];
+        return mapping == HW_OP_UNSUPPORTED ? -EOPNOTSUPP : mapping;
+}
+
+static int
+armpmu_map_raw_event(u64 config)
+{
+        return (int)(config & armpmu->raw_event_mask);
+}
+
+static int
 armpmu_event_set_period(struct perf_event *event,
                         struct hw_perf_event *hwc,
                         int idx)
@@ -197,11 +205,9 @@ armpmu_event_set_period(struct perf_event *event,
 static u64
 armpmu_event_update(struct perf_event *event,
                     struct hw_perf_event *hwc,
-                    int idx)
+                    int idx, int overflow)
 {
-        int shift = 64 - 32;
-        s64 prev_raw_count, new_raw_count;
-        u64 delta;
+        u64 delta, prev_raw_count, new_raw_count;
 
 again:
         prev_raw_count = local64_read(&hwc->prev_count);
@@ -211,8 +217,13 @@ again:
                              new_raw_count) != prev_raw_count)
                 goto again;
 
-        delta = (new_raw_count << shift) - (prev_raw_count << shift);
-        delta >>= shift;
+        new_raw_count &= armpmu->max_period;
+        prev_raw_count &= armpmu->max_period;
+
+        if (overflow)
+                delta = armpmu->max_period - prev_raw_count + new_raw_count + 1;
+        else
+                delta = new_raw_count - prev_raw_count;
 
         local64_add(delta, &event->count);
         local64_sub(delta, &hwc->period_left);
@@ -221,46 +232,56 @@ again:
 }
 
 static void
-armpmu_disable(struct perf_event *event)
+armpmu_read(struct perf_event *event)
 {
-        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
-        int idx = hwc->idx;
-
-        WARN_ON(idx < 0);
-
-        clear_bit(idx, cpuc->active_mask);
-        armpmu->disable(hwc, idx);
 
-        barrier();
-
-        armpmu_event_update(event, hwc, idx);
-        cpuc->events[idx] = NULL;
-        clear_bit(idx, cpuc->used_mask);
+        /* Don't read disabled counters! */
+        if (hwc->idx < 0)
+                return;
 
-        perf_event_update_userpage(event);
+        armpmu_event_update(event, hwc, hwc->idx, 0);
 }
 
 static void
-armpmu_read(struct perf_event *event)
+armpmu_stop(struct perf_event *event, int flags)
 {
         struct hw_perf_event *hwc = &event->hw;
 
-        /* Don't read disabled counters! */
-        if (hwc->idx < 0)
+        if (!armpmu)
                 return;
 
-        armpmu_event_update(event, hwc, hwc->idx);
+        /*
+         * ARM pmu always has to update the counter, so ignore
+         * PERF_EF_UPDATE, see comments in armpmu_start().
+         */
+        if (!(hwc->state & PERF_HES_STOPPED)) {
+                armpmu->disable(hwc, hwc->idx);
+                barrier(); /* why? */
+                armpmu_event_update(event, hwc, hwc->idx, 0);
+                hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
+        }
 }
 
 static void
-armpmu_unthrottle(struct perf_event *event)
+armpmu_start(struct perf_event *event, int flags)
 {
         struct hw_perf_event *hwc = &event->hw;
 
+        if (!armpmu)
+                return;
+
+        /*
+         * ARM pmu always has to reprogram the period, so ignore
+         * PERF_EF_RELOAD, see the comment below.
+         */
+        if (flags & PERF_EF_RELOAD)
+                WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
+
+        hwc->state = 0;
         /*
          * Set the period again. Some counters can't be stopped, so when we
-         * were throttled we simply disabled the IRQ source and the counter
+         * were stopped we simply disabled the IRQ source and the counter
          * may have been left counting. If we don't do this step then we may
          * get an interrupt too soon or *way* too late if the overflow has
          * happened since disabling.
@@ -269,14 +290,33 @@ armpmu_unthrottle(struct perf_event *event)
         armpmu->enable(hwc, hwc->idx);
 }
 
+static void
+armpmu_del(struct perf_event *event, int flags)
+{
+        struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
+        struct hw_perf_event *hwc = &event->hw;
+        int idx = hwc->idx;
+
+        WARN_ON(idx < 0);
+
+        clear_bit(idx, cpuc->active_mask);
+        armpmu_stop(event, PERF_EF_UPDATE);
+        cpuc->events[idx] = NULL;
+        clear_bit(idx, cpuc->used_mask);
+
+        perf_event_update_userpage(event);
+}
+
 static int
-armpmu_enable(struct perf_event *event)
+armpmu_add(struct perf_event *event, int flags)
 {
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
         struct hw_perf_event *hwc = &event->hw;
         int idx;
         int err = 0;
 
+        perf_pmu_disable(event->pmu);
+
         /* If we don't have a space for the counter then finish early. */
         idx = armpmu->get_event_idx(cpuc, hwc);
         if (idx < 0) {
@@ -293,25 +333,19 @@ armpmu_enable(struct perf_event *event)
         cpuc->events[idx] = event;
         set_bit(idx, cpuc->active_mask);
 
-        /* Set the period for the event. */
-        armpmu_event_set_period(event, hwc, idx);
-
-        /* Enable the event. */
-        armpmu->enable(hwc, idx);
+        hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
+        if (flags & PERF_EF_START)
+                armpmu_start(event, PERF_EF_RELOAD);
 
         /* Propagate our changes to the userspace mapping. */
         perf_event_update_userpage(event);
 
 out:
+        perf_pmu_enable(event->pmu);
         return err;
 }
 
-static struct pmu pmu = {
-        .enable     = armpmu_enable,
-        .disable    = armpmu_disable,
-        .unthrottle = armpmu_unthrottle,
-        .read       = armpmu_read,
-};
+static struct pmu pmu;
 
 static int
 validate_event(struct cpu_hw_events *cpuc,
@@ -347,9 +381,18 @@ validate_group(struct perf_event *event)
         return 0;
 }
 
+static irqreturn_t armpmu_platform_irq(int irq, void *dev)
+{
+        struct arm_pmu_platdata *plat = dev_get_platdata(&pmu_device->dev);
+
+        return plat->handle_irq(irq, dev, armpmu->handle_irq);
+}
+
 static int
 armpmu_reserve_hardware(void)
 {
+        struct arm_pmu_platdata *plat;
+        irq_handler_t handle_irq;
         int i, err = -ENODEV, irq;
 
         pmu_device = reserve_pmu(ARM_PMU_DEVICE_CPU);
@@ -360,6 +403,12 @@ armpmu_reserve_hardware(void)
 
         init_pmu(ARM_PMU_DEVICE_CPU);
 
+        plat = dev_get_platdata(&pmu_device->dev);
+        if (plat && plat->handle_irq)
+                handle_irq = armpmu_platform_irq;
+        else
+                handle_irq = armpmu->handle_irq;
+
         if (pmu_device->num_resources < 1) {
                 pr_err("no irqs for PMUs defined\n");
                 return -ENODEV;
@@ -370,7 +419,7 @@ armpmu_reserve_hardware(void)
                 if (irq < 0)
                         continue;
 
-                err = request_irq(irq, armpmu->handle_irq,
+                err = request_irq(irq, handle_irq,
                                   IRQF_DISABLED | IRQF_NOBALANCING,
                                   "armpmu", NULL);
                 if (err) {
@@ -429,11 +478,11 @@ __hw_perf_event_init(struct perf_event *event)
 
         /* Decode the generic type into an ARM event identifier. */
         if (PERF_TYPE_HARDWARE == event->attr.type) {
-                mapping = armpmu->event_map(event->attr.config);
+                mapping = armpmu_map_event(event->attr.config);
         } else if (PERF_TYPE_HW_CACHE == event->attr.type) {
                 mapping = armpmu_map_cache_event(event->attr.config);
         } else if (PERF_TYPE_RAW == event->attr.type) {
-                mapping = armpmu->raw_event(event->attr.config);
+                mapping = armpmu_map_raw_event(event->attr.config);
         } else {
                 pr_debug("event type %x not supported\n", event->attr.type);
                 return -EOPNOTSUPP;
@@ -491,22 +540,26 @@ __hw_perf_event_init(struct perf_event *event)
         return err;
 }
 
-const struct pmu *
-hw_perf_event_init(struct perf_event *event)
+static int armpmu_event_init(struct perf_event *event)
 {
         int err = 0;
 
+        switch (event->attr.type) {
+        case PERF_TYPE_RAW:
+        case PERF_TYPE_HARDWARE:
+        case PERF_TYPE_HW_CACHE:
+                break;
+
+        default:
+                return -ENOENT;
+        }
+
         if (!armpmu)
-                return ERR_PTR(-ENODEV);
+                return -ENODEV;
 
         event->destroy = hw_perf_event_destroy;
 
         if (!atomic_inc_not_zero(&active_events)) {
-                if (atomic_read(&active_events) > perf_max_events) {
-                        atomic_dec(&active_events);
-                        return ERR_PTR(-ENOSPC);
-                }
-
                 mutex_lock(&pmu_reserve_mutex);
                 if (atomic_read(&active_events) == 0) {
                         err = armpmu_reserve_hardware();
@@ -518,20 +571,19 @@ hw_perf_event_init(struct perf_event *event)
         }
 
         if (err)
-                return ERR_PTR(err);
+                return err;
 
         err = __hw_perf_event_init(event);
         if (err)
                 hw_perf_event_destroy(event);
 
-        return err ? ERR_PTR(err) : &pmu;
+        return err;
 }
 
-void
-hw_perf_enable(void)
+static void armpmu_enable(struct pmu *pmu)
 {
         /* Enable all of the perf events on hardware. */
-        int idx;
+        int idx, enabled = 0;
         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 
         if (!armpmu)
@@ -544,2378 +596,47 @@ hw_perf_enable(void)
                         continue;
 
                 armpmu->enable(&event->hw, idx);
+                enabled = 1;
         }
 
-        armpmu->start();
+        if (enabled)
+                armpmu->start();
 }
 
-void
-hw_perf_disable(void)
+static void armpmu_disable(struct pmu *pmu)
 {
         if (armpmu)
                 armpmu->stop();
 }
 
-/*
- * ARMv6 Performance counter handling code.
- *
- * ARMv6 has 2 configurable performance counters and a single cycle counter.
- * They all share a single reset bit but can be written to zero so we can use
- * that for a reset.
- *
- * The counters can't be individually enabled or disabled so when we remove
- * one event and replace it with another we could get spurious counts from the
- * wrong event. However, we can take advantage of the fact that the
- * performance counters can export events to the event bus, and the event bus
- * itself can be monitored. This requires that we *don't* export the events to
- * the event bus. The procedure for disabling a configurable counter is:
- *      - change the counter to count the ETMEXTOUT[0] signal (0x20). This
- *        effectively stops the counter from counting.
- *      - disable the counter's interrupt generation (each counter has it's
- *        own interrupt enable bit).
- * Once stopped, the counter value can be written as 0 to reset.
- *
- * To enable a counter:
- *      - enable the counter's interrupt generation.
- *      - set the new event type.
- *
- * Note: the dedicated cycle counter only counts cycles and can't be
- * enabled/disabled independently of the others. When we want to disable the
- * cycle counter, we have to just disable the interrupt reporting and start
- * ignoring that counter. When re-enabling, we have to reset the value and
- * enable the interrupt.
- */
-
-enum armv6_perf_types {
-        ARMV6_PERFCTR_ICACHE_MISS           = 0x0,
-        ARMV6_PERFCTR_IBUF_STALL            = 0x1,
-        ARMV6_PERFCTR_DDEP_STALL            = 0x2,
-        ARMV6_PERFCTR_ITLB_MISS             = 0x3,
-        ARMV6_PERFCTR_DTLB_MISS             = 0x4,
-        ARMV6_PERFCTR_BR_EXEC               = 0x5,
-        ARMV6_PERFCTR_BR_MISPREDICT         = 0x6,
-        ARMV6_PERFCTR_INSTR_EXEC            = 0x7,
-        ARMV6_PERFCTR_DCACHE_HIT            = 0x9,
-        ARMV6_PERFCTR_DCACHE_ACCESS         = 0xA,
-        ARMV6_PERFCTR_DCACHE_MISS           = 0xB,
-        ARMV6_PERFCTR_DCACHE_WBACK          = 0xC,
-        ARMV6_PERFCTR_SW_PC_CHANGE          = 0xD,
-        ARMV6_PERFCTR_MAIN_TLB_MISS         = 0xF,
-        ARMV6_PERFCTR_EXPL_D_ACCESS         = 0x10,
-        ARMV6_PERFCTR_LSU_FULL_STALL        = 0x11,
-        ARMV6_PERFCTR_WBUF_DRAINED          = 0x12,
-        ARMV6_PERFCTR_CPU_CYCLES            = 0xFF,
-        ARMV6_PERFCTR_NOP                   = 0x20,
-};
-
-enum armv6_counters {
-        ARMV6_CYCLE_COUNTER = 1,
-        ARMV6_COUNTER0,
-        ARMV6_COUNTER1,
-};
-
-/*
- * The hardware events that we support. We do support cache operations but
- * we have harvard caches and no way to combine instruction and data
- * accesses/misses in hardware.
- */
-static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = {
-        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV6_PERFCTR_CPU_CYCLES,
-        [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV6_PERFCTR_INSTR_EXEC,
-        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC,
-        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV6_PERFCTR_BR_MISPREDICT,
-        [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
-};
-
-static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
-                                          [PERF_COUNT_HW_CACHE_OP_MAX]
-                                          [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
-        [C(L1D)] = {
-                /*
-                 * The performance counters don't differentiate between read
-                 * and write accesses/misses so this isn't strictly correct,
-                 * but it's the best we can do. Writes and reads get
-                 * combined.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV6_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DCACHE_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV6_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DCACHE_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(L1I)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ICACHE_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ICACHE_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(LL)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(DTLB)] = {
-                /*
-                 * The ARM performance counters can count micro DTLB misses,
-                 * micro ITLB misses and main TLB misses. There isn't an event
-                 * for TLB misses, so use the micro misses here and if users
-                 * want the main TLB misses they can use a raw counter.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DTLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DTLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(ITLB)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(BPU)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-};
-
-enum armv6mpcore_perf_types {
-        ARMV6MPCORE_PERFCTR_ICACHE_MISS     = 0x0,
-        ARMV6MPCORE_PERFCTR_IBUF_STALL      = 0x1,
-        ARMV6MPCORE_PERFCTR_DDEP_STALL      = 0x2,
-        ARMV6MPCORE_PERFCTR_ITLB_MISS       = 0x3,
-        ARMV6MPCORE_PERFCTR_DTLB_MISS       = 0x4,
-        ARMV6MPCORE_PERFCTR_BR_EXEC         = 0x5,
-        ARMV6MPCORE_PERFCTR_BR_NOTPREDICT   = 0x6,
-        ARMV6MPCORE_PERFCTR_BR_MISPREDICT   = 0x7,
-        ARMV6MPCORE_PERFCTR_INSTR_EXEC      = 0x8,
-        ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS = 0xA,
-        ARMV6MPCORE_PERFCTR_DCACHE_RDMISS   = 0xB,
-        ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS = 0xC,
-        ARMV6MPCORE_PERFCTR_DCACHE_WRMISS   = 0xD,
-        ARMV6MPCORE_PERFCTR_DCACHE_EVICTION = 0xE,
-        ARMV6MPCORE_PERFCTR_SW_PC_CHANGE    = 0xF,
-        ARMV6MPCORE_PERFCTR_MAIN_TLB_MISS   = 0x10,
-        ARMV6MPCORE_PERFCTR_EXPL_MEM_ACCESS = 0x11,
-        ARMV6MPCORE_PERFCTR_LSU_FULL_STALL  = 0x12,
-        ARMV6MPCORE_PERFCTR_WBUF_DRAINED    = 0x13,
-        ARMV6MPCORE_PERFCTR_CPU_CYCLES      = 0xFF,
-};
-
-/*
- * The hardware events that we support. We do support cache operations but
- * we have harvard caches and no way to combine instruction and data
- * accesses/misses in hardware.
- */
-static const unsigned armv6mpcore_perf_map[PERF_COUNT_HW_MAX] = {
-        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV6MPCORE_PERFCTR_CPU_CYCLES,
-        [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV6MPCORE_PERFCTR_INSTR_EXEC,
-        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_BR_EXEC,
-        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV6MPCORE_PERFCTR_BR_MISPREDICT,
-        [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
-};
-
-static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
-                                        [PERF_COUNT_HW_CACHE_OP_MAX]
-                                        [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
-        [C(L1D)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]  =
-                                ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS,
-                        [C(RESULT_MISS)]    =
-                                ARMV6MPCORE_PERFCTR_DCACHE_RDMISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]  =
-                                ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS,
-                        [C(RESULT_MISS)]    =
-                                ARMV6MPCORE_PERFCTR_DCACHE_WRMISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(L1I)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ICACHE_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ICACHE_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(LL)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(DTLB)] = {
-                /*
-                 * The ARM performance counters can count micro DTLB misses,
-                 * micro ITLB misses and main TLB misses. There isn't an event
-                 * for TLB misses, so use the micro misses here and if users
-                 * want the main TLB misses they can use a raw counter.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_DTLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_DTLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(ITLB)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(BPU)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
-                },
-        },
-};
-
-static inline unsigned long
-armv6_pmcr_read(void)
-{
-        u32 val;
-        asm volatile("mrc   p15, 0, %0, c15, c12, 0" : "=r"(val));
-        return val;
-}
-
-static inline void
-armv6_pmcr_write(unsigned long val)
-{
-        asm volatile("mcr   p15, 0, %0, c15, c12, 0" : : "r"(val));
-}
-
-#define ARMV6_PMCR_ENABLE               (1 << 0)
-#define ARMV6_PMCR_CTR01_RESET          (1 << 1)
-#define ARMV6_PMCR_CCOUNT_RESET         (1 << 2)
-#define ARMV6_PMCR_CCOUNT_DIV           (1 << 3)
-#define ARMV6_PMCR_COUNT0_IEN           (1 << 4)
-#define ARMV6_PMCR_COUNT1_IEN           (1 << 5)
-#define ARMV6_PMCR_CCOUNT_IEN           (1 << 6)
-#define ARMV6_PMCR_COUNT0_OVERFLOW      (1 << 8)
-#define ARMV6_PMCR_COUNT1_OVERFLOW      (1 << 9)
-#define ARMV6_PMCR_CCOUNT_OVERFLOW      (1 << 10)
-#define ARMV6_PMCR_EVT_COUNT0_SHIFT     20
-#define ARMV6_PMCR_EVT_COUNT0_MASK      (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT)
-#define ARMV6_PMCR_EVT_COUNT1_SHIFT     12
-#define ARMV6_PMCR_EVT_COUNT1_MASK      (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT)
-
-#define ARMV6_PMCR_OVERFLOWED_MASK \
-        (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \
-         ARMV6_PMCR_CCOUNT_OVERFLOW)
-
-static inline int
-armv6_pmcr_has_overflowed(unsigned long pmcr)
-{
-        return (pmcr & ARMV6_PMCR_OVERFLOWED_MASK);
-}
-
-static inline int
-armv6_pmcr_counter_has_overflowed(unsigned long pmcr,
-                                  enum armv6_counters counter)
-{
-        int ret = 0;
-
-        if (ARMV6_CYCLE_COUNTER == counter)
-                ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW;
-        else if (ARMV6_COUNTER0 == counter)
-                ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW;
-        else if (ARMV6_COUNTER1 == counter)
-                ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW;
-        else
-                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
-
-        return ret;
-}
-
-static inline u32
-armv6pmu_read_counter(int counter)
-{
-        unsigned long value = 0;
-
-        if (ARMV6_CYCLE_COUNTER == counter)
-                asm volatile("mrc   p15, 0, %0, c15, c12, 1" : "=r"(value));
-        else if (ARMV6_COUNTER0 == counter)
-                asm volatile("mrc   p15, 0, %0, c15, c12, 2" : "=r"(value));
-        else if (ARMV6_COUNTER1 == counter)
-                asm volatile("mrc   p15, 0, %0, c15, c12, 3" : "=r"(value));
-        else
-                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
-
-        return value;
-}
-
-static inline void
-armv6pmu_write_counter(int counter,
-                       u32 value)
-{
-        if (ARMV6_CYCLE_COUNTER == counter)
-                asm volatile("mcr   p15, 0, %0, c15, c12, 1" : : "r"(value));
-        else if (ARMV6_COUNTER0 == counter)
-                asm volatile("mcr   p15, 0, %0, c15, c12, 2" : : "r"(value));
-        else if (ARMV6_COUNTER1 == counter)
-                asm volatile("mcr   p15, 0, %0, c15, c12, 3" : : "r"(value));
-        else
-                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
-}
-
-void
-armv6pmu_enable_event(struct hw_perf_event *hwc,
-                      int idx)
-{
-        unsigned long val, mask, evt, flags;
-
-        if (ARMV6_CYCLE_COUNTER == idx) {
-                mask    = 0;
-                evt     = ARMV6_PMCR_CCOUNT_IEN;
-        } else if (ARMV6_COUNTER0 == idx) {
-                mask    = ARMV6_PMCR_EVT_COUNT0_MASK;
-                evt     = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) |
-                          ARMV6_PMCR_COUNT0_IEN;
-        } else if (ARMV6_COUNTER1 == idx) {
-                mask    = ARMV6_PMCR_EVT_COUNT1_MASK;
-                evt     = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) |
-                          ARMV6_PMCR_COUNT1_IEN;
-        } else {
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        /*
-         * Mask out the current event and set the counter to count the event
-         * that we're interested in.
-         */
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = armv6_pmcr_read();
-        val &= ~mask;
-        val |= evt;
-        armv6_pmcr_write(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static irqreturn_t
-armv6pmu_handle_irq(int irq_num,
-                    void *dev)
-{
-        unsigned long pmcr = armv6_pmcr_read();
-        struct perf_sample_data data;
-        struct cpu_hw_events *cpuc;
-        struct pt_regs *regs;
-        int idx;
-
-        if (!armv6_pmcr_has_overflowed(pmcr))
-                return IRQ_NONE;
-
-        regs = get_irq_regs();
-
-        /*
-         * The interrupts are cleared by writing the overflow flags back to
-         * the control register. All of the other bits don't have any effect
-         * if they are rewritten, so write the whole value back.
-         */
-        armv6_pmcr_write(pmcr);
-
-        perf_sample_data_init(&data, 0);
-
-        cpuc = &__get_cpu_var(cpu_hw_events);
-        for (idx = 0; idx <= armpmu->num_events; ++idx) {
-                struct perf_event *event = cpuc->events[idx];
-                struct hw_perf_event *hwc;
-
-                if (!test_bit(idx, cpuc->active_mask))
-                        continue;
-
-                /*
-                 * We have a single interrupt for all counters. Check that
-                 * each counter has overflowed before we process it.
-                 */
-                if (!armv6_pmcr_counter_has_overflowed(pmcr, idx))
-                        continue;
-
-                hwc = &event->hw;
-                armpmu_event_update(event, hwc, idx);
-                data.period = event->hw.last_period;
-                if (!armpmu_event_set_period(event, hwc, idx))
-                        continue;
-
-                if (perf_event_overflow(event, 0, &data, regs))
-                        armpmu->disable(hwc, idx);
-        }
-
-        /*
-         * Handle the pending perf events.
-         *
-         * Note: this call *must* be run with interrupts disabled. For
-         * platforms that can have the PMU interrupts raised as an NMI, this
-         * will not work.
-         */
-        perf_event_do_pending();
-
-        return IRQ_HANDLED;
-}
-
-static void
-armv6pmu_start(void)
-{
-        unsigned long flags, val;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = armv6_pmcr_read();
-        val |= ARMV6_PMCR_ENABLE;
-        armv6_pmcr_write(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-void
-armv6pmu_stop(void)
-{
-        unsigned long flags, val;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = armv6_pmcr_read();
-        val &= ~ARMV6_PMCR_ENABLE;
-        armv6_pmcr_write(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static inline int
-armv6pmu_event_map(int config)
-{
-        int mapping = armv6_perf_map[config];
-        if (HW_OP_UNSUPPORTED == mapping)
-                mapping = -EOPNOTSUPP;
-        return mapping;
-}
-
-static inline int
-armv6mpcore_pmu_event_map(int config)
-{
-        int mapping = armv6mpcore_perf_map[config];
-        if (HW_OP_UNSUPPORTED == mapping)
-                mapping = -EOPNOTSUPP;
-        return mapping;
-}
-
-static u64
-armv6pmu_raw_event(u64 config)
-{
-        return config & 0xff;
-}
-
-static int
-armv6pmu_get_event_idx(struct cpu_hw_events *cpuc,
-                       struct hw_perf_event *event)
-{
-        /* Always place a cycle counter into the cycle counter. */
-        if (ARMV6_PERFCTR_CPU_CYCLES == event->config_base) {
-                if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask))
-                        return -EAGAIN;
-
-                return ARMV6_CYCLE_COUNTER;
-        } else {
-                /*
-                 * For anything other than a cycle counter, try and use
-                 * counter0 and counter1.
-                 */
-                if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask)) {
-                        return ARMV6_COUNTER1;
-                }
-
-                if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask)) {
-                        return ARMV6_COUNTER0;
-                }
-
-                /* The counters are all in use. */
-                return -EAGAIN;
-        }
-}
-
-static void
-armv6pmu_disable_event(struct hw_perf_event *hwc,
-                       int idx)
-{
-        unsigned long val, mask, evt, flags;
-
-        if (ARMV6_CYCLE_COUNTER == idx) {
-                mask    = ARMV6_PMCR_CCOUNT_IEN;
-                evt     = 0;
-        } else if (ARMV6_COUNTER0 == idx) {
-                mask    = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK;
-                evt     = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT;
-        } else if (ARMV6_COUNTER1 == idx) {
-                mask    = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK;
-                evt     = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT;
-        } else {
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        /*
-         * Mask out the current event and set the counter to count the number
-         * of ETM bus signal assertion cycles. The external reporting should
-         * be disabled and so this should never increment.
-         */
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = armv6_pmcr_read();
-        val &= ~mask;
-        val |= evt;
-        armv6_pmcr_write(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void
-armv6mpcore_pmu_disable_event(struct hw_perf_event *hwc,
-                              int idx)
-{
-        unsigned long val, mask, flags, evt = 0;
-
-        if (ARMV6_CYCLE_COUNTER == idx) {
-                mask    = ARMV6_PMCR_CCOUNT_IEN;
-        } else if (ARMV6_COUNTER0 == idx) {
-                mask    = ARMV6_PMCR_COUNT0_IEN;
-        } else if (ARMV6_COUNTER1 == idx) {
-                mask    = ARMV6_PMCR_COUNT1_IEN;
-        } else {
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        /*
-         * Unlike UP ARMv6, we don't have a way of stopping the counters. We
-         * simply disable the interrupt reporting.
-         */
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = armv6_pmcr_read();
-        val &= ~mask;
-        val |= evt;
-        armv6_pmcr_write(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static const struct arm_pmu armv6pmu = {
-        .id                     = ARM_PERF_PMU_ID_V6,
-        .handle_irq             = armv6pmu_handle_irq,
-        .enable                 = armv6pmu_enable_event,
-        .disable                = armv6pmu_disable_event,
-        .event_map              = armv6pmu_event_map,
-        .raw_event              = armv6pmu_raw_event,
-        .read_counter           = armv6pmu_read_counter,
-        .write_counter          = armv6pmu_write_counter,
-        .get_event_idx          = armv6pmu_get_event_idx,
-        .start                  = armv6pmu_start,
-        .stop                   = armv6pmu_stop,
-        .num_events             = 3,
-        .max_period             = (1LLU << 32) - 1,
-};
-
-/*
- * ARMv6mpcore is almost identical to single core ARMv6 with the exception
- * that some of the events have different enumerations and that there is no
- * *hack* to stop the programmable counters. To stop the counters we simply
- * disable the interrupt reporting and update the event. When unthrottling we
- * reset the period and enable the interrupt reporting.
- */
-static const struct arm_pmu armv6mpcore_pmu = {
-        .id                     = ARM_PERF_PMU_ID_V6MP,
-        .handle_irq             = armv6pmu_handle_irq,
-        .enable                 = armv6pmu_enable_event,
-        .disable                = armv6mpcore_pmu_disable_event,
-        .event_map              = armv6mpcore_pmu_event_map,
-        .raw_event              = armv6pmu_raw_event,
-        .read_counter           = armv6pmu_read_counter,
-        .write_counter          = armv6pmu_write_counter,
-        .get_event_idx          = armv6pmu_get_event_idx,
-        .start                  = armv6pmu_start,
-        .stop                   = armv6pmu_stop,
-        .num_events             = 3,
-        .max_period             = (1LLU << 32) - 1,
-};
-
-/*
- * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
- *
- * Copied from ARMv6 code, with the low level code inspired
- *  by the ARMv7 Oprofile code.
- *
- * Cortex-A8 has up to 4 configurable performance counters and
- *  a single cycle counter.
- * Cortex-A9 has up to 31 configurable performance counters and
- *  a single cycle counter.
- *
- * All counters can be enabled/disabled and IRQ masked separately. The cycle
- *  counter and all 4 performance counters together can be reset separately.
- */
-
-/* Common ARMv7 event types */
-enum armv7_perf_types {
-        ARMV7_PERFCTR_PMNC_SW_INCR              = 0x00,
-        ARMV7_PERFCTR_IFETCH_MISS               = 0x01,
-        ARMV7_PERFCTR_ITLB_MISS                 = 0x02,
-        ARMV7_PERFCTR_DCACHE_REFILL             = 0x03,
-        ARMV7_PERFCTR_DCACHE_ACCESS             = 0x04,
-        ARMV7_PERFCTR_DTLB_REFILL               = 0x05,
-        ARMV7_PERFCTR_DREAD                     = 0x06,
-        ARMV7_PERFCTR_DWRITE                    = 0x07,
-
-        ARMV7_PERFCTR_EXC_TAKEN                 = 0x09,
-        ARMV7_PERFCTR_EXC_EXECUTED              = 0x0A,
-        ARMV7_PERFCTR_CID_WRITE                 = 0x0B,
-        /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
-         * It counts:
-         *  - all branch instructions,
-         *  - instructions that explicitly write the PC,
-         *  - exception generating instructions.
-         */
-        ARMV7_PERFCTR_PC_WRITE                  = 0x0C,
-        ARMV7_PERFCTR_PC_IMM_BRANCH             = 0x0D,
-        ARMV7_PERFCTR_UNALIGNED_ACCESS          = 0x0F,
-        ARMV7_PERFCTR_PC_BRANCH_MIS_PRED        = 0x10,
-        ARMV7_PERFCTR_CLOCK_CYCLES              = 0x11,
-
-        ARMV7_PERFCTR_PC_BRANCH_MIS_USED        = 0x12,
-
-        ARMV7_PERFCTR_CPU_CYCLES                = 0xFF
-};
-
-/* ARMv7 Cortex-A8 specific event types */
-enum armv7_a8_perf_types {
-        ARMV7_PERFCTR_INSTR_EXECUTED            = 0x08,
-
-        ARMV7_PERFCTR_PC_PROC_RETURN            = 0x0E,
-
-        ARMV7_PERFCTR_WRITE_BUFFER_FULL         = 0x40,
-        ARMV7_PERFCTR_L2_STORE_MERGED           = 0x41,
-        ARMV7_PERFCTR_L2_STORE_BUFF             = 0x42,
-        ARMV7_PERFCTR_L2_ACCESS                 = 0x43,
-        ARMV7_PERFCTR_L2_CACH_MISS              = 0x44,
-        ARMV7_PERFCTR_AXI_READ_CYCLES           = 0x45,
-        ARMV7_PERFCTR_AXI_WRITE_CYCLES          = 0x46,
-        ARMV7_PERFCTR_MEMORY_REPLAY             = 0x47,
-        ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY   = 0x48,
-        ARMV7_PERFCTR_L1_DATA_MISS              = 0x49,
-        ARMV7_PERFCTR_L1_INST_MISS              = 0x4A,
-        ARMV7_PERFCTR_L1_DATA_COLORING          = 0x4B,
-        ARMV7_PERFCTR_L1_NEON_DATA              = 0x4C,
-        ARMV7_PERFCTR_L1_NEON_CACH_DATA         = 0x4D,
-        ARMV7_PERFCTR_L2_NEON                   = 0x4E,
-        ARMV7_PERFCTR_L2_NEON_HIT               = 0x4F,
-        ARMV7_PERFCTR_L1_INST                   = 0x50,
-        ARMV7_PERFCTR_PC_RETURN_MIS_PRED        = 0x51,
-        ARMV7_PERFCTR_PC_BRANCH_FAILED          = 0x52,
-        ARMV7_PERFCTR_PC_BRANCH_TAKEN           = 0x53,
-        ARMV7_PERFCTR_PC_BRANCH_EXECUTED        = 0x54,
-        ARMV7_PERFCTR_OP_EXECUTED               = 0x55,
-        ARMV7_PERFCTR_CYCLES_INST_STALL         = 0x56,
-        ARMV7_PERFCTR_CYCLES_INST               = 0x57,
-        ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL    = 0x58,
-        ARMV7_PERFCTR_CYCLES_NEON_INST_STALL    = 0x59,
-        ARMV7_PERFCTR_NEON_CYCLES               = 0x5A,
-
-        ARMV7_PERFCTR_PMU0_EVENTS               = 0x70,
-        ARMV7_PERFCTR_PMU1_EVENTS               = 0x71,
-        ARMV7_PERFCTR_PMU_EVENTS                = 0x72,
-};
-
-/* ARMv7 Cortex-A9 specific event types */
-enum armv7_a9_perf_types {
-        ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC     = 0x40,
-        ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC     = 0x41,
-        ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC       = 0x42,
-
-        ARMV7_PERFCTR_COHERENT_LINE_MISS        = 0x50,
-        ARMV7_PERFCTR_COHERENT_LINE_HIT         = 0x51,
-
-        ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES   = 0x60,
-        ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES   = 0x61,
-        ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62,
-        ARMV7_PERFCTR_STREX_EXECUTED_PASSED     = 0x63,
-        ARMV7_PERFCTR_STREX_EXECUTED_FAILED     = 0x64,
-        ARMV7_PERFCTR_DATA_EVICTION             = 0x65,
-        ARMV7_PERFCTR_ISSUE_STAGE_NO_INST       = 0x66,
-        ARMV7_PERFCTR_ISSUE_STAGE_EMPTY         = 0x67,
-        ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE  = 0x68,
-
-        ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E,
-
-        ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST   = 0x70,
-        ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71,
-        ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST  = 0x72,
-        ARMV7_PERFCTR_FP_EXECUTED_INST          = 0x73,
-        ARMV7_PERFCTR_NEON_EXECUTED_INST        = 0x74,
-
-        ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80,
-        ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES  = 0x81,
-        ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES        = 0x82,
-        ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES        = 0x83,
-        ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES  = 0x84,
-        ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES  = 0x85,
-        ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES      = 0x86,
-
-        ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES  = 0x8A,
-        ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B,
-
-        ARMV7_PERFCTR_ISB_INST                  = 0x90,
-        ARMV7_PERFCTR_DSB_INST                  = 0x91,
-        ARMV7_PERFCTR_DMB_INST                  = 0x92,
-        ARMV7_PERFCTR_EXT_INTERRUPTS            = 0x93,
-
-        ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED     = 0xA0,
-        ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED       = 0xA1,
-        ARMV7_PERFCTR_PLE_FIFO_FLUSH            = 0xA2,
-        ARMV7_PERFCTR_PLE_RQST_COMPLETED        = 0xA3,
-        ARMV7_PERFCTR_PLE_FIFO_OVERFLOW         = 0xA4,
-        ARMV7_PERFCTR_PLE_RQST_PROG             = 0xA5
-};
-
-/*
- * Cortex-A8 HW events mapping
- *
- * The hardware events that we support. We do support cache operations but
- * we have harvard caches and no way to combine instruction and data
- * accesses/misses in hardware.
- */
-static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
-        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
-        [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
-        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
-        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
-        [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
-};
-
-static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
-                                          [PERF_COUNT_HW_CACHE_OP_MAX]
-                                          [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
-        [C(L1D)] = {
-                /*
-                 * The performance counters don't differentiate between read
-                 * and write accesses/misses so this isn't strictly correct,
-                 * but it's the best we can do. Writes and reads get
-                 * combined.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(L1I)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(LL)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(DTLB)] = {
-                /*
-                 * Only ITLB misses and DTLB refills are supported.
-                 * If users want the DTLB refills misses a raw counter
-                 * must be used.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(ITLB)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(BPU)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
-                        [C(RESULT_MISS)]
-                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
-                        [C(RESULT_MISS)]
-                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-};
-
-/*
- * Cortex-A9 HW events mapping
- */
-static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
-        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
-        [PERF_COUNT_HW_INSTRUCTIONS]        =
-                                        ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE,
-        [PERF_COUNT_HW_CACHE_REFERENCES]    = ARMV7_PERFCTR_COHERENT_LINE_HIT,
-        [PERF_COUNT_HW_CACHE_MISSES]        = ARMV7_PERFCTR_COHERENT_LINE_MISS,
-        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
-        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
-        [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
-};
-
-static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
-                                          [PERF_COUNT_HW_CACHE_OP_MAX]
-                                          [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
-        [C(L1D)] = {
-                /*
-                 * The performance counters don't differentiate between read
-                 * and write accesses/misses so this isn't strictly correct,
-                 * but it's the best we can do. Writes and reads get
-                 * combined.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(L1I)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(LL)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(DTLB)] = {
-                /*
-                 * Only ITLB misses and DTLB refills are supported.
-                 * If users want the DTLB refills misses a raw counter
-                 * must be used.
-                 */
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(ITLB)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(BPU)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
-                        [C(RESULT_MISS)]
-                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
-                        [C(RESULT_MISS)]
-                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-};
-
-/*
- * Perf Events counters
- */
-enum armv7_counters {
-        ARMV7_CYCLE_COUNTER             = 1,    /* Cycle counter */
-        ARMV7_COUNTER0                  = 2,    /* First event counter */
-};
-
-/*
- * The cycle counter is ARMV7_CYCLE_COUNTER.
- * The first event counter is ARMV7_COUNTER0.
- * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1).
- */
-#define ARMV7_COUNTER_LAST      (ARMV7_COUNTER0 + armpmu->num_events - 1)
-
-/*
- * ARMv7 low level PMNC access
- */
-
-/*
- * Per-CPU PMNC: config reg
- */
-#define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
-#define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
-#define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
-#define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
-#define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
-#define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
-#define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
-#define ARMV7_PMNC_N_MASK       0x1f
-#define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
-
-/*
- * Available counters
- */
-#define ARMV7_CNT0              0       /* First event counter */
-#define ARMV7_CCNT              31      /* Cycle counter */
-
-/* Perf Event to low level counters mapping */
-#define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0)
-
-/*
- * CNTENS: counters enable reg
- */
-#define ARMV7_CNTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
-#define ARMV7_CNTENS_C          (1 << ARMV7_CCNT)
-
-/*
- * CNTENC: counters disable reg
- */
-#define ARMV7_CNTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
-#define ARMV7_CNTENC_C          (1 << ARMV7_CCNT)
-
-/*
- * INTENS: counters overflow interrupt enable reg
- */
-#define ARMV7_INTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
-#define ARMV7_INTENS_C          (1 << ARMV7_CCNT)
-
-/*
- * INTENC: counters overflow interrupt disable reg
- */
-#define ARMV7_INTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
-#define ARMV7_INTENC_C          (1 << ARMV7_CCNT)
-
-/*
- * EVTSEL: Event selection reg
- */
-#define ARMV7_EVTSEL_MASK       0xff            /* Mask for writable bits */
-
-/*
- * SELECT: Counter selection reg
- */
-#define ARMV7_SELECT_MASK       0x1f            /* Mask for writable bits */
-
-/*
- * FLAG: counters overflow flag status reg
- */
-#define ARMV7_FLAG_P(idx)       (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
-#define ARMV7_FLAG_C            (1 << ARMV7_CCNT)
-#define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
-#define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
-
-static inline unsigned long armv7_pmnc_read(void)
-{
-        u32 val;
-        asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
-        return val;
-}
-
-static inline void armv7_pmnc_write(unsigned long val)
-{
-        val &= ARMV7_PMNC_MASK;
-        asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
-}
-
-static inline int armv7_pmnc_has_overflowed(unsigned long pmnc)
-{
-        return pmnc & ARMV7_OVERFLOWED_MASK;
-}
-
-static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc,
-                                        enum armv7_counters counter)
-{
-        int ret;
-
-        if (counter == ARMV7_CYCLE_COUNTER)
-                ret = pmnc & ARMV7_FLAG_C;
-        else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST))
-                ret = pmnc & ARMV7_FLAG_P(counter);
-        else
-                pr_err("CPU%u checking wrong counter %d overflow status\n",
-                        smp_processor_id(), counter);
-
-        return ret;
-}
-
-static inline int armv7_pmnc_select_counter(unsigned int idx)
-{
-        u32 val;
-
-        if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) {
-                pr_err("CPU%u selecting wrong PMNC counter"
-                        " %d\n", smp_processor_id(), idx);
-                return -1;
-        }
-
-        val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK;
-        asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
-
-        return idx;
-}
-
-static inline u32 armv7pmu_read_counter(int idx)
-{
-        unsigned long value = 0;
-
-        if (idx == ARMV7_CYCLE_COUNTER)
-                asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
-        else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
-                if (armv7_pmnc_select_counter(idx) == idx)
-                        asm volatile("mrc p15, 0, %0, c9, c13, 2"
-                                     : "=r" (value));
-        } else
-                pr_err("CPU%u reading wrong counter %d\n",
-                        smp_processor_id(), idx);
-
-        return value;
-}
-
-static inline void armv7pmu_write_counter(int idx, u32 value)
-{
-        if (idx == ARMV7_CYCLE_COUNTER)
-                asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
-        else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
-                if (armv7_pmnc_select_counter(idx) == idx)
-                        asm volatile("mcr p15, 0, %0, c9, c13, 2"
-                                     : : "r" (value));
-        } else
-                pr_err("CPU%u writing wrong counter %d\n",
-                        smp_processor_id(), idx);
-}
-
-static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val)
-{
-        if (armv7_pmnc_select_counter(idx) == idx) {
-                val &= ARMV7_EVTSEL_MASK;
-                asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
-        }
-}
-
-static inline u32 armv7_pmnc_enable_counter(unsigned int idx)
-{
-        u32 val;
-
-        if ((idx != ARMV7_CYCLE_COUNTER) &&
-            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
-                pr_err("CPU%u enabling wrong PMNC counter"
-                        " %d\n", smp_processor_id(), idx);
-                return -1;
-        }
-
-        if (idx == ARMV7_CYCLE_COUNTER)
-                val = ARMV7_CNTENS_C;
-        else
-                val = ARMV7_CNTENS_P(idx);
-
-        asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
-
-        return idx;
-}
-
-static inline u32 armv7_pmnc_disable_counter(unsigned int idx)
-{
-        u32 val;
-
-
-        if ((idx != ARMV7_CYCLE_COUNTER) &&
-            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
-                pr_err("CPU%u disabling wrong PMNC counter"
-                        " %d\n", smp_processor_id(), idx);
-                return -1;
-        }
-
-        if (idx == ARMV7_CYCLE_COUNTER)
-                val = ARMV7_CNTENC_C;
-        else
-                val = ARMV7_CNTENC_P(idx);
-
-        asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
-
-        return idx;
-}
-
-static inline u32 armv7_pmnc_enable_intens(unsigned int idx)
-{
-        u32 val;
-
-        if ((idx != ARMV7_CYCLE_COUNTER) &&
-            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
-                pr_err("CPU%u enabling wrong PMNC counter"
-                        " interrupt enable %d\n", smp_processor_id(), idx);
-                return -1;
-        }
-
-        if (idx == ARMV7_CYCLE_COUNTER)
-                val = ARMV7_INTENS_C;
-        else
-                val = ARMV7_INTENS_P(idx);
-
-        asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
-
-        return idx;
-}
-
-static inline u32 armv7_pmnc_disable_intens(unsigned int idx)
-{
-        u32 val;
-
-        if ((idx != ARMV7_CYCLE_COUNTER) &&
-            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
-                pr_err("CPU%u disabling wrong PMNC counter"
-                        " interrupt enable %d\n", smp_processor_id(), idx);
-                return -1;
-        }
-
-        if (idx == ARMV7_CYCLE_COUNTER)
-                val = ARMV7_INTENC_C;
-        else
-                val = ARMV7_INTENC_P(idx);
-
-        asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
-
-        return idx;
-}
-
-static inline u32 armv7_pmnc_getreset_flags(void)
-{
-        u32 val;
-
-        /* Read */
-        asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
-
-        /* Write to clear flags */
-        val &= ARMV7_FLAG_MASK;
-        asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
-
-        return val;
-}
-
-#ifdef DEBUG
-static void armv7_pmnc_dump_regs(void)
-{
-        u32 val;
-        unsigned int cnt;
-
-        printk(KERN_INFO "PMNC registers dump:\n");
-
-        asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
-        printk(KERN_INFO "PMNC  =0x%08x\n", val);
-
-        asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
-        printk(KERN_INFO "CNTENS=0x%08x\n", val);
-
-        asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
-        printk(KERN_INFO "INTENS=0x%08x\n", val);
-
-        asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
-        printk(KERN_INFO "FLAGS =0x%08x\n", val);
-
-        asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
-        printk(KERN_INFO "SELECT=0x%08x\n", val);
-
-        asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
-        printk(KERN_INFO "CCNT  =0x%08x\n", val);
-
-        for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) {
-                armv7_pmnc_select_counter(cnt);
-                asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
-                printk(KERN_INFO "CNT[%d] count =0x%08x\n",
-                        cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
-                asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
-                printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
-                        cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
-        }
-}
-#endif
-
-void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
-{
-        unsigned long flags;
-
-        /*
-         * Enable counter and interrupt, and set the counter to count
-         * the event that we're interested in.
-         */
-        spin_lock_irqsave(&pmu_lock, flags);
-
-        /*
-         * Disable counter
-         */
-        armv7_pmnc_disable_counter(idx);
-
-        /*
-         * Set event (if destined for PMNx counters)
-         * We don't need to set the event if it's a cycle count
-         */
-        if (idx != ARMV7_CYCLE_COUNTER)
-                armv7_pmnc_write_evtsel(idx, hwc->config_base);
-
-        /*
-         * Enable interrupt for this counter
-         */
-        armv7_pmnc_enable_intens(idx);
-
-        /*
-         * Enable counter
-         */
-        armv7_pmnc_enable_counter(idx);
-
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
-{
-        unsigned long flags;
-
-        /*
-         * Disable counter and interrupt
-         */
-        spin_lock_irqsave(&pmu_lock, flags);
-
-        /*
-         * Disable counter
-         */
-        armv7_pmnc_disable_counter(idx);
-
-        /*
-         * Disable interrupt for this counter
-         */
-        armv7_pmnc_disable_intens(idx);
-
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
-{
-        unsigned long pmnc;
-        struct perf_sample_data data;
-        struct cpu_hw_events *cpuc;
-        struct pt_regs *regs;
-        int idx;
-
-        /*
-         * Get and reset the IRQ flags
-         */
-        pmnc = armv7_pmnc_getreset_flags();
-
-        /*
-         * Did an overflow occur?
-         */
-        if (!armv7_pmnc_has_overflowed(pmnc))
-                return IRQ_NONE;
-
-        /*
-         * Handle the counter(s) overflow(s)
-         */
-        regs = get_irq_regs();
-
-        perf_sample_data_init(&data, 0);
-
-        cpuc = &__get_cpu_var(cpu_hw_events);
-        for (idx = 0; idx <= armpmu->num_events; ++idx) {
-                struct perf_event *event = cpuc->events[idx];
-                struct hw_perf_event *hwc;
-
-                if (!test_bit(idx, cpuc->active_mask))
-                        continue;
-
-                /*
-                 * We have a single interrupt for all counters. Check that
-                 * each counter has overflowed before we process it.
-                 */
-                if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
-                        continue;
-
-                hwc = &event->hw;
-                armpmu_event_update(event, hwc, idx);
-                data.period = event->hw.last_period;
-                if (!armpmu_event_set_period(event, hwc, idx))
-                        continue;
-
-                if (perf_event_overflow(event, 0, &data, regs))
-                        armpmu->disable(hwc, idx);
-        }
-
-        /*
-         * Handle the pending perf events.
-         *
-         * Note: this call *must* be run with interrupts disabled. For
-         * platforms that can have the PMU interrupts raised as an NMI, this
-         * will not work.
-         */
-        perf_event_do_pending();
-
-        return IRQ_HANDLED;
-}
-
-static void armv7pmu_start(void)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        /* Enable all counters */
-        armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void armv7pmu_stop(void)
-{
-        unsigned long flags;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        /* Disable all counters */
-        armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static inline int armv7_a8_pmu_event_map(int config)
-{
-        int mapping = armv7_a8_perf_map[config];
-        if (HW_OP_UNSUPPORTED == mapping)
-                mapping = -EOPNOTSUPP;
-        return mapping;
-}
-
-static inline int armv7_a9_pmu_event_map(int config)
-{
-        int mapping = armv7_a9_perf_map[config];
-        if (HW_OP_UNSUPPORTED == mapping)
-                mapping = -EOPNOTSUPP;
-        return mapping;
-}
-
-static u64 armv7pmu_raw_event(u64 config)
-{
-        return config & 0xff;
-}
-
-static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
-                                  struct hw_perf_event *event)
-{
-        int idx;
-
-        /* Always place a cycle counter into the cycle counter. */
-        if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) {
-                if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask))
-                        return -EAGAIN;
-
-                return ARMV7_CYCLE_COUNTER;
-        } else {
-                /*
-                 * For anything other than a cycle counter, try and use
-                 * the events counters
-                 */
-                for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) {
-                        if (!test_and_set_bit(idx, cpuc->used_mask))
-                                return idx;
-                }
-
-                /* The counters are all in use. */
-                return -EAGAIN;
-        }
-}
-
-static struct arm_pmu armv7pmu = {
-        .handle_irq             = armv7pmu_handle_irq,
-        .enable                 = armv7pmu_enable_event,
-        .disable                = armv7pmu_disable_event,
-        .raw_event              = armv7pmu_raw_event,
-        .read_counter           = armv7pmu_read_counter,
-        .write_counter          = armv7pmu_write_counter,
-        .get_event_idx          = armv7pmu_get_event_idx,
-        .start                  = armv7pmu_start,
-        .stop                   = armv7pmu_stop,
-        .max_period             = (1LLU << 32) - 1,
+static struct pmu pmu = {
+        .pmu_enable     = armpmu_enable,
+        .pmu_disable    = armpmu_disable,
+        .event_init     = armpmu_event_init,
+        .add            = armpmu_add,
+        .del            = armpmu_del,
+        .start          = armpmu_start,
+        .stop           = armpmu_stop,
+        .read           = armpmu_read,
 };
 
-static u32 __init armv7_reset_read_pmnc(void)
-{
-        u32 nb_cnt;
-
-        /* Initialize & Reset PMNC: C and P bits */
-        armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
-
-        /* Read the nb of CNTx counters supported from PMNC */
-        nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
-
-        /* Add the CPU cycles counter and return */
-        return nb_cnt + 1;
-}
+/* Include the PMU-specific implementations. */
+#include "perf_event_xscale.c"
+#include "perf_event_v6.c"
+#include "perf_event_v7.c"
 
 /*
- * ARMv5 [xscale] Performance counter handling code.
- *
- * Based on xscale OProfile code.
- *
- * There are two variants of the xscale PMU that we support:
- *      - xscale1pmu: 2 event counters and a cycle counter
- *      - xscale2pmu: 4 event counters and a cycle counter
- * The two variants share event definitions, but have different
- * PMU structures.
+ * Ensure the PMU has sane values out of reset.
+ * This requires SMP to be available, so exists as a separate initcall.
  */
-
-enum xscale_perf_types {
-        XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
-        XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
-        XSCALE_PERFCTR_DATA_STALL               = 0x02,
-        XSCALE_PERFCTR_ITLB_MISS                = 0x03,
-        XSCALE_PERFCTR_DTLB_MISS                = 0x04,
-        XSCALE_PERFCTR_BRANCH                   = 0x05,
-        XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
-        XSCALE_PERFCTR_INSTRUCTION              = 0x07,
-        XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
-        XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
-        XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
-        XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
-        XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
-        XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
-        XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
-        XSCALE_PERFCTR_BCU_FULL                 = 0x11,
-        XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
-        XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
-        XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
-        XSCALE_PERFCTR_RMW                      = 0x16,
-        /* XSCALE_PERFCTR_CCNT is not hardware defined */
-        XSCALE_PERFCTR_CCNT                     = 0xFE,
-        XSCALE_PERFCTR_UNUSED                   = 0xFF,
-};
-
-enum xscale_counters {
-        XSCALE_CYCLE_COUNTER    = 1,
-        XSCALE_COUNTER0,
-        XSCALE_COUNTER1,
-        XSCALE_COUNTER2,
-        XSCALE_COUNTER3,
-};
-
-static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
-        [PERF_COUNT_HW_CPU_CYCLES]          = XSCALE_PERFCTR_CCNT,
-        [PERF_COUNT_HW_INSTRUCTIONS]        = XSCALE_PERFCTR_INSTRUCTION,
-        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
-        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
-        [PERF_COUNT_HW_BRANCH_MISSES]       = XSCALE_PERFCTR_BRANCH_MISS,
-        [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
-};
-
-static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
-                                           [PERF_COUNT_HW_CACHE_OP_MAX]
-                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
-        [C(L1D)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(L1I)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(LL)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(DTLB)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(ITLB)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-        [C(BPU)] = {
-                [C(OP_READ)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_WRITE)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-                [C(OP_PREFETCH)] = {
-                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
-                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
-                },
-        },
-};
-
-#define XSCALE_PMU_ENABLE       0x001
-#define XSCALE_PMN_RESET        0x002
-#define XSCALE_CCNT_RESET       0x004
-#define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
-#define XSCALE_PMU_CNT64        0x008
-
-static inline int
-xscalepmu_event_map(int config)
-{
-        int mapping = xscale_perf_map[config];
-        if (HW_OP_UNSUPPORTED == mapping)
-                mapping = -EOPNOTSUPP;
-        return mapping;
-}
-
-static u64
-xscalepmu_raw_event(u64 config)
-{
-        return config & 0xff;
-}
-
-#define XSCALE1_OVERFLOWED_MASK 0x700
-#define XSCALE1_CCOUNT_OVERFLOW 0x400
-#define XSCALE1_COUNT0_OVERFLOW 0x100
-#define XSCALE1_COUNT1_OVERFLOW 0x200
-#define XSCALE1_CCOUNT_INT_EN   0x040
-#define XSCALE1_COUNT0_INT_EN   0x010
-#define XSCALE1_COUNT1_INT_EN   0x020
-#define XSCALE1_COUNT0_EVT_SHFT 12
-#define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
-#define XSCALE1_COUNT1_EVT_SHFT 20
-#define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
-
-static inline u32
-xscale1pmu_read_pmnc(void)
-{
-        u32 val;
-        asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
-        return val;
-}
-
-static inline void
-xscale1pmu_write_pmnc(u32 val)
-{
-        /* upper 4bits and 7, 11 are write-as-0 */
-        val &= 0xffff77f;
-        asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
-}
-
-static inline int
-xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
-                                        enum xscale_counters counter)
-{
-        int ret = 0;
-
-        switch (counter) {
-        case XSCALE_CYCLE_COUNTER:
-                ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
-                break;
-        case XSCALE_COUNTER0:
-                ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
-                break;
-        case XSCALE_COUNTER1:
-                ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
-                break;
-        default:
-                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
-        }
-
-        return ret;
-}
-
-static irqreturn_t
-xscale1pmu_handle_irq(int irq_num, void *dev)
-{
-        unsigned long pmnc;
-        struct perf_sample_data data;
-        struct cpu_hw_events *cpuc;
-        struct pt_regs *regs;
-        int idx;
-
-        /*
-         * NOTE: there's an A stepping erratum that states if an overflow
-         *       bit already exists and another occurs, the previous
-         *       Overflow bit gets cleared. There's no workaround.
-         *       Fixed in B stepping or later.
-         */
-        pmnc = xscale1pmu_read_pmnc();
-
-        /*
-         * Write the value back to clear the overflow flags. Overflow
-         * flags remain in pmnc for use below. We also disable the PMU
-         * while we process the interrupt.
-         */
-        xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
-
-        if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
-                return IRQ_NONE;
-
-        regs = get_irq_regs();
-
-        perf_sample_data_init(&data, 0);
-
-        cpuc = &__get_cpu_var(cpu_hw_events);
-        for (idx = 0; idx <= armpmu->num_events; ++idx) {
-                struct perf_event *event = cpuc->events[idx];
-                struct hw_perf_event *hwc;
-
-                if (!test_bit(idx, cpuc->active_mask))
-                        continue;
-
-                if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
-                        continue;
-
-                hwc = &event->hw;
-                armpmu_event_update(event, hwc, idx);
-                data.period = event->hw.last_period;
-                if (!armpmu_event_set_period(event, hwc, idx))
-                        continue;
-
-                if (perf_event_overflow(event, 0, &data, regs))
-                        armpmu->disable(hwc, idx);
-        }
-
-        perf_event_do_pending();
-
-        /*
-         * Re-enable the PMU.
-         */
-        pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
-        xscale1pmu_write_pmnc(pmnc);
-
-        return IRQ_HANDLED;
-}
-
-static void
-xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
-{
-        unsigned long val, mask, evt, flags;
-
-        switch (idx) {
-        case XSCALE_CYCLE_COUNTER:
-                mask = 0;
-                evt = XSCALE1_CCOUNT_INT_EN;
-                break;
-        case XSCALE_COUNTER0:
-                mask = XSCALE1_COUNT0_EVT_MASK;
-                evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
-                        XSCALE1_COUNT0_INT_EN;
-                break;
-        case XSCALE_COUNTER1:
-                mask = XSCALE1_COUNT1_EVT_MASK;
-                evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
-                        XSCALE1_COUNT1_INT_EN;
-                break;
-        default:
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = xscale1pmu_read_pmnc();
-        val &= ~mask;
-        val |= evt;
-        xscale1pmu_write_pmnc(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void
-xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
-{
-        unsigned long val, mask, evt, flags;
-
-        switch (idx) {
-        case XSCALE_CYCLE_COUNTER:
-                mask = XSCALE1_CCOUNT_INT_EN;
-                evt = 0;
-                break;
-        case XSCALE_COUNTER0:
-                mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
-                evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER1:
-                mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
-                evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
-                break;
-        default:
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = xscale1pmu_read_pmnc();
-        val &= ~mask;
-        val |= evt;
-        xscale1pmu_write_pmnc(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static int
-xscale1pmu_get_event_idx(struct cpu_hw_events *cpuc,
-                        struct hw_perf_event *event)
-{
-        if (XSCALE_PERFCTR_CCNT == event->config_base) {
-                if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
-                        return -EAGAIN;
-
-                return XSCALE_CYCLE_COUNTER;
-        } else {
-                if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask)) {
-                        return XSCALE_COUNTER1;
-                }
-
-                if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask)) {
-                        return XSCALE_COUNTER0;
-                }
-
-                return -EAGAIN;
-        }
-}
-
-static void
-xscale1pmu_start(void)
-{
-        unsigned long flags, val;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = xscale1pmu_read_pmnc();
-        val |= XSCALE_PMU_ENABLE;
-        xscale1pmu_write_pmnc(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void
-xscale1pmu_stop(void)
-{
-        unsigned long flags, val;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = xscale1pmu_read_pmnc();
-        val &= ~XSCALE_PMU_ENABLE;
-        xscale1pmu_write_pmnc(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static inline u32
-xscale1pmu_read_counter(int counter)
-{
-        u32 val = 0;
-
-        switch (counter) {
-        case XSCALE_CYCLE_COUNTER:
-                asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
-                break;
-        case XSCALE_COUNTER0:
-                asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
-                break;
-        case XSCALE_COUNTER1:
-                asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
-                break;
-        }
-
-        return val;
-}
-
-static inline void
-xscale1pmu_write_counter(int counter, u32 val)
-{
-        switch (counter) {
-        case XSCALE_CYCLE_COUNTER:
-                asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
-                break;
-        case XSCALE_COUNTER0:
-                asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
-                break;
-        case XSCALE_COUNTER1:
-                asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
-                break;
-        }
-}
-
-static const struct arm_pmu xscale1pmu = {
-        .id             = ARM_PERF_PMU_ID_XSCALE1,
-        .handle_irq     = xscale1pmu_handle_irq,
-        .enable         = xscale1pmu_enable_event,
-        .disable        = xscale1pmu_disable_event,
-        .event_map      = xscalepmu_event_map,
-        .raw_event      = xscalepmu_raw_event,
-        .read_counter   = xscale1pmu_read_counter,
-        .write_counter  = xscale1pmu_write_counter,
-        .get_event_idx  = xscale1pmu_get_event_idx,
-        .start          = xscale1pmu_start,
-        .stop           = xscale1pmu_stop,
-        .num_events     = 3,
-        .max_period     = (1LLU << 32) - 1,
-};
-
-#define XSCALE2_OVERFLOWED_MASK 0x01f
-#define XSCALE2_CCOUNT_OVERFLOW 0x001
-#define XSCALE2_COUNT0_OVERFLOW 0x002
-#define XSCALE2_COUNT1_OVERFLOW 0x004
-#define XSCALE2_COUNT2_OVERFLOW 0x008
-#define XSCALE2_COUNT3_OVERFLOW 0x010
-#define XSCALE2_CCOUNT_INT_EN   0x001
-#define XSCALE2_COUNT0_INT_EN   0x002
-#define XSCALE2_COUNT1_INT_EN   0x004
-#define XSCALE2_COUNT2_INT_EN   0x008
-#define XSCALE2_COUNT3_INT_EN   0x010
-#define XSCALE2_COUNT0_EVT_SHFT 0
-#define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
-#define XSCALE2_COUNT1_EVT_SHFT 8
-#define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
-#define XSCALE2_COUNT2_EVT_SHFT 16
-#define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
-#define XSCALE2_COUNT3_EVT_SHFT 24
-#define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
-
-static inline u32
-xscale2pmu_read_pmnc(void)
-{
-        u32 val;
-        asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
-        /* bits 1-2 and 4-23 are read-unpredictable */
-        return val & 0xff000009;
-}
-
-static inline void
-xscale2pmu_write_pmnc(u32 val)
-{
-        /* bits 4-23 are write-as-0, 24-31 are write ignored */
-        val &= 0xf;
-        asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
-}
-
-static inline u32
-xscale2pmu_read_overflow_flags(void)
-{
-        u32 val;
-        asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
-        return val;
-}
-
-static inline void
-xscale2pmu_write_overflow_flags(u32 val)
-{
-        asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
-}
-
-static inline u32
-xscale2pmu_read_event_select(void)
-{
-        u32 val;
-        asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
-        return val;
-}
-
-static inline void
-xscale2pmu_write_event_select(u32 val)
-{
-        asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
-}
-
-static inline u32
-xscale2pmu_read_int_enable(void)
-{
-        u32 val;
-        asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
-        return val;
-}
-
-static void
-xscale2pmu_write_int_enable(u32 val)
-{
-        asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
-}
-
-static inline int
-xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
-                                        enum xscale_counters counter)
-{
-        int ret = 0;
-
-        switch (counter) {
-        case XSCALE_CYCLE_COUNTER:
-                ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
-                break;
-        case XSCALE_COUNTER0:
-                ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
-                break;
-        case XSCALE_COUNTER1:
-                ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
-                break;
-        case XSCALE_COUNTER2:
-                ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
-                break;
-        case XSCALE_COUNTER3:
-                ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
-                break;
-        default:
-                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
-        }
-
-        return ret;
-}
-
-static irqreturn_t
-xscale2pmu_handle_irq(int irq_num, void *dev)
-{
-        unsigned long pmnc, of_flags;
-        struct perf_sample_data data;
-        struct cpu_hw_events *cpuc;
-        struct pt_regs *regs;
-        int idx;
-
-        /* Disable the PMU. */
-        pmnc = xscale2pmu_read_pmnc();
-        xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
-
-        /* Check the overflow flag register. */
-        of_flags = xscale2pmu_read_overflow_flags();
-        if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
-                return IRQ_NONE;
-
-        /* Clear the overflow bits. */
-        xscale2pmu_write_overflow_flags(of_flags);
-
-        regs = get_irq_regs();
-
-        perf_sample_data_init(&data, 0);
-
-        cpuc = &__get_cpu_var(cpu_hw_events);
-        for (idx = 0; idx <= armpmu->num_events; ++idx) {
-                struct perf_event *event = cpuc->events[idx];
-                struct hw_perf_event *hwc;
-
-                if (!test_bit(idx, cpuc->active_mask))
-                        continue;
-
-                if (!xscale2_pmnc_counter_has_overflowed(pmnc, idx))
-                        continue;
-
-                hwc = &event->hw;
-                armpmu_event_update(event, hwc, idx);
-                data.period = event->hw.last_period;
-                if (!armpmu_event_set_period(event, hwc, idx))
-                        continue;
-
-                if (perf_event_overflow(event, 0, &data, regs))
-                        armpmu->disable(hwc, idx);
-        }
-
-        perf_event_do_pending();
-
-        /*
-         * Re-enable the PMU.
-         */
-        pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
-        xscale2pmu_write_pmnc(pmnc);
-
-        return IRQ_HANDLED;
-}
-
-static void
-xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
-{
-        unsigned long flags, ien, evtsel;
-
-        ien = xscale2pmu_read_int_enable();
-        evtsel = xscale2pmu_read_event_select();
-
-        switch (idx) {
-        case XSCALE_CYCLE_COUNTER:
-                ien |= XSCALE2_CCOUNT_INT_EN;
-                break;
-        case XSCALE_COUNTER0:
-                ien |= XSCALE2_COUNT0_INT_EN;
-                evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
-                evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER1:
-                ien |= XSCALE2_COUNT1_INT_EN;
-                evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
-                evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER2:
-                ien |= XSCALE2_COUNT2_INT_EN;
-                evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
-                evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER3:
-                ien |= XSCALE2_COUNT3_INT_EN;
-                evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
-                evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
-                break;
-        default:
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        xscale2pmu_write_event_select(evtsel);
-        xscale2pmu_write_int_enable(ien);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void
-xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
-{
-        unsigned long flags, ien, evtsel;
-
-        ien = xscale2pmu_read_int_enable();
-        evtsel = xscale2pmu_read_event_select();
-
-        switch (idx) {
-        case XSCALE_CYCLE_COUNTER:
-                ien &= ~XSCALE2_CCOUNT_INT_EN;
-                break;
-        case XSCALE_COUNTER0:
-                ien &= ~XSCALE2_COUNT0_INT_EN;
-                evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
-                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER1:
-                ien &= ~XSCALE2_COUNT1_INT_EN;
-                evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
-                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER2:
-                ien &= ~XSCALE2_COUNT2_INT_EN;
-                evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
-                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
-                break;
-        case XSCALE_COUNTER3:
-                ien &= ~XSCALE2_COUNT3_INT_EN;
-                evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
-                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
-                break;
-        default:
-                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
-                return;
-        }
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        xscale2pmu_write_event_select(evtsel);
-        xscale2pmu_write_int_enable(ien);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static int
-xscale2pmu_get_event_idx(struct cpu_hw_events *cpuc,
-                        struct hw_perf_event *event)
-{
-        int idx = xscale1pmu_get_event_idx(cpuc, event);
-        if (idx >= 0)
-                goto out;
-
-        if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
-                idx = XSCALE_COUNTER3;
-        else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
-                idx = XSCALE_COUNTER2;
-out:
-        return idx;
-}
-
-static void
-xscale2pmu_start(void)
-{
-        unsigned long flags, val;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
-        val |= XSCALE_PMU_ENABLE;
-        xscale2pmu_write_pmnc(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static void
-xscale2pmu_stop(void)
-{
-        unsigned long flags, val;
-
-        spin_lock_irqsave(&pmu_lock, flags);
-        val = xscale2pmu_read_pmnc();
-        val &= ~XSCALE_PMU_ENABLE;
-        xscale2pmu_write_pmnc(val);
-        spin_unlock_irqrestore(&pmu_lock, flags);
-}
-
-static inline u32
-xscale2pmu_read_counter(int counter)
-{
-        u32 val = 0;
-
-        switch (counter) {
-        case XSCALE_CYCLE_COUNTER:
-                asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
-                break;
-        case XSCALE_COUNTER0:
-                asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
-                break;
-        case XSCALE_COUNTER1:
-                asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
-                break;
-        case XSCALE_COUNTER2:
-                asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
-                break;
-        case XSCALE_COUNTER3:
-                asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
-                break;
-        }
-
-        return val;
-}
-
-static inline void
-xscale2pmu_write_counter(int counter, u32 val)
+static int __init
+armpmu_reset(void)
 {
-        switch (counter) {
-        case XSCALE_CYCLE_COUNTER:
-                asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
-                break;
-        case XSCALE_COUNTER0:
-                asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
-                break;
-        case XSCALE_COUNTER1:
-                asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
-                break;
-        case XSCALE_COUNTER2:
-                asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
-                break;
-        case XSCALE_COUNTER3:
-                asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
-                break;
-        }
+        if (armpmu && armpmu->reset)
+                return on_each_cpu(armpmu->reset, NULL, 1);
+        return 0;
 }
-
-static const struct arm_pmu xscale2pmu = {
-        .id             = ARM_PERF_PMU_ID_XSCALE2,
-        .handle_irq     = xscale2pmu_handle_irq,
-        .enable         = xscale2pmu_enable_event,
-        .disable        = xscale2pmu_disable_event,
-        .event_map      = xscalepmu_event_map,
-        .raw_event      = xscalepmu_raw_event,
-        .read_counter   = xscale2pmu_read_counter,
-        .write_counter  = xscale2pmu_write_counter,
-        .get_event_idx  = xscale2pmu_get_event_idx,
-        .start          = xscale2pmu_start,
-        .stop           = xscale2pmu_stop,
-        .num_events     = 5,
-        .max_period     = (1LLU << 32) - 1,
-};
+arch_initcall(armpmu_reset);
 
 static int __init
 init_hw_perf_events(void)
@@ -2930,41 +651,16 @@ init_hw_perf_events(void)
                 case 0xB360:    /* ARM1136 */
                 case 0xB560:    /* ARM1156 */
                 case 0xB760:    /* ARM1176 */
-                        armpmu = &armv6pmu;
-                        memcpy(armpmu_perf_cache_map, armv6_perf_cache_map,
-                                        sizeof(armv6_perf_cache_map));
-                        perf_max_events = armv6pmu.num_events;
+                        armpmu = armv6pmu_init();
                         break;
                 case 0xB020:    /* ARM11mpcore */
-                        armpmu = &armv6mpcore_pmu;
-                        memcpy(armpmu_perf_cache_map,
-                               armv6mpcore_perf_cache_map,
-                               sizeof(armv6mpcore_perf_cache_map));
-                        perf_max_events = armv6mpcore_pmu.num_events;
+                        armpmu = armv6mpcore_pmu_init();
                         break;
                 case 0xC080:    /* Cortex-A8 */
-                        armv7pmu.id = ARM_PERF_PMU_ID_CA8;
-                        memcpy(armpmu_perf_cache_map, armv7_a8_perf_cache_map,
-                                sizeof(armv7_a8_perf_cache_map));
-                        armv7pmu.event_map = armv7_a8_pmu_event_map;
-                        armpmu = &armv7pmu;
-
-                        /* Reset PMNC and read the nb of CNTx counters
-                            supported */
-                        armv7pmu.num_events = armv7_reset_read_pmnc();
-                        perf_max_events = armv7pmu.num_events;
+                        armpmu = armv7_a8_pmu_init();
                         break;
                 case 0xC090:    /* Cortex-A9 */
-                        armv7pmu.id = ARM_PERF_PMU_ID_CA9;
-                        memcpy(armpmu_perf_cache_map, armv7_a9_perf_cache_map,
-                                sizeof(armv7_a9_perf_cache_map));
-                        armv7pmu.event_map = armv7_a9_pmu_event_map;
-                        armpmu = &armv7pmu;
-
-                        /* Reset PMNC and read the nb of CNTx counters
-                            supported */
-                        armv7pmu.num_events = armv7_reset_read_pmnc();
-                        perf_max_events = armv7pmu.num_events;
+                        armpmu = armv7_a9_pmu_init();
                         break;
                 }
         /* Intel CPUs [xscale]. */
@@ -2972,42 +668,30 @@ init_hw_perf_events(void)
                 part_number = (cpuid >> 13) & 0x7;
                 switch (part_number) {
                 case 1:
-                        armpmu = &xscale1pmu;
-                        memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
-                                        sizeof(xscale_perf_cache_map));
-                        perf_max_events = xscale1pmu.num_events;
+                        armpmu = xscale1pmu_init();
                         break;
                 case 2:
-                        armpmu = &xscale2pmu;
-                        memcpy(armpmu_perf_cache_map, xscale_perf_cache_map,
-                                        sizeof(xscale_perf_cache_map));
-                        perf_max_events = xscale2pmu.num_events;
+                        armpmu = xscale2pmu_init();
                         break;
                 }
         }
 
         if (armpmu) {
                 pr_info("enabled with %s PMU driver, %d counters available\n",
-                                arm_pmu_names[armpmu->id], armpmu->num_events);
+                        armpmu->name, armpmu->num_events);
         } else {
                 pr_info("no hardware support available\n");
-                perf_max_events = -1;
         }
 
+        perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW);
+
         return 0;
 }
-arch_initcall(init_hw_perf_events);
+early_initcall(init_hw_perf_events);
 
 /*
  * Callchain handling code.
  */
-static inline void
-callchain_store(struct perf_callchain_entry *entry,
-                u64 ip)
-{
-        if (entry->nr < PERF_MAX_STACK_DEPTH)
-                entry->ip[entry->nr++] = ip;
-}
 
 /*
  * The registers we're interested in are at the end of the variable
@@ -3018,17 +702,17 @@ callchain_store(struct perf_callchain_entry *entry,
  * This code has been adapted from the ARM OProfile support.
  */
 struct frame_tail {
-        struct frame_tail   *fp;
-        unsigned long       sp;
-        unsigned long       lr;
+        struct frame_tail __user *fp;
+        unsigned long sp;
+        unsigned long lr;
 } __attribute__((packed));
 
 /*
  * Get the return address for a single stackframe and return a pointer to the
  * next frame tail.
  */
-static struct frame_tail *
-user_backtrace(struct frame_tail *tail,
+static struct frame_tail __user *
+user_backtrace(struct frame_tail __user *tail,
                struct perf_callchain_entry *entry)
 {
         struct frame_tail buftail;
@@ -3039,32 +723,28 @@ user_backtrace(struct frame_tail *tail,
         if (__copy_from_user_inatomic(&buftail, tail, sizeof(buftail)))
                 return NULL;
 
-        callchain_store(entry, buftail.lr);
+        perf_callchain_store(entry, buftail.lr);
 
         /*
          * Frame pointers should strictly progress back up the stack
          * (towards higher addresses).
          */
-        if (tail >= buftail.fp)
+        if (tail + 1 >= buftail.fp)
                 return NULL;
 
         return buftail.fp - 1;
 }
 
-static void
-perf_callchain_user(struct pt_regs *regs,
-                    struct perf_callchain_entry *entry)
+void
+perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs)
 {
-        struct frame_tail *tail;
-
-        callchain_store(entry, PERF_CONTEXT_USER);
+        struct frame_tail __user *tail;
 
-        if (!user_mode(regs))
-                regs = task_pt_regs(current);
 
-        tail = (struct frame_tail *)regs->ARM_fp - 1;
+        tail = (struct frame_tail __user *)regs->ARM_fp - 1;
 
-        while (tail && !((unsigned long)tail & 0x3))
+        while ((entry->nr < PERF_MAX_STACK_DEPTH) &&
+               tail && !((unsigned long)tail & 0x3))
                 tail = user_backtrace(tail, entry);
 }
 
@@ -3078,56 +758,18 @@ callchain_trace(struct stackframe *fr,
                 void *data)
 {
         struct perf_callchain_entry *entry = data;
-        callchain_store(entry, fr->pc);
+        perf_callchain_store(entry, fr->pc);
         return 0;
 }
 
-static void
-perf_callchain_kernel(struct pt_regs *regs,
-                      struct perf_callchain_entry *entry)
+void
+perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs)
 {
         struct stackframe fr;
 
-        callchain_store(entry, PERF_CONTEXT_KERNEL);
         fr.fp = regs->ARM_fp;
         fr.sp = regs->ARM_sp;
         fr.lr = regs->ARM_lr;
         fr.pc = regs->ARM_pc;
         walk_stackframe(&fr, callchain_trace, entry);
 }
-
-static void
-perf_do_callchain(struct pt_regs *regs,
-                  struct perf_callchain_entry *entry)
-{
-        int is_user;
-
-        if (!regs)
-                return;
-
-        is_user = user_mode(regs);
-
-        if (!current || !current->pid)
-                return;
-
-        if (is_user && current->state != TASK_RUNNING)
-                return;
-
-        if (!is_user)
-                perf_callchain_kernel(regs, entry);
-
-        if (current->mm)
-                perf_callchain_user(regs, entry);
-}
-
-static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
-
-struct perf_callchain_entry *
-perf_callchain(struct pt_regs *regs)
-{
-        struct perf_callchain_entry *entry = &__get_cpu_var(pmc_irq_entry);
-
-        entry->nr = 0;
-        perf_do_callchain(regs, entry);
-        return entry;
-}
diff --git a/arch/arm/kernel/perf_event_v6.c b/arch/arm/kernel/perf_event_v6.c
new file mode 100644
index 000000000000..f1e8dd94afe8
--- /dev/null
+++ b/arch/arm/kernel/perf_event_v6.c
@@ -0,0 +1,672 @@
+/*
+ * ARMv6 Performance counter handling code.
+ *
+ * Copyright (C) 2009 picoChip Designs, Ltd., Jamie Iles
+ *
+ * ARMv6 has 2 configurable performance counters and a single cycle counter.
+ * They all share a single reset bit but can be written to zero so we can use
+ * that for a reset.
+ *
+ * The counters can't be individually enabled or disabled so when we remove
+ * one event and replace it with another we could get spurious counts from the
+ * wrong event. However, we can take advantage of the fact that the
+ * performance counters can export events to the event bus, and the event bus
+ * itself can be monitored. This requires that we *don't* export the events to
+ * the event bus. The procedure for disabling a configurable counter is:
+ *      - change the counter to count the ETMEXTOUT[0] signal (0x20). This
+ *        effectively stops the counter from counting.
+ *      - disable the counter's interrupt generation (each counter has it's
+ *        own interrupt enable bit).
+ * Once stopped, the counter value can be written as 0 to reset.
+ *
+ * To enable a counter:
+ *      - enable the counter's interrupt generation.
+ *      - set the new event type.
+ *
+ * Note: the dedicated cycle counter only counts cycles and can't be
+ * enabled/disabled independently of the others. When we want to disable the
+ * cycle counter, we have to just disable the interrupt reporting and start
+ * ignoring that counter. When re-enabling, we have to reset the value and
+ * enable the interrupt.
+ */
+
+#if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_V6K)
+enum armv6_perf_types {
+        ARMV6_PERFCTR_ICACHE_MISS           = 0x0,
+        ARMV6_PERFCTR_IBUF_STALL            = 0x1,
+        ARMV6_PERFCTR_DDEP_STALL            = 0x2,
+        ARMV6_PERFCTR_ITLB_MISS             = 0x3,
+        ARMV6_PERFCTR_DTLB_MISS             = 0x4,
+        ARMV6_PERFCTR_BR_EXEC               = 0x5,
+        ARMV6_PERFCTR_BR_MISPREDICT         = 0x6,
+        ARMV6_PERFCTR_INSTR_EXEC            = 0x7,
+        ARMV6_PERFCTR_DCACHE_HIT            = 0x9,
+        ARMV6_PERFCTR_DCACHE_ACCESS         = 0xA,
+        ARMV6_PERFCTR_DCACHE_MISS           = 0xB,
+        ARMV6_PERFCTR_DCACHE_WBACK          = 0xC,
+        ARMV6_PERFCTR_SW_PC_CHANGE          = 0xD,
+        ARMV6_PERFCTR_MAIN_TLB_MISS         = 0xF,
+        ARMV6_PERFCTR_EXPL_D_ACCESS         = 0x10,
+        ARMV6_PERFCTR_LSU_FULL_STALL        = 0x11,
+        ARMV6_PERFCTR_WBUF_DRAINED          = 0x12,
+        ARMV6_PERFCTR_CPU_CYCLES            = 0xFF,
+        ARMV6_PERFCTR_NOP                   = 0x20,
+};
+
+enum armv6_counters {
+        ARMV6_CYCLE_COUNTER = 1,
+        ARMV6_COUNTER0,
+        ARMV6_COUNTER1,
+};
+
+/*
+ * The hardware events that we support. We do support cache operations but
+ * we have harvard caches and no way to combine instruction and data
+ * accesses/misses in hardware.
+ */
+static const unsigned armv6_perf_map[PERF_COUNT_HW_MAX] = {
+        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV6_PERFCTR_CPU_CYCLES,
+        [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV6_PERFCTR_INSTR_EXEC,
+        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6_PERFCTR_BR_EXEC,
+        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV6_PERFCTR_BR_MISPREDICT,
+        [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
+};
+
+static const unsigned armv6_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+                                          [PERF_COUNT_HW_CACHE_OP_MAX]
+                                          [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+        [C(L1D)] = {
+                /*
+                 * The performance counters don't differentiate between read
+                 * and write accesses/misses so this isn't strictly correct,
+                 * but it's the best we can do. Writes and reads get
+                 * combined.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV6_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DCACHE_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV6_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DCACHE_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(L1I)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ICACHE_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ICACHE_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(LL)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(DTLB)] = {
+                /*
+                 * The ARM performance counters can count micro DTLB misses,
+                 * micro ITLB misses and main TLB misses. There isn't an event
+                 * for TLB misses, so use the micro misses here and if users
+                 * want the main TLB misses they can use a raw counter.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DTLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_DTLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(ITLB)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV6_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(BPU)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+};
+
+enum armv6mpcore_perf_types {
+        ARMV6MPCORE_PERFCTR_ICACHE_MISS     = 0x0,
+        ARMV6MPCORE_PERFCTR_IBUF_STALL      = 0x1,
+        ARMV6MPCORE_PERFCTR_DDEP_STALL      = 0x2,
+        ARMV6MPCORE_PERFCTR_ITLB_MISS       = 0x3,
+        ARMV6MPCORE_PERFCTR_DTLB_MISS       = 0x4,
+        ARMV6MPCORE_PERFCTR_BR_EXEC         = 0x5,
+        ARMV6MPCORE_PERFCTR_BR_NOTPREDICT   = 0x6,
+        ARMV6MPCORE_PERFCTR_BR_MISPREDICT   = 0x7,
+        ARMV6MPCORE_PERFCTR_INSTR_EXEC      = 0x8,
+        ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS = 0xA,
+        ARMV6MPCORE_PERFCTR_DCACHE_RDMISS   = 0xB,
+        ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS = 0xC,
+        ARMV6MPCORE_PERFCTR_DCACHE_WRMISS   = 0xD,
+        ARMV6MPCORE_PERFCTR_DCACHE_EVICTION = 0xE,
+        ARMV6MPCORE_PERFCTR_SW_PC_CHANGE    = 0xF,
+        ARMV6MPCORE_PERFCTR_MAIN_TLB_MISS   = 0x10,
+        ARMV6MPCORE_PERFCTR_EXPL_MEM_ACCESS = 0x11,
+        ARMV6MPCORE_PERFCTR_LSU_FULL_STALL  = 0x12,
+        ARMV6MPCORE_PERFCTR_WBUF_DRAINED    = 0x13,
+        ARMV6MPCORE_PERFCTR_CPU_CYCLES      = 0xFF,
+};
+
+/*
+ * The hardware events that we support. We do support cache operations but
+ * we have harvard caches and no way to combine instruction and data
+ * accesses/misses in hardware.
+ */
+static const unsigned armv6mpcore_perf_map[PERF_COUNT_HW_MAX] = {
+        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV6MPCORE_PERFCTR_CPU_CYCLES,
+        [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV6MPCORE_PERFCTR_INSTR_EXEC,
+        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV6MPCORE_PERFCTR_BR_EXEC,
+        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV6MPCORE_PERFCTR_BR_MISPREDICT,
+        [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
+};
+
+static const unsigned armv6mpcore_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+                                        [PERF_COUNT_HW_CACHE_OP_MAX]
+                                        [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+        [C(L1D)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]  =
+                                ARMV6MPCORE_PERFCTR_DCACHE_RDACCESS,
+                        [C(RESULT_MISS)]    =
+                                ARMV6MPCORE_PERFCTR_DCACHE_RDMISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]  =
+                                ARMV6MPCORE_PERFCTR_DCACHE_WRACCESS,
+                        [C(RESULT_MISS)]    =
+                                ARMV6MPCORE_PERFCTR_DCACHE_WRMISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(L1I)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ICACHE_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ICACHE_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(LL)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(DTLB)] = {
+                /*
+                 * The ARM performance counters can count micro DTLB misses,
+                 * micro ITLB misses and main TLB misses. There isn't an event
+                 * for TLB misses, so use the micro misses here and if users
+                 * want the main TLB misses they can use a raw counter.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_DTLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_DTLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(ITLB)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = ARMV6MPCORE_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(BPU)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]  = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]    = CACHE_OP_UNSUPPORTED,
+                },
+        },
+};
+
+static inline unsigned long
+armv6_pmcr_read(void)
+{
+        u32 val;
+        asm volatile("mrc   p15, 0, %0, c15, c12, 0" : "=r"(val));
+        return val;
+}
+
+static inline void
+armv6_pmcr_write(unsigned long val)
+{
+        asm volatile("mcr   p15, 0, %0, c15, c12, 0" : : "r"(val));
+}
+
+#define ARMV6_PMCR_ENABLE               (1 << 0)
+#define ARMV6_PMCR_CTR01_RESET          (1 << 1)
+#define ARMV6_PMCR_CCOUNT_RESET         (1 << 2)
+#define ARMV6_PMCR_CCOUNT_DIV           (1 << 3)
+#define ARMV6_PMCR_COUNT0_IEN           (1 << 4)
+#define ARMV6_PMCR_COUNT1_IEN           (1 << 5)
+#define ARMV6_PMCR_CCOUNT_IEN           (1 << 6)
+#define ARMV6_PMCR_COUNT0_OVERFLOW      (1 << 8)
+#define ARMV6_PMCR_COUNT1_OVERFLOW      (1 << 9)
+#define ARMV6_PMCR_CCOUNT_OVERFLOW      (1 << 10)
+#define ARMV6_PMCR_EVT_COUNT0_SHIFT     20
+#define ARMV6_PMCR_EVT_COUNT0_MASK      (0xFF << ARMV6_PMCR_EVT_COUNT0_SHIFT)
+#define ARMV6_PMCR_EVT_COUNT1_SHIFT     12
+#define ARMV6_PMCR_EVT_COUNT1_MASK      (0xFF << ARMV6_PMCR_EVT_COUNT1_SHIFT)
+
+#define ARMV6_PMCR_OVERFLOWED_MASK \
+        (ARMV6_PMCR_COUNT0_OVERFLOW | ARMV6_PMCR_COUNT1_OVERFLOW | \
+         ARMV6_PMCR_CCOUNT_OVERFLOW)
+
+static inline int
+armv6_pmcr_has_overflowed(unsigned long pmcr)
+{
+        return pmcr & ARMV6_PMCR_OVERFLOWED_MASK;
+}
+
+static inline int
+armv6_pmcr_counter_has_overflowed(unsigned long pmcr,
+                                  enum armv6_counters counter)
+{
+        int ret = 0;
+
+        if (ARMV6_CYCLE_COUNTER == counter)
+                ret = pmcr & ARMV6_PMCR_CCOUNT_OVERFLOW;
+        else if (ARMV6_COUNTER0 == counter)
+                ret = pmcr & ARMV6_PMCR_COUNT0_OVERFLOW;
+        else if (ARMV6_COUNTER1 == counter)
+                ret = pmcr & ARMV6_PMCR_COUNT1_OVERFLOW;
+        else
+                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
+
+        return ret;
+}
+
+static inline u32
+armv6pmu_read_counter(int counter)
+{
+        unsigned long value = 0;
+
+        if (ARMV6_CYCLE_COUNTER == counter)
+                asm volatile("mrc   p15, 0, %0, c15, c12, 1" : "=r"(value));
+        else if (ARMV6_COUNTER0 == counter)
+                asm volatile("mrc   p15, 0, %0, c15, c12, 2" : "=r"(value));
+        else if (ARMV6_COUNTER1 == counter)
+                asm volatile("mrc   p15, 0, %0, c15, c12, 3" : "=r"(value));
+        else
+                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
+
+        return value;
+}
+
+static inline void
+armv6pmu_write_counter(int counter,
+                       u32 value)
+{
+        if (ARMV6_CYCLE_COUNTER == counter)
+                asm volatile("mcr   p15, 0, %0, c15, c12, 1" : : "r"(value));
+        else if (ARMV6_COUNTER0 == counter)
+                asm volatile("mcr   p15, 0, %0, c15, c12, 2" : : "r"(value));
+        else if (ARMV6_COUNTER1 == counter)
+                asm volatile("mcr   p15, 0, %0, c15, c12, 3" : : "r"(value));
+        else
+                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
+}
+
+static void
+armv6pmu_enable_event(struct hw_perf_event *hwc,
+                      int idx)
+{
+        unsigned long val, mask, evt, flags;
+
+        if (ARMV6_CYCLE_COUNTER == idx) {
+                mask    = 0;
+                evt     = ARMV6_PMCR_CCOUNT_IEN;
+        } else if (ARMV6_COUNTER0 == idx) {
+                mask    = ARMV6_PMCR_EVT_COUNT0_MASK;
+                evt     = (hwc->config_base << ARMV6_PMCR_EVT_COUNT0_SHIFT) |
+                          ARMV6_PMCR_COUNT0_IEN;
+        } else if (ARMV6_COUNTER1 == idx) {
+                mask    = ARMV6_PMCR_EVT_COUNT1_MASK;
+                evt     = (hwc->config_base << ARMV6_PMCR_EVT_COUNT1_SHIFT) |
+                          ARMV6_PMCR_COUNT1_IEN;
+        } else {
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        /*
+         * Mask out the current event and set the counter to count the event
+         * that we're interested in.
+         */
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = armv6_pmcr_read();
+        val &= ~mask;
+        val |= evt;
+        armv6_pmcr_write(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static irqreturn_t
+armv6pmu_handle_irq(int irq_num,
+                    void *dev)
+{
+        unsigned long pmcr = armv6_pmcr_read();
+        struct perf_sample_data data;
+        struct cpu_hw_events *cpuc;
+        struct pt_regs *regs;
+        int idx;
+
+        if (!armv6_pmcr_has_overflowed(pmcr))
+                return IRQ_NONE;
+
+        regs = get_irq_regs();
+
+        /*
+         * The interrupts are cleared by writing the overflow flags back to
+         * the control register. All of the other bits don't have any effect
+         * if they are rewritten, so write the whole value back.
+         */
+        armv6_pmcr_write(pmcr);
+
+        perf_sample_data_init(&data, 0);
+
+        cpuc = &__get_cpu_var(cpu_hw_events);
+        for (idx = 0; idx <= armpmu->num_events; ++idx) {
+                struct perf_event *event = cpuc->events[idx];
+                struct hw_perf_event *hwc;
+
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+
+                /*
+                 * We have a single interrupt for all counters. Check that
+                 * each counter has overflowed before we process it.
+                 */
+                if (!armv6_pmcr_counter_has_overflowed(pmcr, idx))
+                        continue;
+
+                hwc = &event->hw;
+                armpmu_event_update(event, hwc, idx, 1);
+                data.period = event->hw.last_period;
+                if (!armpmu_event_set_period(event, hwc, idx))
+                        continue;
+
+                if (perf_event_overflow(event, 0, &data, regs))
+                        armpmu->disable(hwc, idx);
+        }
+
+        /*
+         * Handle the pending perf events.
+         *
+         * Note: this call *must* be run with interrupts disabled. For
+         * platforms that can have the PMU interrupts raised as an NMI, this
+         * will not work.
+         */
+        irq_work_run();
+
+        return IRQ_HANDLED;
+}
+
+static void
+armv6pmu_start(void)
+{
+        unsigned long flags, val;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = armv6_pmcr_read();
+        val |= ARMV6_PMCR_ENABLE;
+        armv6_pmcr_write(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void
+armv6pmu_stop(void)
+{
+        unsigned long flags, val;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = armv6_pmcr_read();
+        val &= ~ARMV6_PMCR_ENABLE;
+        armv6_pmcr_write(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static int
+armv6pmu_get_event_idx(struct cpu_hw_events *cpuc,
+                       struct hw_perf_event *event)
+{
+        /* Always place a cycle counter into the cycle counter. */
+        if (ARMV6_PERFCTR_CPU_CYCLES == event->config_base) {
+                if (test_and_set_bit(ARMV6_CYCLE_COUNTER, cpuc->used_mask))
+                        return -EAGAIN;
+
+                return ARMV6_CYCLE_COUNTER;
+        } else {
+                /*
+                 * For anything other than a cycle counter, try and use
+                 * counter0 and counter1.
+                 */
+                if (!test_and_set_bit(ARMV6_COUNTER1, cpuc->used_mask))
+                        return ARMV6_COUNTER1;
+
+                if (!test_and_set_bit(ARMV6_COUNTER0, cpuc->used_mask))
+                        return ARMV6_COUNTER0;
+
+                /* The counters are all in use. */
+                return -EAGAIN;
+        }
+}
+
+static void
+armv6pmu_disable_event(struct hw_perf_event *hwc,
+                       int idx)
+{
+        unsigned long val, mask, evt, flags;
+
+        if (ARMV6_CYCLE_COUNTER == idx) {
+                mask    = ARMV6_PMCR_CCOUNT_IEN;
+                evt     = 0;
+        } else if (ARMV6_COUNTER0 == idx) {
+                mask    = ARMV6_PMCR_COUNT0_IEN | ARMV6_PMCR_EVT_COUNT0_MASK;
+                evt     = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT0_SHIFT;
+        } else if (ARMV6_COUNTER1 == idx) {
+                mask    = ARMV6_PMCR_COUNT1_IEN | ARMV6_PMCR_EVT_COUNT1_MASK;
+                evt     = ARMV6_PERFCTR_NOP << ARMV6_PMCR_EVT_COUNT1_SHIFT;
+        } else {
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        /*
+         * Mask out the current event and set the counter to count the number
+         * of ETM bus signal assertion cycles. The external reporting should
+         * be disabled and so this should never increment.
+         */
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = armv6_pmcr_read();
+        val &= ~mask;
+        val |= evt;
+        armv6_pmcr_write(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void
+armv6mpcore_pmu_disable_event(struct hw_perf_event *hwc,
+                              int idx)
+{
+        unsigned long val, mask, flags, evt = 0;
+
+        if (ARMV6_CYCLE_COUNTER == idx) {
+                mask    = ARMV6_PMCR_CCOUNT_IEN;
+        } else if (ARMV6_COUNTER0 == idx) {
+                mask    = ARMV6_PMCR_COUNT0_IEN;
+        } else if (ARMV6_COUNTER1 == idx) {
+                mask    = ARMV6_PMCR_COUNT1_IEN;
+        } else {
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        /*
+         * Unlike UP ARMv6, we don't have a way of stopping the counters. We
+         * simply disable the interrupt reporting.
+         */
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = armv6_pmcr_read();
+        val &= ~mask;
+        val |= evt;
+        armv6_pmcr_write(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static const struct arm_pmu armv6pmu = {
+        .id                     = ARM_PERF_PMU_ID_V6,
+        .name                   = "v6",
+        .handle_irq             = armv6pmu_handle_irq,
+        .enable                 = armv6pmu_enable_event,
+        .disable                = armv6pmu_disable_event,
+        .read_counter           = armv6pmu_read_counter,
+        .write_counter          = armv6pmu_write_counter,
+        .get_event_idx          = armv6pmu_get_event_idx,
+        .start                  = armv6pmu_start,
+        .stop                   = armv6pmu_stop,
+        .cache_map              = &armv6_perf_cache_map,
+        .event_map              = &armv6_perf_map,
+        .raw_event_mask         = 0xFF,
+        .num_events             = 3,
+        .max_period             = (1LLU << 32) - 1,
+};
+
+static const struct arm_pmu *__init armv6pmu_init(void)
+{
+        return &armv6pmu;
+}
+
+/*
+ * ARMv6mpcore is almost identical to single core ARMv6 with the exception
+ * that some of the events have different enumerations and that there is no
+ * *hack* to stop the programmable counters. To stop the counters we simply
+ * disable the interrupt reporting and update the event. When unthrottling we
+ * reset the period and enable the interrupt reporting.
+ */
+static const struct arm_pmu armv6mpcore_pmu = {
+        .id                     = ARM_PERF_PMU_ID_V6MP,
+        .name                   = "v6mpcore",
+        .handle_irq             = armv6pmu_handle_irq,
+        .enable                 = armv6pmu_enable_event,
+        .disable                = armv6mpcore_pmu_disable_event,
+        .read_counter           = armv6pmu_read_counter,
+        .write_counter          = armv6pmu_write_counter,
+        .get_event_idx          = armv6pmu_get_event_idx,
+        .start                  = armv6pmu_start,
+        .stop                   = armv6pmu_stop,
+        .cache_map              = &armv6mpcore_perf_cache_map,
+        .event_map              = &armv6mpcore_perf_map,
+        .raw_event_mask         = 0xFF,
+        .num_events             = 3,
+        .max_period             = (1LLU << 32) - 1,
+};
+
+static const struct arm_pmu *__init armv6mpcore_pmu_init(void)
+{
+        return &armv6mpcore_pmu;
+}
+#else
+static const struct arm_pmu *__init armv6pmu_init(void)
+{
+        return NULL;
+}
+
+static const struct arm_pmu *__init armv6mpcore_pmu_init(void)
+{
+        return NULL;
+}
+#endif  /* CONFIG_CPU_V6 || CONFIG_CPU_V6K */
diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c
new file mode 100644
index 000000000000..4960686afb58
--- /dev/null
+++ b/arch/arm/kernel/perf_event_v7.c
@@ -0,0 +1,918 @@
+/*
+ * ARMv7 Cortex-A8 and Cortex-A9 Performance Events handling code.
+ *
+ * ARMv7 support: Jean Pihet <jpihet@mvista.com>
+ * 2010 (c) MontaVista Software, LLC.
+ *
+ * Copied from ARMv6 code, with the low level code inspired
+ *  by the ARMv7 Oprofile code.
+ *
+ * Cortex-A8 has up to 4 configurable performance counters and
+ *  a single cycle counter.
+ * Cortex-A9 has up to 31 configurable performance counters and
+ *  a single cycle counter.
+ *
+ * All counters can be enabled/disabled and IRQ masked separately. The cycle
+ *  counter and all 4 performance counters together can be reset separately.
+ */
+
+#ifdef CONFIG_CPU_V7
+/* Common ARMv7 event types */
+enum armv7_perf_types {
+        ARMV7_PERFCTR_PMNC_SW_INCR              = 0x00,
+        ARMV7_PERFCTR_IFETCH_MISS               = 0x01,
+        ARMV7_PERFCTR_ITLB_MISS                 = 0x02,
+        ARMV7_PERFCTR_DCACHE_REFILL             = 0x03,
+        ARMV7_PERFCTR_DCACHE_ACCESS             = 0x04,
+        ARMV7_PERFCTR_DTLB_REFILL               = 0x05,
+        ARMV7_PERFCTR_DREAD                     = 0x06,
+        ARMV7_PERFCTR_DWRITE                    = 0x07,
+
+        ARMV7_PERFCTR_EXC_TAKEN                 = 0x09,
+        ARMV7_PERFCTR_EXC_EXECUTED              = 0x0A,
+        ARMV7_PERFCTR_CID_WRITE                 = 0x0B,
+        /* ARMV7_PERFCTR_PC_WRITE is equivalent to HW_BRANCH_INSTRUCTIONS.
+         * It counts:
+         *  - all branch instructions,
+         *  - instructions that explicitly write the PC,
+         *  - exception generating instructions.
+         */
+        ARMV7_PERFCTR_PC_WRITE                  = 0x0C,
+        ARMV7_PERFCTR_PC_IMM_BRANCH             = 0x0D,
+        ARMV7_PERFCTR_UNALIGNED_ACCESS          = 0x0F,
+        ARMV7_PERFCTR_PC_BRANCH_MIS_PRED        = 0x10,
+        ARMV7_PERFCTR_CLOCK_CYCLES              = 0x11,
+
+        ARMV7_PERFCTR_PC_BRANCH_MIS_USED        = 0x12,
+
+        ARMV7_PERFCTR_CPU_CYCLES                = 0xFF
+};
+
+/* ARMv7 Cortex-A8 specific event types */
+enum armv7_a8_perf_types {
+        ARMV7_PERFCTR_INSTR_EXECUTED            = 0x08,
+
+        ARMV7_PERFCTR_PC_PROC_RETURN            = 0x0E,
+
+        ARMV7_PERFCTR_WRITE_BUFFER_FULL         = 0x40,
+        ARMV7_PERFCTR_L2_STORE_MERGED           = 0x41,
+        ARMV7_PERFCTR_L2_STORE_BUFF             = 0x42,
+        ARMV7_PERFCTR_L2_ACCESS                 = 0x43,
+        ARMV7_PERFCTR_L2_CACH_MISS              = 0x44,
+        ARMV7_PERFCTR_AXI_READ_CYCLES           = 0x45,
+        ARMV7_PERFCTR_AXI_WRITE_CYCLES          = 0x46,
+        ARMV7_PERFCTR_MEMORY_REPLAY             = 0x47,
+        ARMV7_PERFCTR_UNALIGNED_ACCESS_REPLAY   = 0x48,
+        ARMV7_PERFCTR_L1_DATA_MISS              = 0x49,
+        ARMV7_PERFCTR_L1_INST_MISS              = 0x4A,
+        ARMV7_PERFCTR_L1_DATA_COLORING          = 0x4B,
+        ARMV7_PERFCTR_L1_NEON_DATA              = 0x4C,
+        ARMV7_PERFCTR_L1_NEON_CACH_DATA         = 0x4D,
+        ARMV7_PERFCTR_L2_NEON                   = 0x4E,
+        ARMV7_PERFCTR_L2_NEON_HIT               = 0x4F,
+        ARMV7_PERFCTR_L1_INST                   = 0x50,
+        ARMV7_PERFCTR_PC_RETURN_MIS_PRED        = 0x51,
+        ARMV7_PERFCTR_PC_BRANCH_FAILED          = 0x52,
+        ARMV7_PERFCTR_PC_BRANCH_TAKEN           = 0x53,
+        ARMV7_PERFCTR_PC_BRANCH_EXECUTED        = 0x54,
+        ARMV7_PERFCTR_OP_EXECUTED               = 0x55,
+        ARMV7_PERFCTR_CYCLES_INST_STALL         = 0x56,
+        ARMV7_PERFCTR_CYCLES_INST               = 0x57,
+        ARMV7_PERFCTR_CYCLES_NEON_DATA_STALL    = 0x58,
+        ARMV7_PERFCTR_CYCLES_NEON_INST_STALL    = 0x59,
+        ARMV7_PERFCTR_NEON_CYCLES               = 0x5A,
+
+        ARMV7_PERFCTR_PMU0_EVENTS               = 0x70,
+        ARMV7_PERFCTR_PMU1_EVENTS               = 0x71,
+        ARMV7_PERFCTR_PMU_EVENTS                = 0x72,
+};
+
+/* ARMv7 Cortex-A9 specific event types */
+enum armv7_a9_perf_types {
+        ARMV7_PERFCTR_JAVA_HW_BYTECODE_EXEC     = 0x40,
+        ARMV7_PERFCTR_JAVA_SW_BYTECODE_EXEC     = 0x41,
+        ARMV7_PERFCTR_JAZELLE_BRANCH_EXEC       = 0x42,
+
+        ARMV7_PERFCTR_COHERENT_LINE_MISS        = 0x50,
+        ARMV7_PERFCTR_COHERENT_LINE_HIT         = 0x51,
+
+        ARMV7_PERFCTR_ICACHE_DEP_STALL_CYCLES   = 0x60,
+        ARMV7_PERFCTR_DCACHE_DEP_STALL_CYCLES   = 0x61,
+        ARMV7_PERFCTR_TLB_MISS_DEP_STALL_CYCLES = 0x62,
+        ARMV7_PERFCTR_STREX_EXECUTED_PASSED     = 0x63,
+        ARMV7_PERFCTR_STREX_EXECUTED_FAILED     = 0x64,
+        ARMV7_PERFCTR_DATA_EVICTION             = 0x65,
+        ARMV7_PERFCTR_ISSUE_STAGE_NO_INST       = 0x66,
+        ARMV7_PERFCTR_ISSUE_STAGE_EMPTY         = 0x67,
+        ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE  = 0x68,
+
+        ARMV7_PERFCTR_PREDICTABLE_FUNCT_RETURNS = 0x6E,
+
+        ARMV7_PERFCTR_MAIN_UNIT_EXECUTED_INST   = 0x70,
+        ARMV7_PERFCTR_SECOND_UNIT_EXECUTED_INST = 0x71,
+        ARMV7_PERFCTR_LD_ST_UNIT_EXECUTED_INST  = 0x72,
+        ARMV7_PERFCTR_FP_EXECUTED_INST          = 0x73,
+        ARMV7_PERFCTR_NEON_EXECUTED_INST        = 0x74,
+
+        ARMV7_PERFCTR_PLD_FULL_DEP_STALL_CYCLES = 0x80,
+        ARMV7_PERFCTR_DATA_WR_DEP_STALL_CYCLES  = 0x81,
+        ARMV7_PERFCTR_ITLB_MISS_DEP_STALL_CYCLES        = 0x82,
+        ARMV7_PERFCTR_DTLB_MISS_DEP_STALL_CYCLES        = 0x83,
+        ARMV7_PERFCTR_MICRO_ITLB_MISS_DEP_STALL_CYCLES  = 0x84,
+        ARMV7_PERFCTR_MICRO_DTLB_MISS_DEP_STALL_CYCLES  = 0x85,
+        ARMV7_PERFCTR_DMB_DEP_STALL_CYCLES      = 0x86,
+
+        ARMV7_PERFCTR_INTGR_CLK_ENABLED_CYCLES  = 0x8A,
+        ARMV7_PERFCTR_DATA_ENGINE_CLK_EN_CYCLES = 0x8B,
+
+        ARMV7_PERFCTR_ISB_INST                  = 0x90,
+        ARMV7_PERFCTR_DSB_INST                  = 0x91,
+        ARMV7_PERFCTR_DMB_INST                  = 0x92,
+        ARMV7_PERFCTR_EXT_INTERRUPTS            = 0x93,
+
+        ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_COMPLETED     = 0xA0,
+        ARMV7_PERFCTR_PLE_CACHE_LINE_RQST_SKIPPED       = 0xA1,
+        ARMV7_PERFCTR_PLE_FIFO_FLUSH            = 0xA2,
+        ARMV7_PERFCTR_PLE_RQST_COMPLETED        = 0xA3,
+        ARMV7_PERFCTR_PLE_FIFO_OVERFLOW         = 0xA4,
+        ARMV7_PERFCTR_PLE_RQST_PROG             = 0xA5
+};
+
+/*
+ * Cortex-A8 HW events mapping
+ *
+ * The hardware events that we support. We do support cache operations but
+ * we have harvard caches and no way to combine instruction and data
+ * accesses/misses in hardware.
+ */
+static const unsigned armv7_a8_perf_map[PERF_COUNT_HW_MAX] = {
+        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
+        [PERF_COUNT_HW_INSTRUCTIONS]        = ARMV7_PERFCTR_INSTR_EXECUTED,
+        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
+        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
+        [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
+};
+
+static const unsigned armv7_a8_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+                                          [PERF_COUNT_HW_CACHE_OP_MAX]
+                                          [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+        [C(L1D)] = {
+                /*
+                 * The performance counters don't differentiate between read
+                 * and write accesses/misses so this isn't strictly correct,
+                 * but it's the best we can do. Writes and reads get
+                 * combined.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(L1I)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L1_INST,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L1_INST_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(LL)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_L2_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_L2_CACH_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(DTLB)] = {
+                /*
+                 * Only ITLB misses and DTLB refills are supported.
+                 * If users want the DTLB refills misses a raw counter
+                 * must be used.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(ITLB)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(BPU)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
+                        [C(RESULT_MISS)]
+                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
+                        [C(RESULT_MISS)]
+                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+};
+
+/*
+ * Cortex-A9 HW events mapping
+ */
+static const unsigned armv7_a9_perf_map[PERF_COUNT_HW_MAX] = {
+        [PERF_COUNT_HW_CPU_CYCLES]          = ARMV7_PERFCTR_CPU_CYCLES,
+        [PERF_COUNT_HW_INSTRUCTIONS]        =
+                                        ARMV7_PERFCTR_INST_OUT_OF_RENAME_STAGE,
+        [PERF_COUNT_HW_CACHE_REFERENCES]    = ARMV7_PERFCTR_COHERENT_LINE_HIT,
+        [PERF_COUNT_HW_CACHE_MISSES]        = ARMV7_PERFCTR_COHERENT_LINE_MISS,
+        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = ARMV7_PERFCTR_PC_WRITE,
+        [PERF_COUNT_HW_BRANCH_MISSES]       = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
+        [PERF_COUNT_HW_BUS_CYCLES]          = ARMV7_PERFCTR_CLOCK_CYCLES,
+};
+
+static const unsigned armv7_a9_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+                                          [PERF_COUNT_HW_CACHE_OP_MAX]
+                                          [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+        [C(L1D)] = {
+                /*
+                 * The performance counters don't differentiate between read
+                 * and write accesses/misses so this isn't strictly correct,
+                 * but it's the best we can do. Writes and reads get
+                 * combined.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DCACHE_REFILL,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(L1I)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_IFETCH_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(LL)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(DTLB)] = {
+                /*
+                 * Only ITLB misses and DTLB refills are supported.
+                 * If users want the DTLB refills misses a raw counter
+                 * must be used.
+                 */
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_DTLB_REFILL,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(ITLB)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = ARMV7_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(BPU)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
+                        [C(RESULT_MISS)]
+                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = ARMV7_PERFCTR_PC_WRITE,
+                        [C(RESULT_MISS)]
+                                        = ARMV7_PERFCTR_PC_BRANCH_MIS_PRED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+};
+
+/*
+ * Perf Events counters
+ */
+enum armv7_counters {
+        ARMV7_CYCLE_COUNTER             = 1,    /* Cycle counter */
+        ARMV7_COUNTER0                  = 2,    /* First event counter */
+};
+
+/*
+ * The cycle counter is ARMV7_CYCLE_COUNTER.
+ * The first event counter is ARMV7_COUNTER0.
+ * The last event counter is (ARMV7_COUNTER0 + armpmu->num_events - 1).
+ */
+#define ARMV7_COUNTER_LAST      (ARMV7_COUNTER0 + armpmu->num_events - 1)
+
+/*
+ * ARMv7 low level PMNC access
+ */
+
+/*
+ * Per-CPU PMNC: config reg
+ */
+#define ARMV7_PMNC_E            (1 << 0) /* Enable all counters */
+#define ARMV7_PMNC_P            (1 << 1) /* Reset all counters */
+#define ARMV7_PMNC_C            (1 << 2) /* Cycle counter reset */
+#define ARMV7_PMNC_D            (1 << 3) /* CCNT counts every 64th cpu cycle */
+#define ARMV7_PMNC_X            (1 << 4) /* Export to ETM */
+#define ARMV7_PMNC_DP           (1 << 5) /* Disable CCNT if non-invasive debug*/
+#define ARMV7_PMNC_N_SHIFT      11       /* Number of counters supported */
+#define ARMV7_PMNC_N_MASK       0x1f
+#define ARMV7_PMNC_MASK         0x3f     /* Mask for writable bits */
+
+/*
+ * Available counters
+ */
+#define ARMV7_CNT0              0       /* First event counter */
+#define ARMV7_CCNT              31      /* Cycle counter */
+
+/* Perf Event to low level counters mapping */
+#define ARMV7_EVENT_CNT_TO_CNTx (ARMV7_COUNTER0 - ARMV7_CNT0)
+
+/*
+ * CNTENS: counters enable reg
+ */
+#define ARMV7_CNTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
+#define ARMV7_CNTENS_C          (1 << ARMV7_CCNT)
+
+/*
+ * CNTENC: counters disable reg
+ */
+#define ARMV7_CNTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
+#define ARMV7_CNTENC_C          (1 << ARMV7_CCNT)
+
+/*
+ * INTENS: counters overflow interrupt enable reg
+ */
+#define ARMV7_INTENS_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
+#define ARMV7_INTENS_C          (1 << ARMV7_CCNT)
+
+/*
+ * INTENC: counters overflow interrupt disable reg
+ */
+#define ARMV7_INTENC_P(idx)     (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
+#define ARMV7_INTENC_C          (1 << ARMV7_CCNT)
+
+/*
+ * EVTSEL: Event selection reg
+ */
+#define ARMV7_EVTSEL_MASK       0xff            /* Mask for writable bits */
+
+/*
+ * SELECT: Counter selection reg
+ */
+#define ARMV7_SELECT_MASK       0x1f            /* Mask for writable bits */
+
+/*
+ * FLAG: counters overflow flag status reg
+ */
+#define ARMV7_FLAG_P(idx)       (1 << (idx - ARMV7_EVENT_CNT_TO_CNTx))
+#define ARMV7_FLAG_C            (1 << ARMV7_CCNT)
+#define ARMV7_FLAG_MASK         0xffffffff      /* Mask for writable bits */
+#define ARMV7_OVERFLOWED_MASK   ARMV7_FLAG_MASK
+
+static inline unsigned long armv7_pmnc_read(void)
+{
+        u32 val;
+        asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r"(val));
+        return val;
+}
+
+static inline void armv7_pmnc_write(unsigned long val)
+{
+        val &= ARMV7_PMNC_MASK;
+        isb();
+        asm volatile("mcr p15, 0, %0, c9, c12, 0" : : "r"(val));
+}
+
+static inline int armv7_pmnc_has_overflowed(unsigned long pmnc)
+{
+        return pmnc & ARMV7_OVERFLOWED_MASK;
+}
+
+static inline int armv7_pmnc_counter_has_overflowed(unsigned long pmnc,
+                                        enum armv7_counters counter)
+{
+        int ret = 0;
+
+        if (counter == ARMV7_CYCLE_COUNTER)
+                ret = pmnc & ARMV7_FLAG_C;
+        else if ((counter >= ARMV7_COUNTER0) && (counter <= ARMV7_COUNTER_LAST))
+                ret = pmnc & ARMV7_FLAG_P(counter);
+        else
+                pr_err("CPU%u checking wrong counter %d overflow status\n",
+                        smp_processor_id(), counter);
+
+        return ret;
+}
+
+static inline int armv7_pmnc_select_counter(unsigned int idx)
+{
+        u32 val;
+
+        if ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST)) {
+                pr_err("CPU%u selecting wrong PMNC counter"
+                        " %d\n", smp_processor_id(), idx);
+                return -1;
+        }
+
+        val = (idx - ARMV7_EVENT_CNT_TO_CNTx) & ARMV7_SELECT_MASK;
+        asm volatile("mcr p15, 0, %0, c9, c12, 5" : : "r" (val));
+        isb();
+
+        return idx;
+}
+
+static inline u32 armv7pmu_read_counter(int idx)
+{
+        unsigned long value = 0;
+
+        if (idx == ARMV7_CYCLE_COUNTER)
+                asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (value));
+        else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
+                if (armv7_pmnc_select_counter(idx) == idx)
+                        asm volatile("mrc p15, 0, %0, c9, c13, 2"
+                                     : "=r" (value));
+        } else
+                pr_err("CPU%u reading wrong counter %d\n",
+                        smp_processor_id(), idx);
+
+        return value;
+}
+
+static inline void armv7pmu_write_counter(int idx, u32 value)
+{
+        if (idx == ARMV7_CYCLE_COUNTER)
+                asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
+        else if ((idx >= ARMV7_COUNTER0) && (idx <= ARMV7_COUNTER_LAST)) {
+                if (armv7_pmnc_select_counter(idx) == idx)
+                        asm volatile("mcr p15, 0, %0, c9, c13, 2"
+                                     : : "r" (value));
+        } else
+                pr_err("CPU%u writing wrong counter %d\n",
+                        smp_processor_id(), idx);
+}
+
+static inline void armv7_pmnc_write_evtsel(unsigned int idx, u32 val)
+{
+        if (armv7_pmnc_select_counter(idx) == idx) {
+                val &= ARMV7_EVTSEL_MASK;
+                asm volatile("mcr p15, 0, %0, c9, c13, 1" : : "r" (val));
+        }
+}
+
+static inline u32 armv7_pmnc_enable_counter(unsigned int idx)
+{
+        u32 val;
+
+        if ((idx != ARMV7_CYCLE_COUNTER) &&
+            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
+                pr_err("CPU%u enabling wrong PMNC counter"
+                        " %d\n", smp_processor_id(), idx);
+                return -1;
+        }
+
+        if (idx == ARMV7_CYCLE_COUNTER)
+                val = ARMV7_CNTENS_C;
+        else
+                val = ARMV7_CNTENS_P(idx);
+
+        asm volatile("mcr p15, 0, %0, c9, c12, 1" : : "r" (val));
+
+        return idx;
+}
+
+static inline u32 armv7_pmnc_disable_counter(unsigned int idx)
+{
+        u32 val;
+
+
+        if ((idx != ARMV7_CYCLE_COUNTER) &&
+            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
+                pr_err("CPU%u disabling wrong PMNC counter"
+                        " %d\n", smp_processor_id(), idx);
+                return -1;
+        }
+
+        if (idx == ARMV7_CYCLE_COUNTER)
+                val = ARMV7_CNTENC_C;
+        else
+                val = ARMV7_CNTENC_P(idx);
+
+        asm volatile("mcr p15, 0, %0, c9, c12, 2" : : "r" (val));
+
+        return idx;
+}
+
+static inline u32 armv7_pmnc_enable_intens(unsigned int idx)
+{
+        u32 val;
+
+        if ((idx != ARMV7_CYCLE_COUNTER) &&
+            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
+                pr_err("CPU%u enabling wrong PMNC counter"
+                        " interrupt enable %d\n", smp_processor_id(), idx);
+                return -1;
+        }
+
+        if (idx == ARMV7_CYCLE_COUNTER)
+                val = ARMV7_INTENS_C;
+        else
+                val = ARMV7_INTENS_P(idx);
+
+        asm volatile("mcr p15, 0, %0, c9, c14, 1" : : "r" (val));
+
+        return idx;
+}
+
+static inline u32 armv7_pmnc_disable_intens(unsigned int idx)
+{
+        u32 val;
+
+        if ((idx != ARMV7_CYCLE_COUNTER) &&
+            ((idx < ARMV7_COUNTER0) || (idx > ARMV7_COUNTER_LAST))) {
+                pr_err("CPU%u disabling wrong PMNC counter"
+                        " interrupt enable %d\n", smp_processor_id(), idx);
+                return -1;
+        }
+
+        if (idx == ARMV7_CYCLE_COUNTER)
+                val = ARMV7_INTENC_C;
+        else
+                val = ARMV7_INTENC_P(idx);
+
+        asm volatile("mcr p15, 0, %0, c9, c14, 2" : : "r" (val));
+
+        return idx;
+}
+
+static inline u32 armv7_pmnc_getreset_flags(void)
+{
+        u32 val;
+
+        /* Read */
+        asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
+
+        /* Write to clear flags */
+        val &= ARMV7_FLAG_MASK;
+        asm volatile("mcr p15, 0, %0, c9, c12, 3" : : "r" (val));
+
+        return val;
+}
+
+#ifdef DEBUG
+static void armv7_pmnc_dump_regs(void)
+{
+        u32 val;
+        unsigned int cnt;
+
+        printk(KERN_INFO "PMNC registers dump:\n");
+
+        asm volatile("mrc p15, 0, %0, c9, c12, 0" : "=r" (val));
+        printk(KERN_INFO "PMNC  =0x%08x\n", val);
+
+        asm volatile("mrc p15, 0, %0, c9, c12, 1" : "=r" (val));
+        printk(KERN_INFO "CNTENS=0x%08x\n", val);
+
+        asm volatile("mrc p15, 0, %0, c9, c14, 1" : "=r" (val));
+        printk(KERN_INFO "INTENS=0x%08x\n", val);
+
+        asm volatile("mrc p15, 0, %0, c9, c12, 3" : "=r" (val));
+        printk(KERN_INFO "FLAGS =0x%08x\n", val);
+
+        asm volatile("mrc p15, 0, %0, c9, c12, 5" : "=r" (val));
+        printk(KERN_INFO "SELECT=0x%08x\n", val);
+
+        asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r" (val));
+        printk(KERN_INFO "CCNT  =0x%08x\n", val);
+
+        for (cnt = ARMV7_COUNTER0; cnt < ARMV7_COUNTER_LAST; cnt++) {
+                armv7_pmnc_select_counter(cnt);
+                asm volatile("mrc p15, 0, %0, c9, c13, 2" : "=r" (val));
+                printk(KERN_INFO "CNT[%d] count =0x%08x\n",
+                        cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
+                asm volatile("mrc p15, 0, %0, c9, c13, 1" : "=r" (val));
+                printk(KERN_INFO "CNT[%d] evtsel=0x%08x\n",
+                        cnt-ARMV7_EVENT_CNT_TO_CNTx, val);
+        }
+}
+#endif
+
+static void armv7pmu_enable_event(struct hw_perf_event *hwc, int idx)
+{
+        unsigned long flags;
+
+        /*
+         * Enable counter and interrupt, and set the counter to count
+         * the event that we're interested in.
+         */
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+
+        /*
+         * Disable counter
+         */
+        armv7_pmnc_disable_counter(idx);
+
+        /*
+         * Set event (if destined for PMNx counters)
+         * We don't need to set the event if it's a cycle count
+         */
+        if (idx != ARMV7_CYCLE_COUNTER)
+                armv7_pmnc_write_evtsel(idx, hwc->config_base);
+
+        /*
+         * Enable interrupt for this counter
+         */
+        armv7_pmnc_enable_intens(idx);
+
+        /*
+         * Enable counter
+         */
+        armv7_pmnc_enable_counter(idx);
+
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void armv7pmu_disable_event(struct hw_perf_event *hwc, int idx)
+{
+        unsigned long flags;
+
+        /*
+         * Disable counter and interrupt
+         */
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+
+        /*
+         * Disable counter
+         */
+        armv7_pmnc_disable_counter(idx);
+
+        /*
+         * Disable interrupt for this counter
+         */
+        armv7_pmnc_disable_intens(idx);
+
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static irqreturn_t armv7pmu_handle_irq(int irq_num, void *dev)
+{
+        unsigned long pmnc;
+        struct perf_sample_data data;
+        struct cpu_hw_events *cpuc;
+        struct pt_regs *regs;
+        int idx;
+
+        /*
+         * Get and reset the IRQ flags
+         */
+        pmnc = armv7_pmnc_getreset_flags();
+
+        /*
+         * Did an overflow occur?
+         */
+        if (!armv7_pmnc_has_overflowed(pmnc))
+                return IRQ_NONE;
+
+        /*
+         * Handle the counter(s) overflow(s)
+         */
+        regs = get_irq_regs();
+
+        perf_sample_data_init(&data, 0);
+
+        cpuc = &__get_cpu_var(cpu_hw_events);
+        for (idx = 0; idx <= armpmu->num_events; ++idx) {
+                struct perf_event *event = cpuc->events[idx];
+                struct hw_perf_event *hwc;
+
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+
+                /*
+                 * We have a single interrupt for all counters. Check that
+                 * each counter has overflowed before we process it.
+                 */
+                if (!armv7_pmnc_counter_has_overflowed(pmnc, idx))
+                        continue;
+
+                hwc = &event->hw;
+                armpmu_event_update(event, hwc, idx, 1);
+                data.period = event->hw.last_period;
+                if (!armpmu_event_set_period(event, hwc, idx))
+                        continue;
+
+                if (perf_event_overflow(event, 0, &data, regs))
+                        armpmu->disable(hwc, idx);
+        }
+
+        /*
+         * Handle the pending perf events.
+         *
+         * Note: this call *must* be run with interrupts disabled. For
+         * platforms that can have the PMU interrupts raised as an NMI, this
+         * will not work.
+         */
+        irq_work_run();
+
+        return IRQ_HANDLED;
+}
+
+static void armv7pmu_start(void)
+{
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        /* Enable all counters */
+        armv7_pmnc_write(armv7_pmnc_read() | ARMV7_PMNC_E);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void armv7pmu_stop(void)
+{
+        unsigned long flags;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        /* Disable all counters */
+        armv7_pmnc_write(armv7_pmnc_read() & ~ARMV7_PMNC_E);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static int armv7pmu_get_event_idx(struct cpu_hw_events *cpuc,
+                                  struct hw_perf_event *event)
+{
+        int idx;
+
+        /* Always place a cycle counter into the cycle counter. */
+        if (event->config_base == ARMV7_PERFCTR_CPU_CYCLES) {
+                if (test_and_set_bit(ARMV7_CYCLE_COUNTER, cpuc->used_mask))
+                        return -EAGAIN;
+
+                return ARMV7_CYCLE_COUNTER;
+        } else {
+                /*
+                 * For anything other than a cycle counter, try and use
+                 * the events counters
+                 */
+                for (idx = ARMV7_COUNTER0; idx <= armpmu->num_events; ++idx) {
+                        if (!test_and_set_bit(idx, cpuc->used_mask))
+                                return idx;
+                }
+
+                /* The counters are all in use. */
+                return -EAGAIN;
+        }
+}
+
+static void armv7pmu_reset(void *info)
+{
+        u32 idx, nb_cnt = armpmu->num_events;
+
+        /* The counter and interrupt enable registers are unknown at reset. */
+        for (idx = 1; idx < nb_cnt; ++idx)
+                armv7pmu_disable_event(NULL, idx);
+
+        /* Initialize & Reset PMNC: C and P bits */
+        armv7_pmnc_write(ARMV7_PMNC_P | ARMV7_PMNC_C);
+}
+
+static struct arm_pmu armv7pmu = {
+        .handle_irq             = armv7pmu_handle_irq,
+        .enable                 = armv7pmu_enable_event,
+        .disable                = armv7pmu_disable_event,
+        .read_counter           = armv7pmu_read_counter,
+        .write_counter          = armv7pmu_write_counter,
+        .get_event_idx          = armv7pmu_get_event_idx,
+        .start                  = armv7pmu_start,
+        .stop                   = armv7pmu_stop,
+        .reset                  = armv7pmu_reset,
+        .raw_event_mask         = 0xFF,
+        .max_period             = (1LLU << 32) - 1,
+};
+
+static u32 __init armv7_read_num_pmnc_events(void)
+{
+        u32 nb_cnt;
+
+        /* Read the nb of CNTx counters supported from PMNC */
+        nb_cnt = (armv7_pmnc_read() >> ARMV7_PMNC_N_SHIFT) & ARMV7_PMNC_N_MASK;
+
+        /* Add the CPU cycles counter and return */
+        return nb_cnt + 1;
+}
+
+static const struct arm_pmu *__init armv7_a8_pmu_init(void)
+{
+        armv7pmu.id             = ARM_PERF_PMU_ID_CA8;
+        armv7pmu.name           = "ARMv7 Cortex-A8";
+        armv7pmu.cache_map      = &armv7_a8_perf_cache_map;
+        armv7pmu.event_map      = &armv7_a8_perf_map;
+        armv7pmu.num_events     = armv7_read_num_pmnc_events();
+        return &armv7pmu;
+}
+
+static const struct arm_pmu *__init armv7_a9_pmu_init(void)
+{
+        armv7pmu.id             = ARM_PERF_PMU_ID_CA9;
+        armv7pmu.name           = "ARMv7 Cortex-A9";
+        armv7pmu.cache_map      = &armv7_a9_perf_cache_map;
+        armv7pmu.event_map      = &armv7_a9_perf_map;
+        armv7pmu.num_events     = armv7_read_num_pmnc_events();
+        return &armv7pmu;
+}
+#else
+static const struct arm_pmu *__init armv7_a8_pmu_init(void)
+{
+        return NULL;
+}
+
+static const struct arm_pmu *__init armv7_a9_pmu_init(void)
+{
+        return NULL;
+}
+#endif  /* CONFIG_CPU_V7 */
diff --git a/arch/arm/kernel/perf_event_xscale.c b/arch/arm/kernel/perf_event_xscale.c
new file mode 100644
index 000000000000..39affbe4fdb2
--- /dev/null
+++ b/arch/arm/kernel/perf_event_xscale.c
@@ -0,0 +1,807 @@
+/*
+ * ARMv5 [xscale] Performance counter handling code.
+ *
+ * Copyright (C) 2010, ARM Ltd., Will Deacon <will.deacon@arm.com>
+ *
+ * Based on the previous xscale OProfile code.
+ *
+ * There are two variants of the xscale PMU that we support:
+ *      - xscale1pmu: 2 event counters and a cycle counter
+ *      - xscale2pmu: 4 event counters and a cycle counter
+ * The two variants share event definitions, but have different
+ * PMU structures.
+ */
+
+#ifdef CONFIG_CPU_XSCALE
+enum xscale_perf_types {
+        XSCALE_PERFCTR_ICACHE_MISS              = 0x00,
+        XSCALE_PERFCTR_ICACHE_NO_DELIVER        = 0x01,
+        XSCALE_PERFCTR_DATA_STALL               = 0x02,
+        XSCALE_PERFCTR_ITLB_MISS                = 0x03,
+        XSCALE_PERFCTR_DTLB_MISS                = 0x04,
+        XSCALE_PERFCTR_BRANCH                   = 0x05,
+        XSCALE_PERFCTR_BRANCH_MISS              = 0x06,
+        XSCALE_PERFCTR_INSTRUCTION              = 0x07,
+        XSCALE_PERFCTR_DCACHE_FULL_STALL        = 0x08,
+        XSCALE_PERFCTR_DCACHE_FULL_STALL_CONTIG = 0x09,
+        XSCALE_PERFCTR_DCACHE_ACCESS            = 0x0A,
+        XSCALE_PERFCTR_DCACHE_MISS              = 0x0B,
+        XSCALE_PERFCTR_DCACHE_WRITE_BACK        = 0x0C,
+        XSCALE_PERFCTR_PC_CHANGED               = 0x0D,
+        XSCALE_PERFCTR_BCU_REQUEST              = 0x10,
+        XSCALE_PERFCTR_BCU_FULL                 = 0x11,
+        XSCALE_PERFCTR_BCU_DRAIN                = 0x12,
+        XSCALE_PERFCTR_BCU_ECC_NO_ELOG          = 0x14,
+        XSCALE_PERFCTR_BCU_1_BIT_ERR            = 0x15,
+        XSCALE_PERFCTR_RMW                      = 0x16,
+        /* XSCALE_PERFCTR_CCNT is not hardware defined */
+        XSCALE_PERFCTR_CCNT                     = 0xFE,
+        XSCALE_PERFCTR_UNUSED                   = 0xFF,
+};
+
+enum xscale_counters {
+        XSCALE_CYCLE_COUNTER    = 1,
+        XSCALE_COUNTER0,
+        XSCALE_COUNTER1,
+        XSCALE_COUNTER2,
+        XSCALE_COUNTER3,
+};
+
+static const unsigned xscale_perf_map[PERF_COUNT_HW_MAX] = {
+        [PERF_COUNT_HW_CPU_CYCLES]          = XSCALE_PERFCTR_CCNT,
+        [PERF_COUNT_HW_INSTRUCTIONS]        = XSCALE_PERFCTR_INSTRUCTION,
+        [PERF_COUNT_HW_CACHE_REFERENCES]    = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_CACHE_MISSES]        = HW_OP_UNSUPPORTED,
+        [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = XSCALE_PERFCTR_BRANCH,
+        [PERF_COUNT_HW_BRANCH_MISSES]       = XSCALE_PERFCTR_BRANCH_MISS,
+        [PERF_COUNT_HW_BUS_CYCLES]          = HW_OP_UNSUPPORTED,
+};
+
+static const unsigned xscale_perf_cache_map[PERF_COUNT_HW_CACHE_MAX]
+                                           [PERF_COUNT_HW_CACHE_OP_MAX]
+                                           [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
+        [C(L1D)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = XSCALE_PERFCTR_DCACHE_ACCESS,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DCACHE_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(L1I)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ICACHE_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(LL)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(DTLB)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_DTLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(ITLB)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = XSCALE_PERFCTR_ITLB_MISS,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+        [C(BPU)] = {
+                [C(OP_READ)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_WRITE)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+                [C(OP_PREFETCH)] = {
+                        [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
+                        [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
+                },
+        },
+};
+
+#define XSCALE_PMU_ENABLE       0x001
+#define XSCALE_PMN_RESET        0x002
+#define XSCALE_CCNT_RESET       0x004
+#define XSCALE_PMU_RESET        (CCNT_RESET | PMN_RESET)
+#define XSCALE_PMU_CNT64        0x008
+
+#define XSCALE1_OVERFLOWED_MASK 0x700
+#define XSCALE1_CCOUNT_OVERFLOW 0x400
+#define XSCALE1_COUNT0_OVERFLOW 0x100
+#define XSCALE1_COUNT1_OVERFLOW 0x200
+#define XSCALE1_CCOUNT_INT_EN   0x040
+#define XSCALE1_COUNT0_INT_EN   0x010
+#define XSCALE1_COUNT1_INT_EN   0x020
+#define XSCALE1_COUNT0_EVT_SHFT 12
+#define XSCALE1_COUNT0_EVT_MASK (0xff << XSCALE1_COUNT0_EVT_SHFT)
+#define XSCALE1_COUNT1_EVT_SHFT 20
+#define XSCALE1_COUNT1_EVT_MASK (0xff << XSCALE1_COUNT1_EVT_SHFT)
+
+static inline u32
+xscale1pmu_read_pmnc(void)
+{
+        u32 val;
+        asm volatile("mrc p14, 0, %0, c0, c0, 0" : "=r" (val));
+        return val;
+}
+
+static inline void
+xscale1pmu_write_pmnc(u32 val)
+{
+        /* upper 4bits and 7, 11 are write-as-0 */
+        val &= 0xffff77f;
+        asm volatile("mcr p14, 0, %0, c0, c0, 0" : : "r" (val));
+}
+
+static inline int
+xscale1_pmnc_counter_has_overflowed(unsigned long pmnc,
+                                        enum xscale_counters counter)
+{
+        int ret = 0;
+
+        switch (counter) {
+        case XSCALE_CYCLE_COUNTER:
+                ret = pmnc & XSCALE1_CCOUNT_OVERFLOW;
+                break;
+        case XSCALE_COUNTER0:
+                ret = pmnc & XSCALE1_COUNT0_OVERFLOW;
+                break;
+        case XSCALE_COUNTER1:
+                ret = pmnc & XSCALE1_COUNT1_OVERFLOW;
+                break;
+        default:
+                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
+        }
+
+        return ret;
+}
+
+static irqreturn_t
+xscale1pmu_handle_irq(int irq_num, void *dev)
+{
+        unsigned long pmnc;
+        struct perf_sample_data data;
+        struct cpu_hw_events *cpuc;
+        struct pt_regs *regs;
+        int idx;
+
+        /*
+         * NOTE: there's an A stepping erratum that states if an overflow
+         *       bit already exists and another occurs, the previous
+         *       Overflow bit gets cleared. There's no workaround.
+         *       Fixed in B stepping or later.
+         */
+        pmnc = xscale1pmu_read_pmnc();
+
+        /*
+         * Write the value back to clear the overflow flags. Overflow
+         * flags remain in pmnc for use below. We also disable the PMU
+         * while we process the interrupt.
+         */
+        xscale1pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
+
+        if (!(pmnc & XSCALE1_OVERFLOWED_MASK))
+                return IRQ_NONE;
+
+        regs = get_irq_regs();
+
+        perf_sample_data_init(&data, 0);
+
+        cpuc = &__get_cpu_var(cpu_hw_events);
+        for (idx = 0; idx <= armpmu->num_events; ++idx) {
+                struct perf_event *event = cpuc->events[idx];
+                struct hw_perf_event *hwc;
+
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+
+                if (!xscale1_pmnc_counter_has_overflowed(pmnc, idx))
+                        continue;
+
+                hwc = &event->hw;
+                armpmu_event_update(event, hwc, idx, 1);
+                data.period = event->hw.last_period;
+                if (!armpmu_event_set_period(event, hwc, idx))
+                        continue;
+
+                if (perf_event_overflow(event, 0, &data, regs))
+                        armpmu->disable(hwc, idx);
+        }
+
+        irq_work_run();
+
+        /*
+         * Re-enable the PMU.
+         */
+        pmnc = xscale1pmu_read_pmnc() | XSCALE_PMU_ENABLE;
+        xscale1pmu_write_pmnc(pmnc);
+
+        return IRQ_HANDLED;
+}
+
+static void
+xscale1pmu_enable_event(struct hw_perf_event *hwc, int idx)
+{
+        unsigned long val, mask, evt, flags;
+
+        switch (idx) {
+        case XSCALE_CYCLE_COUNTER:
+                mask = 0;
+                evt = XSCALE1_CCOUNT_INT_EN;
+                break;
+        case XSCALE_COUNTER0:
+                mask = XSCALE1_COUNT0_EVT_MASK;
+                evt = (hwc->config_base << XSCALE1_COUNT0_EVT_SHFT) |
+                        XSCALE1_COUNT0_INT_EN;
+                break;
+        case XSCALE_COUNTER1:
+                mask = XSCALE1_COUNT1_EVT_MASK;
+                evt = (hwc->config_base << XSCALE1_COUNT1_EVT_SHFT) |
+                        XSCALE1_COUNT1_INT_EN;
+                break;
+        default:
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = xscale1pmu_read_pmnc();
+        val &= ~mask;
+        val |= evt;
+        xscale1pmu_write_pmnc(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void
+xscale1pmu_disable_event(struct hw_perf_event *hwc, int idx)
+{
+        unsigned long val, mask, evt, flags;
+
+        switch (idx) {
+        case XSCALE_CYCLE_COUNTER:
+                mask = XSCALE1_CCOUNT_INT_EN;
+                evt = 0;
+                break;
+        case XSCALE_COUNTER0:
+                mask = XSCALE1_COUNT0_INT_EN | XSCALE1_COUNT0_EVT_MASK;
+                evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT0_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER1:
+                mask = XSCALE1_COUNT1_INT_EN | XSCALE1_COUNT1_EVT_MASK;
+                evt = XSCALE_PERFCTR_UNUSED << XSCALE1_COUNT1_EVT_SHFT;
+                break;
+        default:
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = xscale1pmu_read_pmnc();
+        val &= ~mask;
+        val |= evt;
+        xscale1pmu_write_pmnc(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static int
+xscale1pmu_get_event_idx(struct cpu_hw_events *cpuc,
+                        struct hw_perf_event *event)
+{
+        if (XSCALE_PERFCTR_CCNT == event->config_base) {
+                if (test_and_set_bit(XSCALE_CYCLE_COUNTER, cpuc->used_mask))
+                        return -EAGAIN;
+
+                return XSCALE_CYCLE_COUNTER;
+        } else {
+                if (!test_and_set_bit(XSCALE_COUNTER1, cpuc->used_mask))
+                        return XSCALE_COUNTER1;
+
+                if (!test_and_set_bit(XSCALE_COUNTER0, cpuc->used_mask))
+                        return XSCALE_COUNTER0;
+
+                return -EAGAIN;
+        }
+}
+
+static void
+xscale1pmu_start(void)
+{
+        unsigned long flags, val;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = xscale1pmu_read_pmnc();
+        val |= XSCALE_PMU_ENABLE;
+        xscale1pmu_write_pmnc(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void
+xscale1pmu_stop(void)
+{
+        unsigned long flags, val;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = xscale1pmu_read_pmnc();
+        val &= ~XSCALE_PMU_ENABLE;
+        xscale1pmu_write_pmnc(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static inline u32
+xscale1pmu_read_counter(int counter)
+{
+        u32 val = 0;
+
+        switch (counter) {
+        case XSCALE_CYCLE_COUNTER:
+                asm volatile("mrc p14, 0, %0, c1, c0, 0" : "=r" (val));
+                break;
+        case XSCALE_COUNTER0:
+                asm volatile("mrc p14, 0, %0, c2, c0, 0" : "=r" (val));
+                break;
+        case XSCALE_COUNTER1:
+                asm volatile("mrc p14, 0, %0, c3, c0, 0" : "=r" (val));
+                break;
+        }
+
+        return val;
+}
+
+static inline void
+xscale1pmu_write_counter(int counter, u32 val)
+{
+        switch (counter) {
+        case XSCALE_CYCLE_COUNTER:
+                asm volatile("mcr p14, 0, %0, c1, c0, 0" : : "r" (val));
+                break;
+        case XSCALE_COUNTER0:
+                asm volatile("mcr p14, 0, %0, c2, c0, 0" : : "r" (val));
+                break;
+        case XSCALE_COUNTER1:
+                asm volatile("mcr p14, 0, %0, c3, c0, 0" : : "r" (val));
+                break;
+        }
+}
+
+static const struct arm_pmu xscale1pmu = {
+        .id             = ARM_PERF_PMU_ID_XSCALE1,
+        .name           = "xscale1",
+        .handle_irq     = xscale1pmu_handle_irq,
+        .enable         = xscale1pmu_enable_event,
+        .disable        = xscale1pmu_disable_event,
+        .read_counter   = xscale1pmu_read_counter,
+        .write_counter  = xscale1pmu_write_counter,
+        .get_event_idx  = xscale1pmu_get_event_idx,
+        .start          = xscale1pmu_start,
+        .stop           = xscale1pmu_stop,
+        .cache_map      = &xscale_perf_cache_map,
+        .event_map      = &xscale_perf_map,
+        .raw_event_mask = 0xFF,
+        .num_events     = 3,
+        .max_period     = (1LLU << 32) - 1,
+};
+
+static const struct arm_pmu *__init xscale1pmu_init(void)
+{
+        return &xscale1pmu;
+}
+
+#define XSCALE2_OVERFLOWED_MASK 0x01f
+#define XSCALE2_CCOUNT_OVERFLOW 0x001
+#define XSCALE2_COUNT0_OVERFLOW 0x002
+#define XSCALE2_COUNT1_OVERFLOW 0x004
+#define XSCALE2_COUNT2_OVERFLOW 0x008
+#define XSCALE2_COUNT3_OVERFLOW 0x010
+#define XSCALE2_CCOUNT_INT_EN   0x001
+#define XSCALE2_COUNT0_INT_EN   0x002
+#define XSCALE2_COUNT1_INT_EN   0x004
+#define XSCALE2_COUNT2_INT_EN   0x008
+#define XSCALE2_COUNT3_INT_EN   0x010
+#define XSCALE2_COUNT0_EVT_SHFT 0
+#define XSCALE2_COUNT0_EVT_MASK (0xff << XSCALE2_COUNT0_EVT_SHFT)
+#define XSCALE2_COUNT1_EVT_SHFT 8
+#define XSCALE2_COUNT1_EVT_MASK (0xff << XSCALE2_COUNT1_EVT_SHFT)
+#define XSCALE2_COUNT2_EVT_SHFT 16
+#define XSCALE2_COUNT2_EVT_MASK (0xff << XSCALE2_COUNT2_EVT_SHFT)
+#define XSCALE2_COUNT3_EVT_SHFT 24
+#define XSCALE2_COUNT3_EVT_MASK (0xff << XSCALE2_COUNT3_EVT_SHFT)
+
+static inline u32
+xscale2pmu_read_pmnc(void)
+{
+        u32 val;
+        asm volatile("mrc p14, 0, %0, c0, c1, 0" : "=r" (val));
+        /* bits 1-2 and 4-23 are read-unpredictable */
+        return val & 0xff000009;
+}
+
+static inline void
+xscale2pmu_write_pmnc(u32 val)
+{
+        /* bits 4-23 are write-as-0, 24-31 are write ignored */
+        val &= 0xf;
+        asm volatile("mcr p14, 0, %0, c0, c1, 0" : : "r" (val));
+}
+
+static inline u32
+xscale2pmu_read_overflow_flags(void)
+{
+        u32 val;
+        asm volatile("mrc p14, 0, %0, c5, c1, 0" : "=r" (val));
+        return val;
+}
+
+static inline void
+xscale2pmu_write_overflow_flags(u32 val)
+{
+        asm volatile("mcr p14, 0, %0, c5, c1, 0" : : "r" (val));
+}
+
+static inline u32
+xscale2pmu_read_event_select(void)
+{
+        u32 val;
+        asm volatile("mrc p14, 0, %0, c8, c1, 0" : "=r" (val));
+        return val;
+}
+
+static inline void
+xscale2pmu_write_event_select(u32 val)
+{
+        asm volatile("mcr p14, 0, %0, c8, c1, 0" : : "r"(val));
+}
+
+static inline u32
+xscale2pmu_read_int_enable(void)
+{
+        u32 val;
+        asm volatile("mrc p14, 0, %0, c4, c1, 0" : "=r" (val));
+        return val;
+}
+
+static void
+xscale2pmu_write_int_enable(u32 val)
+{
+        asm volatile("mcr p14, 0, %0, c4, c1, 0" : : "r" (val));
+}
+
+static inline int
+xscale2_pmnc_counter_has_overflowed(unsigned long of_flags,
+                                        enum xscale_counters counter)
+{
+        int ret = 0;
+
+        switch (counter) {
+        case XSCALE_CYCLE_COUNTER:
+                ret = of_flags & XSCALE2_CCOUNT_OVERFLOW;
+                break;
+        case XSCALE_COUNTER0:
+                ret = of_flags & XSCALE2_COUNT0_OVERFLOW;
+                break;
+        case XSCALE_COUNTER1:
+                ret = of_flags & XSCALE2_COUNT1_OVERFLOW;
+                break;
+        case XSCALE_COUNTER2:
+                ret = of_flags & XSCALE2_COUNT2_OVERFLOW;
+                break;
+        case XSCALE_COUNTER3:
+                ret = of_flags & XSCALE2_COUNT3_OVERFLOW;
+                break;
+        default:
+                WARN_ONCE(1, "invalid counter number (%d)\n", counter);
+        }
+
+        return ret;
+}
+
+static irqreturn_t
+xscale2pmu_handle_irq(int irq_num, void *dev)
+{
+        unsigned long pmnc, of_flags;
+        struct perf_sample_data data;
+        struct cpu_hw_events *cpuc;
+        struct pt_regs *regs;
+        int idx;
+
+        /* Disable the PMU. */
+        pmnc = xscale2pmu_read_pmnc();
+        xscale2pmu_write_pmnc(pmnc & ~XSCALE_PMU_ENABLE);
+
+        /* Check the overflow flag register. */
+        of_flags = xscale2pmu_read_overflow_flags();
+        if (!(of_flags & XSCALE2_OVERFLOWED_MASK))
+                return IRQ_NONE;
+
+        /* Clear the overflow bits. */
+        xscale2pmu_write_overflow_flags(of_flags);
+
+        regs = get_irq_regs();
+
+        perf_sample_data_init(&data, 0);
+
+        cpuc = &__get_cpu_var(cpu_hw_events);
+        for (idx = 0; idx <= armpmu->num_events; ++idx) {
+                struct perf_event *event = cpuc->events[idx];
+                struct hw_perf_event *hwc;
+
+                if (!test_bit(idx, cpuc->active_mask))
+                        continue;
+
+                if (!xscale2_pmnc_counter_has_overflowed(pmnc, idx))
+                        continue;
+
+                hwc = &event->hw;
+                armpmu_event_update(event, hwc, idx, 1);
+                data.period = event->hw.last_period;
+                if (!armpmu_event_set_period(event, hwc, idx))
+                        continue;
+
+                if (perf_event_overflow(event, 0, &data, regs))
+                        armpmu->disable(hwc, idx);
+        }
+
+        irq_work_run();
+
+        /*
+         * Re-enable the PMU.
+         */
+        pmnc = xscale2pmu_read_pmnc() | XSCALE_PMU_ENABLE;
+        xscale2pmu_write_pmnc(pmnc);
+
+        return IRQ_HANDLED;
+}
+
+static void
+xscale2pmu_enable_event(struct hw_perf_event *hwc, int idx)
+{
+        unsigned long flags, ien, evtsel;
+
+        ien = xscale2pmu_read_int_enable();
+        evtsel = xscale2pmu_read_event_select();
+
+        switch (idx) {
+        case XSCALE_CYCLE_COUNTER:
+                ien |= XSCALE2_CCOUNT_INT_EN;
+                break;
+        case XSCALE_COUNTER0:
+                ien |= XSCALE2_COUNT0_INT_EN;
+                evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
+                evtsel |= hwc->config_base << XSCALE2_COUNT0_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER1:
+                ien |= XSCALE2_COUNT1_INT_EN;
+                evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
+                evtsel |= hwc->config_base << XSCALE2_COUNT1_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER2:
+                ien |= XSCALE2_COUNT2_INT_EN;
+                evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
+                evtsel |= hwc->config_base << XSCALE2_COUNT2_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER3:
+                ien |= XSCALE2_COUNT3_INT_EN;
+                evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
+                evtsel |= hwc->config_base << XSCALE2_COUNT3_EVT_SHFT;
+                break;
+        default:
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        xscale2pmu_write_event_select(evtsel);
+        xscale2pmu_write_int_enable(ien);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void
+xscale2pmu_disable_event(struct hw_perf_event *hwc, int idx)
+{
+        unsigned long flags, ien, evtsel;
+
+        ien = xscale2pmu_read_int_enable();
+        evtsel = xscale2pmu_read_event_select();
+
+        switch (idx) {
+        case XSCALE_CYCLE_COUNTER:
+                ien &= ~XSCALE2_CCOUNT_INT_EN;
+                break;
+        case XSCALE_COUNTER0:
+                ien &= ~XSCALE2_COUNT0_INT_EN;
+                evtsel &= ~XSCALE2_COUNT0_EVT_MASK;
+                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT0_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER1:
+                ien &= ~XSCALE2_COUNT1_INT_EN;
+                evtsel &= ~XSCALE2_COUNT1_EVT_MASK;
+                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT1_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER2:
+                ien &= ~XSCALE2_COUNT2_INT_EN;
+                evtsel &= ~XSCALE2_COUNT2_EVT_MASK;
+                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT2_EVT_SHFT;
+                break;
+        case XSCALE_COUNTER3:
+                ien &= ~XSCALE2_COUNT3_INT_EN;
+                evtsel &= ~XSCALE2_COUNT3_EVT_MASK;
+                evtsel |= XSCALE_PERFCTR_UNUSED << XSCALE2_COUNT3_EVT_SHFT;
+                break;
+        default:
+                WARN_ONCE(1, "invalid counter number (%d)\n", idx);
+                return;
+        }
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        xscale2pmu_write_event_select(evtsel);
+        xscale2pmu_write_int_enable(ien);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static int
+xscale2pmu_get_event_idx(struct cpu_hw_events *cpuc,
+                        struct hw_perf_event *event)
+{
+        int idx = xscale1pmu_get_event_idx(cpuc, event);
+        if (idx >= 0)
+                goto out;
+
+        if (!test_and_set_bit(XSCALE_COUNTER3, cpuc->used_mask))
+                idx = XSCALE_COUNTER3;
+        else if (!test_and_set_bit(XSCALE_COUNTER2, cpuc->used_mask))
+                idx = XSCALE_COUNTER2;
+out:
+        return idx;
+}
+
+static void
+xscale2pmu_start(void)
+{
+        unsigned long flags, val;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = xscale2pmu_read_pmnc() & ~XSCALE_PMU_CNT64;
+        val |= XSCALE_PMU_ENABLE;
+        xscale2pmu_write_pmnc(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static void
+xscale2pmu_stop(void)
+{
+        unsigned long flags, val;
+
+        raw_spin_lock_irqsave(&pmu_lock, flags);
+        val = xscale2pmu_read_pmnc();
+        val &= ~XSCALE_PMU_ENABLE;
+        xscale2pmu_write_pmnc(val);
+        raw_spin_unlock_irqrestore(&pmu_lock, flags);
+}
+
+static inline u32
+xscale2pmu_read_counter(int counter)
+{
+        u32 val = 0;
+
+        switch (counter) {
+        case XSCALE_CYCLE_COUNTER:
+                asm volatile("mrc p14, 0, %0, c1, c1, 0" : "=r" (val));
+                break;
+        case XSCALE_COUNTER0:
+                asm volatile("mrc p14, 0, %0, c0, c2, 0" : "=r" (val));
+                break;
+        case XSCALE_COUNTER1:
+                asm volatile("mrc p14, 0, %0, c1, c2, 0" : "=r" (val));
+                break;
+        case XSCALE_COUNTER2:
+                asm volatile("mrc p14, 0, %0, c2, c2, 0" : "=r" (val));
+                break;
+        case XSCALE_COUNTER3:
+                asm volatile("mrc p14, 0, %0, c3, c2, 0" : "=r" (val));
+                break;
+        }
+
+        return val;
+}
+
+static inline void
+xscale2pmu_write_counter(int counter, u32 val)
+{
+        switch (counter) {
+        case XSCALE_CYCLE_COUNTER:
+                asm volatile("mcr p14, 0, %0, c1, c1, 0" : : "r" (val));
+                break;
+        case XSCALE_COUNTER0:
+                asm volatile("mcr p14, 0, %0, c0, c2, 0" : : "r" (val));
+                break;
+        case XSCALE_COUNTER1:
+                asm volatile("mcr p14, 0, %0, c1, c2, 0" : : "r" (val));
+                break;
+        case XSCALE_COUNTER2:
+                asm volatile("mcr p14, 0, %0, c2, c2, 0" : : "r" (val));
+                break;
+        case XSCALE_COUNTER3:
+                asm volatile("mcr p14, 0, %0, c3, c2, 0" : : "r" (val));
+                break;
+        }
+}
+
+static const struct arm_pmu xscale2pmu = {
+        .id             = ARM_PERF_PMU_ID_XSCALE2,
+        .name           = "xscale2",
+        .handle_irq     = xscale2pmu_handle_irq,
+        .enable         = xscale2pmu_enable_event,
+        .disable        = xscale2pmu_disable_event,
+        .read_counter   = xscale2pmu_read_counter,
+        .write_counter  = xscale2pmu_write_counter,
+        .get_event_idx  = xscale2pmu_get_event_idx,
+        .start          = xscale2pmu_start,
+        .stop           = xscale2pmu_stop,
+        .cache_map      = &xscale_perf_cache_map,
+        .event_map      = &xscale_perf_map,
+        .raw_event_mask = 0xFF,
+        .num_events     = 5,
+        .max_period     = (1LLU << 32) - 1,
+};
+
+static const struct arm_pmu *__init xscale2pmu_init(void)
+{
+        return &xscale2pmu;
+}
+#else
+static const struct arm_pmu *__init xscale1pmu_init(void)
+{
+        return NULL;
+}
+
+static const struct arm_pmu *__init xscale2pmu_init(void)
+{
+        return NULL;
+}
+#endif  /* CONFIG_CPU_XSCALE */
diff --git a/arch/arm/kernel/pj4-cp0.c b/arch/arm/kernel/pj4-cp0.c
new file mode 100644
index 000000000000..a4b1b0748fd3
--- /dev/null
+++ b/arch/arm/kernel/pj4-cp0.c
@@ -0,0 +1,94 @@
+/*
+ * linux/arch/arm/kernel/pj4-cp0.c
+ *
+ * PJ4 iWMMXt coprocessor context switching and handling
+ *
+ * Copyright (c) 2010 Marvell International Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/module.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/signal.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <asm/thread_notify.h>
+
+static int iwmmxt_do(struct notifier_block *self, unsigned long cmd, void *t)
+{
+        struct thread_info *thread = t;
+
+        switch (cmd) {
+        case THREAD_NOTIFY_FLUSH:
+                /*
+                 * flush_thread() zeroes thread->fpstate, so no need
+                 * to do anything here.
+                 *
+                 * FALLTHROUGH: Ensure we don't try to overwrite our newly
+                 * initialised state information on the first fault.
+                 */
+
+        case THREAD_NOTIFY_EXIT:
+                iwmmxt_task_release(thread);
+                break;
+
+        case THREAD_NOTIFY_SWITCH:
+                iwmmxt_task_switch(thread);
+                break;
+        }
+
+        return NOTIFY_DONE;
+}
+
+static struct notifier_block iwmmxt_notifier_block = {
+        .notifier_call  = iwmmxt_do,
+};
+
+
+static u32 __init pj4_cp_access_read(void)
+{
+        u32 value;
+
+        __asm__ __volatile__ (
+                "mrc    p15, 0, %0, c1, c0, 2\n\t"
+                : "=r" (value));
+        return value;
+}
+
+static void __init pj4_cp_access_write(u32 value)
+{
+        u32 temp;
+
+        __asm__ __volatile__ (
+                "mcr    p15, 0, %1, c1, c0, 2\n\t"
+                "mrc    p15, 0, %0, c1, c0, 2\n\t"
+                "mov    %0, %0\n\t"
+                "sub    pc, pc, #4\n\t"
+                : "=r" (temp) : "r" (value));
+}
+
+
+/*
+ * Disable CP0/CP1 on boot, and let call_fpe() and the iWMMXt lazy
+ * switch code handle iWMMXt context switching.
+ */
+static int __init pj4_cp0_init(void)
+{
+        u32 cp_access;
+
+        cp_access = pj4_cp_access_read() & ~0xf;
+        pj4_cp_access_write(cp_access);
+
+        printk(KERN_INFO "PJ4 iWMMXt coprocessor enabled.\n");
+        elf_hwcap |= HWCAP_IWMMXT;
+        thread_register_notifier(&iwmmxt_notifier_block);
+
+        return 0;
+}
+
+late_initcall(pj4_cp0_init);
diff --git a/arch/arm/kernel/pmu.c b/arch/arm/kernel/pmu.c
index b8af96ea62e6..2c79eec19262 100644
--- a/arch/arm/kernel/pmu.c
+++ b/arch/arm/kernel/pmu.c
@@ -97,28 +97,34 @@ set_irq_affinity(int irq,
                            irq, cpu);
         return err;
 #else
-        return 0;
+        return -EINVAL;
 #endif
 }
 
 static int
 init_cpu_pmu(void)
 {
-        int i, err = 0;
+        int i, irqs, err = 0;
         struct platform_device *pdev = pmu_devices[ARM_PMU_DEVICE_CPU];
 
-        if (!pdev) {
-                err = -ENODEV;
-                goto out;
-        }
+        if (!pdev)
+                return -ENODEV;
+
+        irqs = pdev->num_resources;
+
+        /*
+         * If we have a single PMU interrupt that we can't shift, assume that
+         * we're running on a uniprocessor machine and continue.
+         */
+        if (irqs == 1 && !irq_can_set_affinity(platform_get_irq(pdev, 0)))
+                return 0;
 
-        for (i = 0; i < pdev->num_resources; ++i) {
+        for (i = 0; i < irqs; ++i) {
                 err = set_irq_affinity(platform_get_irq(pdev, i), i);
                 if (err)
                         break;
         }
 
-out:
         return err;
 }
 
diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c
index 401e38be1f78..5e1e54197227 100644
--- a/arch/arm/kernel/process.c
+++ b/arch/arm/kernel/process.c
@@ -29,6 +29,7 @@
 #include <linux/utsname.h>
 #include <linux/uaccess.h>
 #include <linux/random.h>
+#include <linux/hw_breakpoint.h>
 
 #include <asm/cacheflush.h>
 #include <asm/leds.h>
@@ -135,6 +136,25 @@ EXPORT_SYMBOL(pm_power_off);
 void (*arm_pm_restart)(char str, const char *cmd) = arm_machine_restart;
 EXPORT_SYMBOL_GPL(arm_pm_restart);
 
+static void do_nothing(void *unused)
+{
+}
+
+/*
+ * cpu_idle_wait - Used to ensure that all the CPUs discard old value of
+ * pm_idle and update to new pm_idle value. Required while changing pm_idle
+ * handler on SMP systems.
+ *
+ * Caller must have changed pm_idle to the new value before the call. Old
+ * pm_idle value will not be used by any CPU after the return of this function.
+ */
+void cpu_idle_wait(void)
+{
+        smp_mb();
+        /* kick all the CPUs so that they exit out of pm_idle */
+        smp_call_function(do_nothing, NULL, 1);
+}
+EXPORT_SYMBOL_GPL(cpu_idle_wait);
 
 /*
  * This is our default idle handler.  We need to disable
@@ -317,6 +337,8 @@ void flush_thread(void)
         struct thread_info *thread = current_thread_info();
         struct task_struct *tsk = current;
 
+        flush_ptrace_hw_breakpoint(tsk);
+
         memset(thread->used_cp, 0, sizeof(thread->used_cp));
         memset(&tsk->thread.debug, 0, sizeof(struct debug_info));
         memset(&thread->fpstate, 0, sizeof(union fp_state));
@@ -345,9 +367,13 @@ copy_thread(unsigned long clone_flags, unsigned long stack_start,
         thread->cpu_context.sp = (unsigned long)childregs;
         thread->cpu_context.pc = (unsigned long)ret_from_fork;
 
+        clear_ptrace_hw_breakpoint(p);
+
         if (clone_flags & CLONE_SETTLS)
                 thread->tp_value = regs->ARM_r3;
 
+        thread_notify(THREAD_NOTIFY_COPY, thread);
+
         return 0;
 }
 
@@ -458,3 +484,26 @@ unsigned long arch_randomize_brk(struct mm_struct *mm)
         unsigned long range_end = mm->brk + 0x02000000;
         return randomize_range(mm->brk, range_end, 0) ? : mm->brk;
 }
+
+#ifdef CONFIG_MMU
+/*
+ * The vectors page is always readable from user space for the
+ * atomic helpers and the signal restart code.  Let's declare a mapping
+ * for it so it is visible through ptrace and /proc/<pid>/mem.
+ */
+
+int vectors_user_mapping(void)
+{
+        struct mm_struct *mm = current->mm;
+        return install_special_mapping(mm, 0xffff0000, PAGE_SIZE,
+                                       VM_READ | VM_EXEC |
+                                       VM_MAYREAD | VM_MAYEXEC |
+                                       VM_ALWAYSDUMP | VM_RESERVED,
+                                       NULL);
+}
+
+const char *arch_vma_name(struct vm_area_struct *vma)
+{
+        return (vma->vm_start == 0xffff0000) ? "[vectors]" : NULL;
+}
+#endif
diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c
index f99d489822d5..97260060bf26 100644
--- a/arch/arm/kernel/ptrace.c
+++ b/arch/arm/kernel/ptrace.c
@@ -19,13 +19,14 @@
 #include <linux/init.h>
 #include <linux/signal.h>
 #include <linux/uaccess.h>
+#include <linux/perf_event.h>
+#include <linux/hw_breakpoint.h>
+#include <linux/regset.h>
 
 #include <asm/pgtable.h>
 #include <asm/system.h>
 #include <asm/traps.h>
 
-#include "ptrace.h"
-
 #define REG_PC  15
 #define REG_PSR 16
 /*
@@ -182,389 +183,12 @@ put_user_reg(struct task_struct *task, int offset, long data)
         return ret;
 }
 
-static inline int
-read_u32(struct task_struct *task, unsigned long addr, u32 *res)
-{
-        int ret;
-
-        ret = access_process_vm(task, addr, res, sizeof(*res), 0);
-
-        return ret == sizeof(*res) ? 0 : -EIO;
-}
-
-static inline int
-read_instr(struct task_struct *task, unsigned long addr, u32 *res)
-{
-        int ret;
-
-        if (addr & 1) {
-                u16 val;
-                ret = access_process_vm(task, addr & ~1, &val, sizeof(val), 0);
-                ret = ret == sizeof(val) ? 0 : -EIO;
-                *res = val;
-        } else {
-                u32 val;
-                ret = access_process_vm(task, addr & ~3, &val, sizeof(val), 0);
-                ret = ret == sizeof(val) ? 0 : -EIO;
-                *res = val;
-        }
-        return ret;
-}
-
-/*
- * Get value of register `rn' (in the instruction)
- */
-static unsigned long
-ptrace_getrn(struct task_struct *child, unsigned long insn)
-{
-        unsigned int reg = (insn >> 16) & 15;
-        unsigned long val;
-
-        val = get_user_reg(child, reg);
-        if (reg == 15)
-                val += 8;
-
-        return val;
-}
-
-/*
- * Get value of operand 2 (in an ALU instruction)
- */
-static unsigned long
-ptrace_getaluop2(struct task_struct *child, unsigned long insn)
-{
-        unsigned long val;
-        int shift;
-        int type;
-
-        if (insn & 1 << 25) {
-                val = insn & 255;
-                shift = (insn >> 8) & 15;
-                type = 3;
-        } else {
-                val = get_user_reg (child, insn & 15);
-
-                if (insn & (1 << 4))
-                        shift = (int)get_user_reg (child, (insn >> 8) & 15);
-                else
-                        shift = (insn >> 7) & 31;
-
-                type = (insn >> 5) & 3;
-        }
-
-        switch (type) {
-        case 0: val <<= shift;  break;
-        case 1: val >>= shift;  break;
-        case 2:
-                val = (((signed long)val) >> shift);
-                break;
-        case 3:
-                val = (val >> shift) | (val << (32 - shift));
-                break;
-        }
-        return val;
-}
-
-/*
- * Get value of operand 2 (in a LDR instruction)
- */
-static unsigned long
-ptrace_getldrop2(struct task_struct *child, unsigned long insn)
-{
-        unsigned long val;
-        int shift;
-        int type;
-
-        val = get_user_reg(child, insn & 15);
-        shift = (insn >> 7) & 31;
-        type = (insn >> 5) & 3;
-
-        switch (type) {
-        case 0: val <<= shift;  break;
-        case 1: val >>= shift;  break;
-        case 2:
-                val = (((signed long)val) >> shift);
-                break;
-        case 3:
-                val = (val >> shift) | (val << (32 - shift));
-                break;
-        }
-        return val;
-}
-
-#define OP_MASK 0x01e00000
-#define OP_AND  0x00000000
-#define OP_EOR  0x00200000
-#define OP_SUB  0x00400000
-#define OP_RSB  0x00600000
-#define OP_ADD  0x00800000
-#define OP_ADC  0x00a00000
-#define OP_SBC  0x00c00000
-#define OP_RSC  0x00e00000
-#define OP_ORR  0x01800000
-#define OP_MOV  0x01a00000
-#define OP_BIC  0x01c00000
-#define OP_MVN  0x01e00000
-
-static unsigned long
-get_branch_address(struct task_struct *child, unsigned long pc, unsigned long insn)
-{
-        u32 alt = 0;
-
-        switch (insn & 0x0e000000) {
-        case 0x00000000:
-        case 0x02000000: {
-                /*
-                 * data processing
-                 */
-                long aluop1, aluop2, ccbit;
-
-                if ((insn & 0x0fffffd0) == 0x012fff10) {
-                        /*
-                         * bx or blx
-                         */
-                        alt = get_user_reg(child, insn & 15);
-                        break;
-                }
-
-
-                if ((insn & 0xf000) != 0xf000)
-                        break;
-
-                aluop1 = ptrace_getrn(child, insn);
-                aluop2 = ptrace_getaluop2(child, insn);
-                ccbit  = get_user_reg(child, REG_PSR) & PSR_C_BIT ? 1 : 0;
-
-                switch (insn & OP_MASK) {
-                case OP_AND: alt = aluop1 & aluop2;             break;
-                case OP_EOR: alt = aluop1 ^ aluop2;             break;
-                case OP_SUB: alt = aluop1 - aluop2;             break;
-                case OP_RSB: alt = aluop2 - aluop1;             break;
-                case OP_ADD: alt = aluop1 + aluop2;             break;
-                case OP_ADC: alt = aluop1 + aluop2 + ccbit;     break;
-                case OP_SBC: alt = aluop1 - aluop2 + ccbit;     break;
-                case OP_RSC: alt = aluop2 - aluop1 + ccbit;     break;
-                case OP_ORR: alt = aluop1 | aluop2;             break;
-                case OP_MOV: alt = aluop2;                      break;
-                case OP_BIC: alt = aluop1 & ~aluop2;            break;
-                case OP_MVN: alt = ~aluop2;                     break;
-                }
-                break;
-        }
-
-        case 0x04000000:
-        case 0x06000000:
-                /*
-                 * ldr
-                 */
-                if ((insn & 0x0010f000) == 0x0010f000) {
-                        unsigned long base;
-
-                        base = ptrace_getrn(child, insn);
-                        if (insn & 1 << 24) {
-                                long aluop2;
-
-                                if (insn & 0x02000000)
-                                        aluop2 = ptrace_getldrop2(child, insn);
-                                else
-                                        aluop2 = insn & 0xfff;
-
-                                if (insn & 1 << 23)
-                                        base += aluop2;
-                                else
-                                        base -= aluop2;
-                        }
-                        read_u32(child, base, &alt);
-                }
-                break;
-
-        case 0x08000000:
-                /*
-                 * ldm
-                 */
-                if ((insn & 0x00108000) == 0x00108000) {
-                        unsigned long base;
-                        unsigned int nr_regs;
-
-                        if (insn & (1 << 23)) {
-                                nr_regs = hweight16(insn & 65535) << 2;
-
-                                if (!(insn & (1 << 24)))
-                                        nr_regs -= 4;
-                        } else {
-                                if (insn & (1 << 24))
-                                        nr_regs = -4;
-                                else
-                                        nr_regs = 0;
-                        }
-
-                        base = ptrace_getrn(child, insn);
-
-                        read_u32(child, base + nr_regs, &alt);
-                        break;
-                }
-                break;
-
-        case 0x0a000000: {
-                /*
-                 * bl or b
-                 */
-                signed long displ;
-                /* It's a branch/branch link: instead of trying to
-                 * figure out whether the branch will be taken or not,
-                 * we'll put a breakpoint at both locations.  This is
-                 * simpler, more reliable, and probably not a whole lot
-                 * slower than the alternative approach of emulating the
-                 * branch.
-                 */
-                displ = (insn & 0x00ffffff) << 8;
-                displ = (displ >> 6) + 8;
-                if (displ != 0 && displ != 4)
-                        alt = pc + displ;
-            }
-            break;
-        }
-
-        return alt;
-}
-
-static int
-swap_insn(struct task_struct *task, unsigned long addr,
-          void *old_insn, void *new_insn, int size)
-{
-        int ret;
-
-        ret = access_process_vm(task, addr, old_insn, size, 0);
-        if (ret == size)
-                ret = access_process_vm(task, addr, new_insn, size, 1);
-        return ret;
-}
-
-static void
-add_breakpoint(struct task_struct *task, struct debug_info *dbg, unsigned long addr)
-{
-        int nr = dbg->nsaved;
-
-        if (nr < 2) {
-                u32 new_insn = BREAKINST_ARM;
-                int res;
-
-                res = swap_insn(task, addr, &dbg->bp[nr].insn, &new_insn, 4);
-
-                if (res == 4) {
-                        dbg->bp[nr].address = addr;
-                        dbg->nsaved += 1;
-                }
-        } else
-                printk(KERN_ERR "ptrace: too many breakpoints\n");
-}
-
-/*
- * Clear one breakpoint in the user program.  We copy what the hardware
- * does and use bit 0 of the address to indicate whether this is a Thumb
- * breakpoint or an ARM breakpoint.
- */
-static void clear_breakpoint(struct task_struct *task, struct debug_entry *bp)
-{
-        unsigned long addr = bp->address;
-        union debug_insn old_insn;
-        int ret;
-
-        if (addr & 1) {
-                ret = swap_insn(task, addr & ~1, &old_insn.thumb,
-                                &bp->insn.thumb, 2);
-
-                if (ret != 2 || old_insn.thumb != BREAKINST_THUMB)
-                        printk(KERN_ERR "%s:%d: corrupted Thumb breakpoint at "
-                                "0x%08lx (0x%04x)\n", task->comm,
-                                task_pid_nr(task), addr, old_insn.thumb);
-        } else {
-                ret = swap_insn(task, addr & ~3, &old_insn.arm,
-                                &bp->insn.arm, 4);
-
-                if (ret != 4 || old_insn.arm != BREAKINST_ARM)
-                        printk(KERN_ERR "%s:%d: corrupted ARM breakpoint at "
-                                "0x%08lx (0x%08x)\n", task->comm,
-                                task_pid_nr(task), addr, old_insn.arm);
-        }
-}
-
-void ptrace_set_bpt(struct task_struct *child)
-{
-        struct pt_regs *regs;
-        unsigned long pc;
-        u32 insn;
-        int res;
-
-        regs = task_pt_regs(child);
-        pc = instruction_pointer(regs);
-
-        if (thumb_mode(regs)) {
-                printk(KERN_WARNING "ptrace: can't handle thumb mode\n");
-                return;
-        }
-
-        res = read_instr(child, pc, &insn);
-        if (!res) {
-                struct debug_info *dbg = &child->thread.debug;
-                unsigned long alt;
-
-                dbg->nsaved = 0;
-
-                alt = get_branch_address(child, pc, insn);
-                if (alt)
-                        add_breakpoint(child, dbg, alt);
-
-                /*
-                 * Note that we ignore the result of setting the above
-                 * breakpoint since it may fail.  When it does, this is
-                 * not so much an error, but a forewarning that we may
-                 * be receiving a prefetch abort shortly.
-                 *
-                 * If we don't set this breakpoint here, then we can
-                 * lose control of the thread during single stepping.
-                 */
-                if (!alt || predicate(insn) != PREDICATE_ALWAYS)
-                        add_breakpoint(child, dbg, pc + 4);
-        }
-}
-
-/*
- * Ensure no single-step breakpoint is pending.  Returns non-zero
- * value if child was being single-stepped.
- */
-void ptrace_cancel_bpt(struct task_struct *child)
-{
-        int i, nsaved = child->thread.debug.nsaved;
-
-        child->thread.debug.nsaved = 0;
-
-        if (nsaved > 2) {
-                printk("ptrace_cancel_bpt: bogus nsaved: %d!\n", nsaved);
-                nsaved = 2;
-        }
-
-        for (i = 0; i < nsaved; i++)
-                clear_breakpoint(child, &child->thread.debug.bp[i]);
-}
-
-void user_disable_single_step(struct task_struct *task)
-{
-        task->ptrace &= ~PT_SINGLESTEP;
-        ptrace_cancel_bpt(task);
-}
-
-void user_enable_single_step(struct task_struct *task)
-{
-        task->ptrace |= PT_SINGLESTEP;
-}
-
 /*
  * Called by kernel/ptrace.c when detaching..
  */
 void ptrace_disable(struct task_struct *child)
 {
-        user_disable_single_step(child);
+        /* Nothing to do. */
 }
 
 /*
@@ -574,8 +198,6 @@ void ptrace_break(struct task_struct *tsk, struct pt_regs *regs)
 {
         siginfo_t info;
 
-        ptrace_cancel_bpt(tsk);
-
         info.si_signo = SIGTRAP;
         info.si_errno = 0;
         info.si_code  = TRAP_BRKPT;
@@ -687,58 +309,6 @@ static int ptrace_write_user(struct task_struct *tsk, unsigned long off,
         return put_user_reg(tsk, off >> 2, val);
 }
 
-/*
- * Get all user integer registers.
- */
-static int ptrace_getregs(struct task_struct *tsk, void __user *uregs)
-{
-        struct pt_regs *regs = task_pt_regs(tsk);
-
-        return copy_to_user(uregs, regs, sizeof(struct pt_regs)) ? -EFAULT : 0;
-}
-
-/*
- * Set all user integer registers.
- */
-static int ptrace_setregs(struct task_struct *tsk, void __user *uregs)
-{
-        struct pt_regs newregs;
-        int ret;
-
-        ret = -EFAULT;
-        if (copy_from_user(&newregs, uregs, sizeof(struct pt_regs)) == 0) {
-                struct pt_regs *regs = task_pt_regs(tsk);
-
-                ret = -EINVAL;
-                if (valid_user_regs(&newregs)) {
-                        *regs = newregs;
-                        ret = 0;
-                }
-        }
-
-        return ret;
-}
-
-/*
- * Get the child FPU state.
- */
-static int ptrace_getfpregs(struct task_struct *tsk, void __user *ufp)
-{
-        return copy_to_user(ufp, &task_thread_info(tsk)->fpstate,
-                            sizeof(struct user_fp)) ? -EFAULT : 0;
-}
-
-/*
- * Set the child FPU state.
- */
-static int ptrace_setfpregs(struct task_struct *tsk, void __user *ufp)
-{
-        struct thread_info *thread = task_thread_info(tsk);
-        thread->used_cp[1] = thread->used_cp[2] = 1;
-        return copy_from_user(&thread->fpstate, ufp,
-                              sizeof(struct user_fp)) ? -EFAULT : 0;
-}
-
 #ifdef CONFIG_IWMMXT
 
 /*
@@ -797,63 +367,454 @@ static int ptrace_setcrunchregs(struct task_struct *tsk, void __user *ufp)
 }
 #endif
 
-#ifdef CONFIG_VFP
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
 /*
- * Get the child VFP state.
+ * Convert a virtual register number into an index for a thread_info
+ * breakpoint array. Breakpoints are identified using positive numbers
+ * whilst watchpoints are negative. The registers are laid out as pairs
+ * of (address, control), each pair mapping to a unique hw_breakpoint struct.
+ * Register 0 is reserved for describing resource information.
  */
-static int ptrace_getvfpregs(struct task_struct *tsk, void __user *data)
+static int ptrace_hbp_num_to_idx(long num)
 {
-        struct thread_info *thread = task_thread_info(tsk);
-        union vfp_state *vfp = &thread->vfpstate;
-        struct user_vfp __user *ufp = data;
+        if (num < 0)
+                num = (ARM_MAX_BRP << 1) - num;
+        return (num - 1) >> 1;
+}
 
-        vfp_sync_hwstate(thread);
+/*
+ * Returns the virtual register number for the address of the
+ * breakpoint at index idx.
+ */
+static long ptrace_hbp_idx_to_num(int idx)
+{
+        long mid = ARM_MAX_BRP << 1;
+        long num = (idx << 1) + 1;
+        return num > mid ? mid - num : num;
+}
+
+/*
+ * Handle hitting a HW-breakpoint.
+ */
+static void ptrace_hbptriggered(struct perf_event *bp, int unused,
+                                     struct perf_sample_data *data,
+                                     struct pt_regs *regs)
+{
+        struct arch_hw_breakpoint *bkpt = counter_arch_bp(bp);
+        long num;
+        int i;
+        siginfo_t info;
+
+        for (i = 0; i < ARM_MAX_HBP_SLOTS; ++i)
+                if (current->thread.debug.hbp[i] == bp)
+                        break;
 
-        /* copy the floating point registers */
-        if (copy_to_user(&ufp->fpregs, &vfp->hard.fpregs,
-                         sizeof(vfp->hard.fpregs)))
-                return -EFAULT;
+        num = (i == ARM_MAX_HBP_SLOTS) ? 0 : ptrace_hbp_idx_to_num(i);
 
-        /* copy the status and control register */
-        if (put_user(vfp->hard.fpscr, &ufp->fpscr))
-                return -EFAULT;
+        info.si_signo   = SIGTRAP;
+        info.si_errno   = (int)num;
+        info.si_code    = TRAP_HWBKPT;
+        info.si_addr    = (void __user *)(bkpt->trigger);
 
+        force_sig_info(SIGTRAP, &info, current);
+}
+
+/*
+ * Set ptrace breakpoint pointers to zero for this task.
+ * This is required in order to prevent child processes from unregistering
+ * breakpoints held by their parent.
+ */
+void clear_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+        memset(tsk->thread.debug.hbp, 0, sizeof(tsk->thread.debug.hbp));
+}
+
+/*
+ * Unregister breakpoints from this task and reset the pointers in
+ * the thread_struct.
+ */
+void flush_ptrace_hw_breakpoint(struct task_struct *tsk)
+{
+        int i;
+        struct thread_struct *t = &tsk->thread;
+
+        for (i = 0; i < ARM_MAX_HBP_SLOTS; i++) {
+                if (t->debug.hbp[i]) {
+                        unregister_hw_breakpoint(t->debug.hbp[i]);
+                        t->debug.hbp[i] = NULL;
+                }
+        }
+}
+
+static u32 ptrace_get_hbp_resource_info(void)
+{
+        u8 num_brps, num_wrps, debug_arch, wp_len;
+        u32 reg = 0;
+
+        num_brps        = hw_breakpoint_slots(TYPE_INST);
+        num_wrps        = hw_breakpoint_slots(TYPE_DATA);
+        debug_arch      = arch_get_debug_arch();
+        wp_len          = arch_get_max_wp_len();
+
+        reg             |= debug_arch;
+        reg             <<= 8;
+        reg             |= wp_len;
+        reg             <<= 8;
+        reg             |= num_wrps;
+        reg             <<= 8;
+        reg             |= num_brps;
+
+        return reg;
+}
+
+static struct perf_event *ptrace_hbp_create(struct task_struct *tsk, int type)
+{
+        struct perf_event_attr attr;
+
+        ptrace_breakpoint_init(&attr);
+
+        /* Initialise fields to sane defaults. */
+        attr.bp_addr    = 0;
+        attr.bp_len     = HW_BREAKPOINT_LEN_4;
+        attr.bp_type    = type;
+        attr.disabled   = 1;
+
+        return register_user_hw_breakpoint(&attr, ptrace_hbptriggered, tsk);
+}
+
+static int ptrace_gethbpregs(struct task_struct *tsk, long num,
+                             unsigned long  __user *data)
+{
+        u32 reg;
+        int idx, ret = 0;
+        struct perf_event *bp;
+        struct arch_hw_breakpoint_ctrl arch_ctrl;
+
+        if (num == 0) {
+                reg = ptrace_get_hbp_resource_info();
+        } else {
+                idx = ptrace_hbp_num_to_idx(num);
+                if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) {
+                        ret = -EINVAL;
+                        goto out;
+                }
+
+                bp = tsk->thread.debug.hbp[idx];
+                if (!bp) {
+                        reg = 0;
+                        goto put;
+                }
+
+                arch_ctrl = counter_arch_bp(bp)->ctrl;
+
+                /*
+                 * Fix up the len because we may have adjusted it
+                 * to compensate for an unaligned address.
+                 */
+                while (!(arch_ctrl.len & 0x1))
+                        arch_ctrl.len >>= 1;
+
+                if (num & 0x1)
+                        reg = bp->attr.bp_addr;
+                else
+                        reg = encode_ctrl_reg(arch_ctrl);
+        }
+
+put:
+        if (put_user(reg, data))
+                ret = -EFAULT;
+
+out:
+        return ret;
+}
+
+static int ptrace_sethbpregs(struct task_struct *tsk, long num,
+                             unsigned long __user *data)
+{
+        int idx, gen_len, gen_type, implied_type, ret = 0;
+        u32 user_val;
+        struct perf_event *bp;
+        struct arch_hw_breakpoint_ctrl ctrl;
+        struct perf_event_attr attr;
+
+        if (num == 0)
+                goto out;
+        else if (num < 0)
+                implied_type = HW_BREAKPOINT_RW;
+        else
+                implied_type = HW_BREAKPOINT_X;
+
+        idx = ptrace_hbp_num_to_idx(num);
+        if (idx < 0 || idx >= ARM_MAX_HBP_SLOTS) {
+                ret = -EINVAL;
+                goto out;
+        }
+
+        if (get_user(user_val, data)) {
+                ret = -EFAULT;
+                goto out;
+        }
+
+        bp = tsk->thread.debug.hbp[idx];
+        if (!bp) {
+                bp = ptrace_hbp_create(tsk, implied_type);
+                if (IS_ERR(bp)) {
+                        ret = PTR_ERR(bp);
+                        goto out;
+                }
+                tsk->thread.debug.hbp[idx] = bp;
+        }
+
+        attr = bp->attr;
+
+        if (num & 0x1) {
+                /* Address */
+                attr.bp_addr    = user_val;
+        } else {
+                /* Control */
+                decode_ctrl_reg(user_val, &ctrl);
+                ret = arch_bp_generic_fields(ctrl, &gen_len, &gen_type);
+                if (ret)
+                        goto out;
+
+                if ((gen_type & implied_type) != gen_type) {
+                        ret = -EINVAL;
+                        goto out;
+                }
+
+                attr.bp_len     = gen_len;
+                attr.bp_type    = gen_type;
+                attr.disabled   = !ctrl.enabled;
+        }
+
+        ret = modify_user_hw_breakpoint(bp, &attr);
+out:
+        return ret;
+}
+#endif
+
+/* regset get/set implementations */
+
+static int gpr_get(struct task_struct *target,
+                   const struct user_regset *regset,
+                   unsigned int pos, unsigned int count,
+                   void *kbuf, void __user *ubuf)
+{
+        struct pt_regs *regs = task_pt_regs(target);
+
+        return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
+                                   regs,
+                                   0, sizeof(*regs));
+}
+
+static int gpr_set(struct task_struct *target,
+                   const struct user_regset *regset,
+                   unsigned int pos, unsigned int count,
+                   const void *kbuf, const void __user *ubuf)
+{
+        int ret;
+        struct pt_regs newregs;
+
+        ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+                                 &newregs,
+                                 0, sizeof(newregs));
+        if (ret)
+                return ret;
+
+        if (!valid_user_regs(&newregs))
+                return -EINVAL;
+
+        *task_pt_regs(target) = newregs;
         return 0;
 }
 
+static int fpa_get(struct task_struct *target,
+                   const struct user_regset *regset,
+                   unsigned int pos, unsigned int count,
+                   void *kbuf, void __user *ubuf)
+{
+        return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
+                                   &task_thread_info(target)->fpstate,
+                                   0, sizeof(struct user_fp));
+}
+
+static int fpa_set(struct task_struct *target,
+                   const struct user_regset *regset,
+                   unsigned int pos, unsigned int count,
+                   const void *kbuf, const void __user *ubuf)
+{
+        struct thread_info *thread = task_thread_info(target);
+
+        thread->used_cp[1] = thread->used_cp[2] = 1;
+
+        return user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+                &thread->fpstate,
+                0, sizeof(struct user_fp));
+}
+
+#ifdef CONFIG_VFP
 /*
- * Set the child VFP state.
+ * VFP register get/set implementations.
+ *
+ * With respect to the kernel, struct user_fp is divided into three chunks:
+ * 16 or 32 real VFP registers (d0-d15 or d0-31)
+ *      These are transferred to/from the real registers in the task's
+ *      vfp_hard_struct.  The number of registers depends on the kernel
+ *      configuration.
+ *
+ * 16 or 0 fake VFP registers (d16-d31 or empty)
+ *      i.e., the user_vfp structure has space for 32 registers even if
+ *      the kernel doesn't have them all.
+ *
+ *      vfp_get() reads this chunk as zero where applicable
+ *      vfp_set() ignores this chunk
+ *
+ * 1 word for the FPSCR
+ *
+ * The bounds-checking logic built into user_regset_copyout and friends
+ * means that we can make a simple sequence of calls to map the relevant data
+ * to/from the specified slice of the user regset structure.
  */
-static int ptrace_setvfpregs(struct task_struct *tsk, void __user *data)
+static int vfp_get(struct task_struct *target,
+                   const struct user_regset *regset,
+                   unsigned int pos, unsigned int count,
+                   void *kbuf, void __user *ubuf)
 {
-        struct thread_info *thread = task_thread_info(tsk);
-        union vfp_state *vfp = &thread->vfpstate;
-        struct user_vfp __user *ufp = data;
+        int ret;
+        struct thread_info *thread = task_thread_info(target);
+        struct vfp_hard_struct const *vfp = &thread->vfpstate.hard;
+        const size_t user_fpregs_offset = offsetof(struct user_vfp, fpregs);
+        const size_t user_fpscr_offset = offsetof(struct user_vfp, fpscr);
 
         vfp_sync_hwstate(thread);
 
-        /* copy the floating point registers */
-        if (copy_from_user(&vfp->hard.fpregs, &ufp->fpregs,
-                           sizeof(vfp->hard.fpregs)))
-                return -EFAULT;
+        ret = user_regset_copyout(&pos, &count, &kbuf, &ubuf,
+                                  &vfp->fpregs,
+                                  user_fpregs_offset,
+                                  user_fpregs_offset + sizeof(vfp->fpregs));
+        if (ret)
+                return ret;
 
-        /* copy the status and control register */
-        if (get_user(vfp->hard.fpscr, &ufp->fpscr))
-                return -EFAULT;
+        ret = user_regset_copyout_zero(&pos, &count, &kbuf, &ubuf,
+                                       user_fpregs_offset + sizeof(vfp->fpregs),
+                                       user_fpscr_offset);
+        if (ret)
+                return ret;
 
+        return user_regset_copyout(&pos, &count, &kbuf, &ubuf,
+                                   &vfp->fpscr,
+                                   user_fpscr_offset,
+                                   user_fpscr_offset + sizeof(vfp->fpscr));
+}
+
+/*
+ * For vfp_set() a read-modify-write is done on the VFP registers,
+ * in order to avoid writing back a half-modified set of registers on
+ * failure.
+ */
+static int vfp_set(struct task_struct *target,
+                          const struct user_regset *regset,
+                          unsigned int pos, unsigned int count,
+                          const void *kbuf, const void __user *ubuf)
+{
+        int ret;
+        struct thread_info *thread = task_thread_info(target);
+        struct vfp_hard_struct new_vfp = thread->vfpstate.hard;
+        const size_t user_fpregs_offset = offsetof(struct user_vfp, fpregs);
+        const size_t user_fpscr_offset = offsetof(struct user_vfp, fpscr);
+
+        ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+                                  &new_vfp.fpregs,
+                                  user_fpregs_offset,
+                                  user_fpregs_offset + sizeof(new_vfp.fpregs));
+        if (ret)
+                return ret;
+
+        ret = user_regset_copyin_ignore(&pos, &count, &kbuf, &ubuf,
+                                user_fpregs_offset + sizeof(new_vfp.fpregs),
+                                user_fpscr_offset);
+        if (ret)
+                return ret;
+
+        ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf,
+                                 &new_vfp.fpscr,
+                                 user_fpscr_offset,
+                                 user_fpscr_offset + sizeof(new_vfp.fpscr));
+        if (ret)
+                return ret;
+
+        vfp_sync_hwstate(thread);
+        thread->vfpstate.hard = new_vfp;
         vfp_flush_hwstate(thread);
 
         return 0;
 }
+#endif /* CONFIG_VFP */
+
+enum arm_regset {
+        REGSET_GPR,
+        REGSET_FPR,
+#ifdef CONFIG_VFP
+        REGSET_VFP,
 #endif
+};
 
-long arch_ptrace(struct task_struct *child, long request, long addr, long data)
+static const struct user_regset arm_regsets[] = {
+        [REGSET_GPR] = {
+                .core_note_type = NT_PRSTATUS,
+                .n = ELF_NGREG,
+                .size = sizeof(u32),
+                .align = sizeof(u32),
+                .get = gpr_get,
+                .set = gpr_set
+        },
+        [REGSET_FPR] = {
+                /*
+                 * For the FPA regs in fpstate, the real fields are a mixture
+                 * of sizes, so pretend that the registers are word-sized:
+                 */
+                .core_note_type = NT_PRFPREG,
+                .n = sizeof(struct user_fp) / sizeof(u32),
+                .size = sizeof(u32),
+                .align = sizeof(u32),
+                .get = fpa_get,
+                .set = fpa_set
+        },
+#ifdef CONFIG_VFP
+        [REGSET_VFP] = {
+                /*
+                 * Pretend that the VFP regs are word-sized, since the FPSCR is
+                 * a single word dangling at the end of struct user_vfp:
+                 */
+                .core_note_type = NT_ARM_VFP,
+                .n = ARM_VFPREGS_SIZE / sizeof(u32),
+                .size = sizeof(u32),
+                .align = sizeof(u32),
+                .get = vfp_get,
+                .set = vfp_set
+        },
+#endif /* CONFIG_VFP */
+};
+
+static const struct user_regset_view user_arm_view = {
+        .name = "arm", .e_machine = ELF_ARCH, .ei_osabi = ELF_OSABI,
+        .regsets = arm_regsets, .n = ARRAY_SIZE(arm_regsets)
+};
+
+const struct user_regset_view *task_user_regset_view(struct task_struct *task)
+{
+        return &user_arm_view;
+}
+
+long arch_ptrace(struct task_struct *child, long request,
+                 unsigned long addr, unsigned long data)
 {
         int ret;
+        unsigned long __user *datap = (unsigned long __user *) data;
 
         switch (request) {
                 case PTRACE_PEEKUSR:
-                        ret = ptrace_read_user(child, addr, (unsigned long __user *)data);
+                        ret = ptrace_read_user(child, addr, datap);
                         break;
 
                 case PTRACE_POKEUSR:
@@ -861,34 +822,46 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data)
                         break;
 
                 case PTRACE_GETREGS:
-                        ret = ptrace_getregs(child, (void __user *)data);
+                        ret = copy_regset_to_user(child,
+                                                  &user_arm_view, REGSET_GPR,
+                                                  0, sizeof(struct pt_regs),
+                                                  datap);
                         break;
 
                 case PTRACE_SETREGS:
-                        ret = ptrace_setregs(child, (void __user *)data);
+                        ret = copy_regset_from_user(child,
+                                                    &user_arm_view, REGSET_GPR,
+                                                    0, sizeof(struct pt_regs),
+                                                    datap);
                         break;
 
                 case PTRACE_GETFPREGS:
-                        ret = ptrace_getfpregs(child, (void __user *)data);
+                        ret = copy_regset_to_user(child,
+                                                  &user_arm_view, REGSET_FPR,
+                                                  0, sizeof(union fp_state),
+                                                  datap);
                         break;
-                
+
                 case PTRACE_SETFPREGS:
-                        ret = ptrace_setfpregs(child, (void __user *)data);
+                        ret = copy_regset_from_user(child,
+                                                    &user_arm_view, REGSET_FPR,
+                                                    0, sizeof(union fp_state),
+                                                    datap);
                         break;
 
 #ifdef CONFIG_IWMMXT
                 case PTRACE_GETWMMXREGS:
-                        ret = ptrace_getwmmxregs(child, (void __user *)data);
+                        ret = ptrace_getwmmxregs(child, datap);
                         break;
 
                 case PTRACE_SETWMMXREGS:
-                        ret = ptrace_setwmmxregs(child, (void __user *)data);
+                        ret = ptrace_setwmmxregs(child, datap);
                         break;
 #endif
 
                 case PTRACE_GET_THREAD_AREA:
                         ret = put_user(task_thread_info(child)->tp_value,
-                                       (unsigned long __user *) data);
+                                       datap);
                         break;
 
                 case PTRACE_SET_SYSCALL:
@@ -898,21 +871,46 @@ long arch_ptrace(struct task_struct *child, long request, long addr, long data)
 
 #ifdef CONFIG_CRUNCH
                 case PTRACE_GETCRUNCHREGS:
-                        ret = ptrace_getcrunchregs(child, (void __user *)data);
+                        ret = ptrace_getcrunchregs(child, datap);
                         break;
 
                 case PTRACE_SETCRUNCHREGS:
-                        ret = ptrace_setcrunchregs(child, (void __user *)data);
+                        ret = ptrace_setcrunchregs(child, datap);
                         break;
 #endif
 
 #ifdef CONFIG_VFP
                 case PTRACE_GETVFPREGS:
-                        ret = ptrace_getvfpregs(child, (void __user *)data);
+                        ret = copy_regset_to_user(child,
+                                                  &user_arm_view, REGSET_VFP,
+                                                  0, ARM_VFPREGS_SIZE,
+                                                  datap);
                         break;
 
                 case PTRACE_SETVFPREGS:
-                        ret = ptrace_setvfpregs(child, (void __user *)data);
+                        ret = copy_regset_from_user(child,
+                                                    &user_arm_view, REGSET_VFP,
+                                                    0, ARM_VFPREGS_SIZE,
+                                                    datap);
+                        break;
+#endif
+
+#ifdef CONFIG_HAVE_HW_BREAKPOINT
+                case PTRACE_GETHBPREGS:
+                        if (ptrace_get_breakpoints(child) < 0)
+                                return -ESRCH;
+
+                        ret = ptrace_gethbpregs(child, addr,
+                                                (unsigned long __user *)data);
+                        ptrace_put_breakpoints(child);
+                        break;
+                case PTRACE_SETHBPREGS:
+                        if (ptrace_get_breakpoints(child) < 0)
+                                return -ESRCH;
+
+                        ret = ptrace_sethbpregs(child, addr,
+                                                (unsigned long __user *)data);
+                        ptrace_put_breakpoints(child);
                         break;
 #endif
 
diff --git a/arch/arm/kernel/ptrace.h b/arch/arm/kernel/ptrace.h
deleted file mode 100644
index 3926605b82ea..000000000000
--- a/arch/arm/kernel/ptrace.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- *  linux/arch/arm/kernel/ptrace.h
- *
- *  Copyright (C) 2000-2003 Russell King
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-#include <linux/ptrace.h>
-
-extern void ptrace_cancel_bpt(struct task_struct *);
-extern void ptrace_set_bpt(struct task_struct *);
-extern void ptrace_break(struct task_struct *, struct pt_regs *);
-
-/*
- * Send SIGTRAP if we're single-stepping
- */
-static inline void single_step_trap(struct task_struct *task)
-{
-        if (task->ptrace & PT_SINGLESTEP) {
-                ptrace_cancel_bpt(task);
-                send_sig(SIGTRAP, task, 1);
-        }
-}
-
-static inline void single_step_clear(struct task_struct *task)
-{
-        if (task->ptrace & PT_SINGLESTEP)
-                ptrace_cancel_bpt(task);
-}
-
-static inline void single_step_set(struct task_struct *task)
-{
-        if (task->ptrace & PT_SINGLESTEP)
-                ptrace_set_bpt(task);
-}
diff --git a/arch/arm/kernel/relocate_kernel.S b/arch/arm/kernel/relocate_kernel.S
index fd26f8d65151..9cf4cbf8f95b 100644
--- a/arch/arm/kernel/relocate_kernel.S
+++ b/arch/arm/kernel/relocate_kernel.S
@@ -59,6 +59,8 @@ relocate_new_kernel:
         ldr r2,kexec_boot_atags
         mov pc,lr
 
+        .align
+
         .globl kexec_start_address
 kexec_start_address:
         .long   0x0
diff --git a/arch/arm/kernel/return_address.c b/arch/arm/kernel/return_address.c
index df246da4ceca..0b13a72f855d 100644
--- a/arch/arm/kernel/return_address.c
+++ b/arch/arm/kernel/return_address.c
@@ -9,6 +9,7 @@
  * the Free Software Foundation.
  */
 #include <linux/module.h>
+#include <linux/ftrace.h>
 
 #if defined(CONFIG_FRAME_POINTER) && !defined(CONFIG_ARM_UNWIND)
 #include <linux/sched.h>
diff --git a/arch/arm/kernel/sched_clock.c b/arch/arm/kernel/sched_clock.c
new file mode 100644
index 000000000000..9a46370fe9da
--- /dev/null
+++ b/arch/arm/kernel/sched_clock.c
@@ -0,0 +1,74 @@
+/*
+ * sched_clock.c: support for extending counters to full 64-bit ns counter
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/clocksource.h>
+#include <linux/init.h>
+#include <linux/jiffies.h>
+#include <linux/kernel.h>
+#include <linux/sched.h>
+#include <linux/timer.h>
+
+#include <asm/sched_clock.h>
+
+static void sched_clock_poll(unsigned long wrap_ticks);
+static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0);
+static void (*sched_clock_update_fn)(void);
+
+static void sched_clock_poll(unsigned long wrap_ticks)
+{
+        mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks));
+        sched_clock_update_fn();
+}
+
+void __init init_sched_clock(struct clock_data *cd, void (*update)(void),
+        unsigned int clock_bits, unsigned long rate)
+{
+        unsigned long r, w;
+        u64 res, wrap;
+        char r_unit;
+
+        sched_clock_update_fn = update;
+
+        /* calculate the mult/shift to convert counter ticks to ns. */
+        clocks_calc_mult_shift(&cd->mult, &cd->shift, rate, NSEC_PER_SEC, 0);
+
+        r = rate;
+        if (r >= 4000000) {
+                r /= 1000000;
+                r_unit = 'M';
+        } else {
+                r /= 1000;
+                r_unit = 'k';
+        }
+
+        /* calculate how many ns until we wrap */
+        wrap = cyc_to_ns((1ULL << clock_bits) - 1, cd->mult, cd->shift);
+        do_div(wrap, NSEC_PER_MSEC);
+        w = wrap;
+
+        /* calculate the ns resolution of this counter */
+        res = cyc_to_ns(1ULL, cd->mult, cd->shift);
+        pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n",
+                clock_bits, r, r_unit, res, w);
+
+        /*
+         * Start the timer to keep sched_clock() properly updated and
+         * sets the initial epoch.
+         */
+        sched_clock_timer.data = msecs_to_jiffies(w - (w / 10));
+        update();
+
+        /*
+         * Ensure that sched_clock() starts off at 0ns
+         */
+        cd->epoch_ns = 0;
+}
+
+void __init sched_clock_postinit(void)
+{
+        sched_clock_poll(sched_clock_timer.data);
+}
diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c
index d5231ae7355a..acbb447ac6b5 100644
--- a/arch/arm/kernel/setup.c
+++ b/arch/arm/kernel/setup.c
@@ -20,6 +20,7 @@
 #include <linux/screen_info.h>
 #include <linux/init.h>
 #include <linux/kexec.h>
+#include <linux/of_fdt.h>
 #include <linux/crash_dump.h>
 #include <linux/root_dev.h>
 #include <linux/cpu.h>
@@ -36,11 +37,13 @@
 #include <asm/procinfo.h>
 #include <asm/sections.h>
 #include <asm/setup.h>
+#include <asm/smp_plat.h>
 #include <asm/mach-types.h>
 #include <asm/cacheflush.h>
 #include <asm/cachetype.h>
 #include <asm/tlbflush.h>
 
+#include <asm/prom.h>
 #include <asm/mach/arch.h>
 #include <asm/mach/irq.h>
 #include <asm/mach/time.h>
@@ -70,13 +73,14 @@ __setup("fpe=", fpe_setup);
 #endif
 
 extern void paging_init(struct machine_desc *desc);
+extern void sanity_check_meminfo(void);
 extern void reboot_setup(char *str);
 
 unsigned int processor_id;
 EXPORT_SYMBOL(processor_id);
-unsigned int __machine_arch_type;
+unsigned int __machine_arch_type __read_mostly;
 EXPORT_SYMBOL(__machine_arch_type);
-unsigned int cacheid;
+unsigned int cacheid __read_mostly;
 EXPORT_SYMBOL(cacheid);
 
 unsigned int __atags_pointer __initdata;
@@ -90,24 +94,24 @@ EXPORT_SYMBOL(system_serial_low);
 unsigned int system_serial_high;
 EXPORT_SYMBOL(system_serial_high);
 
-unsigned int elf_hwcap;
+unsigned int elf_hwcap __read_mostly;
 EXPORT_SYMBOL(elf_hwcap);
 
 
 #ifdef MULTI_CPU
-struct processor processor;
+struct processor processor __read_mostly;
 #endif
 #ifdef MULTI_TLB
-struct cpu_tlb_fns cpu_tlb;
+struct cpu_tlb_fns cpu_tlb __read_mostly;
 #endif
 #ifdef MULTI_USER
-struct cpu_user_fns cpu_user;
+struct cpu_user_fns cpu_user __read_mostly;
 #endif
 #ifdef MULTI_CACHE
-struct cpu_cache_fns cpu_cache;
+struct cpu_cache_fns cpu_cache __read_mostly;
 #endif
 #ifdef CONFIG_OUTER_CACHE
-struct outer_cache_fns outer_cache;
+struct outer_cache_fns outer_cache __read_mostly;
 EXPORT_SYMBOL(outer_cache);
 #endif
 
@@ -125,6 +129,7 @@ EXPORT_SYMBOL(elf_platform);
 static const char *cpu_name;
 static const char *machine_name;
 static char __initdata cmd_line[COMMAND_LINE_SIZE];
+struct machine_desc *machine_desc __initdata;
 
 static char default_command_line[COMMAND_LINE_SIZE] __initdata = CONFIG_CMDLINE;
 static union { char c[4]; unsigned long l; } endian_test __initdata = { { 'l', '?', '?', 'b' } };
@@ -224,8 +229,8 @@ int cpu_architecture(void)
                  * Register 0 and check for VMSAv7 or PMSAv7 */
                 asm("mrc        p15, 0, %0, c0, c1, 4"
                     : "=r" (mmfr0));
-                if ((mmfr0 & 0x0000000f) == 0x00000003 ||
-                    (mmfr0 & 0x000000f0) == 0x00000030)
+                if ((mmfr0 & 0x0000000f) >= 0x00000003 ||
+                    (mmfr0 & 0x000000f0) >= 0x00000030)
                         cpu_arch = CPU_ARCH_ARMv7;
                 else if ((mmfr0 & 0x0000000f) == 0x00000002 ||
                          (mmfr0 & 0x000000f0) == 0x00000020)
@@ -238,6 +243,35 @@ int cpu_architecture(void)
         return cpu_arch;
 }
 
+static int cpu_has_aliasing_icache(unsigned int arch)
+{
+        int aliasing_icache;
+        unsigned int id_reg, num_sets, line_size;
+
+        /* arch specifies the register format */
+        switch (arch) {
+        case CPU_ARCH_ARMv7:
+                asm("mcr        p15, 2, %0, c0, c0, 0 @ set CSSELR"
+                    : /* No output operands */
+                    : "r" (1));
+                isb();
+                asm("mrc        p15, 1, %0, c0, c0, 0 @ read CCSIDR"
+                    : "=r" (id_reg));
+                line_size = 4 << ((id_reg & 0x7) + 2);
+                num_sets = ((id_reg >> 13) & 0x7fff) + 1;
+                aliasing_icache = (line_size * num_sets) > PAGE_SIZE;
+                break;
+        case CPU_ARCH_ARMv6:
+                aliasing_icache = read_cpuid_cachetype() & (1 << 11);
+                break;
+        default:
+                /* I-cache aliases will be handled by D-cache aliasing code */
+                aliasing_icache = 0;
+        }
+
+        return aliasing_icache;
+}
+
 static void __init cacheid_init(void)
 {
         unsigned int cachetype = read_cpuid_cachetype();
@@ -249,10 +283,15 @@ static void __init cacheid_init(void)
                         cacheid = CACHEID_VIPT_NONALIASING;
                         if ((cachetype & (3 << 14)) == 1 << 14)
                                 cacheid |= CACHEID_ASID_TAGGED;
-                } else if (cachetype & (1 << 23))
+                        else if (cpu_has_aliasing_icache(CPU_ARCH_ARMv7))
+                                cacheid |= CACHEID_VIPT_I_ALIASING;
+                } else if (cachetype & (1 << 23)) {
                         cacheid = CACHEID_VIPT_ALIASING;
-                else
+                } else {
                         cacheid = CACHEID_VIPT_NONALIASING;
+                        if (cpu_has_aliasing_icache(CPU_ARCH_ARMv6))
+                                cacheid |= CACHEID_VIPT_I_ALIASING;
+                }
         } else {
                 cacheid = CACHEID_VIVT;
         }
@@ -263,7 +302,7 @@ static void __init cacheid_init(void)
                 cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown",
                 cache_is_vivt() ? "VIVT" :
                 icache_is_vivt_asid_tagged() ? "VIVT ASID tagged" :
-                cache_is_vipt_aliasing() ? "VIPT aliasing" :
+                icache_is_vipt_aliasing() ? "VIPT aliasing" :
                 cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown");
 }
 
@@ -272,7 +311,22 @@ static void __init cacheid_init(void)
  * already provide the required functionality.
  */
 extern struct proc_info_list *lookup_processor_type(unsigned int);
-extern struct machine_desc *lookup_machine_type(unsigned int);
+
+void __init early_print(const char *str, ...)
+{
+        extern void printascii(const char *);
+        char buf[256];
+        va_list ap;
+
+        va_start(ap, str);
+        vsnprintf(buf, sizeof(buf), str, ap);
+        va_end(ap);
+
+#ifdef CONFIG_DEBUG_LL
+        printascii(buf);
+#endif
+        printk("%s", buf);
+}
 
 static void __init feat_v6_fixup(void)
 {
@@ -388,32 +442,27 @@ void cpu_init(void)
             : "r14");
 }
 
-static struct machine_desc * __init setup_machine(unsigned int nr)
+void __init dump_machine_table(void)
 {
-        struct machine_desc *list;
+        struct machine_desc *p;
 
-        /*
-         * locate machine in the list of supported machines.
-         */
-        list = lookup_machine_type(nr);
-        if (!list) {
-                printk("Machine configuration botched (nr %d), unable "
-                       "to continue.\n", nr);
-                while (1);
-        }
+        early_print("Available machine support:\n\nID (hex)\tNAME\n");
+        for_each_machine_desc(p)
+                early_print("%08x\t%s\n", p->nr, p->name);
 
-        printk("Machine: %s\n", list->name);
+        early_print("\nPlease check your kernel config and/or bootloader.\n");
 
-        return list;
+        while (true)
+                /* can't use cpu_relax() here as it may require MMU setup */;
 }
 
-static int __init arm_add_memory(unsigned long start, unsigned long size)
+int __init arm_add_memory(phys_addr_t start, unsigned long size)
 {
         struct membank *bank = &meminfo.bank[meminfo.nr_banks];
 
         if (meminfo.nr_banks >= NR_BANKS) {
                 printk(KERN_CRIT "NR_BANKS too low, "
-                        "ignoring memory at %#lx\n", start);
+                        "ignoring memory at 0x%08llx\n", (long long)start);
                 return -EINVAL;
         }
 
@@ -443,7 +492,8 @@ static int __init arm_add_memory(unsigned long start, unsigned long size)
 static int __init early_mem(char *p)
 {
         static int usermem __initdata = 0;
-        unsigned long size, start;
+        unsigned long size;
+        phys_addr_t start;
         char *endp;
 
         /*
@@ -482,25 +532,21 @@ setup_ramdisk(int doload, int prompt, int image_start, unsigned int rd_sz)
 #endif
 }
 
-static void __init
-request_standard_resources(struct meminfo *mi, struct machine_desc *mdesc)
+static void __init request_standard_resources(struct machine_desc *mdesc)
 {
+        struct memblock_region *region;
         struct resource *res;
-        int i;
 
         kernel_code.start   = virt_to_phys(_text);
         kernel_code.end     = virt_to_phys(_etext - 1);
-        kernel_data.start   = virt_to_phys(_data);
+        kernel_data.start   = virt_to_phys(_sdata);
         kernel_data.end     = virt_to_phys(_end - 1);
 
-        for (i = 0; i < mi->nr_banks; i++) {
-                if (mi->bank[i].size == 0)
-                        continue;
-
+        for_each_memblock(memory, region) {
                 res = alloc_bootmem_low(sizeof(*res));
                 res->name  = "System RAM";
-                res->start = mi->bank[i].start;
-                res->end   = mi->bank[i].start + mi->bank[i].size - 1;
+                res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region));
+                res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1;
                 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
 
                 request_resource(&iomem_resource, res);
@@ -614,15 +660,22 @@ static int __init parse_tag_revision(const struct tag *tag)
 
 __tagtable(ATAG_REVISION, parse_tag_revision);
 
-#ifndef CONFIG_CMDLINE_FORCE
 static int __init parse_tag_cmdline(const struct tag *tag)
 {
-        strlcpy(default_command_line, tag->u.cmdline.cmdline, COMMAND_LINE_SIZE);
+#if defined(CONFIG_CMDLINE_EXTEND)
+        strlcat(default_command_line, " ", COMMAND_LINE_SIZE);
+        strlcat(default_command_line, tag->u.cmdline.cmdline,
+                COMMAND_LINE_SIZE);
+#elif defined(CONFIG_CMDLINE_FORCE)
+        pr_warning("Ignoring tag cmdline (using the default kernel command line)\n");
+#else
+        strlcpy(default_command_line, tag->u.cmdline.cmdline,
+                COMMAND_LINE_SIZE);
+#endif
         return 0;
 }
 
 __tagtable(ATAG_CMDLINE, parse_tag_cmdline);
-#endif /* CONFIG_CMDLINE_FORCE */
 
 /*
  * Scan the tag table for this tag, and call its parse function.
@@ -669,17 +722,15 @@ static struct init_tags {
         { tag_size(tag_core), ATAG_CORE },
         { 1, PAGE_SIZE, 0xff },
         { tag_size(tag_mem32), ATAG_MEM },
-        { MEM_SIZE, PHYS_OFFSET },
+        { MEM_SIZE },
         { 0, ATAG_NONE }
 };
 
-static void (*init_machine)(void) __initdata;
-
 static int __init customize_machine(void)
 {
         /* customizes platform devices, or adds new ones */
-        if (init_machine)
-                init_machine();
+        if (machine_desc->init_machine)
+                machine_desc->init_machine();
         return 0;
 }
 arch_initcall(customize_machine);
@@ -733,30 +784,6 @@ static void __init reserve_crashkernel(void)
 static inline void reserve_crashkernel(void) {}
 #endif /* CONFIG_KEXEC */
 
-/*
- * Note: elfcorehdr_addr is not just limited to vmcore. It is also used by
- * is_kdump_kernel() to determine if we are booting after a panic. Hence
- * ifdef it under CONFIG_CRASH_DUMP and not CONFIG_PROC_VMCORE.
- */
-
-#ifdef CONFIG_CRASH_DUMP
-/*
- * elfcorehdr= specifies the location of elf core header stored by the crashed
- * kernel. This option will be passed by kexec loader to the capture kernel.
- */
-static int __init setup_elfcorehdr(char *arg)
-{
-        char *end;
-
-        if (!arg)
-                return -EINVAL;
-
-        elfcorehdr_addr = memparse(arg, &end);
-        return end > arg ? 0 : -EINVAL;
-}
-early_param("elfcorehdr", setup_elfcorehdr);
-#endif /* CONFIG_CRASH_DUMP */
-
 static void __init squash_mem_tags(struct tag *tag)
 {
         for (; tag->hdr.size; tag = tag_next(tag))
@@ -764,25 +791,51 @@ static void __init squash_mem_tags(struct tag *tag)
                         tag->hdr.tag = ATAG_NONE;
 }
 
-void __init setup_arch(char **cmdline_p)
+static struct machine_desc * __init setup_machine_tags(unsigned int nr)
 {
         struct tag *tags = (struct tag *)&init_tags;
-        struct machine_desc *mdesc;
+        struct machine_desc *mdesc = NULL, *p;
         char *from = default_command_line;
 
-        unwind_init();
+        init_tags.mem.start = PHYS_OFFSET;
 
-        setup_processor();
-        mdesc = setup_machine(machine_arch_type);
-        machine_name = mdesc->name;
+        /*
+         * locate machine in the list of supported machines.
+         */
+        for_each_machine_desc(p)
+                if (nr == p->nr) {
+                        printk("Machine: %s\n", p->name);
+                        mdesc = p;
+                        break;
+                }
 
-        if (mdesc->soft_reboot)
-                reboot_setup("s");
+        if (!mdesc) {
+                early_print("\nError: unrecognized/unsupported machine ID"
+                        " (r1 = 0x%08x).\n\n", nr);
+                dump_machine_table(); /* does not return */
+        }
 
         if (__atags_pointer)
                 tags = phys_to_virt(__atags_pointer);
-        else if (mdesc->boot_params)
-                tags = phys_to_virt(mdesc->boot_params);
+        else if (mdesc->boot_params) {
+#ifdef CONFIG_MMU
+                /*
+                 * We still are executing with a minimal MMU mapping created
+                 * with the presumption that the machine default for this
+                 * is located in the first MB of RAM.  Anything else will
+                 * fault and silently hang the kernel at this point.
+                 */
+                if (mdesc->boot_params < PHYS_OFFSET ||
+                    mdesc->boot_params >= PHYS_OFFSET + SZ_1M) {
+                        printk(KERN_WARNING
+                               "Default boot params at physical 0x%08lx out of reach\n",
+                               mdesc->boot_params);
+                } else
+#endif
+                {
+                        tags = phys_to_virt(mdesc->boot_params);
+                }
+        }
 
 #if defined(CONFIG_DEPRECATED_PARAM_STRUCT)
         /*
@@ -792,8 +845,17 @@ void __init setup_arch(char **cmdline_p)
         if (tags->hdr.tag != ATAG_CORE)
                 convert_to_tag_list(tags);
 #endif
-        if (tags->hdr.tag != ATAG_CORE)
+
+        if (tags->hdr.tag != ATAG_CORE) {
+#if defined(CONFIG_OF)
+                /*
+                 * If CONFIG_OF is set, then assume this is a reasonably
+                 * modern system that should pass boot parameters
+                 */
+                early_print("Warning: Neither atags nor dtb found\n");
+#endif
                 tags = (struct tag *)&init_tags;
+        }
 
         if (mdesc->fixup)
                 mdesc->fixup(mdesc, tags, &from, &meminfo);
@@ -805,40 +867,60 @@ void __init setup_arch(char **cmdline_p)
                 parse_tags(tags);
         }
 
+        /* parse_early_param needs a boot_command_line */
+        strlcpy(boot_command_line, from, COMMAND_LINE_SIZE);
+
+        return mdesc;
+}
+
+
+void __init setup_arch(char **cmdline_p)
+{
+        struct machine_desc *mdesc;
+
+        unwind_init();
+
+        setup_processor();
+        mdesc = setup_machine_fdt(__atags_pointer);
+        if (!mdesc)
+                mdesc = setup_machine_tags(machine_arch_type);
+        machine_desc = mdesc;
+        machine_name = mdesc->name;
+
+        if (mdesc->soft_reboot)
+                reboot_setup("s");
+
         init_mm.start_code = (unsigned long) _text;
         init_mm.end_code   = (unsigned long) _etext;
         init_mm.end_data   = (unsigned long) _edata;
         init_mm.brk        = (unsigned long) _end;
 
-        /* parse_early_param needs a boot_command_line */
-        strlcpy(boot_command_line, from, COMMAND_LINE_SIZE);
-
         /* populate cmd_line too for later use, preserving boot_command_line */
         strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE);
         *cmdline_p = cmd_line;
 
         parse_early_param();
 
+        sanity_check_meminfo();
         arm_memblock_init(&meminfo, mdesc);
 
         paging_init(mdesc);
-        request_standard_resources(&meminfo, mdesc);
+        request_standard_resources(mdesc);
+
+        unflatten_device_tree();
 
 #ifdef CONFIG_SMP
-        smp_init_cpus();
+        if (is_smp())
+                smp_init_cpus();
 #endif
         reserve_crashkernel();
 
         cpu_init();
         tcm_init();
 
-        /*
-         * Set up various architecture-specific pointers
-         */
-        arch_nr_irqs = mdesc->nr_irqs;
-        init_arch_irq = mdesc->init_irq;
-        system_timer = mdesc->timer;
-        init_machine = mdesc->init_machine;
+#ifdef CONFIG_MULTI_IRQ_HANDLER
+        handle_arch_irq = mdesc->handle_irq;
+#endif
 
 #ifdef CONFIG_VT
 #if defined(CONFIG_VGA_CONSOLE)
@@ -848,6 +930,9 @@ void __init setup_arch(char **cmdline_p)
 #endif
 #endif
         early_trap_init();
+
+        if (mdesc->init_early)
+                mdesc->init_early();
 }
 
 
diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c
index 907d5a620bca..0340224cf73c 100644
--- a/arch/arm/kernel/signal.c
+++ b/arch/arm/kernel/signal.c
@@ -20,7 +20,6 @@
 #include <asm/unistd.h>
 #include <asm/vfp.h>
 
-#include "ptrace.h"
 #include "signal.h"
 
 #define _BLOCKABLE (~(sigmask(SIGKILL) | sigmask(SIGSTOP)))
@@ -348,8 +347,6 @@ asmlinkage int sys_sigreturn(struct pt_regs *regs)
         if (restore_sigframe(regs, frame))
                 goto badframe;
 
-        single_step_trap(current);
-
         return regs->ARM_r0;
 
 badframe:
@@ -383,8 +380,6 @@ asmlinkage int sys_rt_sigreturn(struct pt_regs *regs)
         if (do_sigaltstack(&frame->sig.uc.uc_stack, NULL, regs->ARM_sp) == -EFAULT)
                 goto badframe;
 
-        single_step_trap(current);
-
         return regs->ARM_r0;
 
 badframe:
@@ -474,7 +469,9 @@ setup_return(struct pt_regs *regs, struct k_sigaction *ka,
         unsigned long handler = (unsigned long)ka->sa.sa_handler;
         unsigned long retcode;
         int thumb = 0;
-        unsigned long cpsr = regs->ARM_cpsr & ~PSR_f;
+        unsigned long cpsr = regs->ARM_cpsr & ~(PSR_f | PSR_E_BIT);
+
+        cpsr |= PSR_ENDSTATE;
 
         /*
          * Maybe we need to deliver a 32-bit signal to a 26-bit task.
@@ -600,19 +597,13 @@ setup_rt_frame(int usig, struct k_sigaction *ka, siginfo_t *info,
         return err;
 }
 
-static inline void setup_syscall_restart(struct pt_regs *regs)
-{
-        regs->ARM_r0 = regs->ARM_ORIG_r0;
-        regs->ARM_pc -= thumb_mode(regs) ? 2 : 4;
-}
-
 /*
  * OK, we're invoking a handler
  */     
 static int
 handle_signal(unsigned long sig, struct k_sigaction *ka,
               siginfo_t *info, sigset_t *oldset,
-              struct pt_regs * regs, int syscall)
+              struct pt_regs * regs)
 {
         struct thread_info *thread = current_thread_info();
         struct task_struct *tsk = current;
@@ -620,26 +611,6 @@ handle_signal(unsigned long sig, struct k_sigaction *ka,
         int ret;
 
         /*
-         * If we were from a system call, check for system call restarting...
-         */
-        if (syscall) {
-                switch (regs->ARM_r0) {
-                case -ERESTART_RESTARTBLOCK:
-                case -ERESTARTNOHAND:
-                        regs->ARM_r0 = -EINTR;
-                        break;
-                case -ERESTARTSYS:
-                        if (!(ka->sa.sa_flags & SA_RESTART)) {
-                                regs->ARM_r0 = -EINTR;
-                                break;
-                        }
-                        /* fallthrough */
-                case -ERESTARTNOINTR:
-                        setup_syscall_restart(regs);
-                }
-        }
-
-        /*
          * translate the signal
          */
         if (usig < 32 && thread->exec_domain && thread->exec_domain->signal_invmap)
@@ -688,6 +659,7 @@ handle_signal(unsigned long sig, struct k_sigaction *ka,
  */
 static void do_signal(struct pt_regs *regs, int syscall)
 {
+        unsigned int retval = 0, continue_addr = 0, restart_addr = 0;
         struct k_sigaction ka;
         siginfo_t info;
         int signr;
@@ -701,20 +673,61 @@ static void do_signal(struct pt_regs *regs, int syscall)
         if (!user_mode(regs))
                 return;
 
+        /*
+         * If we were from a system call, check for system call restarting...
+         */
+        if (syscall) {
+                continue_addr = regs->ARM_pc;
+                restart_addr = continue_addr - (thumb_mode(regs) ? 2 : 4);
+                retval = regs->ARM_r0;
+
+                /*
+                 * Prepare for system call restart.  We do this here so that a
+                 * debugger will see the already changed PSW.
+                 */
+                switch (retval) {
+                case -ERESTARTNOHAND:
+                case -ERESTARTSYS:
+                case -ERESTARTNOINTR:
+                        regs->ARM_r0 = regs->ARM_ORIG_r0;
+                        regs->ARM_pc = restart_addr;
+                        break;
+                case -ERESTART_RESTARTBLOCK:
+                        regs->ARM_r0 = -EINTR;
+                        break;
+                }
+        }
+
         if (try_to_freeze())
                 goto no_signal;
 
-        single_step_clear(current);
-
+        /*
+         * Get the signal to deliver.  When running under ptrace, at this
+         * point the debugger may change all our registers ...
+         */
         signr = get_signal_to_deliver(&info, &ka, regs, NULL);
         if (signr > 0) {
                 sigset_t *oldset;
 
+                /*
+                 * Depending on the signal settings we may need to revert the
+                 * decision to restart the system call.  But skip this if a
+                 * debugger has chosen to restart at a different PC.
+                 */
+                if (regs->ARM_pc == restart_addr) {
+                        if (retval == -ERESTARTNOHAND
+                            || (retval == -ERESTARTSYS
+                                && !(ka.sa.sa_flags & SA_RESTART))) {
+                                regs->ARM_r0 = -EINTR;
+                                regs->ARM_pc = continue_addr;
+                        }
+                }
+
                 if (test_thread_flag(TIF_RESTORE_SIGMASK))
                         oldset = &current->saved_sigmask;
                 else
                         oldset = &current->blocked;
-                if (handle_signal(signr, &ka, &info, oldset, regs, syscall) == 0) {
+                if (handle_signal(signr, &ka, &info, oldset, regs) == 0) {
                         /*
                          * A signal was successfully delivered; the saved
                          * sigmask will have been stored in the signal frame,
@@ -724,16 +737,18 @@ static void do_signal(struct pt_regs *regs, int syscall)
                         if (test_thread_flag(TIF_RESTORE_SIGMASK))
                                 clear_thread_flag(TIF_RESTORE_SIGMASK);
                 }
-                single_step_set(current);
                 return;
         }
 
  no_signal:
-        /*
-         * No signal to deliver to the process - restart the syscall.
-         */
         if (syscall) {
-                if (regs->ARM_r0 == -ERESTART_RESTARTBLOCK) {
+                /*
+                 * Handle restarting a different system call.  As above,
+                 * if a debugger has chosen to restart at a different PC,
+                 * ignore the restart.
+                 */
+                if (retval == -ERESTART_RESTARTBLOCK
+                    && regs->ARM_pc == continue_addr) {
                         if (thumb_mode(regs)) {
                                 regs->ARM_r7 = __NR_restart_syscall - __NR_SYSCALL_BASE;
                                 regs->ARM_pc -= 2;
@@ -756,11 +771,6 @@ static void do_signal(struct pt_regs *regs, int syscall)
 #endif
                         }
                 }
-                if (regs->ARM_r0 == -ERESTARTNOHAND ||
-                    regs->ARM_r0 == -ERESTARTSYS ||
-                    regs->ARM_r0 == -ERESTARTNOINTR) {
-                        setup_syscall_restart(regs);
-                }
 
                 /* If there's no signal to deliver, we just put the saved sigmask
                  * back.
@@ -770,7 +780,6 @@ static void do_signal(struct pt_regs *regs, int syscall)
                         sigprocmask(SIG_SETMASK, &current->saved_sigmask, NULL);
                 }
         }
-        single_step_set(current);
 }
 
 asmlinkage void
diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S
new file mode 100644
index 000000000000..6398ead9d1c0
--- /dev/null
+++ b/arch/arm/kernel/sleep.S
@@ -0,0 +1,142 @@
+#include <linux/linkage.h>
+#include <linux/threads.h>
+#include <asm/asm-offsets.h>
+#include <asm/assembler.h>
+#include <asm/glue-cache.h>
+#include <asm/glue-proc.h>
+#include <asm/system.h>
+        .text
+
+/*
+ * Save CPU state for a suspend
+ *  r1 = v:p offset
+ *  r3 = virtual return function
+ * Note: sp is decremented to allocate space for CPU state on stack
+ * r0-r3,r9,r10,lr corrupted
+ */
+ENTRY(cpu_suspend)
+        mov     r9, lr
+#ifdef MULTI_CPU
+        ldr     r10, =processor
+        mov     r2, sp                  @ current virtual SP
+        ldr     r0, [r10, #CPU_SLEEP_SIZE] @ size of CPU sleep state
+        ldr     ip, [r10, #CPU_DO_RESUME] @ virtual resume function
+        sub     sp, sp, r0              @ allocate CPU state on stack
+        mov     r0, sp                  @ save pointer
+        add     ip, ip, r1              @ convert resume fn to phys
+        stmfd   sp!, {r1, r2, r3, ip}   @ save v:p, virt SP, retfn, phys resume fn
+        ldr     r3, =sleep_save_sp
+        add     r2, sp, r1              @ convert SP to phys
+#ifdef CONFIG_SMP
+        ALT_SMP(mrc p15, 0, lr, c0, c0, 5)
+        ALT_UP(mov lr, #0)
+        and     lr, lr, #15
+        str     r2, [r3, lr, lsl #2]    @ save phys SP
+#else
+        str     r2, [r3]                @ save phys SP
+#endif
+        mov     lr, pc
+        ldr     pc, [r10, #CPU_DO_SUSPEND] @ save CPU state
+#else
+        mov     r2, sp                  @ current virtual SP
+        ldr     r0, =cpu_suspend_size
+        sub     sp, sp, r0              @ allocate CPU state on stack
+        mov     r0, sp                  @ save pointer
+        stmfd   sp!, {r1, r2, r3}       @ save v:p, virt SP, return fn
+        ldr     r3, =sleep_save_sp
+        add     r2, sp, r1              @ convert SP to phys
+#ifdef CONFIG_SMP
+        ALT_SMP(mrc p15, 0, lr, c0, c0, 5)
+        ALT_UP(mov lr, #0)
+        and     lr, lr, #15
+        str     r2, [r3, lr, lsl #2]    @ save phys SP
+#else
+        str     r2, [r3]                @ save phys SP
+#endif
+        bl      cpu_do_suspend
+#endif
+
+        @ flush data cache
+#ifdef MULTI_CACHE
+        ldr     r10, =cpu_cache
+        mov     lr, r9
+        ldr     pc, [r10, #CACHE_FLUSH_KERN_ALL]
+#else
+        mov     lr, r9
+        b       __cpuc_flush_kern_all
+#endif
+ENDPROC(cpu_suspend)
+        .ltorg
+
+/*
+ * r0 = control register value
+ * r1 = v:p offset (preserved by cpu_do_resume)
+ * r2 = phys page table base
+ * r3 = L1 section flags
+ */
+ENTRY(cpu_resume_mmu)
+        adr     r4, cpu_resume_turn_mmu_on
+        mov     r4, r4, lsr #20
+        orr     r3, r3, r4, lsl #20
+        ldr     r5, [r2, r4, lsl #2]    @ save old mapping
+        str     r3, [r2, r4, lsl #2]    @ setup 1:1 mapping for mmu code
+        sub     r2, r2, r1
+        ldr     r3, =cpu_resume_after_mmu
+        bic     r1, r0, #CR_C           @ ensure D-cache is disabled
+        b       cpu_resume_turn_mmu_on
+ENDPROC(cpu_resume_mmu)
+        .ltorg
+        .align  5
+cpu_resume_turn_mmu_on:
+        mcr     p15, 0, r1, c1, c0, 0   @ turn on MMU, I-cache, etc
+        mrc     p15, 0, r1, c0, c0, 0   @ read id reg
+        mov     r1, r1
+        mov     r1, r1
+        mov     pc, r3                  @ jump to virtual address
+ENDPROC(cpu_resume_turn_mmu_on)
+cpu_resume_after_mmu:
+        str     r5, [r2, r4, lsl #2]    @ restore old mapping
+        mcr     p15, 0, r0, c1, c0, 0   @ turn on D-cache
+        mov     pc, lr
+ENDPROC(cpu_resume_after_mmu)
+
+/*
+ * Note: Yes, part of the following code is located into the .data section.
+ *       This is to allow sleep_save_sp to be accessed with a relative load
+ *       while we can't rely on any MMU translation.  We could have put
+ *       sleep_save_sp in the .text section as well, but some setups might
+ *       insist on it to be truly read-only.
+ */
+        .data
+        .align
+ENTRY(cpu_resume)
+#ifdef CONFIG_SMP
+        adr     r0, sleep_save_sp
+        ALT_SMP(mrc p15, 0, r1, c0, c0, 5)
+        ALT_UP(mov r1, #0)
+        and     r1, r1, #15
+        ldr     r0, [r0, r1, lsl #2]    @ stack phys addr
+#else
+        ldr     r0, sleep_save_sp       @ stack phys addr
+#endif
+        setmode PSR_I_BIT | PSR_F_BIT | SVC_MODE, r1  @ set SVC, irqs off
+#ifdef MULTI_CPU
+        @ load v:p, stack, return fn, resume fn
+  ARM(  ldmia   r0!, {r1, sp, lr, pc}   )
+THUMB(  ldmia   r0!, {r1, r2, r3, r4}   )
+THUMB(  mov     sp, r2                  )
+THUMB(  mov     lr, r3                  )
+THUMB(  bx      r4                      )
+#else
+        @ load v:p, stack, return fn
+  ARM(  ldmia   r0!, {r1, sp, lr}       )
+THUMB(  ldmia   r0!, {r1, r2, lr}       )
+THUMB(  mov     sp, r2                  )
+        b       cpu_do_resume
+#endif
+ENDPROC(cpu_resume)
+
+sleep_save_sp:
+        .rept   CONFIG_NR_CPUS
+        .long   0                               @ preserve stack phys ptr here
+        .endr
diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c
index b72fbf3d043c..5a574296ace0 100644
--- a/arch/arm/kernel/smp.c
+++ b/arch/arm/kernel/smp.c
@@ -16,6 +16,7 @@
 #include <linux/cache.h>
 #include <linux/profile.h>
 #include <linux/errno.h>
+#include <linux/ftrace.h>
 #include <linux/mm.h>
 #include <linux/err.h>
 #include <linux/cpu.h>
@@ -24,6 +25,7 @@
 #include <linux/irq.h>
 #include <linux/percpu.h>
 #include <linux/clockchips.h>
+#include <linux/completion.h>
 
 #include <asm/atomic.h>
 #include <asm/cacheflush.h>
@@ -33,10 +35,10 @@
 #include <asm/pgtable.h>
 #include <asm/pgalloc.h>
 #include <asm/processor.h>
+#include <asm/sections.h>
 #include <asm/tlbflush.h>
 #include <asm/ptrace.h>
 #include <asm/localtimer.h>
-#include <asm/smp_plat.h>
 
 #include <litmus/preempt.h>
 
@@ -47,22 +49,8 @@
  */
 struct secondary_data secondary_data;
 
-/*
- * structures for inter-processor calls
- * - A collection of single bit ipi messages.
- */
-struct ipi_data {
-        spinlock_t lock;
-        unsigned long ipi_count;
-        unsigned long bits;
-};
-
-static DEFINE_PER_CPU(struct ipi_data, ipi_data) = {
-        .lock   = SPIN_LOCK_UNLOCKED,
-};
-
 enum ipi_msg_type {
-        IPI_TIMER,
+        IPI_TIMER = 2,
         IPI_RESCHEDULE,
         IPI_CALL_FUNC,
         IPI_CALL_FUNC_SINGLE,
@@ -74,7 +62,6 @@ int __cpuinit __cpu_up(unsigned int cpu)
         struct cpuinfo_arm *ci = &per_cpu(cpu_data, cpu);
         struct task_struct *idle = ci->idle;
         pgd_t *pgd;
-        pmd_t *pmd;
         int ret;
 
         /*
@@ -103,11 +90,16 @@ int __cpuinit __cpu_up(unsigned int cpu)
          * a 1:1 mapping for the physical address of the kernel.
          */
         pgd = pgd_alloc(&init_mm);
-        pmd = pmd_offset(pgd + pgd_index(PHYS_OFFSET), PHYS_OFFSET);
-        *pmd = __pmd((PHYS_OFFSET & PGDIR_MASK) |
-                     PMD_TYPE_SECT | PMD_SECT_AP_WRITE);
-        flush_pmd_entry(pmd);
-        outer_clean_range(__pa(pmd), __pa(pmd + 1));
+        if (!pgd)
+                return -ENOMEM;
+
+        if (PHYS_OFFSET != PAGE_OFFSET) {
+#ifndef CONFIG_HOTPLUG_CPU
+                identity_mapping_add(pgd, __pa(__init_begin), __pa(__init_end));
+#endif
+                identity_mapping_add(pgd, __pa(_stext), __pa(_etext));
+                identity_mapping_add(pgd, __pa(_sdata), __pa(_edata));
+        }
 
         /*
          * We need to tell the secondary core where to find
@@ -115,6 +107,7 @@ int __cpuinit __cpu_up(unsigned int cpu)
          */
         secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
         secondary_data.pgdir = virt_to_phys(pgd);
+        secondary_data.swapper_pg_dir = virt_to_phys(swapper_pg_dir);
         __cpuc_flush_dcache_area(&secondary_data, sizeof(secondary_data));
         outer_clean_range(__pa(&secondary_data), __pa(&secondary_data + 1));
 
@@ -138,29 +131,33 @@ int __cpuinit __cpu_up(unsigned int cpu)
                         barrier();
                 }
 
-                if (!cpu_online(cpu))
+                if (!cpu_online(cpu)) {
+                        pr_crit("CPU%u: failed to come online\n", cpu);
                         ret = -EIO;
+                }
+        } else {
+                pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
         }
 
         secondary_data.stack = NULL;
         secondary_data.pgdir = 0;
 
-        *pmd = __pmd(0);
-        clean_pmd_entry(pmd);
-        pgd_free(&init_mm, pgd);
-
-        if (ret) {
-                printk(KERN_CRIT "CPU%u: processor failed to boot\n", cpu);
-
-                /*
-                 * FIXME: We need to clean up the new idle thread. --rmk
-                 */
+        if (PHYS_OFFSET != PAGE_OFFSET) {
+#ifndef CONFIG_HOTPLUG_CPU
+                identity_mapping_del(pgd, __pa(__init_begin), __pa(__init_end));
+#endif
+                identity_mapping_del(pgd, __pa(_stext), __pa(_etext));
+                identity_mapping_del(pgd, __pa(_sdata), __pa(_edata));
         }
 
+        pgd_free(&init_mm, pgd);
+
         return ret;
 }
 
 #ifdef CONFIG_HOTPLUG_CPU
+static void percpu_timer_stop(void);
+
 /*
  * __cpu_disable runs on the processor to be shutdown.
  */
@@ -188,7 +185,7 @@ int __cpu_disable(void)
         /*
          * Stop the local timer for this CPU.
          */
-        local_timer_stop();
+        percpu_timer_stop();
 
         /*
          * Flush user cache and TLB mappings, and then remove this CPU
@@ -207,12 +204,20 @@ int __cpu_disable(void)
         return 0;
 }
 
+static DECLARE_COMPLETION(cpu_died);
+
 /*
  * called on the thread which is asking for a CPU to be shutdown -
  * waits until shutdown has completed, or it is timed out.
  */
 void __cpu_die(unsigned int cpu)
 {
+        if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) {
+                pr_err("CPU%u: cpu didn't die\n", cpu);
+                return;
+        }
+        printk(KERN_NOTICE "CPU%u: shutdown\n", cpu);
+
         if (!platform_cpu_kill(cpu))
                 printk("CPU%u: unable to kill\n", cpu);
 }
@@ -229,12 +234,17 @@ void __ref cpu_die(void)
 {
         unsigned int cpu = smp_processor_id();
 
-        local_irq_disable();
         idle_task_exit();
 
+        local_irq_disable();
+        mb();
+
+        /* Tell __cpu_die() that this CPU is now safe to dispose of */
+        complete(&cpu_died);
+
         /*
          * actual CPU shutdown procedure is at least platform (if not
-         * CPU) specific
+         * CPU) specific.
          */
         platform_cpu_die(cpu);
 
@@ -244,6 +254,7 @@ void __ref cpu_die(void)
          * to be repeated to undo the effects of taking the CPU offline.
          */
         __asm__("mov    sp, %0\n"
+        "       mov     fp, #0\n"
         "       b       secondary_start_kernel"
                 :
                 : "r" (task_stack_page(current) + THREAD_SIZE - 8));
@@ -251,6 +262,17 @@ void __ref cpu_die(void)
 #endif /* CONFIG_HOTPLUG_CPU */
 
 /*
+ * Called by both boot and secondaries to move global data into
+ * per-processor storage.
+ */
+static void __cpuinit smp_store_cpu_info(unsigned int cpuid)
+{
+        struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
+
+        cpu_info->loops_per_jiffy = loops_per_jiffy;
+}
+
+/*
  * This is the secondary CPU boot entry.  We're using this CPUs
  * idle thread stack, but a set of temporary page tables.
  */
@@ -265,7 +287,6 @@ asmlinkage void __cpuinit secondary_start_kernel(void)
          * All kernel threads share the same mm context; grab a
          * reference and switch to it.
          */
-        atomic_inc(&mm->mm_users);
         atomic_inc(&mm->mm_count);
         current->active_mm = mm;
         cpumask_set_cpu(cpu, mm_cpumask(mm));
@@ -275,6 +296,7 @@ asmlinkage void __cpuinit secondary_start_kernel(void)
 
         cpu_init();
         preempt_disable();
+        trace_hardirqs_off();
 
         /*
          * Give the platform a chance to do its own initialisation.
@@ -298,9 +320,13 @@ asmlinkage void __cpuinit secondary_start_kernel(void)
         smp_store_cpu_info(cpu);
 
         /*
-         * OK, now it's safe to let the boot CPU continue
+         * OK, now it's safe to let the boot CPU continue.  Wait for
+         * the CPU migration code to notice that the CPU is online
+         * before we continue.
          */
         set_cpu_online(cpu, true);
+        while (!cpu_active(cpu))
+                cpu_relax();
 
         /*
          * OK, it's off to the idle thread for us
@@ -308,17 +334,6 @@ asmlinkage void __cpuinit secondary_start_kernel(void)
         cpu_idle();
 }
 
-/*
- * Called by both boot and secondaries to move global data into
- * per-processor storage.
- */
-void __cpuinit smp_store_cpu_info(unsigned int cpuid)
-{
-        struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
-
-        cpu_info->loops_per_jiffy = loops_per_jiffy;
-}
-
 void __init smp_cpus_done(unsigned int max_cpus)
 {
         int cpu;
@@ -341,61 +356,87 @@ void __init smp_prepare_boot_cpu(void)
         per_cpu(cpu_data, cpu).idle = current;
 }
 
-static void send_ipi_message(const struct cpumask *mask, enum ipi_msg_type msg)
+void __init smp_prepare_cpus(unsigned int max_cpus)
 {
-        unsigned long flags;
-        unsigned int cpu;
+        unsigned int ncores = num_possible_cpus();
 
-        local_irq_save(flags);
+        smp_store_cpu_info(smp_processor_id());
 
-        for_each_cpu(cpu, mask) {
-                struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
+        /*
+         * are we trying to boot more cores than exist?
+         */
+        if (max_cpus > ncores)
+                max_cpus = ncores;
 
-                spin_lock(&ipi->lock);
-                ipi->bits |= 1 << msg;
-                spin_unlock(&ipi->lock);
+        if (max_cpus > 1) {
+                /*
+                 * Enable the local timer or broadcast device for the
+                 * boot CPU, but only if we have more than one CPU.
+                 */
+                percpu_timer_setup();
+
+                /*
+                 * Initialise the SCU if there are more than one CPU
+                 * and let them know where to start.
+                 */
+                platform_smp_prepare_cpus(max_cpus);
         }
+}
 
-        /*
-         * Call the platform specific cross-CPU call function.
-         */
-        smp_cross_call(mask);
+static void (*smp_cross_call)(const struct cpumask *, unsigned int);
 
-        local_irq_restore(flags);
+void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
+{
+        smp_cross_call = fn;
 }
 
 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 {
-        send_ipi_message(mask, IPI_CALL_FUNC);
+        smp_cross_call(mask, IPI_CALL_FUNC);
 }
 
 void arch_send_call_function_single_ipi(int cpu)
 {
-        send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
+        smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
 }
 
-void show_ipi_list(struct seq_file *p)
+static const char *ipi_types[NR_IPI] = {
+#define S(x,s)  [x - IPI_TIMER] = s
+        S(IPI_TIMER, "Timer broadcast interrupts"),
+        S(IPI_RESCHEDULE, "Rescheduling interrupts"),
+        S(IPI_CALL_FUNC, "Function call interrupts"),
+        S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
+        S(IPI_CPU_STOP, "CPU stop interrupts"),
+};
+
+void show_ipi_list(struct seq_file *p, int prec)
 {
-        unsigned int cpu;
+        unsigned int cpu, i;
 
-        seq_puts(p, "IPI:");
+        for (i = 0; i < NR_IPI; i++) {
+                seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
 
-        for_each_present_cpu(cpu)
-                seq_printf(p, " %10lu", per_cpu(ipi_data, cpu).ipi_count);
+                for_each_present_cpu(cpu)
+                        seq_printf(p, "%10u ",
+                                   __get_irq_stat(cpu, ipi_irqs[i]));
 
-        seq_putc(p, '\n');
+                seq_printf(p, " %s\n", ipi_types[i]);
+        }
 }
 
-void show_local_irqs(struct seq_file *p)
+u64 smp_irq_stat_cpu(unsigned int cpu)
 {
-        unsigned int cpu;
+        u64 sum = 0;
+        int i;
 
-        seq_printf(p, "LOC: ");
+        for (i = 0; i < NR_IPI; i++)
+                sum += __get_irq_stat(cpu, ipi_irqs[i]);
 
-        for_each_present_cpu(cpu)
-                seq_printf(p, "%10u ", irq_stat[cpu].local_timer_irqs);
+#ifdef CONFIG_LOCAL_TIMERS
+        sum += __get_irq_stat(cpu, local_timer_irqs);
+#endif
 
-        seq_putc(p, '\n');
+        return sum;
 }
 
 /*
@@ -412,36 +453,47 @@ static void ipi_timer(void)
 }
 
 #ifdef CONFIG_LOCAL_TIMERS
-asmlinkage void __exception do_local_timer(struct pt_regs *regs)
+asmlinkage void __exception_irq_entry do_local_timer(struct pt_regs *regs)
 {
         struct pt_regs *old_regs = set_irq_regs(regs);
         int cpu = smp_processor_id();
 
         if (local_timer_ack()) {
-                irq_stat[cpu].local_timer_irqs++;
+                __inc_irq_stat(cpu, local_timer_irqs);
                 ipi_timer();
         }
 
         set_irq_regs(old_regs);
 }
+
+void show_local_irqs(struct seq_file *p, int prec)
+{
+        unsigned int cpu;
+
+        seq_printf(p, "%*s: ", prec, "LOC");
+
+        for_each_present_cpu(cpu)
+                seq_printf(p, "%10u ", __get_irq_stat(cpu, local_timer_irqs));
+
+        seq_printf(p, " Local timer interrupts\n");
+}
 #endif
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 static void smp_timer_broadcast(const struct cpumask *mask)
 {
-        send_ipi_message(mask, IPI_TIMER);
+        smp_cross_call(mask, IPI_TIMER);
 }
 #else
 #define smp_timer_broadcast     NULL
 #endif
 
-#ifndef CONFIG_LOCAL_TIMERS
 static void broadcast_timer_set_mode(enum clock_event_mode mode,
         struct clock_event_device *evt)
 {
 }
 
-static void local_timer_setup(struct clock_event_device *evt)
+static void __cpuinit broadcast_timer_setup(struct clock_event_device *evt)
 {
         evt->name       = "dummy_timer";
         evt->features   = CLOCK_EVT_FEAT_ONESHOT |
@@ -453,7 +505,6 @@ static void local_timer_setup(struct clock_event_device *evt)
 
         clockevents_register_device(evt);
 }
-#endif
 
 void __cpuinit percpu_timer_setup(void)
 {
@@ -463,8 +514,24 @@ void __cpuinit percpu_timer_setup(void)
         evt->cpumask = cpumask_of(cpu);
         evt->broadcast = smp_timer_broadcast;
 
-        local_timer_setup(evt);
+        if (local_timer_setup(evt))
+                broadcast_timer_setup(evt);
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+/*
+ * The generic clock events code purposely does not stop the local timer
+ * on CPU_DEAD/CPU_DEAD_FROZEN hotplug events, so we have to do it
+ * manually here.
+ */
+static void percpu_timer_stop(void)
+{
+        unsigned int cpu = smp_processor_id();
+        struct clock_event_device *evt = &per_cpu(percpu_clockevent, cpu);
+
+        evt->set_mode(CLOCK_EVT_MODE_UNUSED, evt);
 }
+#endif
 
 static DEFINE_SPINLOCK(stop_lock);
 
@@ -492,217 +559,75 @@ static void ipi_cpu_stop(unsigned int cpu)
 
 /*
  * Main handler for inter-processor interrupts
- *
- * For ARM, the ipimask now only identifies a single
- * category of IPI (Bit 1 IPIs have been replaced by a
- * different mechanism):
- *
- *  Bit 0 - Inter-processor function call
  */
-asmlinkage void __exception do_IPI(struct pt_regs *regs)
+asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs)
 {
         unsigned int cpu = smp_processor_id();
-        struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
         struct pt_regs *old_regs = set_irq_regs(regs);
 
-        ipi->ipi_count++;
-
-        for (;;) {
-                unsigned long msgs;
-
-                spin_lock(&ipi->lock);
-                msgs = ipi->bits;
-                ipi->bits = 0;
-                spin_unlock(&ipi->lock);
-
-                if (!msgs)
-                        break;
-
-                do {
-                        unsigned nextmsg;
-
-                        nextmsg = msgs & -msgs;
-                        msgs &= ~nextmsg;
-                        nextmsg = ffz(~nextmsg);
-
-                        switch (nextmsg) {
-                        case IPI_TIMER:
-                                ipi_timer();
-                                break;
-
-                        case IPI_RESCHEDULE:
-                                /*
-                                 * nothing more to do - eveything is
-                                 * done on the interrupt return path
-                                 */
-                                /* LITMUS^RT: take action based on scheduler state */
-                                sched_state_ipi();
-                                break;
-
-                        case IPI_CALL_FUNC:
-                                generic_smp_call_function_interrupt();
-                                break;
-
-                        case IPI_CALL_FUNC_SINGLE:
-                                generic_smp_call_function_single_interrupt();
-                                break;
+        if (ipinr >= IPI_TIMER && ipinr < IPI_TIMER + NR_IPI)
+                __inc_irq_stat(cpu, ipi_irqs[ipinr - IPI_TIMER]);
 
-                        case IPI_CPU_STOP:
-                                ipi_cpu_stop(cpu);
-                                break;
-
-                        default:
-                                printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
-                                       cpu, nextmsg);
-                                break;
-                        }
-                } while (msgs);
+        switch (ipinr) {
+        case IPI_TIMER:
+                ipi_timer();
+                break;
+
+        case IPI_RESCHEDULE:
+                /* LITMUS^RT: take action based on scheduler state */
+                sched_state_ipi();
+                scheduler_ipi();
+                break;
+
+        case IPI_CALL_FUNC:
+                generic_smp_call_function_interrupt();
+                break;
+
+        case IPI_CALL_FUNC_SINGLE:
+                generic_smp_call_function_single_interrupt();
+                break;
+
+        case IPI_CPU_STOP:
+                ipi_cpu_stop(cpu);
+                break;
+
+        default:
+                printk(KERN_CRIT "CPU%u: Unknown IPI message 0x%x\n",
+                       cpu, ipinr);
+                break;
         }
-
         set_irq_regs(old_regs);
 }
 
 void smp_send_reschedule(int cpu)
 {
-        send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
+        smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
 }
 
 void smp_send_stop(void)
 {
-        cpumask_t mask = cpu_online_map;
-        cpu_clear(smp_processor_id(), mask);
-        send_ipi_message(&mask, IPI_CPU_STOP);
-}
+        unsigned long timeout;
 
-/*
- * not supported here
- */
-int setup_profiling_timer(unsigned int multiplier)
-{
-        return -EINVAL;
-}
+        if (num_online_cpus() > 1) {
+                cpumask_t mask = cpu_online_map;
+                cpu_clear(smp_processor_id(), mask);
 
-static void
-on_each_cpu_mask(void (*func)(void *), void *info, int wait,
-                const struct cpumask *mask)
-{
-        preempt_disable();
+                smp_cross_call(&mask, IPI_CPU_STOP);
+        }
 
-        smp_call_function_many(mask, func, info, wait);
-        if (cpumask_test_cpu(smp_processor_id(), mask))
-                func(info);
+        /* Wait up to one second for other CPUs to stop */
+        timeout = USEC_PER_SEC;
+        while (num_online_cpus() > 1 && timeout--)
+                udelay(1);
 
-        preempt_enable();
+        if (num_online_cpus() > 1)
+                pr_warning("SMP: failed to stop secondary CPUs\n");
 }
 
-/**********************************************************************/
-
 /*
- * TLB operations
+ * not supported here
  */
-struct tlb_args {
-        struct vm_area_struct *ta_vma;
-        unsigned long ta_start;
-        unsigned long ta_end;
-};
-
-static inline void ipi_flush_tlb_all(void *ignored)
-{
-        local_flush_tlb_all();
-}
-
-static inline void ipi_flush_tlb_mm(void *arg)
-{
-        struct mm_struct *mm = (struct mm_struct *)arg;
-
-        local_flush_tlb_mm(mm);
-}
-
-static inline void ipi_flush_tlb_page(void *arg)
-{
-        struct tlb_args *ta = (struct tlb_args *)arg;
-
-        local_flush_tlb_page(ta->ta_vma, ta->ta_start);
-}
-
-static inline void ipi_flush_tlb_kernel_page(void *arg)
-{
-        struct tlb_args *ta = (struct tlb_args *)arg;
-
-        local_flush_tlb_kernel_page(ta->ta_start);
-}
-
-static inline void ipi_flush_tlb_range(void *arg)
-{
-        struct tlb_args *ta = (struct tlb_args *)arg;
-
-        local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
-}
-
-static inline void ipi_flush_tlb_kernel_range(void *arg)
-{
-        struct tlb_args *ta = (struct tlb_args *)arg;
-
-        local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
-}
-
-void flush_tlb_all(void)
-{
-        if (tlb_ops_need_broadcast())
-                on_each_cpu(ipi_flush_tlb_all, NULL, 1);
-        else
-                local_flush_tlb_all();
-}
-
-void flush_tlb_mm(struct mm_struct *mm)
-{
-        if (tlb_ops_need_broadcast())
-                on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mm_cpumask(mm));
-        else
-                local_flush_tlb_mm(mm);
-}
-
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
-{
-        if (tlb_ops_need_broadcast()) {
-                struct tlb_args ta;
-                ta.ta_vma = vma;
-                ta.ta_start = uaddr;
-                on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mm_cpumask(vma->vm_mm));
-        } else
-                local_flush_tlb_page(vma, uaddr);
-}
-
-void flush_tlb_kernel_page(unsigned long kaddr)
-{
-        if (tlb_ops_need_broadcast()) {
-                struct tlb_args ta;
-                ta.ta_start = kaddr;
-                on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
-        } else
-                local_flush_tlb_kernel_page(kaddr);
-}
-
-void flush_tlb_range(struct vm_area_struct *vma,
-                     unsigned long start, unsigned long end)
-{
-        if (tlb_ops_need_broadcast()) {
-                struct tlb_args ta;
-                ta.ta_vma = vma;
-                ta.ta_start = start;
-                ta.ta_end = end;
-                on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mm_cpumask(vma->vm_mm));
-        } else
-                local_flush_tlb_range(vma, start, end);
-}
-
-void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+int setup_profiling_timer(unsigned int multiplier)
 {
-        if (tlb_ops_need_broadcast()) {
-                struct tlb_args ta;
-                ta.ta_start = start;
-                ta.ta_end = end;
-                on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
-        } else
-                local_flush_tlb_kernel_range(start, end);
+        return -EINVAL;
 }
diff --git a/arch/arm/kernel/smp_scu.c b/arch/arm/kernel/smp_scu.c
index 9ab4149bd983..a1e757c3439b 100644
--- a/arch/arm/kernel/smp_scu.c
+++ b/arch/arm/kernel/smp_scu.c
@@ -50,3 +50,26 @@ void __init scu_enable(void __iomem *scu_base)
          */
         flush_cache_all();
 }
+
+/*
+ * Set the executing CPUs power mode as defined.  This will be in
+ * preparation for it executing a WFI instruction.
+ *
+ * This function must be called with preemption disabled, and as it
+ * has the side effect of disabling coherency, caches must have been
+ * flushed.  Interrupts must also have been disabled.
+ */
+int scu_power_mode(void __iomem *scu_base, unsigned int mode)
+{
+        unsigned int val;
+        int cpu = smp_processor_id();
+
+        if (mode > 3 || mode == 1 || cpu > 3)
+                return -EINVAL;
+
+        val = __raw_readb(scu_base + SCU_CPU_STATUS + cpu) & ~0x03;
+        val |= mode;
+        __raw_writeb(val, scu_base + SCU_CPU_STATUS + cpu);
+
+        return 0;
+}
diff --git a/arch/arm/kernel/smp_tlb.c b/arch/arm/kernel/smp_tlb.c
new file mode 100644
index 000000000000..7dcb35285be7
--- /dev/null
+++ b/arch/arm/kernel/smp_tlb.c
@@ -0,0 +1,139 @@
+/*
+ *  linux/arch/arm/kernel/smp_tlb.c
+ *
+ *  Copyright (C) 2002 ARM Limited, All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/preempt.h>
+#include <linux/smp.h>
+
+#include <asm/smp_plat.h>
+#include <asm/tlbflush.h>
+
+static void on_each_cpu_mask(void (*func)(void *), void *info, int wait,
+        const struct cpumask *mask)
+{
+        preempt_disable();
+
+        smp_call_function_many(mask, func, info, wait);
+        if (cpumask_test_cpu(smp_processor_id(), mask))
+                func(info);
+
+        preempt_enable();
+}
+
+/**********************************************************************/
+
+/*
+ * TLB operations
+ */
+struct tlb_args {
+        struct vm_area_struct *ta_vma;
+        unsigned long ta_start;
+        unsigned long ta_end;
+};
+
+static inline void ipi_flush_tlb_all(void *ignored)
+{
+        local_flush_tlb_all();
+}
+
+static inline void ipi_flush_tlb_mm(void *arg)
+{
+        struct mm_struct *mm = (struct mm_struct *)arg;
+
+        local_flush_tlb_mm(mm);
+}
+
+static inline void ipi_flush_tlb_page(void *arg)
+{
+        struct tlb_args *ta = (struct tlb_args *)arg;
+
+        local_flush_tlb_page(ta->ta_vma, ta->ta_start);
+}
+
+static inline void ipi_flush_tlb_kernel_page(void *arg)
+{
+        struct tlb_args *ta = (struct tlb_args *)arg;
+
+        local_flush_tlb_kernel_page(ta->ta_start);
+}
+
+static inline void ipi_flush_tlb_range(void *arg)
+{
+        struct tlb_args *ta = (struct tlb_args *)arg;
+
+        local_flush_tlb_range(ta->ta_vma, ta->ta_start, ta->ta_end);
+}
+
+static inline void ipi_flush_tlb_kernel_range(void *arg)
+{
+        struct tlb_args *ta = (struct tlb_args *)arg;
+
+        local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
+}
+
+void flush_tlb_all(void)
+{
+        if (tlb_ops_need_broadcast())
+                on_each_cpu(ipi_flush_tlb_all, NULL, 1);
+        else
+                local_flush_tlb_all();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+        if (tlb_ops_need_broadcast())
+                on_each_cpu_mask(ipi_flush_tlb_mm, mm, 1, mm_cpumask(mm));
+        else
+                local_flush_tlb_mm(mm);
+}
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
+{
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_vma = vma;
+                ta.ta_start = uaddr;
+                on_each_cpu_mask(ipi_flush_tlb_page, &ta, 1, mm_cpumask(vma->vm_mm));
+        } else
+                local_flush_tlb_page(vma, uaddr);
+}
+
+void flush_tlb_kernel_page(unsigned long kaddr)
+{
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_start = kaddr;
+                on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
+        } else
+                local_flush_tlb_kernel_page(kaddr);
+}
+
+void flush_tlb_range(struct vm_area_struct *vma,
+                     unsigned long start, unsigned long end)
+{
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_vma = vma;
+                ta.ta_start = start;
+                ta.ta_end = end;
+                on_each_cpu_mask(ipi_flush_tlb_range, &ta, 1, mm_cpumask(vma->vm_mm));
+        } else
+                local_flush_tlb_range(vma, start, end);
+}
+
+void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+        if (tlb_ops_need_broadcast()) {
+                struct tlb_args ta;
+                ta.ta_start = start;
+                ta.ta_end = end;
+                on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
+        } else
+                local_flush_tlb_kernel_range(start, end);
+}
+
diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c
index 35882fbf37f9..2c277d40cee6 100644
--- a/arch/arm/kernel/smp_twd.c
+++ b/arch/arm/kernel/smp_twd.c
@@ -36,6 +36,7 @@ static void twd_set_mode(enum clock_event_mode mode,
                 /* timer load already set up */
                 ctrl = TWD_TIMER_CONTROL_ENABLE | TWD_TIMER_CONTROL_IT_ENABLE
                         | TWD_TIMER_CONTROL_PERIODIC;
+                __raw_writel(twd_timer_rate / HZ, twd_base + TWD_TIMER_LOAD);
                 break;
         case CLOCK_EVT_MODE_ONESHOT:
                 /* period set, and timer enabled in 'next_event' hook */
@@ -81,7 +82,7 @@ int twd_timer_ack(void)
 
 static void __cpuinit twd_calibrate_rate(void)
 {
-        unsigned long load, count;
+        unsigned long count;
         u64 waitjiffies;
 
         /*
@@ -114,12 +115,8 @@ static void __cpuinit twd_calibrate_rate(void)
                 twd_timer_rate = (0xFFFFFFFFU - count) * (HZ / 5);
 
                 printk("%lu.%02luMHz.\n", twd_timer_rate / 1000000,
-                        (twd_timer_rate / 100000) % 100);
+                        (twd_timer_rate / 10000) % 100);
         }
-
-        load = twd_timer_rate / HZ;
-
-        __raw_writel(load, twd_base + TWD_TIMER_LOAD);
 }
 
 /*
@@ -127,8 +124,6 @@ static void __cpuinit twd_calibrate_rate(void)
  */
 void __cpuinit twd_timer_setup(struct clock_event_device *clk)
 {
-        unsigned long flags;
-
         twd_calibrate_rate();
 
         clk->name = "local_timer";
@@ -143,20 +138,7 @@ void __cpuinit twd_timer_setup(struct clock_event_device *clk)
         clk->min_delta_ns = clockevent_delta2ns(0xf, clk);
 
         /* Make sure our local interrupt controller has this enabled */
-        local_irq_save(flags);
-        irq_to_desc(clk->irq)->status |= IRQ_NOPROBE;
-        get_irq_chip(clk->irq)->unmask(clk->irq);
-        local_irq_restore(flags);
+        gic_enable_ppi(clk->irq);
 
         clockevents_register_device(clk);
 }
-
-#ifdef CONFIG_HOTPLUG_CPU
-/*
- * take a local timer down
- */
-void twd_timer_stop(void)
-{
-        __raw_writel(0, twd_base + TWD_TIMER_CONTROL);
-}
-#endif
diff --git a/arch/arm/kernel/stacktrace.c b/arch/arm/kernel/stacktrace.c
index 20b7411e47fd..381d23a497c1 100644
--- a/arch/arm/kernel/stacktrace.c
+++ b/arch/arm/kernel/stacktrace.c
@@ -28,7 +28,7 @@ int notrace unwind_frame(struct stackframe *frame)
 
         /* only go to a higher address on the stack */
         low = frame->sp;
-        high = ALIGN(low, THREAD_SIZE) + THREAD_SIZE;
+        high = ALIGN(low, THREAD_SIZE);
 
         /* check current frame pointer is within bounds */
         if (fp < (low + 12) || fp + 4 >= high)
@@ -94,10 +94,13 @@ void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace)
         if (tsk != current) {
 #ifdef CONFIG_SMP
                 /*
-                 * What guarantees do we have here that 'tsk'
-                 * is not running on another CPU?
+                 * What guarantees do we have here that 'tsk' is not
+                 * running on another CPU?  For now, ignore it as we
+                 * can't guarantee we won't explode.
                  */
-                BUG();
+                if (trace->nr_entries < trace->max_entries)
+                        trace->entries[trace->nr_entries++] = ULONG_MAX;
+                return;
 #else
                 data.no_sched_functions = 1;
                 frame.fp = thread_saved_fp(tsk);
diff --git a/arch/arm/kernel/swp_emulate.c b/arch/arm/kernel/swp_emulate.c
new file mode 100644
index 000000000000..40ee7e5045e4
--- /dev/null
+++ b/arch/arm/kernel/swp_emulate.c
@@ -0,0 +1,267 @@
+/*
+ *  linux/arch/arm/kernel/swp_emulate.c
+ *
+ *  Copyright (C) 2009 ARM Limited
+ *  __user_* functions adapted from include/asm/uaccess.h
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ *  Implements emulation of the SWP/SWPB instructions using load-exclusive and
+ *  store-exclusive for processors that have them disabled (or future ones that
+ *  might not implement them).
+ *
+ *  Syntax of SWP{B} instruction: SWP{B}<c> <Rt>, <Rt2>, [<Rn>]
+ *  Where: Rt  = destination
+ *         Rt2 = source
+ *         Rn  = address
+ */
+
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/proc_fs.h>
+#include <linux/sched.h>
+#include <linux/syscalls.h>
+#include <linux/perf_event.h>
+
+#include <asm/traps.h>
+#include <asm/uaccess.h>
+
+/*
+ * Error-checking SWP macros implemented using ldrex{b}/strex{b}
+ */
+#define __user_swpX_asm(data, addr, res, temp, B)               \
+        __asm__ __volatile__(                                   \
+        "       mov             %2, %1\n"                       \
+        "0:     ldrex"B"        %1, [%3]\n"                     \
+        "1:     strex"B"        %0, %2, [%3]\n"                 \
+        "       cmp             %0, #0\n"                       \
+        "       movne           %0, %4\n"                       \
+        "2:\n"                                                  \
+        "       .section         .fixup,\"ax\"\n"               \
+        "       .align          2\n"                            \
+        "3:     mov             %0, %5\n"                       \
+        "       b               2b\n"                           \
+        "       .previous\n"                                    \
+        "       .section         __ex_table,\"a\"\n"            \
+        "       .align          3\n"                            \
+        "       .long           0b, 3b\n"                       \
+        "       .long           1b, 3b\n"                       \
+        "       .previous"                                      \
+        : "=&r" (res), "+r" (data), "=&r" (temp)                \
+        : "r" (addr), "i" (-EAGAIN), "i" (-EFAULT)              \
+        : "cc", "memory")
+
+#define __user_swp_asm(data, addr, res, temp) \
+        __user_swpX_asm(data, addr, res, temp, "")
+#define __user_swpb_asm(data, addr, res, temp) \
+        __user_swpX_asm(data, addr, res, temp, "b")
+
+/*
+ * Macros/defines for extracting register numbers from instruction.
+ */
+#define EXTRACT_REG_NUM(instruction, offset) \
+        (((instruction) & (0xf << (offset))) >> (offset))
+#define RN_OFFSET  16
+#define RT_OFFSET  12
+#define RT2_OFFSET  0
+/*
+ * Bit 22 of the instruction encoding distinguishes between
+ * the SWP and SWPB variants (bit set means SWPB).
+ */
+#define TYPE_SWPB (1 << 22)
+
+static unsigned long swpcounter;
+static unsigned long swpbcounter;
+static unsigned long abtcounter;
+static pid_t         previous_pid;
+
+#ifdef CONFIG_PROC_FS
+static int proc_read_status(char *page, char **start, off_t off, int count,
+                            int *eof, void *data)
+{
+        char *p = page;
+        int len;
+
+        p += sprintf(p, "Emulated SWP:\t\t%lu\n", swpcounter);
+        p += sprintf(p, "Emulated SWPB:\t\t%lu\n", swpbcounter);
+        p += sprintf(p, "Aborted SWP{B}:\t\t%lu\n", abtcounter);
+        if (previous_pid != 0)
+                p += sprintf(p, "Last process:\t\t%d\n", previous_pid);
+
+        len = (p - page) - off;
+        if (len < 0)
+                len = 0;
+
+        *eof = (len <= count) ? 1 : 0;
+        *start = page + off;
+
+        return len;
+}
+#endif
+
+/*
+ * Set up process info to signal segmentation fault - called on access error.
+ */
+static void set_segfault(struct pt_regs *regs, unsigned long addr)
+{
+        siginfo_t info;
+
+        if (find_vma(current->mm, addr) == NULL)
+                info.si_code = SEGV_MAPERR;
+        else
+                info.si_code = SEGV_ACCERR;
+
+        info.si_signo = SIGSEGV;
+        info.si_errno = 0;
+        info.si_addr  = (void *) instruction_pointer(regs);
+
+        pr_debug("SWP{B} emulation: access caused memory abort!\n");
+        arm_notify_die("Illegal memory access", regs, &info, 0, 0);
+
+        abtcounter++;
+}
+
+static int emulate_swpX(unsigned int address, unsigned int *data,
+                        unsigned int type)
+{
+        unsigned int res = 0;
+
+        if ((type != TYPE_SWPB) && (address & 0x3)) {
+                /* SWP to unaligned address not permitted */
+                pr_debug("SWP instruction on unaligned pointer!\n");
+                return -EFAULT;
+        }
+
+        while (1) {
+                unsigned long temp;
+
+                /*
+                 * Barrier required between accessing protected resource and
+                 * releasing a lock for it. Legacy code might not have done
+                 * this, and we cannot determine that this is not the case
+                 * being emulated, so insert always.
+                 */
+                smp_mb();
+
+                if (type == TYPE_SWPB)
+                        __user_swpb_asm(*data, address, res, temp);
+                else
+                        __user_swp_asm(*data, address, res, temp);
+
+                if (likely(res != -EAGAIN) || signal_pending(current))
+                        break;
+
+                cond_resched();
+        }
+
+        if (res == 0) {
+                /*
+                 * Barrier also required between acquiring a lock for a
+                 * protected resource and accessing the resource. Inserted for
+                 * same reason as above.
+                 */
+                smp_mb();
+
+                if (type == TYPE_SWPB)
+                        swpbcounter++;
+                else
+                        swpcounter++;
+        }
+
+        return res;
+}
+
+/*
+ * swp_handler logs the id of calling process, dissects the instruction, sanity
+ * checks the memory location, calls emulate_swpX for the actual operation and
+ * deals with fixup/error handling before returning
+ */
+static int swp_handler(struct pt_regs *regs, unsigned int instr)
+{
+        unsigned int address, destreg, data, type;
+        unsigned int res = 0;
+
+        perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, 0, regs, regs->ARM_pc);
+
+        if (current->pid != previous_pid) {
+                pr_debug("\"%s\" (%ld) uses deprecated SWP{B} instruction\n",
+                         current->comm, (unsigned long)current->pid);
+                previous_pid = current->pid;
+        }
+
+        address = regs->uregs[EXTRACT_REG_NUM(instr, RN_OFFSET)];
+        data    = regs->uregs[EXTRACT_REG_NUM(instr, RT2_OFFSET)];
+        destreg = EXTRACT_REG_NUM(instr, RT_OFFSET);
+
+        type = instr & TYPE_SWPB;
+
+        pr_debug("addr in r%d->0x%08x, dest is r%d, source in r%d->0x%08x)\n",
+                 EXTRACT_REG_NUM(instr, RN_OFFSET), address,
+                 destreg, EXTRACT_REG_NUM(instr, RT2_OFFSET), data);
+
+        /* Check access in reasonable access range for both SWP and SWPB */
+        if (!access_ok(VERIFY_WRITE, (address & ~3), 4)) {
+                pr_debug("SWP{B} emulation: access to %p not allowed!\n",
+                         (void *)address);
+                res = -EFAULT;
+        } else {
+                res = emulate_swpX(address, &data, type);
+        }
+
+        if (res == 0) {
+                /*
+                 * On successful emulation, revert the adjustment to the PC
+                 * made in kernel/traps.c in order to resume execution at the
+                 * instruction following the SWP{B}.
+                 */
+                regs->ARM_pc += 4;
+                regs->uregs[destreg] = data;
+        } else if (res == -EFAULT) {
+                /*
+                 * Memory errors do not mean emulation failed.
+                 * Set up signal info to return SEGV, then return OK
+                 */
+                set_segfault(regs, address);
+        }
+
+        return 0;
+}
+
+/*
+ * Only emulate SWP/SWPB executed in ARM state/User mode.
+ * The kernel must be SWP free and SWP{B} does not exist in Thumb/ThumbEE.
+ */
+static struct undef_hook swp_hook = {
+        .instr_mask = 0x0fb00ff0,
+        .instr_val  = 0x01000090,
+        .cpsr_mask  = MODE_MASK | PSR_T_BIT | PSR_J_BIT,
+        .cpsr_val   = USR_MODE,
+        .fn         = swp_handler
+};
+
+/*
+ * Register handler and create status file in /proc/cpu
+ * Invoked as late_initcall, since not needed before init spawned.
+ */
+static int __init swp_emulation_init(void)
+{
+#ifdef CONFIG_PROC_FS
+        struct proc_dir_entry *res;
+
+        res = create_proc_entry("cpu/swp_emulation", S_IRUGO, NULL);
+
+        if (!res)
+                return -ENOMEM;
+
+        res->read_proc = proc_read_status;
+#endif /* CONFIG_PROC_FS */
+
+        printk(KERN_NOTICE "Registering SWP/SWPB emulation handler\n");
+        register_undef_hook(&swp_hook);
+
+        return 0;
+}
+
+late_initcall(swp_emulation_init);
diff --git a/arch/arm/kernel/sys_oabi-compat.c b/arch/arm/kernel/sys_oabi-compat.c
index 4ad8da15ef2b..af0aaebf4de6 100644
--- a/arch/arm/kernel/sys_oabi-compat.c
+++ b/arch/arm/kernel/sys_oabi-compat.c
@@ -311,7 +311,7 @@ asmlinkage long sys_oabi_semtimedop(int semid,
         long err;
         int i;
 
-        if (nsops < 1)
+        if (nsops < 1 || nsops > SEMOPM)
                 return -EINVAL;
         sops = kmalloc(sizeof(*sops) * nsops, GFP_KERNEL);
         if (!sops)
diff --git a/arch/arm/kernel/tcm.c b/arch/arm/kernel/tcm.c
index 26685c2f7a49..f5cf660eefcc 100644
--- a/arch/arm/kernel/tcm.c
+++ b/arch/arm/kernel/tcm.c
@@ -15,7 +15,7 @@
 #include <linux/string.h> /* memcpy */
 #include <asm/cputype.h>
 #include <asm/mach/map.h>
-#include <mach/memory.h>
+#include <asm/memory.h>
 #include "tcm.h"
 
 static struct gen_pool *tcm_pool;
diff --git a/arch/arm/kernel/time.c b/arch/arm/kernel/time.c
index 38c261f9951c..cb634c3e28e9 100644
--- a/arch/arm/kernel/time.c
+++ b/arch/arm/kernel/time.c
@@ -21,7 +21,7 @@
 #include <linux/timex.h>
 #include <linux/errno.h>
 #include <linux/profile.h>
-#include <linux/sysdev.h>
+#include <linux/syscore_ops.h>
 #include <linux/timer.h>
 #include <linux/irq.h>
 
@@ -29,13 +29,15 @@
 
 #include <asm/leds.h>
 #include <asm/thread_info.h>
+#include <asm/sched_clock.h>
 #include <asm/stacktrace.h>
+#include <asm/mach/arch.h>
 #include <asm/mach/time.h>
 
 /*
  * Our system timer.
  */
-struct sys_timer *system_timer;
+static struct sys_timer *system_timer;
 
 #if defined(CONFIG_RTC_DRV_CMOS) || defined(CONFIG_RTC_DRV_CMOS_MODULE)
 /* this needs a better home */
@@ -105,9 +107,7 @@ void timer_tick(void)
 {
         profile_tick(CPU_PROFILING);
         do_leds();
-        write_seqlock(&xtime_lock);
-        do_timer(1);
-        write_sequnlock(&xtime_lock);
+        xtime_update(1);
 #ifndef CONFIG_SMP
         update_process_times(user_mode(get_irq_regs()));
 #endif
@@ -115,51 +115,44 @@ void timer_tick(void)
 #endif
 
 #if defined(CONFIG_PM) && !defined(CONFIG_GENERIC_CLOCKEVENTS)
-static int timer_suspend(struct sys_device *dev, pm_message_t state)
+static int timer_suspend(void)
 {
-        struct sys_timer *timer = container_of(dev, struct sys_timer, dev);
-
-        if (timer->suspend != NULL)
-                timer->suspend();
+        if (system_timer->suspend)
+                system_timer->suspend();
 
         return 0;
 }
 
-static int timer_resume(struct sys_device *dev)
+static void timer_resume(void)
 {
-        struct sys_timer *timer = container_of(dev, struct sys_timer, dev);
-
-        if (timer->resume != NULL)
-                timer->resume();
-
-        return 0;
+        if (system_timer->resume)
+                system_timer->resume();
 }
 #else
 #define timer_suspend NULL
 #define timer_resume NULL
 #endif
 
-static struct sysdev_class timer_sysclass = {
-        .name           = "timer",
+static struct syscore_ops timer_syscore_ops = {
         .suspend        = timer_suspend,
         .resume         = timer_resume,
 };
 
-static int __init timer_init_sysfs(void)
+static int __init timer_init_syscore_ops(void)
 {
-        int ret = sysdev_class_register(&timer_sysclass);
-        if (ret == 0) {
-                system_timer->dev.cls = &timer_sysclass;
-                ret = sysdev_register(&system_timer->dev);
-        }
+        register_syscore_ops(&timer_syscore_ops);
 
-        return ret;
+        return 0;
 }
 
-device_initcall(timer_init_sysfs);
+device_initcall(timer_init_syscore_ops);
 
 void __init time_init(void)
 {
+        system_timer = machine_desc->timer;
         system_timer->init();
+#ifdef CONFIG_HAVE_SCHED_CLOCK
+        sched_clock_postinit();
+#endif
 }
 
diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c
index cda78d59aa31..6807cb1e76dd 100644
--- a/arch/arm/kernel/traps.c
+++ b/arch/arm/kernel/traps.c
@@ -23,6 +23,7 @@
 #include <linux/kexec.h>
 #include <linux/delay.h>
 #include <linux/init.h>
+#include <linux/sched.h>
 
 #include <asm/atomic.h>
 #include <asm/cacheflush.h>
@@ -32,11 +33,12 @@
 #include <asm/unwind.h>
 #include <asm/tls.h>
 
-#include "ptrace.h"
 #include "signal.h"
 
 static const char *handler[]= { "prefetch abort", "data abort", "address exception", "interrupt" };
 
+void *vectors_page;
+
 #ifdef CONFIG_DEBUG_USER
 unsigned int user_debug;
 
@@ -53,10 +55,7 @@ static void dump_mem(const char *, const char *, unsigned long, unsigned long);
 void dump_backtrace_entry(unsigned long where, unsigned long from, unsigned long frame)
 {
 #ifdef CONFIG_KALLSYMS
-        char sym1[KSYM_SYMBOL_LEN], sym2[KSYM_SYMBOL_LEN];
-        sprint_symbol(sym1, where);
-        sprint_symbol(sym2, from);
-        printk("[<%08lx>] (%s) from [<%08lx>] (%s)\n", where, sym1, from, sym2);
+        printk("[<%08lx>] (%pS) from [<%08lx>] (%pS)\n", where, (void *)where, from, (void *)from);
 #else
         printk("Function entered at [<%08lx>] from [<%08lx>]\n", where, from);
 #endif
@@ -140,7 +139,7 @@ static void dump_instr(const char *lvl, struct pt_regs *regs)
         fs = get_fs();
         set_fs(KERNEL_DS);
 
-        for (i = -4; i < 1; i++) {
+        for (i = -4; i < 1 + !!thumb; i++) {
                 unsigned int val, bad;
 
                 if (thumb)
@@ -235,7 +234,6 @@ static int __die(const char *str, int err, struct thread_info *thread, struct pt
 
         printk(KERN_EMERG "Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
                str, err, ++die_counter);
-        sysfs_printk_last_file();
 
         /* trap and error numbers are mostly meaningless on ARM */
         ret = notify_die(DIE_OOPS, str, regs, err, tsk->thread.trap_no, SIGSEGV);
@@ -257,7 +255,7 @@ static int __die(const char *str, int err, struct thread_info *thread, struct pt
         return ret;
 }
 
-DEFINE_SPINLOCK(die_lock);
+static DEFINE_SPINLOCK(die_lock);
 
 /*
  * This function is protected against re-entrancy.
@@ -411,8 +409,7 @@ static int bad_syscall(int n, struct pt_regs *regs)
         struct thread_info *thread = current_thread_info();
         siginfo_t info;
 
-        if (current->personality != PER_LINUX &&
-            current->personality != PER_LINUX_32BIT &&
+        if ((current->personality & PER_MASK) != PER_LINUX &&
             thread->exec_domain->handler) {
                 thread->exec_domain->handler(n, regs);
                 return regs->ARM_r0;
@@ -566,7 +563,7 @@ asmlinkage int arm_syscall(int no, struct pt_regs *regs)
                 if (!pmd_present(*pmd))
                         goto bad_access;
                 pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
-                if (!pte_present(*pte) || !pte_dirty(*pte)) {
+                if (!pte_present(*pte) || !pte_write(*pte) || !pte_dirty(*pte)) {
                         pte_unmap_unlock(pte, ptl);
                         goto bad_access;
                 }
@@ -711,19 +708,19 @@ void __readwrite_bug(const char *fn)
 }
 EXPORT_SYMBOL(__readwrite_bug);
 
-void __pte_error(const char *file, int line, unsigned long val)
+void __pte_error(const char *file, int line, pte_t pte)
 {
-        printk("%s:%d: bad pte %08lx.\n", file, line, val);
+        printk("%s:%d: bad pte %08llx.\n", file, line, (long long)pte_val(pte));
 }
 
-void __pmd_error(const char *file, int line, unsigned long val)
+void __pmd_error(const char *file, int line, pmd_t pmd)
 {
-        printk("%s:%d: bad pmd %08lx.\n", file, line, val);
+        printk("%s:%d: bad pmd %08llx.\n", file, line, (long long)pmd_val(pmd));
 }
 
-void __pgd_error(const char *file, int line, unsigned long val)
+void __pgd_error(const char *file, int line, pgd_t pgd)
 {
-        printk("%s:%d: bad pgd %08lx.\n", file, line, val);
+        printk("%s:%d: bad pgd %08llx.\n", file, line, (long long)pgd_val(pgd));
 }
 
 asmlinkage void __div0(void)
@@ -759,7 +756,11 @@ static void __init kuser_get_tls_init(unsigned long vectors)
 
 void __init early_trap_init(void)
 {
+#if defined(CONFIG_CPU_USE_DOMAINS)
         unsigned long vectors = CONFIG_VECTORS_BASE;
+#else
+        unsigned long vectors = (unsigned long)vectors_page;
+#endif
         extern char __stubs_start[], __stubs_end[];
         extern char __vectors_start[], __vectors_end[];
         extern char __kuser_helper_start[], __kuser_helper_end[];
@@ -783,10 +784,10 @@ void __init early_trap_init(void)
          * Copy signal return handlers into the vector page, and
          * set sigreturn to be a pointer to these.
          */
-        memcpy((void *)KERN_SIGRETURN_CODE, sigreturn_codes,
-               sizeof(sigreturn_codes));
-        memcpy((void *)KERN_RESTART_CODE, syscall_restart_code,
-               sizeof(syscall_restart_code));
+        memcpy((void *)(vectors + KERN_SIGRETURN_CODE - CONFIG_VECTORS_BASE),
+               sigreturn_codes, sizeof(sigreturn_codes));
+        memcpy((void *)(vectors + KERN_RESTART_CODE - CONFIG_VECTORS_BASE),
+               syscall_restart_code, sizeof(syscall_restart_code));
 
         flush_icache_range(vectors, vectors + PAGE_SIZE);
         modify_domain(DOMAIN_USER, DOMAIN_CLIENT);
diff --git a/arch/arm/kernel/unwind.c b/arch/arm/kernel/unwind.c
index dd81a918c106..d2cb0b3c9872 100644
--- a/arch/arm/kernel/unwind.c
+++ b/arch/arm/kernel/unwind.c
@@ -146,6 +146,8 @@ static struct unwind_idx *unwind_find_idx(unsigned long addr)
                             addr < table->end_addr) {
                                 idx = search_index(addr, table->start,
                                                    table->stop - 1);
+                                /* Move-to-front to exploit common traces */
+                                list_move(&table->list, &unwind_tables);
                                 break;
                         }
                 }
@@ -277,7 +279,7 @@ int unwind_frame(struct stackframe *frame)
 
         /* only go to a higher address on the stack */
         low = frame->sp;
-        high = ALIGN(low, THREAD_SIZE) + THREAD_SIZE;
+        high = ALIGN(low, THREAD_SIZE);
 
         pr_debug("%s(pc = %08lx lr = %08lx sp = %08lx)\n", __func__,
                  frame->pc, frame->lr, frame->sp);
diff --git a/arch/arm/kernel/vmlinux.lds.S b/arch/arm/kernel/vmlinux.lds.S
index b16c07914b55..e5287f21badc 100644
--- a/arch/arm/kernel/vmlinux.lds.S
+++ b/arch/arm/kernel/vmlinux.lds.S
@@ -8,6 +8,25 @@
 #include <asm/memory.h>
 #include <asm/page.h>
         
+#define PROC_INFO                                                       \
+        VMLINUX_SYMBOL(__proc_info_begin) = .;                          \
+        *(.proc.info.init)                                              \
+        VMLINUX_SYMBOL(__proc_info_end) = .;
+
+#ifdef CONFIG_HOTPLUG_CPU
+#define ARM_CPU_DISCARD(x)
+#define ARM_CPU_KEEP(x)         x
+#else
+#define ARM_CPU_DISCARD(x)      x
+#define ARM_CPU_KEEP(x)
+#endif
+
+#if defined(CONFIG_SMP_ON_UP) && !defined(CONFIG_DEBUG_SPINLOCK)
+#define ARM_EXIT_KEEP(x)        x
+#else
+#define ARM_EXIT_KEEP(x)
+#endif
+
 OUTPUT_ARCH(arm)
 ENTRY(stext)
 
@@ -30,16 +49,24 @@ SECTIONS
                 _sinittext = .;
                         HEAD_TEXT
                         INIT_TEXT
+                        ARM_EXIT_KEEP(EXIT_TEXT)
                 _einittext = .;
-                __proc_info_begin = .;
-                        *(.proc.info.init)
-                __proc_info_end = .;
+                ARM_CPU_DISCARD(PROC_INFO)
                 __arch_info_begin = .;
                         *(.arch.info.init)
                 __arch_info_end = .;
                 __tagtable_begin = .;
                         *(.taglist.init)
                 __tagtable_end = .;
+#ifdef CONFIG_SMP_ON_UP
+                __smpalt_begin = .;
+                        *(.alt.smp.init)
+                __smpalt_end = .;
+#endif
+
+                __pv_table_begin = .;
+                        *(.pv_table)
+                __pv_table_end = .;
 
                 INIT_SETUP(16)
 
@@ -51,10 +78,11 @@ SECTIONS
 #ifndef CONFIG_XIP_KERNEL
                 __init_begin = _stext;
                 INIT_DATA
+                ARM_EXIT_KEEP(EXIT_DATA)
 #endif
         }
 
-        PERCPU(PAGE_SIZE)
+        PERCPU_SECTION(32)
 
 #ifndef CONFIG_XIP_KERNEL
         . = ALIGN(PAGE_SIZE);
@@ -68,10 +96,8 @@ SECTIONS
         /DISCARD/ : {
                 *(.ARM.exidx.exit.text)
                 *(.ARM.extab.exit.text)
-#ifndef CONFIG_HOTPLUG_CPU
-                *(.ARM.exidx.cpuexit.text)
-                *(.ARM.extab.cpuexit.text)
-#endif
+                ARM_CPU_DISCARD(*(.ARM.exidx.cpuexit.text))
+                ARM_CPU_DISCARD(*(.ARM.extab.cpuexit.text))
 #ifndef CONFIG_HOTPLUG
                 *(.ARM.exidx.devexit.text)
                 *(.ARM.extab.devexit.text)
@@ -87,6 +113,7 @@ SECTIONS
                         __exception_text_start = .;
                         *(.exception.text)
                         __exception_text_end = .;
+                        IRQENTRY_TEXT
                         TEXT_TEXT
                         SCHED_TEXT
                         LOCK_TEXT
@@ -99,13 +126,13 @@ SECTIONS
                         *(.rodata.*)
                         *(.glue_7)
                         *(.glue_7t)
+                . = ALIGN(4);
                 *(.got)                 /* Global offset table          */
+                        ARM_CPU_KEEP(PROC_INFO)
         }
 
         RO_DATA(PAGE_SIZE)
 
-        _etext = .;                     /* End of text and rodata section */
-
 #ifdef CONFIG_ARM_UNWIND
         /*
          * Stack unwinding tables
@@ -123,6 +150,8 @@ SECTIONS
         }
 #endif
 
+        _etext = .;                     /* End of text and rodata section */
+
 #ifdef CONFIG_XIP_KERNEL
         __data_loc = ALIGN(4);          /* location in binary */
         . = PAGE_OFFSET + TEXT_OFFSET;
@@ -145,12 +174,14 @@ SECTIONS
                 . = ALIGN(PAGE_SIZE);
                 __init_begin = .;
                 INIT_DATA
+                ARM_EXIT_KEEP(EXIT_DATA)
                 . = ALIGN(PAGE_SIZE);
                 __init_end = .;
 #endif
 
                 NOSAVE_DATA
                 CACHELINE_ALIGNED_DATA(32)
+                READ_MOSTLY_DATA(32)
 
                 /*
                  * The exception fixup table (might need resorting at runtime)
@@ -229,6 +260,8 @@ SECTIONS
         }
 #endif
 
+        NOTES
+
         BSS_SECTION(0, 0, 0)
         _end = .;
 
@@ -237,6 +270,12 @@ SECTIONS
 
         /* Default discards */
         DISCARDS
+
+#ifndef CONFIG_SMP_ON_UP
+        /DISCARD/ : {
+                *(.alt.smp.init)
+        }
+#endif
 }
 
 /*