12 files changed, 194 insertions, 337 deletions

diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile
index 51230ee6a407..d0130fff20e5 100644
--- a/arch/powerpc/mm/Makefile
+++ b/arch/powerpc/mm/Makefile
@@ -13,9 +13,7 @@ obj-$(CONFIG_PPC_MMU_NOHASH)	+= mmu_context_nohash.o tlb_nohash.o \
                                    tlb_nohash_low.o
 obj-$(CONFIG_PPC_BOOK3E)        += tlb_low_$(CONFIG_WORD_SIZE)e.o
 hash64-$(CONFIG_PPC_NATIVE)     := hash_native_64.o
-obj-$(CONFIG_PPC_STD_MMU_64)    += hash_utils_64.o \
-                                   slb_low.o slb.o stab.o \
-                                   $(hash64-y)
+obj-$(CONFIG_PPC_STD_MMU_64)    += hash_utils_64.o slb_low.o slb.o $(hash64-y)
 obj-$(CONFIG_PPC_STD_MMU_32)    += ppc_mmu_32.o
 obj-$(CONFIG_PPC_STD_MMU)       += hash_low_$(CONFIG_WORD_SIZE).o \
                                    tlb_hash$(CONFIG_WORD_SIZE).o \
diff --git a/arch/powerpc/mm/dma-noncoherent.c b/arch/powerpc/mm/dma-noncoherent.c
index 7b6c10750179..d85e86aac7fb 100644
--- a/arch/powerpc/mm/dma-noncoherent.c
+++ b/arch/powerpc/mm/dma-noncoherent.c
@@ -33,6 +33,7 @@
 #include <linux/export.h>
 
 #include <asm/tlbflush.h>
+#include <asm/dma.h>
 
 #include "mmu_decl.h"
 
diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c
index 88fdd9d25077..daee7f4e5a14 100644
--- a/arch/powerpc/mm/hash_utils_64.c
+++ b/arch/powerpc/mm/hash_utils_64.c
@@ -243,7 +243,7 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend,
 }
 
 #ifdef CONFIG_MEMORY_HOTPLUG
-static int htab_remove_mapping(unsigned long vstart, unsigned long vend,
+int htab_remove_mapping(unsigned long vstart, unsigned long vend,
                       int psize, int ssize)
 {
         unsigned long vaddr;
@@ -821,21 +821,14 @@ static void __init htab_initialize(void)
 
 void __init early_init_mmu(void)
 {
-        /* Setup initial STAB address in the PACA */
-        get_paca()->stab_real = __pa((u64)&initial_stab);
-        get_paca()->stab_addr = (u64)&initial_stab;
-
         /* Initialize the MMU Hash table and create the linear mapping
-         * of memory. Has to be done before stab/slb initialization as
-         * this is currently where the page size encoding is obtained
+         * of memory. Has to be done before SLB initialization as this is
+         * currently where the page size encoding is obtained.
          */
         htab_initialize();
 
-        /* Initialize stab / SLB management */
-        if (mmu_has_feature(MMU_FTR_SLB))
-                slb_initialize();
-        else
-                stab_initialize(get_paca()->stab_real);
+        /* Initialize SLB management */
+        slb_initialize();
 }
 
 #ifdef CONFIG_SMP
@@ -845,13 +838,8 @@ void early_init_mmu_secondary(void)
         if (!firmware_has_feature(FW_FEATURE_LPAR))
                 mtspr(SPRN_SDR1, _SDR1);
 
-        /* Initialize STAB/SLB. We use a virtual address as it works
-         * in real mode on pSeries.
-         */
-        if (mmu_has_feature(MMU_FTR_SLB))
-                slb_initialize();
-        else
-                stab_initialize(get_paca()->stab_addr);
+        /* Initialize SLB */
+        slb_initialize();
 }
 #endif /* CONFIG_SMP */
 
diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c
index e3734edffa69..253b4b971c8a 100644
--- a/arch/powerpc/mm/init_64.c
+++ b/arch/powerpc/mm/init_64.c
@@ -175,9 +175,10 @@ static unsigned long __meminit vmemmap_section_start(unsigned long page)
 static int __meminit vmemmap_populated(unsigned long start, int page_size)
 {
         unsigned long end = start + page_size;
+        start = (unsigned long)(pfn_to_page(vmemmap_section_start(start)));
 
         for (; start < end; start += (PAGES_PER_SECTION * sizeof(struct page)))
-                if (pfn_valid(vmemmap_section_start(start)))
+                if (pfn_valid(page_to_pfn((struct page *)start)))
                         return 1;
 
         return 0;
@@ -212,6 +213,13 @@ static void __meminit vmemmap_create_mapping(unsigned long start,
         for (i = 0; i < page_size; i += PAGE_SIZE)
                 BUG_ON(map_kernel_page(start + i, phys, flags));
 }
+
+#ifdef CONFIG_MEMORY_HOTPLUG
+static void vmemmap_remove_mapping(unsigned long start,
+                                   unsigned long page_size)
+{
+}
+#endif
 #else /* CONFIG_PPC_BOOK3E */
 static void __meminit vmemmap_create_mapping(unsigned long start,
                                              unsigned long page_size,
@@ -223,17 +231,42 @@ static void __meminit vmemmap_create_mapping(unsigned long start,
                                         mmu_kernel_ssize);
         BUG_ON(mapped < 0);
 }
+
+#ifdef CONFIG_MEMORY_HOTPLUG
+extern int htab_remove_mapping(unsigned long vstart, unsigned long vend,
+                        int psize, int ssize);
+
+static void vmemmap_remove_mapping(unsigned long start,
+                                   unsigned long page_size)
+{
+        int mapped = htab_remove_mapping(start, start + page_size,
+                                         mmu_vmemmap_psize,
+                                         mmu_kernel_ssize);
+        BUG_ON(mapped < 0);
+}
+#endif
+
 #endif /* CONFIG_PPC_BOOK3E */
 
 struct vmemmap_backing *vmemmap_list;
+static struct vmemmap_backing *next;
+static int num_left;
+static int num_freed;
 
 static __meminit struct vmemmap_backing * vmemmap_list_alloc(int node)
 {
-        static struct vmemmap_backing *next;
-        static int num_left;
+        struct vmemmap_backing *vmem_back;
+        /* get from freed entries first */
+        if (num_freed) {
+                num_freed--;
+                vmem_back = next;
+                next = next->list;
+
+                return vmem_back;
+        }
 
         /* allocate a page when required and hand out chunks */
-        if (!next || !num_left) {
+        if (!num_left) {
                 next = vmemmap_alloc_block(PAGE_SIZE, node);
                 if (unlikely(!next)) {
                         WARN_ON(1);
@@ -296,10 +329,85 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
         return 0;
 }
 
-void vmemmap_free(unsigned long start, unsigned long end)
+#ifdef CONFIG_MEMORY_HOTPLUG
+static unsigned long vmemmap_list_free(unsigned long start)
 {
+        struct vmemmap_backing *vmem_back, *vmem_back_prev;
+
+        vmem_back_prev = vmem_back = vmemmap_list;
+
+        /* look for it with prev pointer recorded */
+        for (; vmem_back; vmem_back = vmem_back->list) {
+                if (vmem_back->virt_addr == start)
+                        break;
+                vmem_back_prev = vmem_back;
+        }
+
+        if (unlikely(!vmem_back)) {
+                WARN_ON(1);
+                return 0;
+        }
+
+        /* remove it from vmemmap_list */
+        if (vmem_back == vmemmap_list) /* remove head */
+                vmemmap_list = vmem_back->list;
+        else
+                vmem_back_prev->list = vmem_back->list;
+
+        /* next point to this freed entry */
+        vmem_back->list = next;
+        next = vmem_back;
+        num_freed++;
+
+        return vmem_back->phys;
 }
 
+void __ref vmemmap_free(unsigned long start, unsigned long end)
+{
+        unsigned long page_size = 1 << mmu_psize_defs[mmu_vmemmap_psize].shift;
+
+        start = _ALIGN_DOWN(start, page_size);
+
+        pr_debug("vmemmap_free %lx...%lx\n", start, end);
+
+        for (; start < end; start += page_size) {
+                unsigned long addr;
+
+                /*
+                 * the section has already be marked as invalid, so
+                 * vmemmap_populated() true means some other sections still
+                 * in this page, so skip it.
+                 */
+                if (vmemmap_populated(start, page_size))
+                        continue;
+
+                addr = vmemmap_list_free(start);
+                if (addr) {
+                        struct page *page = pfn_to_page(addr >> PAGE_SHIFT);
+
+                        if (PageReserved(page)) {
+                                /* allocated from bootmem */
+                                if (page_size < PAGE_SIZE) {
+                                        /*
+                                         * this shouldn't happen, but if it is
+                                         * the case, leave the memory there
+                                         */
+                                        WARN_ON_ONCE(1);
+                                } else {
+                                        unsigned int nr_pages =
+                                                1 << get_order(page_size);
+                                        while (nr_pages--)
+                                                free_reserved_page(page++);
+                                }
+                        } else
+                                free_pages((unsigned long)(__va(addr)),
+                                                        get_order(page_size));
+
+                        vmemmap_remove_mapping(start, page_size);
+                }
+        }
+}
+#endif
 void register_page_bootmem_memmap(unsigned long section_nr,
                                   struct page *start_page, unsigned long size)
 {
@@ -331,16 +439,16 @@ struct page *realmode_pfn_to_page(unsigned long pfn)
                 if (pg_va < vmem_back->virt_addr)
                         continue;
 
-                /* Check that page struct is not split between real pages */
-                if ((pg_va + sizeof(struct page)) >
-                                (vmem_back->virt_addr + page_size))
-                        return NULL;
-
-                page = (struct page *) (vmem_back->phys + pg_va -
+                /* After vmemmap_list entry free is possible, need check all */
+                if ((pg_va + sizeof(struct page)) <=
+                                (vmem_back->virt_addr + page_size)) {
+                        page = (struct page *) (vmem_back->phys + pg_va -
                                 vmem_back->virt_addr);
-                return page;
+                        return page;
+                }
         }
 
+        /* Probably that page struct is split between real pages */
         return NULL;
 }
 EXPORT_SYMBOL_GPL(realmode_pfn_to_page);
diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 2c8e90f5789e..e0f7a189c48e 100644
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -128,7 +128,8 @@ int arch_add_memory(int nid, u64 start, u64 size)
                 return -EINVAL;
 
         /* this should work for most non-highmem platforms */
-        zone = pgdata->node_zones;
+        zone = pgdata->node_zones +
+                zone_for_memory(nid, start, size, 0);
 
         return __add_pages(nid, zone, start_pfn, nr_pages);
 }
diff --git a/arch/powerpc/mm/mmu_context_hash32.c b/arch/powerpc/mm/mmu_context_hash32.c
index 78fef6726e10..aa5a7fd89461 100644
--- a/arch/powerpc/mm/mmu_context_hash32.c
+++ b/arch/powerpc/mm/mmu_context_hash32.c
@@ -2,7 +2,7 @@
  * This file contains the routines for handling the MMU on those
  * PowerPC implementations where the MMU substantially follows the
  * architecture specification.  This includes the 6xx, 7xx, 7xxx,
- * 8260, and POWER3 implementations but excludes the 8xx and 4xx.
+ * and 8260 implementations but excludes the 8xx and 4xx.
  *  -- paulus
  *
  *  Derived from arch/ppc/mm/init.c:
diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c
index 3b181b22cd46..d3e9a78eaed3 100644
--- a/arch/powerpc/mm/numa.c
+++ b/arch/powerpc/mm/numa.c
@@ -611,8 +611,8 @@ static int cpu_numa_callback(struct notifier_block *nfb, unsigned long action,
         case CPU_UP_CANCELED:
         case CPU_UP_CANCELED_FROZEN:
                 unmap_cpu_from_node(lcpu);
-                break;
                 ret = NOTIFY_OK;
+                break;
 #endif
         }
         return ret;
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index 343a87fa78b5..cf11342bf519 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -41,7 +41,7 @@ unsigned long ioremap_base;
 unsigned long ioremap_bot;
 EXPORT_SYMBOL(ioremap_bot);     /* aka VMALLOC_END */
 
-#if defined(CONFIG_6xx) || defined(CONFIG_POWER3)
+#ifdef CONFIG_6xx
 #define HAVE_BATS       1
 #endif
 
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index f6ce1f111f5b..3b3c4d34c7a0 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -68,7 +68,7 @@
 unsigned long ioremap_bot = IOREMAP_BASE;
 
 #ifdef CONFIG_PPC_MMU_NOHASH
-static void *early_alloc_pgtable(unsigned long size)
+static __ref void *early_alloc_pgtable(unsigned long size)
 {
         void *pt;
 
diff --git a/arch/powerpc/mm/ppc_mmu_32.c b/arch/powerpc/mm/ppc_mmu_32.c
index 11571e118831..5029dc19b517 100644
--- a/arch/powerpc/mm/ppc_mmu_32.c
+++ b/arch/powerpc/mm/ppc_mmu_32.c
@@ -2,7 +2,7 @@
  * This file contains the routines for handling the MMU on those
  * PowerPC implementations where the MMU substantially follows the
  * architecture specification.  This includes the 6xx, 7xx, 7xxx,
- * 8260, and POWER3 implementations but excludes the 8xx and 4xx.
+ * and 8260 implementations but excludes the 8xx and 4xx.
  *  -- paulus
  *
  *  Derived from arch/ppc/mm/init.c:
diff --git a/arch/powerpc/mm/stab.c b/arch/powerpc/mm/stab.c
deleted file mode 100644
index 3f8efa6f2997..000000000000
--- a/arch/powerpc/mm/stab.c
+++ /dev/null
@@ -1,286 +0,0 @@
-/*
- * PowerPC64 Segment Translation Support.
- *
- * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com
- *    Copyright (c) 2001 Dave Engebretsen
- *
- * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM
- *
- *      This program is free software; you can redistribute it and/or
- *      modify it under the terms of the GNU General Public License
- *      as published by the Free Software Foundation; either version
- *      2 of the License, or (at your option) any later version.
- */
-
-#include <linux/memblock.h>
-
-#include <asm/pgtable.h>
-#include <asm/mmu.h>
-#include <asm/mmu_context.h>
-#include <asm/paca.h>
-#include <asm/cputable.h>
-#include <asm/prom.h>
-
-struct stab_entry {
-        unsigned long esid_data;
-        unsigned long vsid_data;
-};
-
-#define NR_STAB_CACHE_ENTRIES 8
-static DEFINE_PER_CPU(long, stab_cache_ptr);
-static DEFINE_PER_CPU(long [NR_STAB_CACHE_ENTRIES], stab_cache);
-
-/*
- * Create a segment table entry for the given esid/vsid pair.
- */
-static int make_ste(unsigned long stab, unsigned long esid, unsigned long vsid)
-{
-        unsigned long esid_data, vsid_data;
-        unsigned long entry, group, old_esid, castout_entry, i;
-        unsigned int global_entry;
-        struct stab_entry *ste, *castout_ste;
-        unsigned long kernel_segment = (esid << SID_SHIFT) >= PAGE_OFFSET;
-
-        vsid_data = vsid << STE_VSID_SHIFT;
-        esid_data = esid << SID_SHIFT | STE_ESID_KP | STE_ESID_V;
-        if (! kernel_segment)
-                esid_data |= STE_ESID_KS;
-
-        /* Search the primary group first. */
-        global_entry = (esid & 0x1f) << 3;
-        ste = (struct stab_entry *)(stab | ((esid & 0x1f) << 7));
-
-        /* Find an empty entry, if one exists. */
-        for (group = 0; group < 2; group++) {
-                for (entry = 0; entry < 8; entry++, ste++) {
-                        if (!(ste->esid_data & STE_ESID_V)) {
-                                ste->vsid_data = vsid_data;
-                                eieio();
-                                ste->esid_data = esid_data;
-                                return (global_entry | entry);
-                        }
-                }
-                /* Now search the secondary group. */
-                global_entry = ((~esid) & 0x1f) << 3;
-                ste = (struct stab_entry *)(stab | (((~esid) & 0x1f) << 7));
-        }
-
-        /*
-         * Could not find empty entry, pick one with a round robin selection.
-         * Search all entries in the two groups.
-         */
-        castout_entry = get_paca()->stab_rr;
-        for (i = 0; i < 16; i++) {
-                if (castout_entry < 8) {
-                        global_entry = (esid & 0x1f) << 3;
-                        ste = (struct stab_entry *)(stab | ((esid & 0x1f) << 7));
-                        castout_ste = ste + castout_entry;
-                } else {
-                        global_entry = ((~esid) & 0x1f) << 3;
-                        ste = (struct stab_entry *)(stab | (((~esid) & 0x1f) << 7));
-                        castout_ste = ste + (castout_entry - 8);
-                }
-
-                /* Dont cast out the first kernel segment */
-                if ((castout_ste->esid_data & ESID_MASK) != PAGE_OFFSET)
-                        break;
-
-                castout_entry = (castout_entry + 1) & 0xf;
-        }
-
-        get_paca()->stab_rr = (castout_entry + 1) & 0xf;
-
-        /* Modify the old entry to the new value. */
-
-        /* Force previous translations to complete. DRENG */
-        asm volatile("isync" : : : "memory");
-
-        old_esid = castout_ste->esid_data >> SID_SHIFT;
-        castout_ste->esid_data = 0;             /* Invalidate old entry */
-
-        asm volatile("sync" : : : "memory");    /* Order update */
-
-        castout_ste->vsid_data = vsid_data;
-        eieio();                                /* Order update */
-        castout_ste->esid_data = esid_data;
-
-        asm volatile("slbie  %0" : : "r" (old_esid << SID_SHIFT));
-        /* Ensure completion of slbie */
-        asm volatile("sync" : : : "memory");
-
-        return (global_entry | (castout_entry & 0x7));
-}
-
-/*
- * Allocate a segment table entry for the given ea and mm
- */
-static int __ste_allocate(unsigned long ea, struct mm_struct *mm)
-{
-        unsigned long vsid;
-        unsigned char stab_entry;
-        unsigned long offset;
-
-        /* Kernel or user address? */
-        if (is_kernel_addr(ea)) {
-                vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
-        } else {
-                if ((ea >= TASK_SIZE_USER64) || (! mm))
-                        return 1;
-
-                vsid = get_vsid(mm->context.id, ea, MMU_SEGSIZE_256M);
-        }
-
-        stab_entry = make_ste(get_paca()->stab_addr, GET_ESID(ea), vsid);
-
-        if (!is_kernel_addr(ea)) {
-                offset = __get_cpu_var(stab_cache_ptr);
-                if (offset < NR_STAB_CACHE_ENTRIES)
-                        __get_cpu_var(stab_cache[offset++]) = stab_entry;
-                else
-                        offset = NR_STAB_CACHE_ENTRIES+1;
-                __get_cpu_var(stab_cache_ptr) = offset;
-
-                /* Order update */
-                asm volatile("sync":::"memory");
-        }
-
-        return 0;
-}
-
-int ste_allocate(unsigned long ea)
-{
-        return __ste_allocate(ea, current->mm);
-}
-
-/*
- * Do the segment table work for a context switch: flush all user
- * entries from the table, then preload some probably useful entries
- * for the new task
- */
-void switch_stab(struct task_struct *tsk, struct mm_struct *mm)
-{
-        struct stab_entry *stab = (struct stab_entry *) get_paca()->stab_addr;
-        struct stab_entry *ste;
-        unsigned long offset;
-        unsigned long pc = KSTK_EIP(tsk);
-        unsigned long stack = KSTK_ESP(tsk);
-        unsigned long unmapped_base;
-
-        /* Force previous translations to complete. DRENG */
-        asm volatile("isync" : : : "memory");
-
-        /*
-         * We need interrupts hard-disabled here, not just soft-disabled,
-         * so that a PMU interrupt can't occur, which might try to access
-         * user memory (to get a stack trace) and possible cause an STAB miss
-         * which would update the stab_cache/stab_cache_ptr per-cpu variables.
-         */
-        hard_irq_disable();
-
-        offset = __get_cpu_var(stab_cache_ptr);
-        if (offset <= NR_STAB_CACHE_ENTRIES) {
-                int i;
-
-                for (i = 0; i < offset; i++) {
-                        ste = stab + __get_cpu_var(stab_cache[i]);
-                        ste->esid_data = 0; /* invalidate entry */
-                }
-        } else {
-                unsigned long entry;
-
-                /* Invalidate all entries. */
-                ste = stab;
-
-                /* Never flush the first entry. */
-                ste += 1;
-                for (entry = 1;
-                     entry < (HW_PAGE_SIZE / sizeof(struct stab_entry));
-                     entry++, ste++) {
-                        unsigned long ea;
-                        ea = ste->esid_data & ESID_MASK;
-                        if (!is_kernel_addr(ea)) {
-                                ste->esid_data = 0;
-                        }
-                }
-        }
-
-        asm volatile("sync; slbia; sync":::"memory");
-
-        __get_cpu_var(stab_cache_ptr) = 0;
-
-        /* Now preload some entries for the new task */
-        if (test_tsk_thread_flag(tsk, TIF_32BIT))
-                unmapped_base = TASK_UNMAPPED_BASE_USER32;
-        else
-                unmapped_base = TASK_UNMAPPED_BASE_USER64;
-
-        __ste_allocate(pc, mm);
-
-        if (GET_ESID(pc) == GET_ESID(stack))
-                return;
-
-        __ste_allocate(stack, mm);
-
-        if ((GET_ESID(pc) == GET_ESID(unmapped_base))
-            || (GET_ESID(stack) == GET_ESID(unmapped_base)))
-                return;
-
-        __ste_allocate(unmapped_base, mm);
-
-        /* Order update */
-        asm volatile("sync" : : : "memory");
-}
-
-/*
- * Allocate segment tables for secondary CPUs.  These must all go in
- * the first (bolted) segment, so that do_stab_bolted won't get a
- * recursive segment miss on the segment table itself.
- */
-void __init stabs_alloc(void)
-{
-        int cpu;
-
-        if (mmu_has_feature(MMU_FTR_SLB))
-                return;
-
-        for_each_possible_cpu(cpu) {
-                unsigned long newstab;
-
-                if (cpu == 0)
-                        continue; /* stab for CPU 0 is statically allocated */
-
-                newstab = memblock_alloc_base(HW_PAGE_SIZE, HW_PAGE_SIZE,
-                                         1<<SID_SHIFT);
-                newstab = (unsigned long)__va(newstab);
-
-                memset((void *)newstab, 0, HW_PAGE_SIZE);
-
-                paca[cpu].stab_addr = newstab;
-                paca[cpu].stab_real = __pa(newstab);
-                printk(KERN_INFO "Segment table for CPU %d at 0x%llx "
-                       "virtual, 0x%llx absolute\n",
-                       cpu, paca[cpu].stab_addr, paca[cpu].stab_real);
-        }
-}
-
-/*
- * Build an entry for the base kernel segment and put it into
- * the segment table or SLB.  All other segment table or SLB
- * entries are faulted in.
- */
-void stab_initialize(unsigned long stab)
-{
-        unsigned long vsid = get_kernel_vsid(PAGE_OFFSET, MMU_SEGSIZE_256M);
-        unsigned long stabreal;
-
-        asm volatile("isync; slbia; isync":::"memory");
-        make_ste(stab, GET_ESID(PAGE_OFFSET), vsid);
-
-        /* Order update */
-        asm volatile("sync":::"memory");
-
-        /* Set ASR */
-        stabreal = get_paca()->stab_real | 0x1ul;
-
-        mtspr(SPRN_ASR, stabreal);
-}
diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index 356e8b41fb09..89bf95bd63b1 100644
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -296,9 +296,12 @@ itlb_miss_fault_bolted:
  * r14 = page table base
  * r13 = PACA
  * r11 = tlb_per_core ptr
- * r10 = cpu number
+ * r10 = crap (free to use)
  */
 tlb_miss_common_e6500:
+        crmove  cr2*4+2,cr0*4+2         /* cr2.eq != 0 if kernel address */
+
+BEGIN_FTR_SECTION               /* CPU_FTR_SMT */
         /*
          * Search if we already have an indirect entry for that virtual
          * address, and if we do, bail out.
@@ -309,6 +312,7 @@ tlb_miss_common_e6500:
         lhz     r10,PACAPACAINDEX(r13)
         cmpdi   r15,0
         cmpdi   cr1,r15,1       /* set cr1.eq = 0 for non-recursive */
+        addi    r10,r10,1
         bne     2f
         stbcx.  r10,0,r11
         bne     1b
@@ -322,18 +326,62 @@ tlb_miss_common_e6500:
         b       1b
         .previous
 
+        /*
+         * Erratum A-008139 says that we can't use tlbwe to change
+         * an indirect entry in any way (including replacing or
+         * invalidating) if the other thread could be in the process
+         * of a lookup.  The workaround is to invalidate the entry
+         * with tlbilx before overwriting.
+         */
+
+        lbz     r15,TCD_ESEL_NEXT(r11)
+        rlwinm  r10,r15,16,0xff0000
+        oris    r10,r10,MAS0_TLBSEL(1)@h
+        mtspr   SPRN_MAS0,r10
+        isync
+        tlbre
+        mfspr   r15,SPRN_MAS1
+        andis.  r15,r15,MAS1_VALID@h
+        beq     5f
+
+BEGIN_FTR_SECTION_NESTED(532)
+        mfspr   r10,SPRN_MAS8
+        rlwinm  r10,r10,0,0x80000fff  /* tgs,tlpid -> sgs,slpid */
+        mtspr   SPRN_MAS5,r10
+END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
+
+        mfspr   r10,SPRN_MAS1
+        rlwinm  r15,r10,0,0x3fff0000  /* tid -> spid */
+        rlwimi  r15,r10,20,0x00000003 /* ind,ts -> sind,sas */
+        mfspr   r10,SPRN_MAS6
+        mtspr   SPRN_MAS6,r15
+
         mfspr   r15,SPRN_MAS2
+        isync
+        tlbilxva 0,r15
+        isync
+
+        mtspr   SPRN_MAS6,r10
+
+5:
+BEGIN_FTR_SECTION_NESTED(532)
+        li      r10,0
+        mtspr   SPRN_MAS8,r10
+        mtspr   SPRN_MAS5,r10
+END_FTR_SECTION_NESTED(CPU_FTR_EMB_HV,CPU_FTR_EMB_HV,532)
 
         tlbsx   0,r16
         mfspr   r10,SPRN_MAS1
-        andis.  r10,r10,MAS1_VALID@h
+        andis.  r15,r10,MAS1_VALID@h
         bne     tlb_miss_done_e6500
-
-        /* Undo MAS-damage from the tlbsx */
+FTR_SECTION_ELSE
         mfspr   r10,SPRN_MAS1
+ALT_FTR_SECTION_END_IFSET(CPU_FTR_SMT)
+
         oris    r10,r10,MAS1_VALID@h
-        mtspr   SPRN_MAS1,r10
-        mtspr   SPRN_MAS2,r15
+        beq     cr2,4f
+        rlwinm  r10,r10,0,16,1          /* Clear TID */
+4:      mtspr   SPRN_MAS1,r10
 
         /* Now, we need to walk the page tables. First check if we are in
          * range.
@@ -394,11 +442,13 @@ tlb_miss_common_e6500:
 
 tlb_miss_done_e6500:
         .macro  tlb_unlock_e6500
+BEGIN_FTR_SECTION
         beq     cr1,1f          /* no unlock if lock was recursively grabbed */
         li      r15,0
         isync
         stb     r15,0(r11)
 1:
+END_FTR_SECTION_IFSET(CPU_FTR_SMT)
         .endm
 
         tlb_unlock_e6500
@@ -407,12 +457,9 @@ tlb_miss_done_e6500:
         rfi
 
 tlb_miss_kernel_e6500:
-        mfspr   r10,SPRN_MAS1
         ld      r14,PACA_KERNELPGD(r13)
-        cmpldi  cr0,r15,8               /* Check for vmalloc region */
-        rlwinm  r10,r10,0,16,1          /* Clear TID */
-        mtspr   SPRN_MAS1,r10
-        beq+    tlb_miss_common_e6500
+        cmpldi  cr1,r15,8               /* Check for vmalloc region */
+        beq+    cr1,tlb_miss_common_e6500
 
 tlb_miss_fault_e6500:
         tlb_unlock_e6500