3 files changed, 45 insertions, 15 deletions

diff --git a/arch/sparc64/kernel/entry.S b/arch/sparc64/kernel/entry.S
index c9b0d7af64..ea257e8283 100644
--- a/arch/sparc64/kernel/entry.S
+++ b/arch/sparc64/kernel/entry.S
@@ -2593,3 +2593,15 @@ sun4v_mmustat_info:
         retl
          nop
         .size   sun4v_mmustat_info, .-sun4v_mmustat_info
+
+        .globl  sun4v_mmu_demap_all
+        .type   sun4v_mmu_demap_all,#function
+sun4v_mmu_demap_all:
+        clr     %o0
+        clr     %o1
+        mov     HV_MMU_ALL, %o2
+        mov     HV_FAST_MMU_DEMAP_ALL, %o5
+        ta      HV_FAST_TRAP
+        retl
+         nop
+        .size   sun4v_mmu_demap_all, .-sun4v_mmu_demap_all
diff --git a/arch/sparc64/kernel/smp.c b/arch/sparc64/kernel/smp.c
index 894b506f96..c39944927f 100644
--- a/arch/sparc64/kernel/smp.c
+++ b/arch/sparc64/kernel/smp.c
@@ -476,7 +476,7 @@ static inline void spitfire_xcall_deliver(u64 data0, u64 data1, u64 data2, cpuma
  */
 static void cheetah_xcall_deliver(u64 data0, u64 data1, u64 data2, cpumask_t mask)
 {
-        u64 pstate, ver;
+        u64 pstate, ver, busy_mask;
         int nack_busy_id, is_jbus, need_more;
 
         if (cpus_empty(mask))
@@ -508,14 +508,20 @@ retry:
                                "i" (ASI_INTR_W));
 
         nack_busy_id = 0;
+        busy_mask = 0;
         {
                 int i;
 
                 for_each_cpu_mask(i, mask) {
                         u64 target = (i << 14) | 0x70;
 
-                        if (!is_jbus)
+                        if (is_jbus) {
+                                busy_mask |= (0x1UL << (i * 2));
+                        } else {
                                 target |= (nack_busy_id << 24);
+                                busy_mask |= (0x1UL <<
+                                              (nack_busy_id * 2));
+                        }
                         __asm__ __volatile__(
                                 "stxa   %%g0, [%0] %1\n\t"
                                 "membar #Sync\n\t"
@@ -531,15 +537,16 @@ retry:
 
         /* Now, poll for completion. */
         {
-                u64 dispatch_stat;
+                u64 dispatch_stat, nack_mask;
                 long stuck;
 
                 stuck = 100000 * nack_busy_id;
+                nack_mask = busy_mask << 1;
                 do {
                         __asm__ __volatile__("ldxa      [%%g0] %1, %0"
                                              : "=r" (dispatch_stat)
                                              : "i" (ASI_INTR_DISPATCH_STAT));
-                        if (dispatch_stat == 0UL) {
+                        if (!(dispatch_stat & (busy_mask | nack_mask))) {
                                 __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
                                                      : : "r" (pstate));
                                 if (unlikely(need_more)) {
@@ -556,12 +563,12 @@ retry:
                         }
                         if (!--stuck)
                                 break;
-                } while (dispatch_stat & 0x5555555555555555UL);
+                } while (dispatch_stat & busy_mask);
 
                 __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
                                      : : "r" (pstate));
 
-                if ((dispatch_stat & ~(0x5555555555555555UL)) == 0) {
+                if (dispatch_stat & busy_mask) {
                         /* Busy bits will not clear, continue instead
                          * of freezing up on this cpu.
                          */
diff --git a/arch/sparc64/mm/init.c b/arch/sparc64/mm/init.c
index e18ccf8522..fbeb55d71e 100644
--- a/arch/sparc64/mm/init.c
+++ b/arch/sparc64/mm/init.c
@@ -1133,14 +1133,9 @@ static void __init mark_kpte_bitmap(unsigned long start, unsigned long end)
         }
 }
 
-static void __init kernel_physical_mapping_init(void)
+static void __init init_kpte_bitmap(void)
 {
         unsigned long i;
-#ifdef CONFIG_DEBUG_PAGEALLOC
-        unsigned long mem_alloced = 0UL;
-#endif
-
-        read_obp_memory("reg", &pall[0], &pall_ents);
 
         for (i = 0; i < pall_ents; i++) {
                 unsigned long phys_start, phys_end;
@@ -1149,14 +1144,24 @@ static void __init kernel_physical_mapping_init(void)
                 phys_end = phys_start + pall[i].reg_size;
 
                 mark_kpte_bitmap(phys_start, phys_end);
+        }
+}
 
+static void __init kernel_physical_mapping_init(void)
+{
 #ifdef CONFIG_DEBUG_PAGEALLOC
+        unsigned long i, mem_alloced = 0UL;
+
+        for (i = 0; i < pall_ents; i++) {
+                unsigned long phys_start, phys_end;
+
+                phys_start = pall[i].phys_addr;
+                phys_end = phys_start + pall[i].reg_size;
+
                 mem_alloced += kernel_map_range(phys_start, phys_end,
                                                 PAGE_KERNEL);
-#endif
         }
 
-#ifdef CONFIG_DEBUG_PAGEALLOC
         printk("Allocated %ld bytes for kernel page tables.\n",
                mem_alloced);
 
@@ -1398,6 +1403,10 @@ void __init paging_init(void)
         
         inherit_prom_mappings();
         
+        read_obp_memory("reg", &pall[0], &pall_ents);
+
+        init_kpte_bitmap();
+
         /* Ok, we can use our TLB miss and window trap handlers safely.  */
         setup_tba();
 
@@ -1904,7 +1913,9 @@ void __flush_tlb_all(void)
                              "wrpr      %0, %1, %%pstate"
                              : "=r" (pstate)
                              : "i" (PSTATE_IE));
-        if (tlb_type == spitfire) {
+        if (tlb_type == hypervisor) {
+                sun4v_mmu_demap_all();
+        } else if (tlb_type == spitfire) {
                 for (i = 0; i < 64; i++) {
                         /* Spitfire Errata #32 workaround */
                         /* NOTE: Always runs on spitfire, so no