4 files changed, 76 insertions, 28 deletions

diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h
index b049abf9902f..0ff92fa22064 100644
--- a/arch/sparc/include/asm/pgtable_64.h
+++ b/arch/sparc/include/asm/pgtable_64.h
@@ -726,11 +726,17 @@ extern unsigned long pte_file(pte_t);
 extern pte_t pgoff_to_pte(unsigned long);
 #define PTE_FILE_MAX_BITS       (64UL - PAGE_SHIFT - 1UL)
 
-extern unsigned long *sparc64_valid_addr_bitmap;
+extern unsigned long sparc64_valid_addr_bitmap[];
 
 /* Needs to be defined here and not in linux/mm.h, as it is arch dependent */
-#define kern_addr_valid(addr)   \
-        (test_bit(__pa((unsigned long)(addr))>>22, sparc64_valid_addr_bitmap))
+static inline bool kern_addr_valid(unsigned long addr)
+{
+        unsigned long paddr = __pa(addr);
+
+        if ((paddr >> 41UL) != 0UL)
+                return false;
+        return test_bit(paddr >> 22, sparc64_valid_addr_bitmap);
+}
 
 extern int page_in_phys_avail(unsigned long paddr);
 
diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
index cef8defcd7a9..3ea6e8cde8c5 100644
--- a/arch/sparc/kernel/ktlb.S
+++ b/arch/sparc/kernel/ktlb.S
@@ -151,12 +151,46 @@ kvmap_dtlb_4v:
          * Must preserve %g1 and %g6 (TAG).
          */
 kvmap_dtlb_tsb4m_miss:
-        sethi           %hi(kpte_linear_bitmap), %g2
-        or              %g2, %lo(kpte_linear_bitmap), %g2
+        /* Clear the PAGE_OFFSET top virtual bits, shift
+         * down to get PFN, and make sure PFN is in range.
+         */
+        sllx            %g4, 21, %g5
 
-        /* Clear the PAGE_OFFSET top virtual bits, then shift
-         * down to get a 256MB physical address index.
+        /* Check to see if we know about valid memory at the 4MB
+         * chunk this physical address will reside within.
          */
+        srlx            %g5, 21 + 41, %g2
+        brnz,pn         %g2, kvmap_dtlb_longpath
+         nop
+
+        /* This unconditional branch and delay-slot nop gets patched
+         * by the sethi sequence once the bitmap is properly setup.
+         */
+        .globl          valid_addr_bitmap_insn
+valid_addr_bitmap_insn:
+        ba,pt           %xcc, 2f
+         nop
+        .subsection     2
+        .globl          valid_addr_bitmap_patch
+valid_addr_bitmap_patch:
+        sethi           %hi(sparc64_valid_addr_bitmap), %g7
+        or              %g7, %lo(sparc64_valid_addr_bitmap), %g7
+        .previous
+
+        srlx            %g5, 21 + 22, %g2
+        srlx            %g2, 6, %g5
+        and             %g2, 63, %g2
+        sllx            %g5, 3, %g5
+        ldx             [%g7 + %g5], %g5
+        mov             1, %g7
+        sllx            %g7, %g2, %g7
+        andcc           %g5, %g7, %g0
+        be,pn           %xcc, kvmap_dtlb_longpath
+
+2:       sethi          %hi(kpte_linear_bitmap), %g2
+        or              %g2, %lo(kpte_linear_bitmap), %g2
+
+        /* Get the 256MB physical address index. */
         sllx            %g4, 21, %g5
         mov             1, %g7
         srlx            %g5, 21 + 28, %g5
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index ed6be6ba2f4e..a70a5e1904d9 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -145,7 +145,8 @@ static void __init read_obp_memory(const char *property,
              cmp_p64, NULL);
 }
 
-unsigned long *sparc64_valid_addr_bitmap __read_mostly;
+unsigned long sparc64_valid_addr_bitmap[VALID_ADDR_BITMAP_BYTES /
+                                        sizeof(unsigned long)];
 EXPORT_SYMBOL(sparc64_valid_addr_bitmap);
 
 /* Kernel physical address base and size in bytes.  */
@@ -1874,7 +1875,7 @@ static int pavail_rescan_ents __initdata;
  * memory list again, and make sure it provides at least as much
  * memory as 'pavail' does.
  */
-static void __init setup_valid_addr_bitmap_from_pavail(void)
+static void __init setup_valid_addr_bitmap_from_pavail(unsigned long *bitmap)
 {
         int i;
 
@@ -1897,8 +1898,7 @@ static void __init setup_valid_addr_bitmap_from_pavail(void)
 
                                 if (new_start <= old_start &&
                                     new_end >= (old_start + PAGE_SIZE)) {
-                                        set_bit(old_start >> 22,
-                                                sparc64_valid_addr_bitmap);
+                                        set_bit(old_start >> 22, bitmap);
                                         goto do_next_page;
                                 }
                         }
@@ -1919,20 +1919,21 @@ static void __init setup_valid_addr_bitmap_from_pavail(void)
         }
 }
 
+static void __init patch_tlb_miss_handler_bitmap(void)
+{
+        extern unsigned int valid_addr_bitmap_insn[];
+        extern unsigned int valid_addr_bitmap_patch[];
+
+        valid_addr_bitmap_insn[1] = valid_addr_bitmap_patch[1];
+        mb();
+        valid_addr_bitmap_insn[0] = valid_addr_bitmap_patch[0];
+        flushi(&valid_addr_bitmap_insn[0]);
+}
+
 void __init mem_init(void)
 {
         unsigned long codepages, datapages, initpages;
         unsigned long addr, last;
-        int i;
-
-        i = last_valid_pfn >> ((22 - PAGE_SHIFT) + 6);
-        i += 1;
-        sparc64_valid_addr_bitmap = (unsigned long *) alloc_bootmem(i << 3);
-        if (sparc64_valid_addr_bitmap == NULL) {
-                prom_printf("mem_init: Cannot alloc valid_addr_bitmap.\n");
-                prom_halt();
-        }
-        memset(sparc64_valid_addr_bitmap, 0, i << 3);
 
         addr = PAGE_OFFSET + kern_base;
         last = PAGE_ALIGN(kern_size) + addr;
@@ -1941,15 +1942,19 @@ void __init mem_init(void)
                 addr += PAGE_SIZE;
         }
 
-        setup_valid_addr_bitmap_from_pavail();
+        setup_valid_addr_bitmap_from_pavail(sparc64_valid_addr_bitmap);
+        patch_tlb_miss_handler_bitmap();
 
         high_memory = __va(last_valid_pfn << PAGE_SHIFT);
 
 #ifdef CONFIG_NEED_MULTIPLE_NODES
-        for_each_online_node(i) {
-                if (NODE_DATA(i)->node_spanned_pages != 0) {
-                        totalram_pages +=
-                                free_all_bootmem_node(NODE_DATA(i));
+        {
+                int i;
+                for_each_online_node(i) {
+                        if (NODE_DATA(i)->node_spanned_pages != 0) {
+                                totalram_pages +=
+                                        free_all_bootmem_node(NODE_DATA(i));
+                        }
                 }
         }
 #else
diff --git a/arch/sparc/mm/init_64.h b/arch/sparc/mm/init_64.h
index 16063870a489..c2f772dbd556 100644
--- a/arch/sparc/mm/init_64.h
+++ b/arch/sparc/mm/init_64.h
@@ -5,10 +5,13 @@
  * marked non-static so that assembler code can get at them.
  */
 
-#define MAX_PHYS_ADDRESS        (1UL << 42UL)
-#define KPTE_BITMAP_CHUNK_SZ    (256UL * 1024UL * 1024UL)
+#define MAX_PHYS_ADDRESS        (1UL << 41UL)
+#define KPTE_BITMAP_CHUNK_SZ            (256UL * 1024UL * 1024UL)
 #define KPTE_BITMAP_BYTES       \
         ((MAX_PHYS_ADDRESS / KPTE_BITMAP_CHUNK_SZ) / 8)
+#define VALID_ADDR_BITMAP_CHUNK_SZ      (4UL * 1024UL * 1024UL)
+#define VALID_ADDR_BITMAP_BYTES \
+        ((MAX_PHYS_ADDRESS / VALID_ADDR_BITMAP_CHUNK_SZ) / 8)
 
 extern unsigned long kern_linear_pte_xor[2];
 extern unsigned long kpte_linear_bitmap[KPTE_BITMAP_BYTES / sizeof(unsigned long)];