sparc64: Fix physical memory management regressions with large max_phys_bits.

If max_phys_bits needs to be > 43 (f.e. for T4 chips), things like
DEBUG_PAGEALLOC stop working because the 3-level page tables only
can cover up to 43 bits.

Another problem is that when we increased MAX_PHYS_ADDRESS_BITS up to
47, several statically allocated tables became enormous.

Compounding this is that we will need to support up to 49 bits of
physical addressing for M7 chips.

The two tables in question are sparc64_valid_addr_bitmap and
kpte_linear_bitmap.

The first holds a bitmap, with 1 bit for each 4MB chunk of physical
memory, indicating whether that chunk actually exists in the machine
and is valid.

The second table is a set of 2-bit values which tell how large of a
mapping (4MB, 256MB, 2GB, 16GB, respectively) we can use at each 256MB
chunk of ram in the system.

These tables are huge and take up an enormous amount of the BSS
section of the sparc64 kernel image.  Specifically, the
sparc64_valid_addr_bitmap is 4MB, and the kpte_linear_bitmap is 128K.

So let's solve the space wastage and the DEBUG_PAGEALLOC problem
at the same time, by using the kernel page tables (as designed) to
manage this information.

We have to keep using large mappings when DEBUG_PAGEALLOC is disabled,
and we do this by encoding huge PMDs and PUDs.

On a T4-2 with 256GB of ram the kernel page table takes up 16K with
DEBUG_PAGEALLOC disabled and 256MB with it enabled.  Furthermore, this
memory is dynamically allocated at run time rather than coded
statically into the kernel image.

Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Bob Picco <bob.picco@oracle.com>

2 files changed, 9 insertions, 104 deletions

diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S
index 605d49204580..94a1e6648bd0 100644
--- a/arch/sparc/kernel/ktlb.S
+++ b/arch/sparc/kernel/ktlb.S
@@ -47,14 +47,6 @@ kvmap_itlb_vmalloc_addr:
         KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)
 
         TSB_LOCK_TAG(%g1, %g2, %g7)
-
-        /* Load and check PTE.  */
-        ldxa            [%g5] ASI_PHYS_USE_EC, %g5
-        mov             1, %g7
-        sllx            %g7, TSB_TAG_INVALID_BIT, %g7
-        brgez,a,pn      %g5, kvmap_itlb_longpath
-         TSB_STORE(%g1, %g7)
-
         TSB_WRITE(%g1, %g5, %g6)
 
         /* fallthrough to TLB load */
@@ -118,6 +110,12 @@ kvmap_dtlb_obp:
         ba,pt           %xcc, kvmap_dtlb_load
          nop
 
+kvmap_linear_early:
+        sethi           %hi(kern_linear_pte_xor), %g7
+        ldx             [%g7 + %lo(kern_linear_pte_xor)], %g2
+        ba,pt           %xcc, kvmap_dtlb_tsb4m_load
+         xor            %g2, %g4, %g5
+
         .align          32
 kvmap_dtlb_tsb4m_load:
         TSB_LOCK_TAG(%g1, %g2, %g7)
@@ -146,105 +144,17 @@ kvmap_dtlb_4v:
         /* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
         KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
 #endif
-        /* TSB entry address left in %g1, lookup linear PTE.
-         * Must preserve %g1 and %g6 (TAG).
-         */
-kvmap_dtlb_tsb4m_miss:
-        /* Clear the PAGE_OFFSET top virtual bits, shift
-         * down to get PFN, and make sure PFN is in range.
-         */
-661:    sllx            %g4, 0, %g5
-        .section        .page_offset_shift_patch, "ax"
-        .word           661b
-        .previous
-
-        /* Check to see if we know about valid memory at the 4MB
-         * chunk this physical address will reside within.
+        /* Linear mapping TSB lookup failed.  Fallthrough to kernel
+         * page table based lookup.
          */
-661:    srlx            %g5, MAX_PHYS_ADDRESS_BITS, %g2
-        .section        .page_offset_shift_patch, "ax"
-        .word           661b
-        .previous
-
-        brnz,pn         %g2, kvmap_dtlb_longpath
-         nop
-
-        /* This unconditional branch and delay-slot nop gets patched
-         * by the sethi sequence once the bitmap is properly setup.
-         */
-        .globl          valid_addr_bitmap_insn
-valid_addr_bitmap_insn:
-        ba,pt           %xcc, 2f
-         nop
-        .subsection     2
-        .globl          valid_addr_bitmap_patch
-valid_addr_bitmap_patch:
-        sethi           %hi(sparc64_valid_addr_bitmap), %g7
-        or              %g7, %lo(sparc64_valid_addr_bitmap), %g7
-        .previous
-
-661:    srlx            %g5, ILOG2_4MB, %g2
-        .section        .page_offset_shift_patch, "ax"
-        .word           661b
-        .previous
-
-        srlx            %g2, 6, %g5
-        and             %g2, 63, %g2
-        sllx            %g5, 3, %g5
-        ldx             [%g7 + %g5], %g5
-        mov             1, %g7
-        sllx            %g7, %g2, %g7
-        andcc           %g5, %g7, %g0
-        be,pn           %xcc, kvmap_dtlb_longpath
-
-2:       sethi          %hi(kpte_linear_bitmap), %g2
-
-        /* Get the 256MB physical address index. */
-661:    sllx            %g4, 0, %g5
-        .section        .page_offset_shift_patch, "ax"
-        .word           661b
-        .previous
-
-        or              %g2, %lo(kpte_linear_bitmap), %g2
-
-661:    srlx            %g5, ILOG2_256MB, %g5
-        .section        .page_offset_shift_patch, "ax"
-        .word           661b
-        .previous
-
-        and             %g5, (32 - 1), %g7
-
-        /* Divide by 32 to get the offset into the bitmask.  */
-        srlx            %g5, 5, %g5
-        add             %g7, %g7, %g7
-        sllx            %g5, 3, %g5
-
-        /* kern_linear_pte_xor[(mask >> shift) & 3)] */
-        ldx             [%g2 + %g5], %g2
-        srlx            %g2, %g7, %g7
-        sethi           %hi(kern_linear_pte_xor), %g5
-        and             %g7, 3, %g7
-        or              %g5, %lo(kern_linear_pte_xor), %g5
-        sllx            %g7, 3, %g7
-        ldx             [%g5 + %g7], %g2
-
         .globl          kvmap_linear_patch
 kvmap_linear_patch:
-        ba,pt           %xcc, kvmap_dtlb_tsb4m_load
-         xor            %g2, %g4, %g5
+        ba,a,pt         %xcc, kvmap_linear_early
 
 kvmap_dtlb_vmalloc_addr:
         KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
 
         TSB_LOCK_TAG(%g1, %g2, %g7)
-
-        /* Load and check PTE.  */
-        ldxa            [%g5] ASI_PHYS_USE_EC, %g5
-        mov             1, %g7
-        sllx            %g7, TSB_TAG_INVALID_BIT, %g7
-        brgez,a,pn      %g5, kvmap_dtlb_longpath
-         TSB_STORE(%g1, %g7)
-
         TSB_WRITE(%g1, %g5, %g6)
 
         /* fallthrough to TLB load */
diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S
index 932ff90fd760..0bacceb19150 100644
--- a/arch/sparc/kernel/vmlinux.lds.S
+++ b/arch/sparc/kernel/vmlinux.lds.S
@@ -122,11 +122,6 @@ SECTIONS
                 *(.swapper_4m_tsb_phys_patch)
                 __swapper_4m_tsb_phys_patch_end = .;
         }
-        .page_offset_shift_patch : {
-                __page_offset_shift_patch = .;
-                *(.page_offset_shift_patch)
-                __page_offset_shift_patch_end = .;
-        }
         .popc_3insn_patch : {
                 __popc_3insn_patch = .;
                 *(.popc_3insn_patch)