sparc64: Encode huge PMDs using PTE encoding.

Now that we have 64-bits for PMDs we can stop using special encodings
for the huge PMD values, and just put real PTEs in there.

We allocate a _PAGE_PMD_HUGE bit to distinguish between plain PMDs and
huge ones.  It is the same for both 4U and 4V PTE layouts.

We also use _PAGE_SPECIAL to indicate the splitting state, since a
huge PMD cannot also be special.

All of the PMD --> PTE translation code disappears, and most of the
huge PMD bit modifications and tests just degenerate into the PTE
operations.  In particular USER_PGTABLE_CHECK_PMD_HUGE becomes
trivial.

As a side effect, normal PMDs don't shift the physical address around.
This also speeds up the page table walks in the TLB miss paths since
they don't have to do the shifts any more.

Another non-trivial aspect is that pte_modify() has to be changed
to preserve the _PAGE_PMD_HUGE bits as well as the page size field
of the pte.

Signed-off-by: David S. Miller <davem@davemloft.net>

1 files changed, 12 insertions, 80 deletions

diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h
index cc0432f15817..2230f80d9fe3 100644
--- a/arch/sparc/include/asm/tsb.h
+++ b/arch/sparc/include/asm/tsb.h
@@ -147,100 +147,34 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         brz,pn          REG1, FAIL_LABEL; \
          sllx           VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
-        sllx            REG1, PGD_PADDR_SHIFT, REG1; \
         andn            REG2, 0x7, REG2; \
         ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
         brz,pn          REG1, FAIL_LABEL; \
          sllx           VADDR, 64 - PMD_SHIFT, REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
-        sllx            REG1, PMD_PADDR_SHIFT, REG1; \
         andn            REG2, 0x7, REG2; \
         add             REG1, REG2, REG1;
 
-        /* These macros exists only to make the PMD translator below
-         * easier to read.  It hides the ELF section switch for the
-         * sun4v code patching.
-         */
-#define OR_PTE_BIT_1INSN(REG, NAME)                     \
-661:    or              REG, _PAGE_##NAME##_4U, REG;    \
-        .section        .sun4v_1insn_patch, "ax";       \
-        .word           661b;                           \
-        or              REG, _PAGE_##NAME##_4V, REG;    \
-        .previous;
-
-#define OR_PTE_BIT_2INSN(REG, TMP, NAME)                \
-661:    sethi           %hi(_PAGE_##NAME##_4U), TMP;    \
-        or              REG, TMP, REG;                  \
-        .section        .sun4v_2insn_patch, "ax";       \
-        .word           661b;                           \
-        mov             -1, TMP;                        \
-        or              REG, _PAGE_##NAME##_4V, REG;    \
-        .previous;
-
-        /* Load into REG the PTE value for VALID, CACHE, and SZHUGE.
-         *
-         * We are fabricating an 8MB page using 2 4MB HW pages here.
-         */
-#define BUILD_PTE_VALID_SZHUGE_CACHE(VADDR, PADDR_BITS, REG)               \
-        sethi           %hi(4 * 1024 * 1024), REG;                         \
-        andn            PADDR_BITS, REG, PADDR_BITS;                       \
-        and             VADDR, REG, REG;                                   \
-        or              PADDR_BITS, REG, PADDR_BITS;                       \
-661:    sethi           %uhi(_PAGE_VALID|_PAGE_SZHUGE_4U), REG;            \
-        .section        .sun4v_1insn_patch, "ax";                          \
-        .word           661b;                                              \
-        sethi           %uhi(_PAGE_VALID), REG;                            \
-        .previous;                                                         \
-        sllx            REG, 32, REG;                                      \
-661:    or              REG, _PAGE_CP_4U|_PAGE_CV_4U, REG;                 \
-        .section        .sun4v_1insn_patch, "ax";                          \
-        .word           661b;                                              \
-        or              REG, _PAGE_CP_4V|_PAGE_CV_4V|_PAGE_SZHUGE_4V, REG; \
-        .previous;
-
         /* PMD has been loaded into REG1, interpret the value, seeing
          * if it is a HUGE PMD or a normal one.  If it is not valid
          * then jump to FAIL_LABEL.  If it is a HUGE PMD, and it
          * translates to a valid PTE, branch to PTE_LABEL.
          *
-         * We translate the PMD by hand, one bit at a time,
-         * constructing the huge PTE.
-         *
-         * So we construct the PTE in REG2 as follows:
-         *
-         * 1) Extract the PMD PFN from REG1 and place it into REG2.
-         *
-         * 2) Translate PMD protection bits in REG1 into REG2, one bit
-         *    at a time using andcc tests on REG1 and OR's into REG2.
-         *
-         *    Only two bits to be concerned with here, EXEC and WRITE.
-         *    Now REG1 is freed up and we can use it as a temporary.
-         *
-         * 3) Construct the VALID, CACHE, and page size PTE bits in
-         *    REG1, OR with REG2 to form final PTE.
+         * We have to propagate the 4MB bit of the virtual address
+         * because we are fabricating 8MB pages using 4MB hw pages.
          */
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
-        brz,pn          REG1, FAIL_LABEL;                                     \
-         andcc          REG1, PMD_ISHUGE, %g0;                                \
-        be,pt           %xcc, 700f;                                           \
-         and            REG1, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED, REG2;       \
-        cmp             REG2, PMD_HUGE_PRESENT|PMD_HUGE_ACCESSED;             \
-        bne,pn          %xcc, FAIL_LABEL;                                     \
-         andn           REG1, PMD_HUGE_PROTBITS, REG2;                        \
-        sllx            REG2, PMD_PADDR_SHIFT, REG2;                          \
-        /* REG2 now holds PFN << PAGE_SHIFT */                                \
-        andcc           REG1, PMD_HUGE_WRITE, %g0;                            \
-        bne,a,pt        %xcc, 1f;                                             \
-         OR_PTE_BIT_1INSN(REG2, W);                                           \
-1:      andcc           REG1, PMD_HUGE_EXEC, %g0;                             \
-        be,pt           %xcc, 1f;                                             \
-         nop;                                                                 \
-        OR_PTE_BIT_2INSN(REG2, REG1, EXEC);                                   \
-        /* REG1 can now be clobbered, build final PTE */                      \
-1:      BUILD_PTE_VALID_SZHUGE_CACHE(VADDR, REG2, REG1);                      \
-        ba,pt           %xcc, PTE_LABEL;                                      \
-         or             REG1, REG2, REG1;                                     \
+        brz,pn          REG1, FAIL_LABEL;               \
+         sethi          %uhi(_PAGE_PMD_HUGE), REG2;     \
+        sllx            REG2, 32, REG2;                 \
+        andcc           REG1, REG2, %g0;                \
+        be,pt           %xcc, 700f;                     \
+         sethi          %hi(4 * 1024 * 1024), REG2;     \
+        andn            REG1, REG2, REG1;               \
+        and             VADDR, REG2, REG2;              \
+        brlz,pt         REG1, PTE_LABEL;                \
+         or             REG1, REG2, REG1;               \
 700:
 #else
 #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
@@ -265,13 +199,11 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
         brz,pn          REG1, FAIL_LABEL; \
          sllx           VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
-        sllx            REG1, PGD_PADDR_SHIFT, REG1; \
         andn            REG2, 0x7, REG2; \
         ldxa            [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
         USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
         sllx            VADDR, 64 - PMD_SHIFT, REG2; \
         srlx            REG2, 64 - PAGE_SHIFT, REG2; \
-        sllx            REG1, PMD_PADDR_SHIFT, REG1; \
         andn            REG2, 0x7, REG2; \
         add             REG1, REG2, REG1; \
         ldxa            [REG1] ASI_PHYS_USE_EC, REG1; \