[POWERPC] Use 1TB segments

This makes the kernel use 1TB segments for all kernel mappings and for
user addresses of 1TB and above, on machines which support them
(currently POWER5+, POWER6 and PA6T).

We detect that the machine supports 1TB segments by looking at the
ibm,processor-segment-sizes property in the device tree.

We don't currently use 1TB segments for user addresses < 1T, since
that would effectively prevent 32-bit processes from using huge pages
unless we also had a way to revert to using 256MB segments.  That
would be possible but would involve extra complications (such as
keeping track of which segment size was used when HPTEs were inserted)
and is not addressed here.

Parts of this patch were originally written by Ben Herrenschmidt.

Signed-off-by: Paul Mackerras <paulus@samba.org>

1 files changed, 65 insertions, 8 deletions

diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S
index 35eabfb50723..ad253b959030 100644
--- a/arch/powerpc/mm/hash_low_64.S
+++ b/arch/powerpc/mm/hash_low_64.S
@@ -54,7 +54,7 @@
 
 /*
  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
- *               pte_t *ptep, unsigned long trap, int local)
+ *               pte_t *ptep, unsigned long trap, int local, int ssize)
  *
  * Adds a 4K page to the hash table in a segment of 4K pages only
  */
@@ -66,6 +66,7 @@ _GLOBAL(__hash_page_4K)
         /* Save all params that we need after a function call */
         std     r6,STK_PARM(r6)(r1)
         std     r8,STK_PARM(r8)(r1)
+        std     r9,STK_PARM(r9)(r1)
         
         /* Add _PAGE_PRESENT to access */
         ori     r4,r4,_PAGE_PRESENT
@@ -117,6 +118,10 @@ _GLOBAL(__hash_page_4K)
          * r4 (access) is re-useable, we use it for the new HPTE flags
          */
 
+BEGIN_FTR_SECTION
+        cmpdi   r9,0                    /* check segment size */
+        bne     3f
+END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
         /* Calc va and put it in r29 */
         rldicr  r29,r5,28,63-28
         rldicl  r3,r3,0,36
@@ -126,9 +131,20 @@ _GLOBAL(__hash_page_4K)
         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
         xor     r28,r5,r0
+        b       4f
+
+3:      /* Calc VA and hash in r29 and r28 for 1T segment */
+        sldi    r29,r5,40               /* vsid << 40 */
+        clrldi  r3,r3,24                /* ea & 0xffffffffff */
+        rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
+        clrldi  r5,r5,40                /* vsid & 0xffffff */
+        rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
+        xor     r28,r28,r5
+        or      r29,r3,r29              /* VA */
+        xor     r28,r28,r0              /* hash */
 
         /* Convert linux PTE bits into HW equivalents */
-        andi.   r3,r30,0x1fe            /* Get basic set of flags */
+4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
@@ -183,6 +199,7 @@ htab_insert_pte:
         mr      r4,r29                  /* Retreive va */
         li      r7,0                    /* !bolted, !secondary */
         li      r8,MMU_PAGE_4K          /* page size */
+        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 _GLOBAL(htab_call_hpte_insert1)
         bl      .                       /* Patched by htab_finish_init() */
         cmpdi   0,r3,0
@@ -205,6 +222,7 @@ _GLOBAL(htab_call_hpte_insert1)
         mr      r4,r29                  /* Retreive va */
         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
         li      r8,MMU_PAGE_4K          /* page size */
+        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 _GLOBAL(htab_call_hpte_insert2)
         bl      .                       /* Patched by htab_finish_init() */
         cmpdi   0,r3,0
@@ -273,7 +291,8 @@ htab_modify_pte:
         /* Call ppc_md.hpte_updatepp */
         mr      r5,r29                  /* va */
         li      r6,MMU_PAGE_4K          /* page size */
-        ld      r7,STK_PARM(r8)(r1)     /* get "local" param */
+        ld      r7,STK_PARM(r9)(r1)     /* segment size */
+        ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 _GLOBAL(htab_call_hpte_updatepp)
         bl      .                       /* Patched by htab_finish_init() */
 
@@ -325,6 +344,7 @@ _GLOBAL(__hash_page_4K)
         /* Save all params that we need after a function call */
         std     r6,STK_PARM(r6)(r1)
         std     r8,STK_PARM(r8)(r1)
+        std     r9,STK_PARM(r9)(r1)
 
         /* Add _PAGE_PRESENT to access */
         ori     r4,r4,_PAGE_PRESENT
@@ -383,18 +403,33 @@ _GLOBAL(__hash_page_4K)
         /* Load the hidx index */
         rldicl  r25,r3,64-12,60
 
+BEGIN_FTR_SECTION
+        cmpdi   r9,0                    /* check segment size */
+        bne     3f
+END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
         /* Calc va and put it in r29 */
         rldicr  r29,r5,28,63-28         /* r29 = (vsid << 28) */
         rldicl  r3,r3,0,36              /* r3 = (ea & 0x0fffffff) */
-        or      r29,r3,r29              /* r29 = va
+        or      r29,r3,r29              /* r29 = va */
 
         /* Calculate hash value for primary slot and store it in r28 */
         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
         rldicl  r0,r3,64-12,48          /* (ea >> 12) & 0xffff */
         xor     r28,r5,r0
+        b       4f
+
+3:      /* Calc VA and hash in r29 and r28 for 1T segment */
+        sldi    r29,r5,40               /* vsid << 40 */
+        clrldi  r3,r3,24                /* ea & 0xffffffffff */
+        rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
+        clrldi  r5,r5,40                /* vsid & 0xffffff */
+        rldicl  r0,r3,64-12,36          /* (ea >> 12) & 0xfffffff */
+        xor     r28,r28,r5
+        or      r29,r3,r29              /* VA */
+        xor     r28,r28,r0              /* hash */
 
         /* Convert linux PTE bits into HW equivalents */
-        andi.   r3,r30,0x1fe            /* Get basic set of flags */
+4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
@@ -462,6 +497,7 @@ htab_special_pfn:
         mr      r4,r29                  /* Retreive va */
         li      r7,0                    /* !bolted, !secondary */
         li      r8,MMU_PAGE_4K          /* page size */
+        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 _GLOBAL(htab_call_hpte_insert1)
         bl      .                       /* patched by htab_finish_init() */
         cmpdi   0,r3,0
@@ -488,6 +524,7 @@ _GLOBAL(htab_call_hpte_insert1)
         mr      r4,r29                  /* Retreive va */
         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
         li      r8,MMU_PAGE_4K          /* page size */
+        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 _GLOBAL(htab_call_hpte_insert2)
         bl      .                       /* patched by htab_finish_init() */
         cmpdi   0,r3,0
@@ -586,7 +623,8 @@ htab_modify_pte:
         /* Call ppc_md.hpte_updatepp */
         mr      r5,r29                  /* va */
         li      r6,MMU_PAGE_4K          /* page size */
-        ld      r7,STK_PARM(r8)(r1)     /* get "local" param */
+        ld      r7,STK_PARM(r9)(r1)     /* segment size */
+        ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 _GLOBAL(htab_call_hpte_updatepp)
         bl      .                       /* patched by htab_finish_init() */
 
@@ -634,6 +672,7 @@ _GLOBAL(__hash_page_64K)
         /* Save all params that we need after a function call */
         std     r6,STK_PARM(r6)(r1)
         std     r8,STK_PARM(r8)(r1)
+        std     r9,STK_PARM(r9)(r1)
 
         /* Add _PAGE_PRESENT to access */
         ori     r4,r4,_PAGE_PRESENT
@@ -690,6 +729,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
          * r4 (access) is re-useable, we use it for the new HPTE flags
          */
 
+BEGIN_FTR_SECTION
+        cmpdi   r9,0                    /* check segment size */
+        bne     3f
+END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
         /* Calc va and put it in r29 */
         rldicr  r29,r5,28,63-28
         rldicl  r3,r3,0,36
@@ -699,9 +742,20 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE)
         rldicl  r5,r5,0,25              /* vsid & 0x0000007fffffffff */
         rldicl  r0,r3,64-16,52          /* (ea >> 16) & 0xfff */
         xor     r28,r5,r0
+        b       4f
+
+3:      /* Calc VA and hash in r29 and r28 for 1T segment */
+        sldi    r29,r5,40               /* vsid << 40 */
+        clrldi  r3,r3,24                /* ea & 0xffffffffff */
+        rldic   r28,r5,25,25            /* (vsid << 25) & 0x7fffffffff */
+        clrldi  r5,r5,40                /* vsid & 0xffffff */
+        rldicl  r0,r3,64-16,40          /* (ea >> 16) & 0xffffff */
+        xor     r28,r28,r5
+        or      r29,r3,r29              /* VA */
+        xor     r28,r28,r0              /* hash */
 
         /* Convert linux PTE bits into HW equivalents */
-        andi.   r3,r30,0x1fe            /* Get basic set of flags */
+4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
@@ -756,6 +810,7 @@ ht64_insert_pte:
         mr      r4,r29                  /* Retreive va */
         li      r7,0                    /* !bolted, !secondary */
         li      r8,MMU_PAGE_64K
+        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 _GLOBAL(ht64_call_hpte_insert1)
         bl      .                       /* patched by htab_finish_init() */
         cmpdi   0,r3,0
@@ -778,6 +833,7 @@ _GLOBAL(ht64_call_hpte_insert1)
         mr      r4,r29                  /* Retreive va */
         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
         li      r8,MMU_PAGE_64K
+        ld      r9,STK_PARM(r9)(r1)     /* segment size */
 _GLOBAL(ht64_call_hpte_insert2)
         bl      .                       /* patched by htab_finish_init() */
         cmpdi   0,r3,0
@@ -846,7 +902,8 @@ ht64_modify_pte:
         /* Call ppc_md.hpte_updatepp */
         mr      r5,r29                  /* va */
         li      r6,MMU_PAGE_64K
-        ld      r7,STK_PARM(r8)(r1)     /* get "local" param */
+        ld      r7,STK_PARM(r9)(r1)     /* segment size */
+        ld      r8,STK_PARM(r8)(r1)     /* get "local" param */
 _GLOBAL(ht64_call_hpte_updatepp)
         bl      .                       /* patched by htab_finish_init() */