[POWERPC] Use 1TB segments

This makes the kernel use 1TB segments for all kernel mappings and for
user addresses of 1TB and above, on machines which support them
(currently POWER5+, POWER6 and PA6T).

We detect that the machine supports 1TB segments by looking at the
ibm,processor-segment-sizes property in the device tree.

We don't currently use 1TB segments for user addresses < 1T, since
that would effectively prevent 32-bit processes from using huge pages
unless we also had a way to revert to using 256MB segments.  That
would be possible but would involve extra complications (such as
keeping track of which segment size was used when HPTEs were inserted)
and is not addressed here.

Parts of this patch were originally written by Ben Herrenschmidt.

Signed-off-by: Paul Mackerras <paulus@samba.org>

6 files changed, 86 insertions, 72 deletions

diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c
index b5a21177bb32..c83c3e3f5178 100644
--- a/arch/powerpc/platforms/cell/spu_base.c
+++ b/arch/powerpc/platforms/cell/spu_base.c
@@ -168,7 +168,7 @@ static int __spu_trap_data_seg(struct spu *spu, unsigned long ea)
 #else
                 psize = mm->context.user_psize;
 #endif
-                vsid = (get_vsid(mm->context.id, ea) << SLB_VSID_SHIFT) |
+                vsid = (get_vsid(mm->context.id, ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) |
                                 SLB_VSID_USER;
                 break;
         case VMALLOC_REGION_ID:
@@ -176,12 +176,12 @@ static int __spu_trap_data_seg(struct spu *spu, unsigned long ea)
                         psize = mmu_vmalloc_psize;
                 else
                         psize = mmu_io_psize;
-                vsid = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) |
+                vsid = (get_kernel_vsid(ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) |
                         SLB_VSID_KERNEL;
                 break;
         case KERNEL_REGION_ID:
                 psize = mmu_linear_psize;
-                vsid = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) |
+                vsid = (get_kernel_vsid(ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) |
                         SLB_VSID_KERNEL;
                 break;
         default:
diff --git a/arch/powerpc/platforms/cell/spufs/switch.c b/arch/powerpc/platforms/cell/spufs/switch.c
index de7e5ee451d2..3d64c81cc6e2 100644
--- a/arch/powerpc/platforms/cell/spufs/switch.c
+++ b/arch/powerpc/platforms/cell/spufs/switch.c
@@ -699,7 +699,7 @@ static inline void get_kernel_slb(u64 ea, u64 slb[2])
                 llp = mmu_psize_defs[mmu_linear_psize].sllp;
         else
                 llp = mmu_psize_defs[mmu_virtual_psize].sllp;
-        slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) |
+        slb[0] = (get_kernel_vsid(ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) |
                 SLB_VSID_KERNEL | llp;
         slb[1] = (ea & ESID_MASK) | SLB_ESID_V;
 }
diff --git a/arch/powerpc/platforms/celleb/htab.c b/arch/powerpc/platforms/celleb/htab.c
index 5e75c77ea8f3..fbf27c74ebda 100644
--- a/arch/powerpc/platforms/celleb/htab.c
+++ b/arch/powerpc/platforms/celleb/htab.c
@@ -90,7 +90,7 @@ static inline unsigned int beat_read_mask(unsigned hpte_group)
 static long beat_lpar_hpte_insert(unsigned long hpte_group,
                                   unsigned long va, unsigned long pa,
                                   unsigned long rflags, unsigned long vflags,
-                                  int psize)
+                                  int psize, int ssize)
 {
         unsigned long lpar_rc;
         unsigned long slot;
@@ -105,7 +105,8 @@ static long beat_lpar_hpte_insert(unsigned long hpte_group,
                         "rflags=%lx, vflags=%lx, psize=%d)\n",
                 hpte_group, va, pa, rflags, vflags, psize);
 
-        hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID;
+        hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) |
+                vflags | HPTE_V_VALID;
         hpte_r = hpte_encode_r(pa, psize) | rflags;
 
         if (!(vflags & HPTE_V_BOLTED))
@@ -184,12 +185,12 @@ static void beat_lpar_hptab_clear(void)
 static long beat_lpar_hpte_updatepp(unsigned long slot,
                                     unsigned long newpp,
                                     unsigned long va,
-                                    int psize, int local)
+                                    int psize, int ssize, int local)
 {
         unsigned long lpar_rc;
         unsigned long dummy0, dummy1, want_v;
 
-        want_v = hpte_encode_v(va, psize);
+        want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
 
         DBG_LOW("    update: "
                 "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ",
@@ -225,8 +226,8 @@ static long beat_lpar_hpte_find(unsigned long va, int psize)
         long slot;
         unsigned long want_v, hpte_v;
 
-        hash = hpt_hash(va, mmu_psize_defs[psize].shift);
-        want_v = hpte_encode_v(va, psize);
+        hash = hpt_hash(va, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M);
+        want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
 
         for (j = 0; j < 2; j++) {
                 slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
@@ -251,11 +252,11 @@ static long beat_lpar_hpte_find(unsigned long va, int psize)
 
 static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
                                           unsigned long ea,
-                                          int psize)
+                                          int psize, int ssize)
 {
         unsigned long lpar_rc, slot, vsid, va, dummy0, dummy1;
 
-        vsid = get_kernel_vsid(ea);
+        vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
         va = (vsid << 28) | (ea & 0x0fffffff);
 
         spin_lock(&beat_htab_lock);
@@ -270,7 +271,7 @@ static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
 }
 
 static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
-                                         int psize, int local)
+                                         int psize, int ssize, int local)
 {
         unsigned long want_v;
         unsigned long lpar_rc;
@@ -279,7 +280,7 @@ static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
 
         DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
                 slot, va, psize, local);
-        want_v = hpte_encode_v(va, psize);
+        want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
 
         spin_lock_irqsave(&beat_htab_lock, flags);
         dummy1 = beat_lpar_hpte_getword0(slot);
@@ -310,7 +311,7 @@ void __init hpte_init_beat(void)
 static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
                                   unsigned long va, unsigned long pa,
                                   unsigned long rflags, unsigned long vflags,
-                                  int psize)
+                                  int psize, int ssize)
 {
         unsigned long lpar_rc;
         unsigned long slot;
@@ -325,7 +326,8 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
                         "rflags=%lx, vflags=%lx, psize=%d)\n",
                 hpte_group, va, pa, rflags, vflags, psize);
 
-        hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID;
+        hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) |
+                vflags | HPTE_V_VALID;
         hpte_r = hpte_encode_r(pa, psize) | rflags;
 
         if (!(vflags & HPTE_V_BOLTED))
@@ -363,13 +365,13 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
 static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
                                     unsigned long newpp,
                                     unsigned long va,
-                                    int psize, int local)
+                                    int psize, int ssize, int local)
 {
         unsigned long lpar_rc;
         unsigned long want_v;
         unsigned long pss;
 
-        want_v = hpte_encode_v(va, psize);
+        want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
         pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
 
         DBG_LOW("    update: "
@@ -391,7 +393,7 @@ static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
 }
 
 static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va,
-                                         int psize, int local)
+                                         int psize, int ssize, int local)
 {
         unsigned long want_v;
         unsigned long lpar_rc;
@@ -399,7 +401,7 @@ static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va,
 
         DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d\n",
                 slot, va, psize, local);
-        want_v = hpte_encode_v(va, psize);
+        want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
         pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc;
 
         lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss);
diff --git a/arch/powerpc/platforms/iseries/htab.c b/arch/powerpc/platforms/iseries/htab.c
index b4e2c7a038e1..15a7097e5dd7 100644
--- a/arch/powerpc/platforms/iseries/htab.c
+++ b/arch/powerpc/platforms/iseries/htab.c
@@ -86,7 +86,8 @@ long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va,
         }
 
 
-        lhpte.v = hpte_encode_v(va, MMU_PAGE_4K) | vflags | HPTE_V_VALID;
+        lhpte.v = hpte_encode_v(va, MMU_PAGE_4K, MMU_SEGSIZE_256M) |
+                vflags | HPTE_V_VALID;
         lhpte.r = hpte_encode_r(phys_to_abs(pa), MMU_PAGE_4K) | rflags;
 
         /* Now fill in the actual HPTE */
@@ -142,7 +143,7 @@ static long iSeries_hpte_remove(unsigned long hpte_group)
  *      bits 61..63 : PP2,PP1,PP0
  */
 static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
-                                  unsigned long va, int psize, int local)
+                        unsigned long va, int psize, int ssize, int local)
 {
         struct hash_pte hpte;
         unsigned long want_v;
@@ -150,7 +151,7 @@ static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
         iSeries_hlock(slot);
 
         HvCallHpt_get(&hpte, slot);
-        want_v = hpte_encode_v(va, MMU_PAGE_4K);
+        want_v = hpte_encode_v(va, MMU_PAGE_4K, MMU_SEGSIZE_256M);
 
         if (HPTE_V_COMPARE(hpte.v, want_v) && (hpte.v & HPTE_V_VALID)) {
                 /*
@@ -205,14 +206,14 @@ static long iSeries_hpte_find(unsigned long vpn)
  * No need to lock here because we should be the only user.
  */
 static void iSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
-                                        int psize)
+                                        int psize, int ssize)
 {
         unsigned long vsid,va,vpn;
         long slot;
 
         BUG_ON(psize != MMU_PAGE_4K);
 
-        vsid = get_kernel_vsid(ea);
+        vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M);
         va = (vsid << 28) | (ea & 0x0fffffff);
         vpn = va >> HW_PAGE_SHIFT;
         slot = iSeries_hpte_find(vpn);
@@ -222,7 +223,7 @@ static void iSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
 }
 
 static void iSeries_hpte_invalidate(unsigned long slot, unsigned long va,
-                                    int psize, int local)
+                                    int psize, int ssize, int local)
 {
         unsigned long hpte_v;
         unsigned long avpn = va >> 23;
diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c
index 5d2e176a1b18..7382f195c4f8 100644
--- a/arch/powerpc/platforms/ps3/htab.c
+++ b/arch/powerpc/platforms/ps3/htab.c
@@ -60,7 +60,8 @@ static void _debug_dump_hpte(unsigned long pa, unsigned long va,
 }
 
 static long ps3_hpte_insert(unsigned long hpte_group, unsigned long va,
-        unsigned long pa, unsigned long rflags, unsigned long vflags, int psize)
+        unsigned long pa, unsigned long rflags, unsigned long vflags,
+        int psize, int ssize)
 {
         unsigned long slot;
         struct hash_pte lhpte;
@@ -72,7 +73,8 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long va,
 
         vflags &= ~HPTE_V_SECONDARY; /* this bit is ignored */
 
-        lhpte.v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID;
+        lhpte.v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) |
+                vflags | HPTE_V_VALID;
         lhpte.r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize) | rflags;
 
         p_pteg = hpte_group / HPTES_PER_GROUP;
@@ -167,14 +169,14 @@ static long ps3_hpte_remove(unsigned long hpte_group)
 }
 
 static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp,
-        unsigned long va, int psize, int local)
+        unsigned long va, int psize, int ssize, int local)
 {
         unsigned long flags;
         unsigned long result;
         unsigned long pteg, bit;
         unsigned long hpte_v, want_v;
 
-        want_v = hpte_encode_v(va, psize);
+        want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M);
 
         spin_lock_irqsave(&ps3_bolttab_lock, flags);
 
@@ -205,13 +207,13 @@ static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp,
 }
 
 static void ps3_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
-        int psize)
+        int psize, int ssize)
 {
         panic("ps3_hpte_updateboltedpp() not implemented");
 }
 
 static void ps3_hpte_invalidate(unsigned long slot, unsigned long va,
-        int psize, int local)
+        int psize, int ssize, int local)
 {
         unsigned long flags;
         unsigned long result;
diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c
index ea327ca345c6..9a455d46379d 100644
--- a/arch/powerpc/platforms/pseries/lpar.c
+++ b/arch/powerpc/platforms/pseries/lpar.c
@@ -284,7 +284,7 @@ void vpa_init(int cpu)
 static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
                               unsigned long va, unsigned long pa,
                               unsigned long rflags, unsigned long vflags,
-                              int psize)
+                              int psize, int ssize)
 {
         unsigned long lpar_rc;
         unsigned long flags;
@@ -296,7 +296,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group,
                         "rflags=%lx, vflags=%lx, psize=%d)\n",
                 hpte_group, va, pa, rflags, vflags, psize);
 
-        hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID;
+        hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID;
         hpte_r = hpte_encode_r(pa, psize) | rflags;
 
         if (!(vflags & HPTE_V_BOLTED))
@@ -392,6 +392,22 @@ static void pSeries_lpar_hptab_clear(void)
 }
 
 /*
+ * This computes the AVPN and B fields of the first dword of a HPTE,
+ * for use when we want to match an existing PTE.  The bottom 7 bits
+ * of the returned value are zero.
+ */
+static inline unsigned long hpte_encode_avpn(unsigned long va, int psize,
+                                             int ssize)
+{
+        unsigned long v;
+
+        v = (va >> 23) & ~(mmu_psize_defs[psize].avpnm);
+        v <<= HPTE_V_AVPN_SHIFT;
+        v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT;
+        return v;
+}
+
+/*
  * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and
  * the low 3 bits of flags happen to line up.  So no transform is needed.
  * We can probably optimize here and assume the high bits of newpp are
@@ -400,18 +416,18 @@ static void pSeries_lpar_hptab_clear(void)
 static long pSeries_lpar_hpte_updatepp(unsigned long slot,
                                        unsigned long newpp,
                                        unsigned long va,
-                                       int psize, int local)
+                                       int psize, int ssize, int local)
 {
         unsigned long lpar_rc;
         unsigned long flags = (newpp & 7) | H_AVPN;
         unsigned long want_v;
 
-        want_v = hpte_encode_v(va, psize);
+        want_v = hpte_encode_avpn(va, psize, ssize);
 
         DBG_LOW("    update: avpnv=%016lx, hash=%016lx, f=%x, psize: %d ... ",
-                want_v & HPTE_V_AVPN, slot, flags, psize);
+                want_v, slot, flags, psize);
 
-        lpar_rc = plpar_pte_protect(flags, slot, want_v & HPTE_V_AVPN);
+        lpar_rc = plpar_pte_protect(flags, slot, want_v);
 
         if (lpar_rc == H_NOT_FOUND) {
                 DBG_LOW("not found !\n");
@@ -444,32 +460,25 @@ static unsigned long pSeries_lpar_hpte_getword0(unsigned long slot)
         return dword0;
 }
 
-static long pSeries_lpar_hpte_find(unsigned long va, int psize)
+static long pSeries_lpar_hpte_find(unsigned long va, int psize, int ssize)
 {
         unsigned long hash;
-        unsigned long i, j;
+        unsigned long i;
         long slot;
         unsigned long want_v, hpte_v;
 
-        hash = hpt_hash(va, mmu_psize_defs[psize].shift);
-        want_v = hpte_encode_v(va, psize);
-
-        for (j = 0; j < 2; j++) {
-                slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
-                for (i = 0; i < HPTES_PER_GROUP; i++) {
-                        hpte_v = pSeries_lpar_hpte_getword0(slot);
-
-                        if (HPTE_V_COMPARE(hpte_v, want_v)
-                            && (hpte_v & HPTE_V_VALID)
-                            && (!!(hpte_v & HPTE_V_SECONDARY) == j)) {
-                                /* HPTE matches */
-                                if (j)
-                                        slot = -slot;
-                                return slot;
-                        }
-                        ++slot;
-                }
-                hash = ~hash;
+        hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize);
+        want_v = hpte_encode_avpn(va, psize, ssize);
+
+        /* Bolted entries are always in the primary group */
+        slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
+        for (i = 0; i < HPTES_PER_GROUP; i++) {
+                hpte_v = pSeries_lpar_hpte_getword0(slot);
+
+                if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID))
+                        /* HPTE matches */
+                        return slot;
+                ++slot;
         }
 
         return -1;
@@ -477,14 +486,14 @@ static long pSeries_lpar_hpte_find(unsigned long va, int psize)
 
 static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp,
                                              unsigned long ea,
-                                             int psize)
+                                             int psize, int ssize)
 {
         unsigned long lpar_rc, slot, vsid, va, flags;
 
-        vsid = get_kernel_vsid(ea);
-        va = (vsid << 28) | (ea & 0x0fffffff);
+        vsid = get_kernel_vsid(ea, ssize);
+        va = hpt_va(ea, vsid, ssize);
 
-        slot = pSeries_lpar_hpte_find(va, psize);
+        slot = pSeries_lpar_hpte_find(va, psize, ssize);
         BUG_ON(slot == -1);
 
         flags = newpp & 7;
@@ -494,7 +503,7 @@ static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp,
 }
 
 static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
-                                         int psize, int local)
+                                         int psize, int ssize, int local)
 {
         unsigned long want_v;
         unsigned long lpar_rc;
@@ -503,9 +512,8 @@ static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va,
         DBG_LOW("    inval : slot=%lx, va=%016lx, psize: %d, local: %d",
                 slot, va, psize, local);
 
-        want_v = hpte_encode_v(va, psize);
-        lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v & HPTE_V_AVPN,
-                                   &dummy1, &dummy2);
+        want_v = hpte_encode_avpn(va, psize, ssize);
+        lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2);
         if (lpar_rc == H_NOT_FOUND)
                 return;
 
@@ -533,18 +541,19 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
         unsigned long va;
         unsigned long hash, index, shift, hidx, slot;
         real_pte_t pte;
-        int psize;
+        int psize, ssize;
 
         if (lock_tlbie)
                 spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags);
 
         psize = batch->psize;
+        ssize = batch->ssize;
         pix = 0;
         for (i = 0; i < number; i++) {
                 va = batch->vaddr[i];
                 pte = batch->pte[i];
                 pte_iterate_hashed_subpages(pte, psize, va, index, shift) {
-                        hash = hpt_hash(va, shift);
+                        hash = hpt_hash(va, shift, ssize);
                         hidx = __rpte_to_hidx(pte, index);
                         if (hidx & _PTEIDX_SECONDARY)
                                 hash = ~hash;
@@ -552,11 +561,11 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local)
                         slot += hidx & _PTEIDX_GROUP_IX;
                         if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) {
                                 pSeries_lpar_hpte_invalidate(slot, va, psize,
-                                                             local);
+                                                             ssize, local);
                         } else {
                                 param[pix] = HBR_REQUEST | HBR_AVPN | slot;
-                                param[pix+1] = hpte_encode_v(va, psize) &
-                                        HPTE_V_AVPN;
+                                param[pix+1] = hpte_encode_avpn(va, psize,
+                                                                ssize);
                                 pix += 2;
                                 if (pix == 8) {
                                         rc = plpar_hcall9(H_BULK_REMOVE, param,