[PATCH] Four level pagetables for ppc64

Implement 4-level pagetables for ppc64

This patch implements full four-level page tables for ppc64, thereby
extending the usable user address range to 44 bits (16T).

The patch uses a full page for the tables at the bottom and top level,
and a quarter page for the intermediate levels.  It uses full 64-bit
pointers at every level, thus also increasing the addressable range of
physical memory.  This patch also tweaks the VSID allocation to allow
matching range for user addresses (this halves the number of available
contexts) and adds some #if and BUILD_BUG sanity checks.

Signed-off-by: David Gibson <dwg@au1.ibm.com>
Signed-off-by: Paul Mackerras <paulus@samba.org>

6 files changed, 158 insertions, 192 deletions

diff --git a/arch/ppc64/mm/hash_utils.c b/arch/ppc64/mm/hash_utils.c
index 623b5d130c31..65d6e8527948 100644
--- a/arch/ppc64/mm/hash_utils.c
+++ b/arch/ppc64/mm/hash_utils.c
@@ -302,7 +302,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
         int local = 0;
         cpumask_t tmp;
 
-        if ((ea & ~REGION_MASK) > EADDR_MASK)
+        if ((ea & ~REGION_MASK) >= PGTABLE_RANGE)
                 return 1;
 
         switch (REGION_ID(ea)) {
diff --git a/arch/ppc64/mm/hugetlbpage.c b/arch/ppc64/mm/hugetlbpage.c
index f9524602818d..a13e44230a6f 100644
--- a/arch/ppc64/mm/hugetlbpage.c
+++ b/arch/ppc64/mm/hugetlbpage.c
@@ -27,124 +27,91 @@
 
 #include <linux/sysctl.h>
 
-#define HUGEPGDIR_SHIFT         (HPAGE_SHIFT + PAGE_SHIFT - 3)
-#define HUGEPGDIR_SIZE          (1UL << HUGEPGDIR_SHIFT)
-#define HUGEPGDIR_MASK          (~(HUGEPGDIR_SIZE-1))
-
-#define HUGEPTE_INDEX_SIZE      9
-#define HUGEPGD_INDEX_SIZE      10
-
-#define PTRS_PER_HUGEPTE        (1 << HUGEPTE_INDEX_SIZE)
-#define PTRS_PER_HUGEPGD        (1 << HUGEPGD_INDEX_SIZE)
-
-static inline int hugepgd_index(unsigned long addr)
-{
-        return (addr & ~REGION_MASK) >> HUGEPGDIR_SHIFT;
-}
-
-static pud_t *hugepgd_offset(struct mm_struct *mm, unsigned long addr)
+/* Modelled after find_linux_pte() */
+pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 {
-        int index;
+        pgd_t *pg;
+        pud_t *pu;
+        pmd_t *pm;
+        pte_t *pt;
 
-        if (! mm->context.huge_pgdir)
-                return NULL;
+        BUG_ON(! in_hugepage_area(mm->context, addr));
 
+        addr &= HPAGE_MASK;
+
+        pg = pgd_offset(mm, addr);
+        if (!pgd_none(*pg)) {
+                pu = pud_offset(pg, addr);
+                if (!pud_none(*pu)) {
+                        pm = pmd_offset(pu, addr);
+                        pt = (pte_t *)pm;
+                        BUG_ON(!pmd_none(*pm)
+                               && !(pte_present(*pt) && pte_huge(*pt)));
+                        return pt;
+                }
+        }
 
-        index = hugepgd_index(addr);
-        BUG_ON(index >= PTRS_PER_HUGEPGD);
-        return (pud_t *)(mm->context.huge_pgdir + index);
+        return NULL;
 }
 
-static inline pte_t *hugepte_offset(pud_t *dir, unsigned long addr)
+pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
 {
-        int index;
-
-        if (pud_none(*dir))
-                return NULL;
-
-        index = (addr >> HPAGE_SHIFT) % PTRS_PER_HUGEPTE;
-        return (pte_t *)pud_page(*dir) + index;
-}
+        pgd_t *pg;
+        pud_t *pu;
+        pmd_t *pm;
+        pte_t *pt;
 
-static pud_t *hugepgd_alloc(struct mm_struct *mm, unsigned long addr)
-{
         BUG_ON(! in_hugepage_area(mm->context, addr));
 
-        if (! mm->context.huge_pgdir) {
-                pgd_t *new;
-                spin_unlock(&mm->page_table_lock);
-                /* Don't use pgd_alloc(), because we want __GFP_REPEAT */
-                new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
-                BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
-                spin_lock(&mm->page_table_lock);
-
-                /*
-                 * Because we dropped the lock, we should re-check the
-                 * entry, as somebody else could have populated it..
-                 */
-                if (mm->context.huge_pgdir)
-                        pgd_free(new);
-                else
-                        mm->context.huge_pgdir = new;
-        }
-        return hugepgd_offset(mm, addr);
-}
+        addr &= HPAGE_MASK;
 
-static pte_t *hugepte_alloc(struct mm_struct *mm, pud_t *dir, unsigned long addr)
-{
-        if (! pud_present(*dir)) {
-                pte_t *new;
+        pg = pgd_offset(mm, addr);
+        pu = pud_alloc(mm, pg, addr);
 
-                spin_unlock(&mm->page_table_lock);
-                new = kmem_cache_alloc(zero_cache, GFP_KERNEL | __GFP_REPEAT);
-                BUG_ON(memcmp(new, empty_zero_page, PAGE_SIZE));
-                spin_lock(&mm->page_table_lock);
-                /*
-                 * Because we dropped the lock, we should re-check the
-                 * entry, as somebody else could have populated it..
-                 */
-                if (pud_present(*dir)) {
-                        if (new)
-                                kmem_cache_free(zero_cache, new);
-                } else {
-                        struct page *ptepage;
-
-                        if (! new)
-                                return NULL;
-                        ptepage = virt_to_page(new);
-                        ptepage->mapping = (void *) mm;
-                        ptepage->index = addr & HUGEPGDIR_MASK;
-                        pud_populate(mm, dir, new);
+        if (pu) {
+                pm = pmd_alloc(mm, pu, addr);
+                if (pm) {
+                        pt = (pte_t *)pm;
+                        BUG_ON(!pmd_none(*pm)
+                               && !(pte_present(*pt) && pte_huge(*pt)));
+                        return pt;
                 }
         }
 
-        return hugepte_offset(dir, addr);
+        return NULL;
 }
 
-pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
-{
-        pud_t *pud;
+#define HUGEPTE_BATCH_SIZE      (HPAGE_SIZE / PMD_SIZE)
 
-        BUG_ON(! in_hugepage_area(mm->context, addr));
+void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
+                     pte_t *ptep, pte_t pte)
+{
+        int i;
 
-        pud = hugepgd_offset(mm, addr);
-        if (! pud)
-                return NULL;
+        if (pte_present(*ptep)) {
+                pte_clear(mm, addr, ptep);
+                flush_tlb_pending();
+        }
 
-        return hugepte_offset(pud, addr);
+        for (i = 0; i < HUGEPTE_BATCH_SIZE; i++) {
+                *ptep = __pte(pte_val(pte) & ~_PAGE_HPTEFLAGS);
+                ptep++;
+        }
 }
 
-pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr)
+pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr,
+                              pte_t *ptep)
 {
-        pud_t *pud;
+        unsigned long old = pte_update(ptep, ~0UL);
+        int i;
 
-        BUG_ON(! in_hugepage_area(mm->context, addr));
+        if (old & _PAGE_HASHPTE)
+                hpte_update(mm, addr, old, 0);
 
-        pud = hugepgd_alloc(mm, addr);
-        if (! pud)
-                return NULL;
+        for (i = 1; i < HUGEPTE_BATCH_SIZE; i++)
+                ptep[i] = __pte(0);
 
-        return hugepte_alloc(mm, pud, addr);
+        return __pte(old);
 }
 
 /*
@@ -541,42 +508,6 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
         }
 }
 
-void hugetlb_mm_free_pgd(struct mm_struct *mm)
-{
-        int i;
-        pgd_t *pgdir;
-
-        spin_lock(&mm->page_table_lock);
-
-        pgdir = mm->context.huge_pgdir;
-        if (! pgdir)
-                goto out;
-
-        mm->context.huge_pgdir = NULL;
-
-        /* cleanup any hugepte pages leftover */
-        for (i = 0; i < PTRS_PER_HUGEPGD; i++) {
-                pud_t *pud = (pud_t *)(pgdir + i);
-
-                if (! pud_none(*pud)) {
-                        pte_t *pte = (pte_t *)pud_page(*pud);
-                        struct page *ptepage = virt_to_page(pte);
-
-                        ptepage->mapping = NULL;
-
-                        BUG_ON(memcmp(pte, empty_zero_page, PAGE_SIZE));
-                        kmem_cache_free(zero_cache, pte);
-                }
-                pud_clear(pud);
-        }
-
-        BUG_ON(memcmp(pgdir, empty_zero_page, PAGE_SIZE));
-        kmem_cache_free(zero_cache, pgdir);
-
- out:
-        spin_unlock(&mm->page_table_lock);
-}
-
 int hash_huge_page(struct mm_struct *mm, unsigned long access,
                    unsigned long ea, unsigned long vsid, int local)
 {
diff --git a/arch/ppc64/mm/imalloc.c b/arch/ppc64/mm/imalloc.c
index b6e75b891ac0..c65b87b92756 100644
--- a/arch/ppc64/mm/imalloc.c
+++ b/arch/ppc64/mm/imalloc.c
@@ -31,7 +31,7 @@ static int get_free_im_addr(unsigned long size, unsigned long *im_addr)
                         break;
                 if ((unsigned long)tmp->addr >= ioremap_bot)
                         addr = tmp->size + (unsigned long) tmp->addr;
-                if (addr > IMALLOC_END-size) 
+                if (addr >= IMALLOC_END-size)
                         return 1;
         }
         *im_addr = addr;
diff --git a/arch/ppc64/mm/init.c b/arch/ppc64/mm/init.c
index e58a24d42879..87f256df8de5 100644
--- a/arch/ppc64/mm/init.c
+++ b/arch/ppc64/mm/init.c
@@ -66,6 +66,14 @@
 #include <asm/vdso.h>
 #include <asm/imalloc.h>
 
+#if PGTABLE_RANGE > USER_VSID_RANGE
+#warning Limited user VSID range means pagetable space is wasted
+#endif
+
+#if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE)
+#warning TASK_SIZE is smaller than it needs to be.
+#endif
+
 int mem_init_done;
 unsigned long ioremap_bot = IMALLOC_BASE;
 static unsigned long phbs_io_bot = PHBS_IO_BASE;
@@ -226,7 +234,7 @@ void __iomem * __ioremap(unsigned long addr, unsigned long size,
          * Before that, we map using addresses going
          * up from ioremap_bot.  imalloc will use
          * the addresses from ioremap_bot through
-         * IMALLOC_END (0xE000001fffffffff)
+         * IMALLOC_END
          * 
          */
         pa = addr & PAGE_MASK;
@@ -417,12 +425,6 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
         int index;
         int err;
 
-#ifdef CONFIG_HUGETLB_PAGE
-        /* We leave htlb_segs as it was, but for a fork, we need to
-         * clear the huge_pgdir. */
-        mm->context.huge_pgdir = NULL;
-#endif
-
 again:
         if (!idr_pre_get(&mmu_context_idr, GFP_KERNEL))
                 return -ENOMEM;
@@ -453,8 +455,6 @@ void destroy_context(struct mm_struct *mm)
         spin_unlock(&mmu_context_lock);
 
         mm->context.id = NO_CONTEXT;
-
-        hugetlb_mm_free_pgd(mm);
 }
 
 /*
@@ -833,23 +833,43 @@ void __iomem * reserve_phb_iospace(unsigned long size)
         return virt_addr;
 }
 
-kmem_cache_t *zero_cache;
-
-static void zero_ctor(void *pte, kmem_cache_t *cache, unsigned long flags)
+static void zero_ctor(void *addr, kmem_cache_t *cache, unsigned long flags)
 {
-        memset(pte, 0, PAGE_SIZE);
+        memset(addr, 0, kmem_cache_size(cache));
 }
 
+static const int pgtable_cache_size[2] = {
+        PTE_TABLE_SIZE, PMD_TABLE_SIZE
+};
+static const char *pgtable_cache_name[ARRAY_SIZE(pgtable_cache_size)] = {
+        "pgd_pte_cache", "pud_pmd_cache",
+};
+
+kmem_cache_t *pgtable_cache[ARRAY_SIZE(pgtable_cache_size)];
+
 void pgtable_cache_init(void)
 {
-        zero_cache = kmem_cache_create("zero",
-                                PAGE_SIZE,
-                                0,
-                                SLAB_HWCACHE_ALIGN | SLAB_MUST_HWCACHE_ALIGN,
-                                zero_ctor,
-                                NULL);
-        if (!zero_cache)
-                panic("pgtable_cache_init(): could not create zero_cache!\n");
+        int i;
+
+        BUILD_BUG_ON(PTE_TABLE_SIZE != pgtable_cache_size[PTE_CACHE_NUM]);
+        BUILD_BUG_ON(PMD_TABLE_SIZE != pgtable_cache_size[PMD_CACHE_NUM]);
+        BUILD_BUG_ON(PUD_TABLE_SIZE != pgtable_cache_size[PUD_CACHE_NUM]);
+        BUILD_BUG_ON(PGD_TABLE_SIZE != pgtable_cache_size[PGD_CACHE_NUM]);
+
+        for (i = 0; i < ARRAY_SIZE(pgtable_cache_size); i++) {
+                int size = pgtable_cache_size[i];
+                const char *name = pgtable_cache_name[i];
+
+                pgtable_cache[i] = kmem_cache_create(name,
+                                                     size, size,
+                                                     SLAB_HWCACHE_ALIGN
+                                                     | SLAB_MUST_HWCACHE_ALIGN,
+                                                     zero_ctor,
+                                                     NULL);
+                if (! pgtable_cache[i])
+                        panic("pgtable_cache_init(): could not create %s!\n",
+                              name);
+        }
 }
 
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long addr,
diff --git a/arch/ppc64/mm/slb_low.S b/arch/ppc64/mm/slb_low.S
index 8379d678f70f..f20fc52483a7 100644
--- a/arch/ppc64/mm/slb_low.S
+++ b/arch/ppc64/mm/slb_low.S
@@ -91,7 +91,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_16M_PAGE)
 0:      /* user address: proto-VSID = context<<15 | ESID */
         li      r11,SLB_VSID_USER
 
-        srdi.   r9,r3,13
+        srdi.   r9,r3,USER_ESID_BITS
         bne-    8f                      /* invalid ea bits set */
 
 #ifdef CONFIG_HUGETLB_PAGE
diff --git a/arch/ppc64/mm/tlb.c b/arch/ppc64/mm/tlb.c
index 26f0172c4527..d8a6593a13f0 100644
--- a/arch/ppc64/mm/tlb.c
+++ b/arch/ppc64/mm/tlb.c
@@ -41,7 +41,58 @@ DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
 DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
 unsigned long pte_freelist_forced_free;
 
-void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage)
+struct pte_freelist_batch
+{
+        struct rcu_head rcu;
+        unsigned int    index;
+        pgtable_free_t  tables[0];
+};
+
+DEFINE_PER_CPU(struct pte_freelist_batch *, pte_freelist_cur);
+unsigned long pte_freelist_forced_free;
+
+#define PTE_FREELIST_SIZE \
+        ((PAGE_SIZE - sizeof(struct pte_freelist_batch)) \
+          / sizeof(pgtable_free_t))
+
+#ifdef CONFIG_SMP
+static void pte_free_smp_sync(void *arg)
+{
+        /* Do nothing, just ensure we sync with all CPUs */
+}
+#endif
+
+/* This is only called when we are critically out of memory
+ * (and fail to get a page in pte_free_tlb).
+ */
+static void pgtable_free_now(pgtable_free_t pgf)
+{
+        pte_freelist_forced_free++;
+
+        smp_call_function(pte_free_smp_sync, NULL, 0, 1);
+
+        pgtable_free(pgf);
+}
+
+static void pte_free_rcu_callback(struct rcu_head *head)
+{
+        struct pte_freelist_batch *batch =
+                container_of(head, struct pte_freelist_batch, rcu);
+        unsigned int i;
+
+        for (i = 0; i < batch->index; i++)
+                pgtable_free(batch->tables[i]);
+
+        free_page((unsigned long)batch);
+}
+
+static void pte_free_submit(struct pte_freelist_batch *batch)
+{
+        INIT_RCU_HEAD(&batch->rcu);
+        call_rcu(&batch->rcu, pte_free_rcu_callback);
+}
+
+void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf)
 {
         /* This is safe as we are holding page_table_lock */
         cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id());
@@ -49,19 +100,19 @@ void __pte_free_tlb(struct mmu_gather *tlb, struct page *ptepage)
 
         if (atomic_read(&tlb->mm->mm_users) < 2 ||
             cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) {
-                pte_free(ptepage);
+                pgtable_free(pgf);
                 return;
         }
 
         if (*batchp == NULL) {
                 *batchp = (struct pte_freelist_batch *)__get_free_page(GFP_ATOMIC);
                 if (*batchp == NULL) {
-                        pte_free_now(ptepage);
+                        pgtable_free_now(pgf);
                         return;
                 }
                 (*batchp)->index = 0;
         }
-        (*batchp)->pages[(*batchp)->index++] = ptepage;
+        (*batchp)->tables[(*batchp)->index++] = pgf;
         if ((*batchp)->index == PTE_FREELIST_SIZE) {
                 pte_free_submit(*batchp);
                 *batchp = NULL;
@@ -132,42 +183,6 @@ void __flush_tlb_pending(struct ppc64_tlb_batch *batch)
         put_cpu();
 }
 
-#ifdef CONFIG_SMP
-static void pte_free_smp_sync(void *arg)
-{
-        /* Do nothing, just ensure we sync with all CPUs */
-}
-#endif
-
-/* This is only called when we are critically out of memory
- * (and fail to get a page in pte_free_tlb).
- */
-void pte_free_now(struct page *ptepage)
-{
-        pte_freelist_forced_free++;
-
-        smp_call_function(pte_free_smp_sync, NULL, 0, 1);
-
-        pte_free(ptepage);
-}
-
-static void pte_free_rcu_callback(struct rcu_head *head)
-{
-        struct pte_freelist_batch *batch =
-                container_of(head, struct pte_freelist_batch, rcu);
-        unsigned int i;
-
-        for (i = 0; i < batch->index; i++)
-                pte_free(batch->pages[i]);
-        free_page((unsigned long)batch);
-}
-
-void pte_free_submit(struct pte_freelist_batch *batch)
-{
-        INIT_RCU_HEAD(&batch->rcu);
-        call_rcu(&batch->rcu, pte_free_rcu_callback);
-}
-
 void pte_free_finish(void)
 {
         /* This is safe as we are holding page_table_lock */