5 files changed, 306 insertions, 84 deletions

diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c
index 413d29263304..2976dba1ebaf 100644
--- a/arch/sparc/mm/fault_64.c
+++ b/arch/sparc/mm/fault_64.c
@@ -465,13 +465,13 @@ good_area:
         up_read(&mm->mmap_sem);
 
         mm_rss = get_mm_rss(mm);
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE));
 #endif
         if (unlikely(mm_rss >
                      mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit))
                 tsb_grow(mm, MM_TSB_BASE, mm_rss);
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         mm_rss = mm->context.huge_pte_count;
         if (unlikely(mm_rss >
                      mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit))
diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c
index 07e14535375c..f76f83d5ac63 100644
--- a/arch/sparc/mm/hugetlbpage.c
+++ b/arch/sparc/mm/hugetlbpage.c
@@ -303,53 +303,3 @@ struct page *follow_huge_pmd(struct mm_struct *mm, unsigned long address,
 {
         return NULL;
 }
-
-static void context_reload(void *__data)
-{
-        struct mm_struct *mm = __data;
-
-        if (mm == current->mm)
-                load_secondary_context(mm);
-}
-
-void hugetlb_prefault_arch_hook(struct mm_struct *mm)
-{
-        struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE];
-
-        if (likely(tp->tsb != NULL))
-                return;
-
-        tsb_grow(mm, MM_TSB_HUGE, 0);
-        tsb_context_switch(mm);
-        smp_tsb_sync(mm);
-
-        /* On UltraSPARC-III+ and later, configure the second half of
-         * the Data-TLB for huge pages.
-         */
-        if (tlb_type == cheetah_plus) {
-                unsigned long ctx;
-
-                spin_lock(&ctx_alloc_lock);
-                ctx = mm->context.sparc64_ctx_val;
-                ctx &= ~CTX_PGSZ_MASK;
-                ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
-                ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
-
-                if (ctx != mm->context.sparc64_ctx_val) {
-                        /* When changing the page size fields, we
-                         * must perform a context flush so that no
-                         * stale entries match.  This flush must
-                         * occur with the original context register
-                         * settings.
-                         */
-                        do_flush_tlb_mm(mm);
-
-                        /* Reload the context register of all processors
-                         * also executing in this address space.
-                         */
-                        mm->context.sparc64_ctx_val = ctx;
-                        on_each_cpu(context_reload, mm, 0);
-                }
-                spin_unlock(&ctx_alloc_lock);
-        }
-}
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index 12ef4ea60c88..9e28a118e6a4 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -306,12 +306,24 @@ static void flush_dcache(unsigned long pfn)
         }
 }
 
+/* mm->context.lock must be held */
+static void __update_mmu_tsb_insert(struct mm_struct *mm, unsigned long tsb_index,
+                                    unsigned long tsb_hash_shift, unsigned long address,
+                                    unsigned long tte)
+{
+        struct tsb *tsb = mm->context.tsb_block[tsb_index].tsb;
+        unsigned long tag;
+
+        tsb += ((address >> tsb_hash_shift) &
+                (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
+        tag = (address >> 22UL);
+        tsb_insert(tsb, tag, tte);
+}
+
 void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *ptep)
 {
+        unsigned long tsb_index, tsb_hash_shift, flags;
         struct mm_struct *mm;
-        struct tsb *tsb;
-        unsigned long tag, flags;
-        unsigned long tsb_index, tsb_hash_shift;
         pte_t pte = *ptep;
 
         if (tlb_type != hypervisor) {
@@ -328,7 +340,7 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
 
         spin_lock_irqsave(&mm->context.lock, flags);
 
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL) {
                 if ((tlb_type == hypervisor &&
                      (pte_val(pte) & _PAGE_SZALL_4V) == _PAGE_SZHUGE_4V) ||
@@ -340,11 +352,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t *
         }
 #endif
 
-        tsb = mm->context.tsb_block[tsb_index].tsb;
-        tsb += ((address >> tsb_hash_shift) &
-                (mm->context.tsb_block[tsb_index].tsb_nentries - 1UL));
-        tag = (address >> 22UL);
-        tsb_insert(tsb, tag, pte_val(pte));
+        __update_mmu_tsb_insert(mm, tsb_index, tsb_hash_shift,
+                                address, pte_val(pte));
 
         spin_unlock_irqrestore(&mm->context.lock, flags);
 }
@@ -2568,3 +2577,180 @@ void pgtable_free(void *table, bool is_page)
         else
                 kmem_cache_free(pgtable_cache, table);
 }
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static pmd_t pmd_set_protbits(pmd_t pmd, pgprot_t pgprot, bool for_modify)
+{
+        if (pgprot_val(pgprot) & _PAGE_VALID)
+                pmd_val(pmd) |= PMD_HUGE_PRESENT;
+        if (tlb_type == hypervisor) {
+                if (pgprot_val(pgprot) & _PAGE_WRITE_4V)
+                        pmd_val(pmd) |= PMD_HUGE_WRITE;
+                if (pgprot_val(pgprot) & _PAGE_EXEC_4V)
+                        pmd_val(pmd) |= PMD_HUGE_EXEC;
+
+                if (!for_modify) {
+                        if (pgprot_val(pgprot) & _PAGE_ACCESSED_4V)
+                                pmd_val(pmd) |= PMD_HUGE_ACCESSED;
+                        if (pgprot_val(pgprot) & _PAGE_MODIFIED_4V)
+                                pmd_val(pmd) |= PMD_HUGE_DIRTY;
+                }
+        } else {
+                if (pgprot_val(pgprot) & _PAGE_WRITE_4U)
+                        pmd_val(pmd) |= PMD_HUGE_WRITE;
+                if (pgprot_val(pgprot) & _PAGE_EXEC_4U)
+                        pmd_val(pmd) |= PMD_HUGE_EXEC;
+
+                if (!for_modify) {
+                        if (pgprot_val(pgprot) & _PAGE_ACCESSED_4U)
+                                pmd_val(pmd) |= PMD_HUGE_ACCESSED;
+                        if (pgprot_val(pgprot) & _PAGE_MODIFIED_4U)
+                                pmd_val(pmd) |= PMD_HUGE_DIRTY;
+                }
+        }
+
+        return pmd;
+}
+
+pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot)
+{
+        pmd_t pmd;
+
+        pmd_val(pmd) = (page_nr << ((PAGE_SHIFT - PMD_PADDR_SHIFT)));
+        pmd_val(pmd) |= PMD_ISHUGE;
+        pmd = pmd_set_protbits(pmd, pgprot, false);
+        return pmd;
+}
+
+pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+        pmd_val(pmd) &= ~(PMD_HUGE_PRESENT |
+                          PMD_HUGE_WRITE |
+                          PMD_HUGE_EXEC);
+        pmd = pmd_set_protbits(pmd, newprot, true);
+        return pmd;
+}
+
+pgprot_t pmd_pgprot(pmd_t entry)
+{
+        unsigned long pte = 0;
+
+        if (pmd_val(entry) & PMD_HUGE_PRESENT)
+                pte |= _PAGE_VALID;
+
+        if (tlb_type == hypervisor) {
+                if (pmd_val(entry) & PMD_HUGE_PRESENT)
+                        pte |= _PAGE_PRESENT_4V;
+                if (pmd_val(entry) & PMD_HUGE_EXEC)
+                        pte |= _PAGE_EXEC_4V;
+                if (pmd_val(entry) & PMD_HUGE_WRITE)
+                        pte |= _PAGE_W_4V;
+                if (pmd_val(entry) & PMD_HUGE_ACCESSED)
+                        pte |= _PAGE_ACCESSED_4V;
+                if (pmd_val(entry) & PMD_HUGE_DIRTY)
+                        pte |= _PAGE_MODIFIED_4V;
+                pte |= _PAGE_CP_4V|_PAGE_CV_4V;
+        } else {
+                if (pmd_val(entry) & PMD_HUGE_PRESENT)
+                        pte |= _PAGE_PRESENT_4U;
+                if (pmd_val(entry) & PMD_HUGE_EXEC)
+                        pte |= _PAGE_EXEC_4U;
+                if (pmd_val(entry) & PMD_HUGE_WRITE)
+                        pte |= _PAGE_W_4U;
+                if (pmd_val(entry) & PMD_HUGE_ACCESSED)
+                        pte |= _PAGE_ACCESSED_4U;
+                if (pmd_val(entry) & PMD_HUGE_DIRTY)
+                        pte |= _PAGE_MODIFIED_4U;
+                pte |= _PAGE_CP_4U|_PAGE_CV_4U;
+        }
+
+        return __pgprot(pte);
+}
+
+void update_mmu_cache_pmd(struct vm_area_struct *vma, unsigned long addr,
+                          pmd_t *pmd)
+{
+        unsigned long pte, flags;
+        struct mm_struct *mm;
+        pmd_t entry = *pmd;
+        pgprot_t prot;
+
+        if (!pmd_large(entry) || !pmd_young(entry))
+                return;
+
+        pte = (pmd_val(entry) & ~PMD_HUGE_PROTBITS);
+        pte <<= PMD_PADDR_SHIFT;
+        pte |= _PAGE_VALID;
+
+        prot = pmd_pgprot(entry);
+
+        if (tlb_type == hypervisor)
+                pgprot_val(prot) |= _PAGE_SZHUGE_4V;
+        else
+                pgprot_val(prot) |= _PAGE_SZHUGE_4U;
+
+        pte |= pgprot_val(prot);
+
+        mm = vma->vm_mm;
+
+        spin_lock_irqsave(&mm->context.lock, flags);
+
+        if (mm->context.tsb_block[MM_TSB_HUGE].tsb != NULL)
+                __update_mmu_tsb_insert(mm, MM_TSB_HUGE, HPAGE_SHIFT,
+                                        addr, pte);
+
+        spin_unlock_irqrestore(&mm->context.lock, flags);
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
+static void context_reload(void *__data)
+{
+        struct mm_struct *mm = __data;
+
+        if (mm == current->mm)
+                load_secondary_context(mm);
+}
+
+void hugetlb_setup(struct mm_struct *mm)
+{
+        struct tsb_config *tp = &mm->context.tsb_block[MM_TSB_HUGE];
+
+        if (likely(tp->tsb != NULL))
+                return;
+
+        tsb_grow(mm, MM_TSB_HUGE, 0);
+        tsb_context_switch(mm);
+        smp_tsb_sync(mm);
+
+        /* On UltraSPARC-III+ and later, configure the second half of
+         * the Data-TLB for huge pages.
+         */
+        if (tlb_type == cheetah_plus) {
+                unsigned long ctx;
+
+                spin_lock(&ctx_alloc_lock);
+                ctx = mm->context.sparc64_ctx_val;
+                ctx &= ~CTX_PGSZ_MASK;
+                ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT;
+                ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT;
+
+                if (ctx != mm->context.sparc64_ctx_val) {
+                        /* When changing the page size fields, we
+                         * must perform a context flush so that no
+                         * stale entries match.  This flush must
+                         * occur with the original context register
+                         * settings.
+                         */
+                        do_flush_tlb_mm(mm);
+
+                        /* Reload the context register of all processors
+                         * also executing in this address space.
+                         */
+                        mm->context.sparc64_ctx_val = ctx;
+                        on_each_cpu(context_reload, mm, 0);
+                }
+                spin_unlock(&ctx_alloc_lock);
+        }
+}
+#endif
diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c
index b1f279cd00bf..3e8fec391fe0 100644
--- a/arch/sparc/mm/tlb.c
+++ b/arch/sparc/mm/tlb.c
@@ -43,16 +43,37 @@ void flush_tlb_pending(void)
         put_cpu_var(tlb_batch);
 }
 
-void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
-                   pte_t *ptep, pte_t orig, int fullmm)
+static void tlb_batch_add_one(struct mm_struct *mm, unsigned long vaddr,
+                              bool exec)
 {
         struct tlb_batch *tb = &get_cpu_var(tlb_batch);
         unsigned long nr;
 
         vaddr &= PAGE_MASK;
-        if (pte_exec(orig))
+        if (exec)
                 vaddr |= 0x1UL;
 
+        nr = tb->tlb_nr;
+
+        if (unlikely(nr != 0 && mm != tb->mm)) {
+                flush_tlb_pending();
+                nr = 0;
+        }
+
+        if (nr == 0)
+                tb->mm = mm;
+
+        tb->vaddrs[nr] = vaddr;
+        tb->tlb_nr = ++nr;
+        if (nr >= TLB_BATCH_NR)
+                flush_tlb_pending();
+
+        put_cpu_var(tlb_batch);
+}
+
+void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
+                   pte_t *ptep, pte_t orig, int fullmm)
+{
         if (tlb_type != hypervisor &&
             pte_dirty(orig)) {
                 unsigned long paddr, pfn = pte_pfn(orig);
@@ -77,26 +98,91 @@ void tlb_batch_add(struct mm_struct *mm, unsigned long vaddr,
         }
 
 no_cache_flush:
+        if (!fullmm)
+                tlb_batch_add_one(mm, vaddr, pte_exec(orig));
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static void tlb_batch_pmd_scan(struct mm_struct *mm, unsigned long vaddr,
+                               pmd_t pmd, bool exec)
+{
+        unsigned long end;
+        pte_t *pte;
+
+        pte = pte_offset_map(&pmd, vaddr);
+        end = vaddr + HPAGE_SIZE;
+        while (vaddr < end) {
+                if (pte_val(*pte) & _PAGE_VALID)
+                        tlb_batch_add_one(mm, vaddr, exec);
+                pte++;
+                vaddr += PAGE_SIZE;
+        }
+        pte_unmap(pte);
+}
 
-        if (fullmm) {
-                put_cpu_var(tlb_batch);
+void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+                pmd_t *pmdp, pmd_t pmd)
+{
+        pmd_t orig = *pmdp;
+
+        *pmdp = pmd;
+
+        if (mm == &init_mm)
                 return;
+
+        if ((pmd_val(pmd) ^ pmd_val(orig)) & PMD_ISHUGE) {
+                if (pmd_val(pmd) & PMD_ISHUGE)
+                        mm->context.huge_pte_count++;
+                else
+                        mm->context.huge_pte_count--;
+                if (mm->context.huge_pte_count == 1)
+                        hugetlb_setup(mm);
         }
 
-        nr = tb->tlb_nr;
+        if (!pmd_none(orig)) {
+                bool exec = ((pmd_val(orig) & PMD_HUGE_EXEC) != 0);
 
-        if (unlikely(nr != 0 && mm != tb->mm)) {
-                flush_tlb_pending();
-                nr = 0;
+                addr &= HPAGE_MASK;
+                if (pmd_val(orig) & PMD_ISHUGE)
+                        tlb_batch_add_one(mm, addr, exec);
+                else
+                        tlb_batch_pmd_scan(mm, addr, orig, exec);
         }
+}
 
-        if (nr == 0)
-                tb->mm = mm;
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
+{
+        struct list_head *lh = (struct list_head *) pgtable;
 
-        tb->vaddrs[nr] = vaddr;
-        tb->tlb_nr = ++nr;
-        if (nr >= TLB_BATCH_NR)
-                flush_tlb_pending();
+        assert_spin_locked(&mm->page_table_lock);
 
-        put_cpu_var(tlb_batch);
+        /* FIFO */
+        if (!mm->pmd_huge_pte)
+                INIT_LIST_HEAD(lh);
+        else
+                list_add(lh, (struct list_head *) mm->pmd_huge_pte);
+        mm->pmd_huge_pte = pgtable;
+}
+
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
+{
+        struct list_head *lh;
+        pgtable_t pgtable;
+
+        assert_spin_locked(&mm->page_table_lock);
+
+        /* FIFO */
+        pgtable = mm->pmd_huge_pte;
+        lh = (struct list_head *) pgtable;
+        if (list_empty(lh))
+                mm->pmd_huge_pte = NULL;
+        else {
+                mm->pmd_huge_pte = (pgtable_t) lh->next;
+                list_del(lh);
+        }
+        pte_val(pgtable[0]) = 0;
+        pte_val(pgtable[1]) = 0;
+
+        return pgtable;
 }
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c
index a35ee832baf3..7f6474347491 100644
--- a/arch/sparc/mm/tsb.c
+++ b/arch/sparc/mm/tsb.c
@@ -78,7 +78,7 @@ void flush_tsb_user(struct tlb_batch *tb)
                 base = __pa(base);
         __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
 
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
                 base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
                 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
@@ -93,7 +93,7 @@ void flush_tsb_user(struct tlb_batch *tb)
 #define HV_PGSZ_IDX_BASE        HV_PGSZ_IDX_8K
 #define HV_PGSZ_MASK_BASE       HV_PGSZ_MASK_8K
 
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
 #define HV_PGSZ_IDX_HUGE        HV_PGSZ_IDX_4MB
 #define HV_PGSZ_MASK_HUGE       HV_PGSZ_MASK_4MB
 #endif
@@ -190,7 +190,7 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign
                 case MM_TSB_BASE:
                         hp->pgsz_idx = HV_PGSZ_IDX_BASE;
                         break;
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
                 case MM_TSB_HUGE:
                         hp->pgsz_idx = HV_PGSZ_IDX_HUGE;
                         break;
@@ -205,7 +205,7 @@ static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsign
                 case MM_TSB_BASE:
                         hp->pgsz_mask = HV_PGSZ_MASK_BASE;
                         break;
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
                 case MM_TSB_HUGE:
                         hp->pgsz_mask = HV_PGSZ_MASK_HUGE;
                         break;
@@ -427,7 +427,7 @@ retry_tsb_alloc:
 
 int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 {
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         unsigned long huge_pte_count;
 #endif
         unsigned int i;
@@ -436,7 +436,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
 
         mm->context.sparc64_ctx_val = 0UL;
 
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         /* We reset it to zero because the fork() page copying
          * will re-increment the counters as the parent PTEs are
          * copied into the child address space.
@@ -459,7 +459,7 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
          */
         tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
 
-#ifdef CONFIG_HUGETLB_PAGE
+#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
         if (unlikely(huge_pte_count))
                 tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
 #endif