9 files changed, 341 insertions, 19 deletions

diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index c8af429991d9..ceff7aef2477 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -68,6 +68,7 @@ config S390
         select HAVE_FTRACE_MCOUNT_RECORD
         select HAVE_C_RECORDMCOUNT
         select HAVE_SYSCALL_TRACEPOINTS
+        select SYSCTL_EXCEPTION_TRACE
         select HAVE_DYNAMIC_FTRACE
         select HAVE_FUNCTION_GRAPH_TRACER
         select HAVE_REGS_AND_STACK_ACCESS_API
@@ -80,6 +81,7 @@ config S390
         select HAVE_IRQ_WORK
         select HAVE_PERF_EVENTS
         select ARCH_HAVE_NMI_SAFE_CMPXCHG
+        select HAVE_DEBUG_KMEMLEAK
         select HAVE_KERNEL_GZIP
         select HAVE_KERNEL_BZIP2
         select HAVE_KERNEL_LZMA
@@ -126,6 +128,7 @@ config S390
         select ARCH_INLINE_WRITE_UNLOCK_BH
         select ARCH_INLINE_WRITE_UNLOCK_IRQ
         select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE
+        select HAVE_UID16 if 32BIT
         select ARCH_WANT_IPC_PARSE_VERSION
         select GENERIC_SMP_IDLE_THREAD
         select GENERIC_TIME_VSYSCALL
diff --git a/arch/s390/include/asm/hugetlb.h b/arch/s390/include/asm/hugetlb.h
index 2d6e6e380564..593753ee07f3 100644
--- a/arch/s390/include/asm/hugetlb.h
+++ b/arch/s390/include/asm/hugetlb.h
@@ -33,6 +33,7 @@ static inline int prepare_hugepage_range(struct file *file,
 }
 
 #define hugetlb_prefault_arch_hook(mm)          do { } while (0)
+#define arch_clear_hugepage_flags(page)         do { } while (0)
 
 int arch_prepare_hugepage(struct page *page);
 void arch_release_hugepage(struct page *page);
@@ -77,23 +78,6 @@ static inline void __pmd_csp(pmd_t *pmdp)
                 "       csp %1,%3"
                 : "=m" (*pmdp)
                 : "d" (reg2), "d" (reg3), "d" (reg4), "m" (*pmdp) : "cc");
-        pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
-}
-
-static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
-{
-        unsigned long sto = (unsigned long) pmdp -
-                                pmd_index(address) * sizeof(pmd_t);
-
-        if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
-                asm volatile(
-                        "       .insn   rrf,0xb98e0000,%2,%3,0,0"
-                        : "=m" (*pmdp)
-                        : "m" (*pmdp), "a" (sto),
-                          "a" ((address & HPAGE_MASK))
-                );
-        }
-        pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
 }
 
 static inline void huge_ptep_invalidate(struct mm_struct *mm,
@@ -105,6 +89,7 @@ static inline void huge_ptep_invalidate(struct mm_struct *mm,
                 __pmd_idte(address, pmdp);
         else
                 __pmd_csp(pmdp);
+        pmd_val(*pmdp) = _SEGMENT_ENTRY_INV | _SEGMENT_ENTRY;
 }
 
 static inline pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 6bd7d7483017..979fe3dc0788 100644
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -42,6 +42,7 @@ extern void fault_init(void);
  * tables contain all the necessary information.
  */
 #define update_mmu_cache(vma, address, ptep)     do { } while (0)
+#define update_mmu_cache_pmd(vma, address, ptep) do { } while (0)
 
 /*
  * ZERO_PAGE is a global shared page that is always zero; used
@@ -347,6 +348,12 @@ extern struct page *vmemmap;
 
 #define _SEGMENT_ENTRY_LARGE    0x400   /* STE-format control, large page   */
 #define _SEGMENT_ENTRY_CO       0x100   /* change-recording override   */
+#define _SEGMENT_ENTRY_SPLIT_BIT 0      /* THP splitting bit number */
+#define _SEGMENT_ENTRY_SPLIT    (1UL << _SEGMENT_ENTRY_SPLIT_BIT)
+
+/* Set of bits not changed in pmd_modify */
+#define _SEGMENT_CHG_MASK       (_SEGMENT_ENTRY_ORIGIN | _SEGMENT_ENTRY_LARGE \
+                                 | _SEGMENT_ENTRY_SPLIT | _SEGMENT_ENTRY_CO)
 
 /* Page status table bits for virtualization */
 #define RCP_ACC_BITS    0xf000000000000000UL
@@ -506,6 +513,30 @@ static inline int pmd_bad(pmd_t pmd)
         return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
 }
 
+#define __HAVE_ARCH_PMDP_SPLITTING_FLUSH
+extern void pmdp_splitting_flush(struct vm_area_struct *vma,
+                                 unsigned long addr, pmd_t *pmdp);
+
+#define  __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
+extern int pmdp_set_access_flags(struct vm_area_struct *vma,
+                                 unsigned long address, pmd_t *pmdp,
+                                 pmd_t entry, int dirty);
+
+#define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH
+extern int pmdp_clear_flush_young(struct vm_area_struct *vma,
+                                  unsigned long address, pmd_t *pmdp);
+
+#define __HAVE_ARCH_PMD_WRITE
+static inline int pmd_write(pmd_t pmd)
+{
+        return (pmd_val(pmd) & _SEGMENT_ENTRY_RO) == 0;
+}
+
+static inline int pmd_young(pmd_t pmd)
+{
+        return 0;
+}
+
 static inline int pte_none(pte_t pte)
 {
         return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
@@ -1159,6 +1190,185 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
 #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
 #define pte_unmap(pte) do { } while (0)
 
+static inline void __pmd_idte(unsigned long address, pmd_t *pmdp)
+{
+        unsigned long sto = (unsigned long) pmdp -
+                            pmd_index(address) * sizeof(pmd_t);
+
+        if (!(pmd_val(*pmdp) & _SEGMENT_ENTRY_INV)) {
+                asm volatile(
+                        "       .insn   rrf,0xb98e0000,%2,%3,0,0"
+                        : "=m" (*pmdp)
+                        : "m" (*pmdp), "a" (sto),
+                          "a" ((address & HPAGE_MASK))
+                        : "cc"
+                );
+        }
+}
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+#define __HAVE_ARCH_PGTABLE_DEPOSIT
+extern void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable);
+
+#define __HAVE_ARCH_PGTABLE_WITHDRAW
+extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm);
+
+static inline int pmd_trans_splitting(pmd_t pmd)
+{
+        return pmd_val(pmd) & _SEGMENT_ENTRY_SPLIT;
+}
+
+static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
+                              pmd_t *pmdp, pmd_t entry)
+{
+        *pmdp = entry;
+}
+
+static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot)
+{
+        unsigned long pgprot_pmd = 0;
+
+        if (pgprot_val(pgprot) & _PAGE_INVALID) {
+                if (pgprot_val(pgprot) & _PAGE_SWT)
+                        pgprot_pmd |= _HPAGE_TYPE_NONE;
+                pgprot_pmd |= _SEGMENT_ENTRY_INV;
+        }
+        if (pgprot_val(pgprot) & _PAGE_RO)
+                pgprot_pmd |= _SEGMENT_ENTRY_RO;
+        return pgprot_pmd;
+}
+
+static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
+{
+        pmd_val(pmd) &= _SEGMENT_CHG_MASK;
+        pmd_val(pmd) |= massage_pgprot_pmd(newprot);
+        return pmd;
+}
+
+static inline pmd_t pmd_mkhuge(pmd_t pmd)
+{
+        pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
+        return pmd;
+}
+
+static inline pmd_t pmd_mkwrite(pmd_t pmd)
+{
+        pmd_val(pmd) &= ~_SEGMENT_ENTRY_RO;
+        return pmd;
+}
+
+static inline pmd_t pmd_wrprotect(pmd_t pmd)
+{
+        pmd_val(pmd) |= _SEGMENT_ENTRY_RO;
+        return pmd;
+}
+
+static inline pmd_t pmd_mkdirty(pmd_t pmd)
+{
+        /* No dirty bit in the segment table entry. */
+        return pmd;
+}
+
+static inline pmd_t pmd_mkold(pmd_t pmd)
+{
+        /* No referenced bit in the segment table entry. */
+        return pmd;
+}
+
+static inline pmd_t pmd_mkyoung(pmd_t pmd)
+{
+        /* No referenced bit in the segment table entry. */
+        return pmd;
+}
+
+#define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG
+static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma,
+                                            unsigned long address, pmd_t *pmdp)
+{
+        unsigned long pmd_addr = pmd_val(*pmdp) & HPAGE_MASK;
+        long tmp, rc;
+        int counter;
+
+        rc = 0;
+        if (MACHINE_HAS_RRBM) {
+                counter = PTRS_PER_PTE >> 6;
+                asm volatile(
+                        "0:     .insn   rre,0xb9ae0000,%0,%3\n" /* rrbm */
+                        "       ogr     %1,%0\n"
+                        "       la      %3,0(%4,%3)\n"
+                        "       brct    %2,0b\n"
+                        : "=&d" (tmp), "+&d" (rc), "+d" (counter),
+                          "+a" (pmd_addr)
+                        : "a" (64 * 4096UL) : "cc");
+                rc = !!rc;
+        } else {
+                counter = PTRS_PER_PTE;
+                asm volatile(
+                        "0:     rrbe    0,%2\n"
+                        "       la      %2,0(%3,%2)\n"
+                        "       brc     12,1f\n"
+                        "       lhi     %0,1\n"
+                        "1:     brct    %1,0b\n"
+                        : "+d" (rc), "+d" (counter), "+a" (pmd_addr)
+                        : "a" (4096UL) : "cc");
+        }
+        return rc;
+}
+
+#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
+static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
+                                       unsigned long address, pmd_t *pmdp)
+{
+        pmd_t pmd = *pmdp;
+
+        __pmd_idte(address, pmdp);
+        pmd_clear(pmdp);
+        return pmd;
+}
+
+#define __HAVE_ARCH_PMDP_CLEAR_FLUSH
+static inline pmd_t pmdp_clear_flush(struct vm_area_struct *vma,
+                                     unsigned long address, pmd_t *pmdp)
+{
+        return pmdp_get_and_clear(vma->vm_mm, address, pmdp);
+}
+
+#define __HAVE_ARCH_PMDP_INVALIDATE
+static inline void pmdp_invalidate(struct vm_area_struct *vma,
+                                   unsigned long address, pmd_t *pmdp)
+{
+        __pmd_idte(address, pmdp);
+}
+
+static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot)
+{
+        pmd_t __pmd;
+        pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot);
+        return __pmd;
+}
+
+#define pfn_pmd(pfn, pgprot)    mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot))
+#define mk_pmd(page, pgprot)    pfn_pmd(page_to_pfn(page), (pgprot))
+
+static inline int pmd_trans_huge(pmd_t pmd)
+{
+        return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE;
+}
+
+static inline int has_transparent_hugepage(void)
+{
+        return MACHINE_HAS_HPAGE ? 1 : 0;
+}
+
+static inline unsigned long pmd_pfn(pmd_t pmd)
+{
+        if (pmd_trans_huge(pmd))
+                return pmd_val(pmd) >> HPAGE_SHIFT;
+        else
+                return pmd_val(pmd) >> PAGE_SHIFT;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
 /*
  * 31 bit swap entry format:
  * A page-table entry has some bits we have to treat in a special way.
diff --git a/arch/s390/include/asm/setup.h b/arch/s390/include/asm/setup.h
index 87b47ca954f1..8cfd731a18d8 100644
--- a/arch/s390/include/asm/setup.h
+++ b/arch/s390/include/asm/setup.h
@@ -81,6 +81,7 @@ extern unsigned int s390_user_mode;
 #define MACHINE_FLAG_SPP        (1UL << 13)
 #define MACHINE_FLAG_TOPOLOGY   (1UL << 14)
 #define MACHINE_FLAG_TE         (1UL << 15)
+#define MACHINE_FLAG_RRBM       (1UL << 16)
 
 #define MACHINE_IS_VM           (S390_lowcore.machine_flags & MACHINE_FLAG_VM)
 #define MACHINE_IS_KVM          (S390_lowcore.machine_flags & MACHINE_FLAG_KVM)
@@ -99,7 +100,8 @@ extern unsigned int s390_user_mode;
 #define MACHINE_HAS_PFMF        (0)
 #define MACHINE_HAS_SPP         (0)
 #define MACHINE_HAS_TOPOLOGY    (0)
-#define MACHINE_HAS_TE                (0)
+#define MACHINE_HAS_TE          (0)
+#define MACHINE_HAS_RRBM        (0)
 #else /* CONFIG_64BIT */
 #define MACHINE_HAS_IEEE        (1)
 #define MACHINE_HAS_CSP         (1)
@@ -112,6 +114,7 @@ extern unsigned int s390_user_mode;
 #define MACHINE_HAS_SPP         (S390_lowcore.machine_flags & MACHINE_FLAG_SPP)
 #define MACHINE_HAS_TOPOLOGY    (S390_lowcore.machine_flags & MACHINE_FLAG_TOPOLOGY)
 #define MACHINE_HAS_TE          (S390_lowcore.machine_flags & MACHINE_FLAG_TE)
+#define MACHINE_HAS_RRBM        (S390_lowcore.machine_flags & MACHINE_FLAG_RRBM)
 #endif /* CONFIG_64BIT */
 
 #define ZFCPDUMP_HSA_SIZE       (32UL<<20)
diff --git a/arch/s390/include/asm/tlb.h b/arch/s390/include/asm/tlb.h
index 06e5acbc84bd..b75d7d686684 100644
--- a/arch/s390/include/asm/tlb.h
+++ b/arch/s390/include/asm/tlb.h
@@ -137,6 +137,7 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
 #define tlb_start_vma(tlb, vma)                 do { } while (0)
 #define tlb_end_vma(tlb, vma)                   do { } while (0)
 #define tlb_remove_tlb_entry(tlb, ptep, addr)   do { } while (0)
+#define tlb_remove_pmd_tlb_entry(tlb, pmdp, addr)       do { } while (0)
 #define tlb_migrate_finish(mm)                  do { } while (0)
 
 #endif /* _S390_TLB_H */
diff --git a/arch/s390/kernel/early.c b/arch/s390/kernel/early.c
index 7f4717675c19..00d114445068 100644
--- a/arch/s390/kernel/early.c
+++ b/arch/s390/kernel/early.c
@@ -388,6 +388,8 @@ static __init void detect_machine_facilities(void)
                 S390_lowcore.machine_flags |= MACHINE_FLAG_SPP;
         if (test_facility(50) && test_facility(73))
                 S390_lowcore.machine_flags |= MACHINE_FLAG_TE;
+        if (test_facility(66))
+                S390_lowcore.machine_flags |= MACHINE_FLAG_RRBM;
 #endif
 }
 
diff --git a/arch/s390/mm/fault.c b/arch/s390/mm/fault.c
index ac9122ca1152..04ad4001a289 100644
--- a/arch/s390/mm/fault.c
+++ b/arch/s390/mm/fault.c
@@ -367,6 +367,7 @@ retry:
                         /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
                          * of starvation. */
                         flags &= ~FAULT_FLAG_ALLOW_RETRY;
+                        flags |= FAULT_FLAG_TRIED;
                         down_read(&mm->mmap_sem);
                         goto retry;
                 }
diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c
index eeaf8023851f..60acb93a4680 100644
--- a/arch/s390/mm/gup.c
+++ b/arch/s390/mm/gup.c
@@ -115,7 +115,16 @@ static inline int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr,
                 pmd = *pmdp;
                 barrier();
                 next = pmd_addr_end(addr, end);
-                if (pmd_none(pmd))
+                /*
+                 * The pmd_trans_splitting() check below explains why
+                 * pmdp_splitting_flush() has to serialize with
+                 * smp_call_function() against our disabled IRQs, to stop
+                 * this gup-fast code from running while we set the
+                 * splitting bit in the pmd. Returning zero will take
+                 * the slow path that will call wait_split_huge_page()
+                 * if the pmd is still in splitting state.
+                 */
+                if (pmd_none(pmd) || pmd_trans_splitting(pmd))
                         return 0;
                 if (unlikely(pmd_huge(pmd))) {
                         if (!gup_huge_pmd(pmdp, pmd, addr, next,
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index b402991e43d7..c8188a18af05 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -787,6 +787,30 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
                 tlb_table_flush(tlb);
 }
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+void thp_split_vma(struct vm_area_struct *vma)
+{
+        unsigned long addr;
+        struct page *page;
+
+        for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
+                page = follow_page(vma, addr, FOLL_SPLIT);
+        }
+}
+
+void thp_split_mm(struct mm_struct *mm)
+{
+        struct vm_area_struct *vma = mm->mmap;
+
+        while (vma != NULL) {
+                thp_split_vma(vma);
+                vma->vm_flags &= ~VM_HUGEPAGE;
+                vma->vm_flags |= VM_NOHUGEPAGE;
+                vma = vma->vm_next;
+        }
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
+
 /*
  * switch on pgstes for its userspace process (for kvm)
  */
@@ -824,6 +848,12 @@ int s390_enable_sie(void)
         if (!mm)
                 return -ENOMEM;
 
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+        /* split thp mappings and disable thp for future mappings */
+        thp_split_mm(mm);
+        mm->def_flags |= VM_NOHUGEPAGE;
+#endif
+
         /* Now lets check again if something happened */
         task_lock(tsk);
         if (!tsk->mm || atomic_read(&tsk->mm->mm_users) > 1 ||
@@ -866,3 +896,81 @@ bool kernel_page_present(struct page *page)
         return cc == 0;
 }
 #endif /* CONFIG_HIBERNATION && CONFIG_DEBUG_PAGEALLOC */
+
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address,
+                           pmd_t *pmdp)
+{
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+        /* No need to flush TLB
+         * On s390 reference bits are in storage key and never in TLB */
+        return pmdp_test_and_clear_young(vma, address, pmdp);
+}
+
+int pmdp_set_access_flags(struct vm_area_struct *vma,
+                          unsigned long address, pmd_t *pmdp,
+                          pmd_t entry, int dirty)
+{
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+
+        if (pmd_same(*pmdp, entry))
+                return 0;
+        pmdp_invalidate(vma, address, pmdp);
+        set_pmd_at(vma->vm_mm, address, pmdp, entry);
+        return 1;
+}
+
+static void pmdp_splitting_flush_sync(void *arg)
+{
+        /* Simply deliver the interrupt */
+}
+
+void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long address,
+                          pmd_t *pmdp)
+{
+        VM_BUG_ON(address & ~HPAGE_PMD_MASK);
+        if (!test_and_set_bit(_SEGMENT_ENTRY_SPLIT_BIT,
+                              (unsigned long *) pmdp)) {
+                /* need to serialize against gup-fast (IRQ disabled) */
+                smp_call_function(pmdp_splitting_flush_sync, NULL, 1);
+        }
+}
+
+void pgtable_trans_huge_deposit(struct mm_struct *mm, pgtable_t pgtable)
+{
+        struct list_head *lh = (struct list_head *) pgtable;
+
+        assert_spin_locked(&mm->page_table_lock);
+
+        /* FIFO */
+        if (!mm->pmd_huge_pte)
+                INIT_LIST_HEAD(lh);
+        else
+                list_add(lh, (struct list_head *) mm->pmd_huge_pte);
+        mm->pmd_huge_pte = pgtable;
+}
+
+pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm)
+{
+        struct list_head *lh;
+        pgtable_t pgtable;
+        pte_t *ptep;
+
+        assert_spin_locked(&mm->page_table_lock);
+
+        /* FIFO */
+        pgtable = mm->pmd_huge_pte;
+        lh = (struct list_head *) pgtable;
+        if (list_empty(lh))
+                mm->pmd_huge_pte = NULL;
+        else {
+                mm->pmd_huge_pte = (pgtable_t) lh->next;
+                list_del(lh);
+        }
+        ptep = (pte_t *) pgtable;
+        pte_val(*ptep) = _PAGE_TYPE_EMPTY;
+        ptep++;
+        pte_val(*ptep) = _PAGE_TYPE_EMPTY;
+        return pgtable;
+}
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */