1 files changed, 49 insertions, 37 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3afb5cbe1312..078832cf3636 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -67,6 +67,7 @@ static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1;
 
 static int khugepaged(void *none);
 static int khugepaged_slab_init(void);
+static void khugepaged_slab_exit(void);
 
 #define MM_SLOTS_HASH_BITS 10
 static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
@@ -109,9 +110,6 @@ static int set_recommended_min_free_kbytes(void)
         int nr_zones = 0;
         unsigned long recommended_min;
 
-        if (!khugepaged_enabled())
-                return 0;
-
         for_each_populated_zone(zone)
                 nr_zones++;
 
@@ -143,9 +141,8 @@ static int set_recommended_min_free_kbytes(void)
         setup_per_zone_wmarks();
         return 0;
 }
-late_initcall(set_recommended_min_free_kbytes);
 
-static int start_khugepaged(void)
+static int start_stop_khugepaged(void)
 {
         int err = 0;
         if (khugepaged_enabled()) {
@@ -156,6 +153,7 @@ static int start_khugepaged(void)
                         pr_err("khugepaged: kthread_run(khugepaged) failed\n");
                         err = PTR_ERR(khugepaged_thread);
                         khugepaged_thread = NULL;
+                        goto fail;
                 }
 
                 if (!list_empty(&khugepaged_scan.mm_head))
@@ -166,7 +164,7 @@ static int start_khugepaged(void)
                 kthread_stop(khugepaged_thread);
                 khugepaged_thread = NULL;
         }
-
+fail:
         return err;
 }
 
@@ -183,7 +181,7 @@ static struct page *get_huge_zero_page(void)
         struct page *zero_page;
 retry:
         if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
-                return ACCESS_ONCE(huge_zero_page);
+                return READ_ONCE(huge_zero_page);
 
         zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
                         HPAGE_PMD_ORDER);
@@ -202,7 +200,7 @@ retry:
         /* We take additional reference here. It will be put back by shrinker */
         atomic_set(&huge_zero_refcount, 2);
         preempt_enable();
-        return ACCESS_ONCE(huge_zero_page);
+        return READ_ONCE(huge_zero_page);
 }
 
 static void put_huge_zero_page(void)
@@ -300,7 +298,7 @@ static ssize_t enabled_store(struct kobject *kobj,
                 int err;
 
                 mutex_lock(&khugepaged_mutex);
-                err = start_khugepaged();
+                err = start_stop_khugepaged();
                 mutex_unlock(&khugepaged_mutex);
 
                 if (err)
@@ -634,27 +632,38 @@ static int __init hugepage_init(void)
 
         err = hugepage_init_sysfs(&hugepage_kobj);
         if (err)
-                return err;
+                goto err_sysfs;
 
         err = khugepaged_slab_init();
         if (err)
-                goto out;
+                goto err_slab;
 
-        register_shrinker(&huge_zero_page_shrinker);
+        err = register_shrinker(&huge_zero_page_shrinker);
+        if (err)
+                goto err_hzp_shrinker;
 
         /*
          * By default disable transparent hugepages on smaller systems,
          * where the extra memory used could hurt more than TLB overhead
          * is likely to save.  The admin can still enable it through /sys.
          */
-        if (totalram_pages < (512 << (20 - PAGE_SHIFT)))
+        if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
                 transparent_hugepage_flags = 0;
+                return 0;
+        }
 
-        start_khugepaged();
+        err = start_stop_khugepaged();
+        if (err)
+                goto err_khugepaged;
 
         return 0;
-out:
+err_khugepaged:
+        unregister_shrinker(&huge_zero_page_shrinker);
+err_hzp_shrinker:
+        khugepaged_slab_exit();
+err_slab:
         hugepage_exit_sysfs(hugepage_kobj);
+err_sysfs:
         return err;
 }
 subsys_initcall(hugepage_init);
@@ -708,7 +717,7 @@ static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
 static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
                                         struct vm_area_struct *vma,
                                         unsigned long haddr, pmd_t *pmd,
-                                        struct page *page)
+                                        struct page *page, gfp_t gfp)
 {
         struct mem_cgroup *memcg;
         pgtable_t pgtable;
@@ -716,7 +725,7 @@ static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
 
         VM_BUG_ON_PAGE(!PageCompound(page), page);
 
-        if (mem_cgroup_try_charge(page, mm, GFP_TRANSHUGE, &memcg))
+        if (mem_cgroup_try_charge(page, mm, gfp, &memcg))
                 return VM_FAULT_OOM;
 
         pgtable = pte_alloc_one(mm, haddr);
@@ -822,7 +831,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 count_vm_event(THP_FAULT_FALLBACK);
                 return VM_FAULT_FALLBACK;
         }
-        if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page))) {
+        if (unlikely(__do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page, gfp))) {
                 put_page(page);
                 count_vm_event(THP_FAULT_FALLBACK);
                 return VM_FAULT_FALLBACK;
@@ -1080,6 +1089,7 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
         unsigned long haddr;
         unsigned long mmun_start;       /* For mmu_notifiers */
         unsigned long mmun_end;         /* For mmu_notifiers */
+        gfp_t huge_gfp;                 /* for allocation and charge */
 
         ptl = pmd_lockptr(mm, pmd);
         VM_BUG_ON_VMA(!vma->anon_vma, vma);
@@ -1106,10 +1116,8 @@ int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
 alloc:
         if (transparent_hugepage_enabled(vma) &&
             !transparent_hugepage_debug_cow()) {
-                gfp_t gfp;
-
-                gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
-                new_page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
+                huge_gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma), 0);
+                new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
         } else
                 new_page = NULL;
 
@@ -1130,8 +1138,7 @@ alloc:
                 goto out;
         }
 
-        if (unlikely(mem_cgroup_try_charge(new_page, mm,
-                                           GFP_TRANSHUGE, &memcg))) {
+        if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg))) {
                 put_page(new_page);
                 if (page) {
                         split_huge_page(page);
@@ -1976,6 +1983,11 @@ static int __init khugepaged_slab_init(void)
         return 0;
 }
 
+static void __init khugepaged_slab_exit(void)
+{
+        kmem_cache_destroy(mm_slot_cache);
+}
+
 static inline struct mm_slot *alloc_mm_slot(void)
 {
         if (!mm_slot_cache)     /* initialization failed */
@@ -2323,19 +2335,13 @@ static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
         return true;
 }
 
-static struct page
-*khugepaged_alloc_page(struct page **hpage, struct mm_struct *mm,
+static struct page *
+khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
                        struct vm_area_struct *vma, unsigned long address,
                        int node)
 {
-        gfp_t flags;
-
         VM_BUG_ON_PAGE(*hpage, *hpage);
 
-        /* Only allocate from the target node */
-        flags = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) |
-                __GFP_THISNODE;
-
         /*
          * Before allocating the hugepage, release the mmap_sem read lock.
          * The allocation can take potentially a long time if it involves
@@ -2344,7 +2350,7 @@ static struct page
          */
         up_read(&mm->mmap_sem);
 
-        *hpage = alloc_pages_exact_node(node, flags, HPAGE_PMD_ORDER);
+        *hpage = alloc_pages_exact_node(node, gfp, HPAGE_PMD_ORDER);
         if (unlikely(!*hpage)) {
                 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
                 *hpage = ERR_PTR(-ENOMEM);
@@ -2397,13 +2403,14 @@ static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
         return true;
 }
 
-static struct page
-*khugepaged_alloc_page(struct page **hpage, struct mm_struct *mm,
+static struct page *
+khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
                        struct vm_area_struct *vma, unsigned long address,
                        int node)
 {
         up_read(&mm->mmap_sem);
         VM_BUG_ON(!*hpage);
+
         return  *hpage;
 }
 #endif
@@ -2438,16 +2445,21 @@ static void collapse_huge_page(struct mm_struct *mm,
         struct mem_cgroup *memcg;
         unsigned long mmun_start;       /* For mmu_notifiers */
         unsigned long mmun_end;         /* For mmu_notifiers */
+        gfp_t gfp;
 
         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 
+        /* Only allocate from the target node */
+        gfp = alloc_hugepage_gfpmask(khugepaged_defrag(), __GFP_OTHER_NODE) |
+                __GFP_THISNODE;
+
         /* release the mmap_sem read lock. */
-        new_page = khugepaged_alloc_page(hpage, mm, vma, address, node);
+        new_page = khugepaged_alloc_page(hpage, gfp, mm, vma, address, node);
         if (!new_page)
                 return;
 
         if (unlikely(mem_cgroup_try_charge(new_page, mm,
-                                           GFP_TRANSHUGE, &memcg)))
+                                           gfp, &memcg)))
                 return;
 
         /*