12 files changed, 91 insertions, 23 deletions

diff --git a/mm/gup.c b/mm/gup.c
index cd62c8c90d4a..0ca1df9075ab 100644
--- a/mm/gup.c
+++ b/mm/gup.c
@@ -3,7 +3,6 @@
 #include <linux/err.h>
 #include <linux/spinlock.h>
 
-#include <linux/hugetlb.h>
 #include <linux/mm.h>
 #include <linux/pagemap.h>
 #include <linux/rmap.h>
@@ -12,6 +11,7 @@
 
 #include <linux/sched.h>
 #include <linux/rwsem.h>
+#include <linux/hugetlb.h>
 #include <asm/pgtable.h>
 
 #include "internal.h"
@@ -875,6 +875,49 @@ static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
         return 1;
 }
 
+static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
+                        unsigned long end, int write,
+                        struct page **pages, int *nr)
+{
+        int refs;
+        struct page *head, *page, *tail;
+
+        if (write && !pgd_write(orig))
+                return 0;
+
+        refs = 0;
+        head = pgd_page(orig);
+        page = head + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
+        tail = page;
+        do {
+                VM_BUG_ON_PAGE(compound_head(page) != head, page);
+                pages[*nr] = page;
+                (*nr)++;
+                page++;
+                refs++;
+        } while (addr += PAGE_SIZE, addr != end);
+
+        if (!page_cache_add_speculative(head, refs)) {
+                *nr -= refs;
+                return 0;
+        }
+
+        if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
+                *nr -= refs;
+                while (refs--)
+                        put_page(head);
+                return 0;
+        }
+
+        while (refs--) {
+                if (PageTail(tail))
+                        get_huge_page_tail(tail);
+                tail++;
+        }
+
+        return 1;
+}
+
 static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                 int write, struct page **pages, int *nr)
 {
@@ -902,6 +945,14 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
                                 pages, nr))
                                 return 0;
 
+                } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
+                        /*
+                         * architecture have different format for hugetlbfs
+                         * pmd format and THP pmd format
+                         */
+                        if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
+                                         PMD_SHIFT, next, write, pages, nr))
+                                return 0;
                 } else if (!gup_pte_range(pmd, addr, next, write, pages, nr))
                                 return 0;
         } while (pmdp++, addr = next, addr != end);
@@ -909,22 +960,26 @@ static int gup_pmd_range(pud_t pud, unsigned long addr, unsigned long end,
         return 1;
 }
 
-static int gup_pud_range(pgd_t *pgdp, unsigned long addr, unsigned long end,
-                int write, struct page **pages, int *nr)
+static int gup_pud_range(pgd_t pgd, unsigned long addr, unsigned long end,
+                         int write, struct page **pages, int *nr)
 {
         unsigned long next;
         pud_t *pudp;
 
-        pudp = pud_offset(pgdp, addr);
+        pudp = pud_offset(&pgd, addr);
         do {
                 pud_t pud = ACCESS_ONCE(*pudp);
 
                 next = pud_addr_end(addr, end);
                 if (pud_none(pud))
                         return 0;
-                if (pud_huge(pud)) {
+                if (unlikely(pud_huge(pud))) {
                         if (!gup_huge_pud(pud, pudp, addr, next, write,
-                                        pages, nr))
+                                          pages, nr))
+                                return 0;
+                } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
+                        if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
+                                         PUD_SHIFT, next, write, pages, nr))
                                 return 0;
                 } else if (!gup_pmd_range(pud, addr, next, write, pages, nr))
                         return 0;
@@ -970,10 +1025,20 @@ int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
         local_irq_save(flags);
         pgdp = pgd_offset(mm, addr);
         do {
+                pgd_t pgd = ACCESS_ONCE(*pgdp);
+
                 next = pgd_addr_end(addr, end);
-                if (pgd_none(*pgdp))
+                if (pgd_none(pgd))
                         break;
-                else if (!gup_pud_range(pgdp, addr, next, write, pages, &nr))
+                if (unlikely(pgd_huge(pgd))) {
+                        if (!gup_huge_pgd(pgd, pgdp, addr, next, write,
+                                          pages, &nr))
+                                break;
+                } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
+                        if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
+                                         PGDIR_SHIFT, next, write, pages, &nr))
+                                break;
+                } else if (!gup_pud_range(pgd, addr, next, write, pages, &nr))
                         break;
         } while (pgdp++, addr = next, addr != end);
         local_irq_restore(flags);
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 5b2c6875fc38..46f96c23cc27 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -804,7 +804,7 @@ int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 return VM_FAULT_OOM;
         if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
                 return VM_FAULT_OOM;
-        if (!(flags & FAULT_FLAG_WRITE) &&
+        if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) &&
                         transparent_hugepage_use_zero_page()) {
                 spinlock_t *ptl;
                 pgtable_t pgtable;
@@ -1399,7 +1399,8 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
                  * pgtable_trans_huge_withdraw after finishing pmdp related
                  * operations.
                  */
-                orig_pmd = pmdp_get_and_clear(tlb->mm, addr, pmd);
+                orig_pmd = pmdp_get_and_clear_full(tlb->mm, addr, pmd,
+                                                   tlb->fullmm);
                 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
                 pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
                 if (is_huge_zero_pmd(orig_pmd)) {
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 30cd96879152..919b86a2164d 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -582,7 +582,7 @@ retry_cpuset:
 
         for_each_zone_zonelist_nodemask(zone, z, zonelist,
                                                 MAX_NR_ZONES - 1, nodemask) {
-                if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask(h))) {
+                if (cpuset_zone_allowed(zone, htlb_alloc_mask(h))) {
                         page = dequeue_huge_page_node(h, zone_to_nid(zone));
                         if (page) {
                                 if (avoid_reserve)
diff --git a/mm/memory.c b/mm/memory.c
index 0b3f6c71620d..4b5a282e1107 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2627,7 +2627,7 @@ static int do_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
                 return VM_FAULT_SIGBUS;
 
         /* Use the zero-page for reads */
-        if (!(flags & FAULT_FLAG_WRITE)) {
+        if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm)) {
                 entry = pte_mkspecial(pfn_pte(my_zero_pfn(address),
                                                 vma->vm_page_prot));
                 page_table = pte_offset_map_lock(mm, pmd, address, &ptl);
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 3b014d326151..864bba992735 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -233,7 +233,7 @@ static enum oom_constraint constrained_alloc(struct zonelist *zonelist,
         /* Check this allocation failure is caused by cpuset's wall function */
         for_each_zone_zonelist_nodemask(zone, z, zonelist,
                         high_zoneidx, nodemask)
-                if (!cpuset_zone_allowed_softwall(zone, gfp_mask))
+                if (!cpuset_zone_allowed(zone, gfp_mask))
                         cpuset_limited = true;
 
         if (cpuset_limited) {
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a7198c065999..df542feaac3b 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1990,7 +1990,7 @@ zonelist_scan:
 
         /*
          * Scan zonelist, looking for a zone with enough free.
-         * See also __cpuset_node_allowed_softwall() comment in kernel/cpuset.c.
+         * See also __cpuset_node_allowed() comment in kernel/cpuset.c.
          */
         for_each_zone_zonelist_nodemask(zone, z, zonelist,
                                                 high_zoneidx, nodemask) {
@@ -2001,7 +2001,7 @@ zonelist_scan:
                                 continue;
                 if (cpusets_enabled() &&
                         (alloc_flags & ALLOC_CPUSET) &&
-                        !cpuset_zone_allowed_softwall(zone, gfp_mask))
+                        !cpuset_zone_allowed(zone, gfp_mask))
                                 continue;
                 /*
                  * Distribute pages in proportion to the individual
@@ -2529,7 +2529,7 @@ gfp_to_alloc_flags(gfp_t gfp_mask)
                         alloc_flags |= ALLOC_HARDER;
                 /*
                  * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the
-                 * comment for __cpuset_node_allowed_softwall().
+                 * comment for __cpuset_node_allowed().
                  */
                 alloc_flags &= ~ALLOC_CPUSET;
         } else if (unlikely(rt_task(current)) && !in_interrupt())
diff --git a/mm/percpu.c b/mm/percpu.c
index 014bab65e0ff..d39e2f4e335c 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -1591,7 +1591,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
                         if (cpu == NR_CPUS)
                                 continue;
 
-                        PCPU_SETUP_BUG_ON(cpu > nr_cpu_ids);
+                        PCPU_SETUP_BUG_ON(cpu >= nr_cpu_ids);
                         PCPU_SETUP_BUG_ON(!cpu_possible(cpu));
                         PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX);
 
diff --git a/mm/slab.c b/mm/slab.c
index 79e15f0a2a6e..fee275b5b6b7 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3015,7 +3015,7 @@ retry:
         for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
                 nid = zone_to_nid(zone);
 
-                if (cpuset_zone_allowed_hardwall(zone, flags) &&
+                if (cpuset_zone_allowed(zone, flags | __GFP_HARDWALL) &&
                         get_node(cache, nid) &&
                         get_node(cache, nid)->free_objects) {
                                 obj = ____cache_alloc_node(cache,
diff --git a/mm/slub.c b/mm/slub.c
index 386bbed76e94..765c5884d03d 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1665,7 +1665,8 @@ static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
 
                         n = get_node(s, zone_to_nid(zone));
 
-                        if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
+                        if (n && cpuset_zone_allowed(zone,
+                                                     flags | __GFP_HARDWALL) &&
                                         n->nr_partial > s->min_partial) {
                                 object = get_partial_node(s, n, c, flags);
                                 if (object) {
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4636d9e822c1..a384339bf718 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -2405,7 +2405,8 @@ static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
                  * to global LRU.
                  */
                 if (global_reclaim(sc)) {
-                        if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
+                        if (!cpuset_zone_allowed(zone,
+                                                 GFP_KERNEL | __GFP_HARDWALL))
                                 continue;
 
                         lru_pages += zone_reclaimable_pages(zone);
@@ -3388,7 +3389,7 @@ void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
         if (!populated_zone(zone))
                 return;
 
-        if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
+        if (!cpuset_zone_allowed(zone, GFP_KERNEL | __GFP_HARDWALL))
                 return;
         pgdat = zone->zone_pgdat;
         if (pgdat->kswapd_max_order < order) {
diff --git a/mm/zbud.c b/mm/zbud.c
index ecf1dbef6983..ec71b37fb06c 100644
--- a/mm/zbud.c
+++ b/mm/zbud.c
@@ -619,5 +619,5 @@ module_init(init_zbud);
 module_exit(exit_zbud);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Seth Jennings <sjenning@linux.vnet.ibm.com>");
+MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
 MODULE_DESCRIPTION("Buddy Allocator for Compressed Pages");
diff --git a/mm/zswap.c b/mm/zswap.c
index ea064c1a09ba..c1543061a192 100644
--- a/mm/zswap.c
+++ b/mm/zswap.c
@@ -951,5 +951,5 @@ error:
 late_initcall(init_zswap);
 
 MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Seth Jennings <sjenning@linux.vnet.ibm.com>");
+MODULE_AUTHOR("Seth Jennings <sjennings@variantweb.net>");
 MODULE_DESCRIPTION("Compressed cache for swap pages");