1 files changed, 88 insertions, 27 deletions

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 35527173cf50..c1b7414c7bef 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -5,6 +5,7 @@
 
 #ifdef __KERNEL__
 
+#include <linux/mmdebug.h>
 #include <linux/gfp.h>
 #include <linux/bug.h>
 #include <linux/list.h>
@@ -57,6 +58,15 @@ extern int sysctl_legacy_va_layout;
 extern unsigned long sysctl_user_reserve_kbytes;
 extern unsigned long sysctl_admin_reserve_kbytes;
 
+extern int sysctl_overcommit_memory;
+extern int sysctl_overcommit_ratio;
+extern unsigned long sysctl_overcommit_kbytes;
+
+extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
+                                    size_t *, loff_t *);
+extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
+                                    size_t *, loff_t *);
+
 #define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
 
 /* to align the pointer to the (next) page boundary */
@@ -165,7 +175,7 @@ extern unsigned int kobjsize(const void *objp);
  * Special vmas that are non-mergable, non-mlock()able.
  * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
  */
-#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP)
+#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
 
 /*
  * mapping from the currently active vm_flags protection bits (the
@@ -294,7 +304,7 @@ static inline int get_freepage_migratetype(struct page *page)
  */
 static inline int put_page_testzero(struct page *page)
 {
-        VM_BUG_ON(atomic_read(&page->_count) == 0);
+        VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);
         return atomic_dec_and_test(&page->_count);
 }
 
@@ -355,7 +365,7 @@ static inline int is_vmalloc_or_module_addr(const void *x)
 static inline void compound_lock(struct page *page)
 {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-        VM_BUG_ON(PageSlab(page));
+        VM_BUG_ON_PAGE(PageSlab(page), page);
         bit_spin_lock(PG_compound_lock, &page->flags);
 #endif
 }
@@ -363,7 +373,7 @@ static inline void compound_lock(struct page *page)
 static inline void compound_unlock(struct page *page)
 {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-        VM_BUG_ON(PageSlab(page));
+        VM_BUG_ON_PAGE(PageSlab(page), page);
         bit_spin_unlock(PG_compound_lock, &page->flags);
 #endif
 }
@@ -389,8 +399,18 @@ static inline void compound_unlock_irqrestore(struct page *page,
 
 static inline struct page *compound_head(struct page *page)
 {
-        if (unlikely(PageTail(page)))
-                return page->first_page;
+        if (unlikely(PageTail(page))) {
+                struct page *head = page->first_page;
+
+                /*
+                 * page->first_page may be a dangling pointer to an old
+                 * compound page, so recheck that it is still a tail
+                 * page before returning.
+                 */
+                smp_rmb();
+                if (likely(PageTail(page)))
+                        return head;
+        }
         return page;
 }
 
@@ -414,15 +434,44 @@ static inline int page_count(struct page *page)
         return atomic_read(&compound_head(page)->_count);
 }
 
+#ifdef CONFIG_HUGETLB_PAGE
+extern int PageHeadHuge(struct page *page_head);
+#else /* CONFIG_HUGETLB_PAGE */
+static inline int PageHeadHuge(struct page *page_head)
+{
+        return 0;
+}
+#endif /* CONFIG_HUGETLB_PAGE */
+
+static inline bool __compound_tail_refcounted(struct page *page)
+{
+        return !PageSlab(page) && !PageHeadHuge(page);
+}
+
+/*
+ * This takes a head page as parameter and tells if the
+ * tail page reference counting can be skipped.
+ *
+ * For this to be safe, PageSlab and PageHeadHuge must remain true on
+ * any given page where they return true here, until all tail pins
+ * have been released.
+ */
+static inline bool compound_tail_refcounted(struct page *page)
+{
+        VM_BUG_ON_PAGE(!PageHead(page), page);
+        return __compound_tail_refcounted(page);
+}
+
 static inline void get_huge_page_tail(struct page *page)
 {
         /*
-         * __split_huge_page_refcount() cannot run
-         * from under us.
+         * __split_huge_page_refcount() cannot run from under us.
          */
-        VM_BUG_ON(page_mapcount(page) < 0);
-        VM_BUG_ON(atomic_read(&page->_count) != 0);
-        atomic_inc(&page->_mapcount);
+        VM_BUG_ON_PAGE(!PageTail(page), page);
+        VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
+        VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);
+        if (compound_tail_refcounted(page->first_page))
+                atomic_inc(&page->_mapcount);
 }
 
 extern bool __get_page_tail(struct page *page);
@@ -436,7 +485,7 @@ static inline void get_page(struct page *page)
          * Getting a normal page or the head of a compound page
          * requires to already have an elevated page->_count.
          */
-        VM_BUG_ON(atomic_read(&page->_count) <= 0);
+        VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
         atomic_inc(&page->_count);
 }
 
@@ -473,13 +522,13 @@ static inline int PageBuddy(struct page *page)
 
 static inline void __SetPageBuddy(struct page *page)
 {
-        VM_BUG_ON(atomic_read(&page->_mapcount) != -1);
+        VM_BUG_ON_PAGE(atomic_read(&page->_mapcount) != -1, page);
         atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
 }
 
 static inline void __ClearPageBuddy(struct page *page)
 {
-        VM_BUG_ON(!PageBuddy(page));
+        VM_BUG_ON_PAGE(!PageBuddy(page), page);
         atomic_set(&page->_mapcount, -1);
 }
 
@@ -718,7 +767,7 @@ static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
 #ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
 static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
 {
-        return xchg(&page->_last_cpupid, cpupid);
+        return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
 }
 
 static inline int page_cpupid_last(struct page *page)
@@ -727,7 +776,7 @@ static inline int page_cpupid_last(struct page *page)
 }
 static inline void page_cpupid_reset_last(struct page *page)
 {
-        page->_last_cpupid = -1;
+        page->_last_cpupid = -1 & LAST_CPUPID_MASK;
 }
 #else
 static inline int page_cpupid_last(struct page *page)
@@ -846,11 +895,14 @@ static __always_inline void *lowmem_page_address(const struct page *page)
 #endif
 
 #if defined(WANT_PAGE_VIRTUAL)
-#define page_address(page) ((page)->virtual)
-#define set_page_address(page, address)                 \
-        do {                                            \
-                (page)->virtual = (address);            \
-        } while(0)
+static inline void *page_address(const struct page *page)
+{
+        return page->virtual;
+}
+static inline void set_page_address(struct page *page, void *address)
+{
+        page->virtual = address;
+}
 #define page_address_init()  do { } while(0)
 #endif
 
@@ -984,7 +1036,6 @@ extern void pagefault_out_of_memory(void);
  * various contexts.
  */
 #define SHOW_MEM_FILTER_NODES           (0x0001u)       /* disallowed nodes */
-#define SHOW_MEM_FILTER_PAGE_COUNT      (0x0002u)       /* page type count */
 
 extern void show_free_areas(unsigned int flags);
 extern bool skip_free_areas_node(unsigned int flags, int nid);
@@ -1318,6 +1369,7 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
 
 #if USE_SPLIT_PTE_PTLOCKS
 #if ALLOC_SPLIT_PTLOCKS
+void __init ptlock_cache_init(void);
 extern bool ptlock_alloc(struct page *page);
 extern void ptlock_free(struct page *page);
 
@@ -1326,6 +1378,10 @@ static inline spinlock_t *ptlock_ptr(struct page *page)
         return page->ptl;
 }
 #else /* ALLOC_SPLIT_PTLOCKS */
+static inline void ptlock_cache_init(void)
+{
+}
+
 static inline bool ptlock_alloc(struct page *page)
 {
         return true;
@@ -1356,7 +1412,7 @@ static inline bool ptlock_init(struct page *page)
          * slab code uses page->slab_cache and page->first_page (for tail
          * pages), which share storage with page->ptl.
          */
-        VM_BUG_ON(*(unsigned long *)&page->ptl);
+        VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
         if (!ptlock_alloc(page))
                 return false;
         spin_lock_init(ptlock_ptr(page));
@@ -1378,10 +1434,17 @@ static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
 {
         return &mm->page_table_lock;
 }
+static inline void ptlock_cache_init(void) {}
 static inline bool ptlock_init(struct page *page) { return true; }
 static inline void pte_lock_deinit(struct page *page) {}
 #endif /* USE_SPLIT_PTE_PTLOCKS */
 
+static inline void pgtable_init(void)
+{
+        ptlock_cache_init();
+        pgtable_cache_init();
+}
+
 static inline bool pgtable_page_ctor(struct page *page)
 {
         inc_zone_page_state(page, NR_PAGETABLE);
@@ -1440,7 +1503,7 @@ static inline bool pgtable_pmd_page_ctor(struct page *page)
 static inline void pgtable_pmd_page_dtor(struct page *page)
 {
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-        VM_BUG_ON(page->pmd_huge_pte);
+        VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
 #endif
         ptlock_free(page);
 }
@@ -1842,7 +1905,7 @@ static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
 }
 #endif
 
-#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
+#ifdef CONFIG_NUMA_BALANCING
 unsigned long change_prot_numa(struct vm_area_struct *vma,
                         unsigned long start, unsigned long end);
 #endif
@@ -1977,8 +2040,6 @@ extern void shake_page(struct page *p, int access);
 extern atomic_long_t num_poisoned_pages;
 extern int soft_offline_page(struct page *page, int flags);
 
-extern void dump_page(struct page *page);
-
 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
 extern void clear_huge_page(struct page *page,
                             unsigned long addr,