1 files changed, 229 insertions, 46 deletions

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8b6e55ee8855..0548eb201e05 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -50,6 +50,10 @@ extern int sysctl_legacy_va_layout;
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 
+#ifndef __pa_symbol
+#define __pa_symbol(x)  __pa(RELOC_HIDE((unsigned long)(x), 0))
+#endif
+
 extern unsigned long sysctl_user_reserve_kbytes;
 extern unsigned long sysctl_admin_reserve_kbytes;
 
@@ -297,12 +301,26 @@ static inline int put_page_testzero(struct page *page)
 /*
  * Try to grab a ref unless the page has a refcount of zero, return false if
  * that is the case.
+ * This can be called when MMU is off so it must not access
+ * any of the virtual mappings.
  */
 static inline int get_page_unless_zero(struct page *page)
 {
         return atomic_inc_not_zero(&page->_count);
 }
 
+/*
+ * Try to drop a ref unless the page has a refcount of one, return false if
+ * that is the case.
+ * This is to make sure that the refcount won't become zero after this drop.
+ * This can be called when MMU is off so it must not access
+ * any of the virtual mappings.
+ */
+static inline int put_page_unless_one(struct page *page)
+{
+        return atomic_add_unless(&page->_count, -1, 1);
+}
+
 extern int page_is_ram(unsigned long pfn);
 
 /* Support for virtually mapped pages */
@@ -581,11 +599,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
  * sets it, so none of the operations on it need to be atomic.
  */
 
-/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NID] | ... | FLAGS | */
+/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
 #define SECTIONS_PGOFF          ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
 #define NODES_PGOFF             (SECTIONS_PGOFF - NODES_WIDTH)
 #define ZONES_PGOFF             (NODES_PGOFF - ZONES_WIDTH)
-#define LAST_NID_PGOFF          (ZONES_PGOFF - LAST_NID_WIDTH)
+#define LAST_CPUPID_PGOFF       (ZONES_PGOFF - LAST_CPUPID_WIDTH)
 
 /*
  * Define the bit shifts to access each section.  For non-existent
@@ -595,7 +613,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
 #define SECTIONS_PGSHIFT        (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
 #define NODES_PGSHIFT           (NODES_PGOFF * (NODES_WIDTH != 0))
 #define ZONES_PGSHIFT           (ZONES_PGOFF * (ZONES_WIDTH != 0))
-#define LAST_NID_PGSHIFT        (LAST_NID_PGOFF * (LAST_NID_WIDTH != 0))
+#define LAST_CPUPID_PGSHIFT     (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
 
 /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
 #ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -617,7 +635,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
 #define ZONES_MASK              ((1UL << ZONES_WIDTH) - 1)
 #define NODES_MASK              ((1UL << NODES_WIDTH) - 1)
 #define SECTIONS_MASK           ((1UL << SECTIONS_WIDTH) - 1)
-#define LAST_NID_MASK           ((1UL << LAST_NID_WIDTH) - 1)
+#define LAST_CPUPID_MASK        ((1UL << LAST_CPUPID_WIDTH) - 1)
 #define ZONEID_MASK             ((1UL << ZONEID_SHIFT) - 1)
 
 static inline enum zone_type page_zonenum(const struct page *page)
@@ -661,51 +679,117 @@ static inline int page_to_nid(const struct page *page)
 #endif
 
 #ifdef CONFIG_NUMA_BALANCING
-#ifdef LAST_NID_NOT_IN_PAGE_FLAGS
-static inline int page_nid_xchg_last(struct page *page, int nid)
+static inline int cpu_pid_to_cpupid(int cpu, int pid)
 {
-        return xchg(&page->_last_nid, nid);
+        return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
 }
 
-static inline int page_nid_last(struct page *page)
+static inline int cpupid_to_pid(int cpupid)
 {
-        return page->_last_nid;
+        return cpupid & LAST__PID_MASK;
 }
-static inline void page_nid_reset_last(struct page *page)
+
+static inline int cpupid_to_cpu(int cpupid)
 {
-        page->_last_nid = -1;
+        return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
 }
-#else
-static inline int page_nid_last(struct page *page)
+
+static inline int cpupid_to_nid(int cpupid)
 {
-        return (page->flags >> LAST_NID_PGSHIFT) & LAST_NID_MASK;
+        return cpu_to_node(cpupid_to_cpu(cpupid));
 }
 
-extern int page_nid_xchg_last(struct page *page, int nid);
+static inline bool cpupid_pid_unset(int cpupid)
+{
+        return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
+}
 
-static inline void page_nid_reset_last(struct page *page)
+static inline bool cpupid_cpu_unset(int cpupid)
 {
-        int nid = (1 << LAST_NID_SHIFT) - 1;
+        return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
+}
 
-        page->flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT);
-        page->flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT;
+static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
+{
+        return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
+}
+
+#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
+#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
+static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
+{
+        return xchg(&page->_last_cpupid, cpupid);
+}
+
+static inline int page_cpupid_last(struct page *page)
+{
+        return page->_last_cpupid;
+}
+static inline void page_cpupid_reset_last(struct page *page)
+{
+        page->_last_cpupid = -1;
 }
-#endif /* LAST_NID_NOT_IN_PAGE_FLAGS */
 #else
-static inline int page_nid_xchg_last(struct page *page, int nid)
+static inline int page_cpupid_last(struct page *page)
+{
+        return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
+}
+
+extern int page_cpupid_xchg_last(struct page *page, int cpupid);
+
+static inline void page_cpupid_reset_last(struct page *page)
 {
-        return page_to_nid(page);
+        int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
+
+        page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
+        page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
+}
+#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
+#else /* !CONFIG_NUMA_BALANCING */
+static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
+{
+        return page_to_nid(page); /* XXX */
 }
 
-static inline int page_nid_last(struct page *page)
+static inline int page_cpupid_last(struct page *page)
 {
-        return page_to_nid(page);
+        return page_to_nid(page); /* XXX */
 }
 
-static inline void page_nid_reset_last(struct page *page)
+static inline int cpupid_to_nid(int cpupid)
 {
+        return -1;
+}
+
+static inline int cpupid_to_pid(int cpupid)
+{
+        return -1;
 }
-#endif
+
+static inline int cpupid_to_cpu(int cpupid)
+{
+        return -1;
+}
+
+static inline int cpu_pid_to_cpupid(int nid, int pid)
+{
+        return -1;
+}
+
+static inline bool cpupid_pid_unset(int cpupid)
+{
+        return 1;
+}
+
+static inline void page_cpupid_reset_last(struct page *page)
+{
+}
+
+static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
+{
+        return false;
+}
+#endif /* CONFIG_NUMA_BALANCING */
 
 static inline struct zone *page_zone(const struct page *page)
 {
@@ -1232,32 +1316,85 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
 }
 #endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
 
-#if USE_SPLIT_PTLOCKS
-/*
- * We tuck a spinlock to guard each pagetable page into its struct page,
- * at page->private, with BUILD_BUG_ON to make sure that this will not
- * overflow into the next struct page (as it might with DEBUG_SPINLOCK).
- * When freeing, reset page->mapping so free_pages_check won't complain.
- */
-#define __pte_lockptr(page)     &((page)->ptl)
-#define pte_lock_init(_page)    do {                                    \
-        spin_lock_init(__pte_lockptr(_page));                           \
-} while (0)
-#define pte_lock_deinit(page)   ((page)->mapping = NULL)
-#define pte_lockptr(mm, pmd)    ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
-#else   /* !USE_SPLIT_PTLOCKS */
+#if USE_SPLIT_PTE_PTLOCKS
+#if BLOATED_SPINLOCKS
+void __init ptlock_cache_init(void);
+extern bool ptlock_alloc(struct page *page);
+extern void ptlock_free(struct page *page);
+
+static inline spinlock_t *ptlock_ptr(struct page *page)
+{
+        return page->ptl;
+}
+#else /* BLOATED_SPINLOCKS */
+static inline void ptlock_cache_init(void) {}
+static inline bool ptlock_alloc(struct page *page)
+{
+        return true;
+}
+
+static inline void ptlock_free(struct page *page)
+{
+}
+
+static inline spinlock_t *ptlock_ptr(struct page *page)
+{
+        return &page->ptl;
+}
+#endif /* BLOATED_SPINLOCKS */
+
+static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
+{
+        return ptlock_ptr(pmd_page(*pmd));
+}
+
+static inline bool ptlock_init(struct page *page)
+{
+        /*
+         * prep_new_page() initialize page->private (and therefore page->ptl)
+         * with 0. Make sure nobody took it in use in between.
+         *
+         * It can happen if arch try to use slab for page table allocation:
+         * slab code uses page->slab_cache and page->first_page (for tail
+         * pages), which share storage with page->ptl.
+         */
+        VM_BUG_ON(*(unsigned long *)&page->ptl);
+        if (!ptlock_alloc(page))
+                return false;
+        spin_lock_init(ptlock_ptr(page));
+        return true;
+}
+
+/* Reset page->mapping so free_pages_check won't complain. */
+static inline void pte_lock_deinit(struct page *page)
+{
+        page->mapping = NULL;
+        ptlock_free(page);
+}
+
+#else   /* !USE_SPLIT_PTE_PTLOCKS */
 /*
  * We use mm->page_table_lock to guard all pagetable pages of the mm.
  */
-#define pte_lock_init(page)     do {} while (0)
-#define pte_lock_deinit(page)   do {} while (0)
-#define pte_lockptr(mm, pmd)    ({(void)(pmd); &(mm)->page_table_lock;})
-#endif /* USE_SPLIT_PTLOCKS */
+static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
+{
+        return &mm->page_table_lock;
+}
+static inline void ptlock_cache_init(void) {}
+static inline bool ptlock_init(struct page *page) { return true; }
+static inline void pte_lock_deinit(struct page *page) {}
+#endif /* USE_SPLIT_PTE_PTLOCKS */
+
+static inline void pgtable_init(void)
+{
+        ptlock_cache_init();
+        pgtable_cache_init();
+}
 
-static inline void pgtable_page_ctor(struct page *page)
+static inline bool pgtable_page_ctor(struct page *page)
 {
-        pte_lock_init(page);
         inc_zone_page_state(page, NR_PAGETABLE);
+        return ptlock_init(page);
 }
 
 static inline void pgtable_page_dtor(struct page *page)
@@ -1294,6 +1431,52 @@ static inline void pgtable_page_dtor(struct page *page)
         ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
                 NULL: pte_offset_kernel(pmd, address))
 
+#if USE_SPLIT_PMD_PTLOCKS
+
+static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
+{
+        return ptlock_ptr(virt_to_page(pmd));
+}
+
+static inline bool pgtable_pmd_page_ctor(struct page *page)
+{
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+        page->pmd_huge_pte = NULL;
+#endif
+        return ptlock_init(page);
+}
+
+static inline void pgtable_pmd_page_dtor(struct page *page)
+{
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+        VM_BUG_ON(page->pmd_huge_pte);
+#endif
+        ptlock_free(page);
+}
+
+#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte)
+
+#else
+
+static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
+{
+        return &mm->page_table_lock;
+}
+
+static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
+static inline void pgtable_pmd_page_dtor(struct page *page) {}
+
+#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
+
+#endif
+
+static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
+{
+        spinlock_t *ptl = pmd_lockptr(mm, pmd);
+        spin_lock(ptl);
+        return ptl;
+}
+
 extern void free_area_init(unsigned long * zones_size);
 extern void free_area_init_node(int nid, unsigned long * zones_size,
                 unsigned long zone_start_pfn, unsigned long *zholes_size);