1 files changed, 61 insertions, 30 deletions

diff --git a/mm/rmap.c b/mm/rmap.c
index c95d2ba27a0b..f21f4a1d6a1c 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -177,6 +177,10 @@ static void anon_vma_chain_link(struct vm_area_struct *vma,
         list_add(&avc->same_vma, &vma->anon_vma_chain);
 
         anon_vma_lock(anon_vma);
+        /*
+         * It's critical to add new vmas to the tail of the anon_vma,
+         * see comment in huge_memory.c:__split_huge_page().
+         */
         list_add_tail(&avc->same_anon_vma, &anon_vma->head);
         anon_vma_unlock(anon_vma);
 }
@@ -360,7 +364,7 @@ void page_unlock_anon_vma(struct anon_vma *anon_vma)
  * Returns virtual address or -EFAULT if page's index/offset is not
  * within the range mapped the @vma.
  */
-static inline unsigned long
+inline unsigned long
 vma_address(struct page *page, struct vm_area_struct *vma)
 {
         pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
@@ -435,6 +439,8 @@ pte_t *__page_check_address(struct page *page, struct mm_struct *mm,
         pmd = pmd_offset(pud, address);
         if (!pmd_present(*pmd))
                 return NULL;
+        if (pmd_trans_huge(*pmd))
+                return NULL;
 
         pte = pte_offset_map(pmd, address);
         /* Make a quick check before getting the lock */
@@ -489,35 +495,17 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                         unsigned long *vm_flags)
 {
         struct mm_struct *mm = vma->vm_mm;
-        pte_t *pte;
-        spinlock_t *ptl;
         int referenced = 0;
 
-        pte = page_check_address(page, mm, address, &ptl, 0);
-        if (!pte)
-                goto out;
-
         /*
          * Don't want to elevate referenced for mlocked page that gets this far,
          * in order that it progresses to try_to_unmap and is moved to the
          * unevictable list.
          */
         if (vma->vm_flags & VM_LOCKED) {
-                *mapcount = 1;  /* break early from loop */
+                *mapcount = 0;  /* break early from loop */
                 *vm_flags |= VM_LOCKED;
-                goto out_unmap;
-        }
-
-        if (ptep_clear_flush_young_notify(vma, address, pte)) {
-                /*
-                 * Don't treat a reference through a sequentially read
-                 * mapping as such.  If the page has been used in
-                 * another mapping, we will catch it; if this other
-                 * mapping is already gone, the unmap path will have
-                 * set PG_referenced or activated the page.
-                 */
-                if (likely(!VM_SequentialReadHint(vma)))
-                        referenced++;
+                goto out;
         }
 
         /* Pretend the page is referenced if the task has the
@@ -526,9 +514,39 @@ int page_referenced_one(struct page *page, struct vm_area_struct *vma,
                         rwsem_is_locked(&mm->mmap_sem))
                 referenced++;
 
-out_unmap:
+        if (unlikely(PageTransHuge(page))) {
+                pmd_t *pmd;
+
+                spin_lock(&mm->page_table_lock);
+                pmd = page_check_address_pmd(page, mm, address,
+                                             PAGE_CHECK_ADDRESS_PMD_FLAG);
+                if (pmd && !pmd_trans_splitting(*pmd) &&
+                    pmdp_clear_flush_young_notify(vma, address, pmd))
+                        referenced++;
+                spin_unlock(&mm->page_table_lock);
+        } else {
+                pte_t *pte;
+                spinlock_t *ptl;
+
+                pte = page_check_address(page, mm, address, &ptl, 0);
+                if (!pte)
+                        goto out;
+
+                if (ptep_clear_flush_young_notify(vma, address, pte)) {
+                        /*
+                         * Don't treat a reference through a sequentially read
+                         * mapping as such.  If the page has been used in
+                         * another mapping, we will catch it; if this other
+                         * mapping is already gone, the unmap path will have
+                         * set PG_referenced or activated the page.
+                         */
+                        if (likely(!VM_SequentialReadHint(vma)))
+                                referenced++;
+                }
+                pte_unmap_unlock(pte, ptl);
+        }
+
         (*mapcount)--;
-        pte_unmap_unlock(pte, ptl);
 
         if (referenced)
                 *vm_flags |= vma->vm_flags;
@@ -864,8 +882,13 @@ void do_page_add_anon_rmap(struct page *page,
         struct vm_area_struct *vma, unsigned long address, int exclusive)
 {
         int first = atomic_inc_and_test(&page->_mapcount);
-        if (first)
-                __inc_zone_page_state(page, NR_ANON_PAGES);
+        if (first) {
+                if (!PageTransHuge(page))
+                        __inc_zone_page_state(page, NR_ANON_PAGES);
+                else
+                        __inc_zone_page_state(page,
+                                              NR_ANON_TRANSPARENT_HUGEPAGES);
+        }
         if (unlikely(PageKsm(page)))
                 return;
 
@@ -893,7 +916,10 @@ void page_add_new_anon_rmap(struct page *page,
         VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
         SetPageSwapBacked(page);
         atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */
-        __inc_zone_page_state(page, NR_ANON_PAGES);
+        if (!PageTransHuge(page))
+                __inc_zone_page_state(page, NR_ANON_PAGES);
+        else
+                __inc_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
         __page_set_anon_rmap(page, vma, address, 1);
         if (page_evictable(page, vma))
                 lru_cache_add_lru(page, LRU_ACTIVE_ANON);
@@ -911,7 +937,7 @@ void page_add_file_rmap(struct page *page)
 {
         if (atomic_inc_and_test(&page->_mapcount)) {
                 __inc_zone_page_state(page, NR_FILE_MAPPED);
-                mem_cgroup_update_file_mapped(page, 1);
+                mem_cgroup_inc_page_stat(page, MEMCG_NR_FILE_MAPPED);
         }
 }
 
@@ -946,10 +972,14 @@ void page_remove_rmap(struct page *page)
                 return;
         if (PageAnon(page)) {
                 mem_cgroup_uncharge_page(page);
-                __dec_zone_page_state(page, NR_ANON_PAGES);
+                if (!PageTransHuge(page))
+                        __dec_zone_page_state(page, NR_ANON_PAGES);
+                else
+                        __dec_zone_page_state(page,
+                                              NR_ANON_TRANSPARENT_HUGEPAGES);
         } else {
                 __dec_zone_page_state(page, NR_FILE_MAPPED);
-                mem_cgroup_update_file_mapped(page, -1);
+                mem_cgroup_dec_page_stat(page, MEMCG_NR_FILE_MAPPED);
         }
         /*
          * It would be tidy to reset the PageAnon mapping here,
@@ -1202,7 +1232,7 @@ static int try_to_unmap_cluster(unsigned long cursor, unsigned int *mapcount,
         return ret;
 }
 
-static bool is_vma_temporary_stack(struct vm_area_struct *vma)
+bool is_vma_temporary_stack(struct vm_area_struct *vma)
 {
         int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP);
 
@@ -1400,6 +1430,7 @@ int try_to_unmap(struct page *page, enum ttu_flags flags)
         int ret;
 
         BUG_ON(!PageLocked(page));
+        VM_BUG_ON(!PageHuge(page) && PageTransHuge(page));
 
         if (unlikely(PageKsm(page)))
                 ret = try_to_unmap_ksm(page, flags);