thp: fix anon memory statistics with transparent hugepages

Count each transparent hugepage as HPAGE_PMD_NR pages in the LRU
statistics, so the Active(anon) and Inactive(anon) statistics in
/proc/meminfo are correct.

Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

3 files changed, 17 insertions, 6 deletions

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 892d8a17a7e5..f4f6041176a4 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1143,6 +1143,7 @@ static void __split_huge_page_refcount(struct page *page)
         int i;
         unsigned long head_index = page->index;
         struct zone *zone = page_zone(page);
+        int zonestat;
 
         /* prevent PageLRU to go away from under us, and freeze lru stats */
         spin_lock_irq(&zone->lru_lock);
@@ -1207,6 +1208,15 @@ static void __split_huge_page_refcount(struct page *page)
         __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
         __mod_zone_page_state(zone, NR_ANON_PAGES, HPAGE_PMD_NR);
 
+        /*
+         * A hugepage counts for HPAGE_PMD_NR pages on the LRU statistics,
+         * so adjust those appropriately if this page is on the LRU.
+         */
+        if (PageLRU(page)) {
+                zonestat = NR_LRU_BASE + page_lru(page);
+                __mod_zone_page_state(zone, zonestat, -(HPAGE_PMD_NR-1));
+        }
+
         ClearPageCompound(page);
         compound_unlock(page);
         spin_unlock_irq(&zone->lru_lock);
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index a1bb59d4c9d0..f4ea3410fb4d 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1091,7 +1091,7 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
                 case 0:
                         list_move(&page->lru, dst);
                         mem_cgroup_del_lru(page);
-                        nr_taken++;
+                        nr_taken += hpage_nr_pages(page);
                         break;
                 case -EBUSY:
                         /* we don't affect global LRU but rotate in our LRU */
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f5b762ae23a2..0882014d2ce5 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1045,7 +1045,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                 case 0:
                         list_move(&page->lru, dst);
                         mem_cgroup_del_lru(page);
-                        nr_taken++;
+                        nr_taken += hpage_nr_pages(page);
                         break;
 
                 case -EBUSY:
@@ -1103,7 +1103,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
                         if (__isolate_lru_page(cursor_page, mode, file) == 0) {
                                 list_move(&cursor_page->lru, dst);
                                 mem_cgroup_del_lru(cursor_page);
-                                nr_taken++;
+                                nr_taken += hpage_nr_pages(page);
                                 nr_lumpy_taken++;
                                 if (PageDirty(cursor_page))
                                         nr_lumpy_dirty++;
@@ -1158,14 +1158,15 @@ static unsigned long clear_active_flags(struct list_head *page_list,
         struct page *page;
 
         list_for_each_entry(page, page_list, lru) {
+                int numpages = hpage_nr_pages(page);
                 lru = page_lru_base_type(page);
                 if (PageActive(page)) {
                         lru += LRU_ACTIVE;
                         ClearPageActive(page);
-                        nr_active++;
+                        nr_active += numpages;
                 }
                 if (count)
-                        count[lru]++;
+                        count[lru] += numpages;
         }
 
         return nr_active;
@@ -1483,7 +1484,7 @@ static void move_active_pages_to_lru(struct zone *zone,
 
                 list_move(&page->lru, &zone->lru[lru].list);
                 mem_cgroup_add_lru_list(page, lru);
-                pgmoved++;
+                pgmoved += hpage_nr_pages(page);
 
                 if (!pagevec_add(&pvec, page) || list_empty(list)) {
                         spin_unlock_irq(&zone->lru_lock);