Memory controller: memory accounting

Add the accounting hooks.  The accounting is carried out for RSS and Page
Cache (unmapped) pages.  There is now a common limit and accounting for both.
The RSS accounting is accounted at page_add_*_rmap() and page_remove_rmap()
time.  Page cache is accounted at add_to_page_cache(),
__delete_from_page_cache().  Swap cache is also accounted for.

Each page's page_cgroup is protected with the last bit of the
page_cgroup pointer, this makes handling of race conditions involving
simultaneous mappings of a page easier.  A reference count is kept in the
page_cgroup to deal with cases where a page might be unmapped from the RSS
of all tasks, but still lives in the page cache.

Credits go to Vaidyanathan Srinivasan for helping with reference counting work
of the page cgroup.  Almost all of the page cache accounting code has help
from Vaidyanathan Srinivasan.

[hugh@veritas.com: fix swapoff breakage]
[akpm@linux-foundation.org: fix locking]
Signed-off-by: Vaidyanathan Srinivasan <svaidy@linux.vnet.ibm.com>
Signed-off-by: Balbir Singh <balbir@linux.vnet.ibm.com>
Cc: Pavel Emelianov <xemul@openvz.org>
Cc: Paul Menage <menage@google.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: "Eric W. Biederman" <ebiederm@xmission.com>
Cc: Nick Piggin <nickpiggin@yahoo.com.au>
Cc: Kirill Korotaev <dev@sw.ru>
Cc: Herbert Poetzl <herbert@13thfloor.at>
Cc: David Rientjes <rientjes@google.com>
Cc: <Valdis.Kletnieks@vt.edu>
Signed-off-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 24 insertions, 17 deletions

diff --git a/mm/swapfile.c b/mm/swapfile.c
index afae7b1f680b..fddc4cc4149b 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -27,6 +27,7 @@
 #include <linux/mutex.h>
 #include <linux/capability.h>
 #include <linux/syscalls.h>
+#include <linux/memcontrol.h>
 
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
@@ -506,9 +507,12 @@ unsigned int count_swap_pages(int type, int free)
  * just let do_wp_page work it out if a write is requested later - to
  * force COW, vm_page_prot omits write permission from any private vma.
  */
-static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
+static int unuse_pte(struct vm_area_struct *vma, pte_t *pte,
                 unsigned long addr, swp_entry_t entry, struct page *page)
 {
+        if (mem_cgroup_charge(page, vma->vm_mm))
+                return -ENOMEM;
+
         inc_mm_counter(vma->vm_mm, anon_rss);
         get_page(page);
         set_pte_at(vma->vm_mm, addr, pte,
@@ -520,6 +524,7 @@ static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
          * immediately swapped out again after swapon.
          */
         activate_page(page);
+        return 1;
 }
 
 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
@@ -529,7 +534,7 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
         pte_t swp_pte = swp_entry_to_pte(entry);
         pte_t *pte;
         spinlock_t *ptl;
-        int found = 0;
+        int ret = 0;
 
         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
         do {
@@ -538,13 +543,12 @@ static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                  * Test inline before going to call unuse_pte.
                  */
                 if (unlikely(pte_same(*pte, swp_pte))) {
-                        unuse_pte(vma, pte++, addr, entry, page);
-                        found = 1;
+                        ret = unuse_pte(vma, pte++, addr, entry, page);
                         break;
                 }
         } while (pte++, addr += PAGE_SIZE, addr != end);
         pte_unmap_unlock(pte - 1, ptl);
-        return found;
+        return ret;
 }
 
 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
@@ -553,14 +557,16 @@ static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
 {
         pmd_t *pmd;
         unsigned long next;
+        int ret;
 
         pmd = pmd_offset(pud, addr);
         do {
                 next = pmd_addr_end(addr, end);
                 if (pmd_none_or_clear_bad(pmd))
                         continue;
-                if (unuse_pte_range(vma, pmd, addr, next, entry, page))
-                        return 1;
+                ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
+                if (ret)
+                        return ret;
         } while (pmd++, addr = next, addr != end);
         return 0;
 }
@@ -571,14 +577,16 @@ static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
 {
         pud_t *pud;
         unsigned long next;
+        int ret;
 
         pud = pud_offset(pgd, addr);
         do {
                 next = pud_addr_end(addr, end);
                 if (pud_none_or_clear_bad(pud))
                         continue;
-                if (unuse_pmd_range(vma, pud, addr, next, entry, page))
-                        return 1;
+                ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
+                if (ret)
+                        return ret;
         } while (pud++, addr = next, addr != end);
         return 0;
 }
@@ -588,6 +596,7 @@ static int unuse_vma(struct vm_area_struct *vma,
 {
         pgd_t *pgd;
         unsigned long addr, end, next;
+        int ret;
 
         if (page->mapping) {
                 addr = page_address_in_vma(page, vma);
@@ -605,8 +614,9 @@ static int unuse_vma(struct vm_area_struct *vma,
                 next = pgd_addr_end(addr, end);
                 if (pgd_none_or_clear_bad(pgd))
                         continue;
-                if (unuse_pud_range(vma, pgd, addr, next, entry, page))
-                        return 1;
+                ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
+                if (ret)
+                        return ret;
         } while (pgd++, addr = next, addr != end);
         return 0;
 }
@@ -615,6 +625,7 @@ static int unuse_mm(struct mm_struct *mm,
                                 swp_entry_t entry, struct page *page)
 {
         struct vm_area_struct *vma;
+        int ret = 0;
 
         if (!down_read_trylock(&mm->mmap_sem)) {
                 /*
@@ -627,15 +638,11 @@ static int unuse_mm(struct mm_struct *mm,
                 lock_page(page);
         }
         for (vma = mm->mmap; vma; vma = vma->vm_next) {
-                if (vma->anon_vma && unuse_vma(vma, entry, page))
+                if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
                         break;
         }
         up_read(&mm->mmap_sem);
-        /*
-         * Currently unuse_mm cannot fail, but leave error handling
-         * at call sites for now, since we change it from time to time.
-         */
-        return 0;
+        return (ret < 0)? ret: 0;
 }
 
 /*