1 files changed, 35 insertions, 17 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 356d4964fe95..a1bb59d4c9d0 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2094,23 +2094,10 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
  * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
  * USED state. If already USED, uncharge and return.
  */
-
-static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
-                                       struct page_cgroup *pc,
-                                       enum charge_type ctype,
-                                       int page_size)
+static void ____mem_cgroup_commit_charge(struct mem_cgroup *mem,
+                                         struct page_cgroup *pc,
+                                         enum charge_type ctype)
 {
-        /* try_charge() can return NULL to *memcg, taking care of it. */
-        if (!mem)
-                return;
-
-        lock_page_cgroup(pc);
-        if (unlikely(PageCgroupUsed(pc))) {
-                unlock_page_cgroup(pc);
-                mem_cgroup_cancel_charge(mem, page_size);
-                return;
-        }
-
         pc->mem_cgroup = mem;
         /*
          * We access a page_cgroup asynchronously without lock_page_cgroup().
@@ -2135,6 +2122,33 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
         }
 
         mem_cgroup_charge_statistics(mem, pc, true);
+}
+
+static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
+                                       struct page_cgroup *pc,
+                                       enum charge_type ctype,
+                                       int page_size)
+{
+        int i;
+        int count = page_size >> PAGE_SHIFT;
+
+        /* try_charge() can return NULL to *memcg, taking care of it. */
+        if (!mem)
+                return;
+
+        lock_page_cgroup(pc);
+        if (unlikely(PageCgroupUsed(pc))) {
+                unlock_page_cgroup(pc);
+                mem_cgroup_cancel_charge(mem, page_size);
+                return;
+        }
+
+        /*
+         * we don't need page_cgroup_lock about tail pages, becase they are not
+         * accessed by any other context at this point.
+         */
+        for (i = 0; i < count; i++)
+                ____mem_cgroup_commit_charge(mem, pc + i, ctype);
 
         unlock_page_cgroup(pc);
         /*
@@ -2532,6 +2546,8 @@ direct_uncharge:
 static struct mem_cgroup *
 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 {
+        int i;
+        int count;
         struct page_cgroup *pc;
         struct mem_cgroup *mem = NULL;
         int page_size = PAGE_SIZE;
@@ -2545,6 +2561,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
         if (PageTransHuge(page))
                 page_size <<= compound_order(page);
 
+        count = page_size >> PAGE_SHIFT;
         /*
          * Check if our page_cgroup is valid
          */
@@ -2577,7 +2594,8 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
                 break;
         }
 
-        mem_cgroup_charge_statistics(mem, pc, false);
+        for (i = 0; i < count; i++)
+                mem_cgroup_charge_statistics(mem, pc + i, false);
 
         ClearPageCgroupUsed(pc);
         /*