1 files changed, 60 insertions, 0 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6f5c0c517c49..26e2999af608 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -3260,6 +3260,60 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
         }
 }
 
+static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
+{
+        unsigned long stat[MEMCG_NR_STAT];
+        struct mem_cgroup *mi;
+        int node, cpu, i;
+
+        for (i = 0; i < MEMCG_NR_STAT; i++)
+                stat[i] = 0;
+
+        for_each_online_cpu(cpu)
+                for (i = 0; i < MEMCG_NR_STAT; i++)
+                        stat[i] += raw_cpu_read(memcg->vmstats_percpu->stat[i]);
+
+        for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+                for (i = 0; i < MEMCG_NR_STAT; i++)
+                        atomic_long_add(stat[i], &mi->vmstats[i]);
+
+        for_each_node(node) {
+                struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
+                struct mem_cgroup_per_node *pi;
+
+                for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                        stat[i] = 0;
+
+                for_each_online_cpu(cpu)
+                        for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                                stat[i] += raw_cpu_read(
+                                        pn->lruvec_stat_cpu->count[i]);
+
+                for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
+                        for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
+                                atomic_long_add(stat[i], &pi->lruvec_stat[i]);
+        }
+}
+
+static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
+{
+        unsigned long events[NR_VM_EVENT_ITEMS];
+        struct mem_cgroup *mi;
+        int cpu, i;
+
+        for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                events[i] = 0;
+
+        for_each_online_cpu(cpu)
+                for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                        events[i] += raw_cpu_read(
+                                memcg->vmstats_percpu->events[i]);
+
+        for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+                for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                        atomic_long_add(events[i], &mi->vmevents[i]);
+}
+
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_online_kmem(struct mem_cgroup *memcg)
 {
@@ -4682,6 +4736,12 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
         int node;
 
+        /*
+         * Flush percpu vmstats and vmevents to guarantee the value correctness
+         * on parent's and all ancestor levels.
+         */
+        memcg_flush_percpu_vmstats(memcg);
+        memcg_flush_percpu_vmevents(memcg);
         for_each_node(node)
                 free_mem_cgroup_per_node_info(memcg, node);
         free_percpu(memcg->vmstats_percpu);