1 files changed, 131 insertions, 15 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index cdbb7a84cb6e..9ec5e12486a7 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -752,15 +752,13 @@ void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
         /* Update memcg */
         __mod_memcg_state(memcg, idx, val);
 
+        /* Update lruvec */
+        __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
+
         x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
         if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
                 struct mem_cgroup_per_node *pi;
 
-                /*
-                 * Batch local counters to keep them in sync with
-                 * the hierarchical ones.
-                 */
-                __this_cpu_add(pn->lruvec_stat_local->count[idx], x);
                 for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
                         atomic_long_add(x, &pi->lruvec_stat[idx]);
                 x = 0;
@@ -768,6 +766,26 @@ void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
         __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
 }
 
+void __mod_lruvec_slab_state(void *p, enum node_stat_item idx, int val)
+{
+        struct page *page = virt_to_head_page(p);
+        pg_data_t *pgdat = page_pgdat(page);
+        struct mem_cgroup *memcg;
+        struct lruvec *lruvec;
+
+        rcu_read_lock();
+        memcg = memcg_from_slab_page(page);
+
+        /* Untracked pages have no memcg, no lruvec. Update only the node */
+        if (!memcg || memcg == root_mem_cgroup) {
+                __mod_node_page_state(pgdat, idx, val);
+        } else {
+                lruvec = mem_cgroup_lruvec(pgdat, memcg);
+                __mod_lruvec_state(lruvec, idx, val);
+        }
+        rcu_read_unlock();
+}
+
 /**
  * __count_memcg_events - account VM events in a cgroup
  * @memcg: the memory cgroup
@@ -1130,26 +1148,45 @@ void mem_cgroup_iter_break(struct mem_cgroup *root,
                 css_put(&prev->css);
 }
 
-static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
+static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
+                                        struct mem_cgroup *dead_memcg)
 {
-        struct mem_cgroup *memcg = dead_memcg;
         struct mem_cgroup_reclaim_iter *iter;
         struct mem_cgroup_per_node *mz;
         int nid;
         int i;
 
-        for (; memcg; memcg = parent_mem_cgroup(memcg)) {
-                for_each_node(nid) {
-                        mz = mem_cgroup_nodeinfo(memcg, nid);
-                        for (i = 0; i <= DEF_PRIORITY; i++) {
-                                iter = &mz->iter[i];
-                                cmpxchg(&iter->position,
-                                        dead_memcg, NULL);
-                        }
+        for_each_node(nid) {
+                mz = mem_cgroup_nodeinfo(from, nid);
+                for (i = 0; i <= DEF_PRIORITY; i++) {
+                        iter = &mz->iter[i];
+                        cmpxchg(&iter->position,
+                                dead_memcg, NULL);
                 }
         }
 }
 
+static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
+{
+        struct mem_cgroup *memcg = dead_memcg;
+        struct mem_cgroup *last;
+
+        do {
+                __invalidate_reclaim_iterators(memcg, dead_memcg);
+                last = memcg;
+        } while ((memcg = parent_mem_cgroup(memcg)));
+
+        /*
+         * When cgruop1 non-hierarchy mode is used,
+         * parent_mem_cgroup() does not walk all the way up to the
+         * cgroup root (root_mem_cgroup). So we have to handle
+         * dead_memcg from cgroup root separately.
+         */
+        if (last != root_mem_cgroup)
+                __invalidate_reclaim_iterators(root_mem_cgroup,
+                                                dead_memcg);
+}
+
 /**
  * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
  * @memcg: hierarchy root
@@ -3221,6 +3258,72 @@ static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
         }
 }
 
+static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg, bool slab_only)
+{
+        unsigned long stat[MEMCG_NR_STAT];
+        struct mem_cgroup *mi;
+        int node, cpu, i;
+        int min_idx, max_idx;
+
+        if (slab_only) {
+                min_idx = NR_SLAB_RECLAIMABLE;
+                max_idx = NR_SLAB_UNRECLAIMABLE;
+        } else {
+                min_idx = 0;
+                max_idx = MEMCG_NR_STAT;
+        }
+
+        for (i = min_idx; i < max_idx; i++)
+                stat[i] = 0;
+
+        for_each_online_cpu(cpu)
+                for (i = min_idx; i < max_idx; i++)
+                        stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
+
+        for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+                for (i = min_idx; i < max_idx; i++)
+                        atomic_long_add(stat[i], &mi->vmstats[i]);
+
+        if (!slab_only)
+                max_idx = NR_VM_NODE_STAT_ITEMS;
+
+        for_each_node(node) {
+                struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
+                struct mem_cgroup_per_node *pi;
+
+                for (i = min_idx; i < max_idx; i++)
+                        stat[i] = 0;
+
+                for_each_online_cpu(cpu)
+                        for (i = min_idx; i < max_idx; i++)
+                                stat[i] += per_cpu(
+                                        pn->lruvec_stat_cpu->count[i], cpu);
+
+                for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
+                        for (i = min_idx; i < max_idx; i++)
+                                atomic_long_add(stat[i], &pi->lruvec_stat[i]);
+        }
+}
+
+static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
+{
+        unsigned long events[NR_VM_EVENT_ITEMS];
+        struct mem_cgroup *mi;
+        int cpu, i;
+
+        for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                events[i] = 0;
+
+        for_each_online_cpu(cpu)
+                for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                        events[i] += per_cpu(memcg->vmstats_percpu->events[i],
+                                             cpu);
+
+        for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
+                for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
+                        atomic_long_add(events[i], &mi->vmevents[i]);
+}
+
 #ifdef CONFIG_MEMCG_KMEM
 static int memcg_online_kmem(struct mem_cgroup *memcg)
 {
@@ -3270,7 +3373,14 @@ static void memcg_offline_kmem(struct mem_cgroup *memcg)
         if (!parent)
                 parent = root_mem_cgroup;
 
+        /*
+         * Deactivate and reparent kmem_caches. Then flush percpu
+         * slab statistics to have precise values at the parent and
+         * all ancestor levels. It's required to keep slab stats
+         * accurate after the reparenting of kmem_caches.
+         */
         memcg_deactivate_kmem_caches(memcg, parent);
+        memcg_flush_percpu_vmstats(memcg, true);
 
         kmemcg_id = memcg->kmemcg_id;
         BUG_ON(kmemcg_id < 0);
@@ -4643,6 +4753,12 @@ static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
         int node;
 
+        /*
+         * Flush percpu vmstats and vmevents to guarantee the value correctness
+         * on parent's and all ancestor levels.
+         */
+        memcg_flush_percpu_vmstats(memcg, false);
+        memcg_flush_percpu_vmevents(memcg);
         for_each_node(node)
                 free_mem_cgroup_per_node_info(memcg, node);
         free_percpu(memcg->vmstats_percpu);