3 files changed, 302 insertions, 111 deletions

diff --git a/mm/highmem.c b/mm/highmem.c
index 781e754a75ac..693394daa2ed 100644
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -29,6 +29,11 @@
 #include <linux/kgdb.h>
 #include <asm/tlbflush.h>
 
+
+#if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32)
+DEFINE_PER_CPU(int, __kmap_atomic_idx);
+#endif
+
 /*
  * Virtual_count is not a pure "count".
  *  0 means that it is not mapped, and has not been mapped
@@ -43,7 +48,6 @@ unsigned long totalhigh_pages __read_mostly;
 EXPORT_SYMBOL(totalhigh_pages);
 
 
-DEFINE_PER_CPU(int, __kmap_atomic_idx);
 EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx);
 
 unsigned int nr_free_highpages (void)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 9be3cf8a5da4..9a99cfaf0a19 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -89,7 +89,10 @@ enum mem_cgroup_stat_index {
         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
         MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
-        MEM_CGROUP_EVENTS,      /* incremented at every  pagein/pageout */
+        MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
+        /* incremented at every  pagein/pageout */
+        MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
+        MEM_CGROUP_ON_MOVE,     /* someone is moving account between groups */
 
         MEM_CGROUP_STAT_NSTATS,
 };
@@ -254,6 +257,12 @@ struct mem_cgroup {
          * percpu counter.
          */
         struct mem_cgroup_stat_cpu *stat;
+        /*
+         * used when a cpu is offlined or other synchronizations
+         * See mem_cgroup_read_stat().
+         */
+        struct mem_cgroup_stat_cpu nocpu_base;
+        spinlock_t pcp_counter_lock;
 };
 
 /* Stuffs for move charges at task migration. */
@@ -530,14 +539,40 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
         return mz;
 }
 
+/*
+ * Implementation Note: reading percpu statistics for memcg.
+ *
+ * Both of vmstat[] and percpu_counter has threshold and do periodic
+ * synchronization to implement "quick" read. There are trade-off between
+ * reading cost and precision of value. Then, we may have a chance to implement
+ * a periodic synchronizion of counter in memcg's counter.
+ *
+ * But this _read() function is used for user interface now. The user accounts
+ * memory usage by memory cgroup and he _always_ requires exact value because
+ * he accounts memory. Even if we provide quick-and-fuzzy read, we always
+ * have to visit all online cpus and make sum. So, for now, unnecessary
+ * synchronization is not implemented. (just implemented for cpu hotplug)
+ *
+ * If there are kernel internal actions which can make use of some not-exact
+ * value, and reading all cpu value can be performance bottleneck in some
+ * common workload, threashold and synchonization as vmstat[] should be
+ * implemented.
+ */
 static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
                 enum mem_cgroup_stat_index idx)
 {
         int cpu;
         s64 val = 0;
 
-        for_each_possible_cpu(cpu)
+        get_online_cpus();
+        for_each_online_cpu(cpu)
                 val += per_cpu(mem->stat->count[idx], cpu);
+#ifdef CONFIG_HOTPLUG_CPU
+        spin_lock(&mem->pcp_counter_lock);
+        val += mem->nocpu_base.count[idx];
+        spin_unlock(&mem->pcp_counter_lock);
+#endif
+        put_online_cpus();
         return val;
 }
 
@@ -659,40 +694,83 @@ static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
         return mem;
 }
 
-/*
- * Call callback function against all cgroup under hierarchy tree.
- */
-static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
-                          int (*func)(struct mem_cgroup *, void *))
+/* The caller has to guarantee "mem" exists before calling this */
+static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
 {
-        int found, ret, nextid;
         struct cgroup_subsys_state *css;
-        struct mem_cgroup *mem;
-
-        if (!root->use_hierarchy)
-                return (*func)(root, data);
+        int found;
 
-        nextid = 1;
-        do {
-                ret = 0;
+        if (!mem) /* ROOT cgroup has the smallest ID */
+                return root_mem_cgroup; /*css_put/get against root is ignored*/
+        if (!mem->use_hierarchy) {
+                if (css_tryget(&mem->css))
+                        return mem;
+                return NULL;
+        }
+        rcu_read_lock();
+        /*
+         * searching a memory cgroup which has the smallest ID under given
+         * ROOT cgroup. (ID >= 1)
+         */
+        css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
+        if (css && css_tryget(css))
+                mem = container_of(css, struct mem_cgroup, css);
+        else
                 mem = NULL;
+        rcu_read_unlock();
+        return mem;
+}
+
+static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
+                                        struct mem_cgroup *root,
+                                        bool cond)
+{
+        int nextid = css_id(&iter->css) + 1;
+        int found;
+        int hierarchy_used;
+        struct cgroup_subsys_state *css;
+
+        hierarchy_used = iter->use_hierarchy;
 
+        css_put(&iter->css);
+        /* If no ROOT, walk all, ignore hierarchy */
+        if (!cond || (root && !hierarchy_used))
+                return NULL;
+
+        if (!root)
+                root = root_mem_cgroup;
+
+        do {
+                iter = NULL;
                 rcu_read_lock();
-                css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
-                                   &found);
+
+                css = css_get_next(&mem_cgroup_subsys, nextid,
+                                &root->css, &found);
                 if (css && css_tryget(css))
-                        mem = container_of(css, struct mem_cgroup, css);
+                        iter = container_of(css, struct mem_cgroup, css);
                 rcu_read_unlock();
-
-                if (mem) {
-                        ret = (*func)(mem, data);
-                        css_put(&mem->css);
-                }
+                /* If css is NULL, no more cgroups will be found */
                 nextid = found + 1;
-        } while (!ret && css);
+        } while (css && !iter);
 
-        return ret;
+        return iter;
 }
+/*
+ * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
+ * be careful that "break" loop is not allowed. We have reference count.
+ * Instead of that modify "cond" to be false and "continue" to exit the loop.
+ */
+#define for_each_mem_cgroup_tree_cond(iter, root, cond) \
+        for (iter = mem_cgroup_start_loop(root);\
+             iter != NULL;\
+             iter = mem_cgroup_get_next(iter, root, cond))
+
+#define for_each_mem_cgroup_tree(iter, root) \
+        for_each_mem_cgroup_tree_cond(iter, root, true)
+
+#define for_each_mem_cgroup_all(iter) \
+        for_each_mem_cgroup_tree_cond(iter, NULL, true)
+
 
 static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
 {
@@ -1051,7 +1129,52 @@ static unsigned int get_swappiness(struct mem_cgroup *memcg)
         return swappiness;
 }
 
-/* A routine for testing mem is not under move_account */
+static void mem_cgroup_start_move(struct mem_cgroup *mem)
+{
+        int cpu;
+
+        get_online_cpus();
+        spin_lock(&mem->pcp_counter_lock);
+        for_each_online_cpu(cpu)
+                per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
+        mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
+        spin_unlock(&mem->pcp_counter_lock);
+        put_online_cpus();
+
+        synchronize_rcu();
+}
+
+static void mem_cgroup_end_move(struct mem_cgroup *mem)
+{
+        int cpu;
+
+        if (!mem)
+                return;
+        get_online_cpus();
+        spin_lock(&mem->pcp_counter_lock);
+        for_each_online_cpu(cpu)
+                per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
+        mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
+        spin_unlock(&mem->pcp_counter_lock);
+        put_online_cpus();
+}
+/*
+ * 2 routines for checking "mem" is under move_account() or not.
+ *
+ * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
+ *                        for avoiding race in accounting. If true,
+ *                        pc->mem_cgroup may be overwritten.
+ *
+ * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
+ *                        under hierarchy of moving cgroups. This is for
+ *                        waiting at hith-memory prressure caused by "move".
+ */
+
+static bool mem_cgroup_stealed(struct mem_cgroup *mem)
+{
+        VM_BUG_ON(!rcu_read_lock_held());
+        return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
+}
 
 static bool mem_cgroup_under_move(struct mem_cgroup *mem)
 {
@@ -1092,13 +1215,6 @@ static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
         return false;
 }
 
-static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
-{
-        int *val = data;
-        (*val)++;
-        return 0;
-}
-
 /**
  * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
  * @memcg: The memory cgroup that went over limit
@@ -1173,7 +1289,10 @@ done:
 static int mem_cgroup_count_children(struct mem_cgroup *mem)
 {
         int num = 0;
-        mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
+        struct mem_cgroup *iter;
+
+        for_each_mem_cgroup_tree(iter, mem)
+                num++;
         return num;
 }
 
@@ -1322,49 +1441,39 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
         return total;
 }
 
-static int mem_cgroup_oom_lock_cb(struct mem_cgroup *mem, void *data)
-{
-        int *val = (int *)data;
-        int x;
-        /*
-         * Logically, we can stop scanning immediately when we find
-         * a memcg is already locked. But condidering unlock ops and
-         * creation/removal of memcg, scan-all is simple operation.
-         */
-        x = atomic_inc_return(&mem->oom_lock);
-        *val = max(x, *val);
-        return 0;
-}
 /*
  * Check OOM-Killer is already running under our hierarchy.
  * If someone is running, return false.
  */
 static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
 {
-        int lock_count = 0;
+        int x, lock_count = 0;
+        struct mem_cgroup *iter;
 
-        mem_cgroup_walk_tree(mem, &lock_count, mem_cgroup_oom_lock_cb);
+        for_each_mem_cgroup_tree(iter, mem) {
+                x = atomic_inc_return(&iter->oom_lock);
+                lock_count = max(x, lock_count);
+        }
 
         if (lock_count == 1)
                 return true;
         return false;
 }
 
-static int mem_cgroup_oom_unlock_cb(struct mem_cgroup *mem, void *data)
+static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
 {
+        struct mem_cgroup *iter;
+
         /*
          * When a new child is created while the hierarchy is under oom,
          * mem_cgroup_oom_lock() may not be called. We have to use
          * atomic_add_unless() here.
          */
-        atomic_add_unless(&mem->oom_lock, -1, 0);
+        for_each_mem_cgroup_tree(iter, mem)
+                atomic_add_unless(&iter->oom_lock, -1, 0);
         return 0;
 }
 
-static void mem_cgroup_oom_unlock(struct mem_cgroup *mem)
-{
-        mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_unlock_cb);
-}
 
 static DEFINE_MUTEX(memcg_oom_mutex);
 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
@@ -1462,34 +1571,73 @@ bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
 /*
  * Currently used to update mapped file statistics, but the routine can be
  * generalized to update other statistics as well.
+ *
+ * Notes: Race condition
+ *
+ * We usually use page_cgroup_lock() for accessing page_cgroup member but
+ * it tends to be costly. But considering some conditions, we doesn't need
+ * to do so _always_.
+ *
+ * Considering "charge", lock_page_cgroup() is not required because all
+ * file-stat operations happen after a page is attached to radix-tree. There
+ * are no race with "charge".
+ *
+ * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
+ * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
+ * if there are race with "uncharge". Statistics itself is properly handled
+ * by flags.
+ *
+ * Considering "move", this is an only case we see a race. To make the race
+ * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
+ * possibility of race condition. If there is, we take a lock.
  */
-void mem_cgroup_update_file_mapped(struct page *page, int val)
+
+static void mem_cgroup_update_file_stat(struct page *page, int idx, int val)
 {
         struct mem_cgroup *mem;
-        struct page_cgroup *pc;
+        struct page_cgroup *pc = lookup_page_cgroup(page);
+        bool need_unlock = false;
 
-        pc = lookup_page_cgroup(page);
         if (unlikely(!pc))
                 return;
 
-        lock_page_cgroup(pc);
+        rcu_read_lock();
         mem = pc->mem_cgroup;
-        if (!mem || !PageCgroupUsed(pc))
-                goto done;
+        if (unlikely(!mem || !PageCgroupUsed(pc)))
+                goto out;
+        /* pc->mem_cgroup is unstable ? */
+        if (unlikely(mem_cgroup_stealed(mem))) {
+                /* take a lock against to access pc->mem_cgroup */
+                lock_page_cgroup(pc);
+                need_unlock = true;
+                mem = pc->mem_cgroup;
+                if (!mem || !PageCgroupUsed(pc))
+                        goto out;
+        }
 
-        /*
-         * Preemption is already disabled. We can use __this_cpu_xxx
-         */
-        if (val > 0) {
-                __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
-                SetPageCgroupFileMapped(pc);
-        } else {
-                __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
-                ClearPageCgroupFileMapped(pc);
+        this_cpu_add(mem->stat->count[idx], val);
+
+        switch (idx) {
+        case MEM_CGROUP_STAT_FILE_MAPPED:
+                if (val > 0)
+                        SetPageCgroupFileMapped(pc);
+                else if (!page_mapped(page))
+                        ClearPageCgroupFileMapped(pc);
+                break;
+        default:
+                BUG();
         }
 
-done:
-        unlock_page_cgroup(pc);
+out:
+        if (unlikely(need_unlock))
+                unlock_page_cgroup(pc);
+        rcu_read_unlock();
+        return;
+}
+
+void mem_cgroup_update_file_mapped(struct page *page, int val)
+{
+        mem_cgroup_update_file_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, val);
 }
 
 /*
@@ -1605,15 +1753,55 @@ static void drain_all_stock_sync(void)
         atomic_dec(&memcg_drain_count);
 }
 
-static int __cpuinit memcg_stock_cpu_callback(struct notifier_block *nb,
+/*
+ * This function drains percpu counter value from DEAD cpu and
+ * move it to local cpu. Note that this function can be preempted.
+ */
+static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
+{
+        int i;
+
+        spin_lock(&mem->pcp_counter_lock);
+        for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
+                s64 x = per_cpu(mem->stat->count[i], cpu);
+
+                per_cpu(mem->stat->count[i], cpu) = 0;
+                mem->nocpu_base.count[i] += x;
+        }
+        /* need to clear ON_MOVE value, works as a kind of lock. */
+        per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
+        spin_unlock(&mem->pcp_counter_lock);
+}
+
+static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
+{
+        int idx = MEM_CGROUP_ON_MOVE;
+
+        spin_lock(&mem->pcp_counter_lock);
+        per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
+        spin_unlock(&mem->pcp_counter_lock);
+}
+
+static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
                                         unsigned long action,
                                         void *hcpu)
 {
         int cpu = (unsigned long)hcpu;
         struct memcg_stock_pcp *stock;
+        struct mem_cgroup *iter;
+
+        if ((action == CPU_ONLINE)) {
+                for_each_mem_cgroup_all(iter)
+                        synchronize_mem_cgroup_on_move(iter, cpu);
+                return NOTIFY_OK;
+        }
 
-        if (action != CPU_DEAD)
+        if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
                 return NOTIFY_OK;
+
+        for_each_mem_cgroup_all(iter)
+                mem_cgroup_drain_pcp_counter(iter, cpu);
+
         stock = &per_cpu(memcg_stock, cpu);
         drain_stock(stock);
         return NOTIFY_OK;
@@ -3038,6 +3226,7 @@ move_account:
                 lru_add_drain_all();
                 drain_all_stock_sync();
                 ret = 0;
+                mem_cgroup_start_move(mem);
                 for_each_node_state(node, N_HIGH_MEMORY) {
                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
                                 enum lru_list l;
@@ -3051,6 +3240,7 @@ move_account:
                         if (ret)
                                 break;
                 }
+                mem_cgroup_end_move(mem);
                 memcg_oom_recover(mem);
                 /* it seems parent cgroup doesn't have enough mem */
                 if (ret == -ENOMEM)
@@ -3137,33 +3327,25 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
         return retval;
 }
 
-struct mem_cgroup_idx_data {
-        s64 val;
-        enum mem_cgroup_stat_index idx;
-};
 
-static int
-mem_cgroup_get_idx_stat(struct mem_cgroup *mem, void *data)
+static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
+                                enum mem_cgroup_stat_index idx)
 {
-        struct mem_cgroup_idx_data *d = data;
-        d->val += mem_cgroup_read_stat(mem, d->idx);
-        return 0;
-}
+        struct mem_cgroup *iter;
+        s64 val = 0;
 
-static void
-mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
-                                enum mem_cgroup_stat_index idx, s64 *val)
-{
-        struct mem_cgroup_idx_data d;
-        d.idx = idx;
-        d.val = 0;
-        mem_cgroup_walk_tree(mem, &d, mem_cgroup_get_idx_stat);
-        *val = d.val;
+        /* each per cpu's value can be minus.Then, use s64 */
+        for_each_mem_cgroup_tree(iter, mem)
+                val += mem_cgroup_read_stat(iter, idx);
+
+        if (val < 0) /* race ? */
+                val = 0;
+        return val;
 }
 
 static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
 {
-        u64 idx_val, val;
+        u64 val;
 
         if (!mem_cgroup_is_root(mem)) {
                 if (!swap)
@@ -3172,16 +3354,12 @@ static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
                         return res_counter_read_u64(&mem->memsw, RES_USAGE);
         }
 
-        mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE, &idx_val);
-        val = idx_val;
-        mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS, &idx_val);
-        val += idx_val;
+        val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
+        val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
 
-        if (swap) {
-                mem_cgroup_get_recursive_idx_stat(mem,
-                                MEM_CGROUP_STAT_SWAPOUT, &idx_val);
-                val += idx_val;
-        }
+        if (swap)
+                val += mem_cgroup_get_recursive_idx_stat(mem,
+                                MEM_CGROUP_STAT_SWAPOUT);
 
         return val << PAGE_SHIFT;
 }
@@ -3389,9 +3567,9 @@ struct {
 };
 
 
-static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
+static void
+mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
 {
-        struct mcs_total_stat *s = data;
         s64 val;
 
         /* per cpu stat */
@@ -3421,13 +3599,15 @@ static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
         s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
         val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
         s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
-        return 0;
 }
 
 static void
 mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
 {
-        mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
+        struct mem_cgroup *iter;
+
+        for_each_mem_cgroup_tree(iter, mem)
+                mem_cgroup_get_local_stat(iter, s);
 }
 
 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
@@ -3604,7 +3784,7 @@ static int compare_thresholds(const void *a, const void *b)
         return _a->threshold - _b->threshold;
 }
 
-static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem, void *data)
+static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
 {
         struct mem_cgroup_eventfd_list *ev;
 
@@ -3615,7 +3795,10 @@ static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem, void *data)
 
 static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
 {
-        mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_notify_cb);
+        struct mem_cgroup *iter;
+
+        for_each_mem_cgroup_tree(iter, mem)
+                mem_cgroup_oom_notify_cb(iter);
 }
 
 static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
@@ -4032,6 +4215,7 @@ static struct mem_cgroup *mem_cgroup_alloc(void)
                         vfree(mem);
                 mem = NULL;
         }
+        spin_lock_init(&mem->pcp_counter_lock);
         return mem;
 }
 
@@ -4158,7 +4342,7 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                                                 &per_cpu(memcg_stock, cpu);
                         INIT_WORK(&stock->work, drain_local_stock);
                 }
-                hotcpu_notifier(memcg_stock_cpu_callback, 0);
+                hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
         } else {
                 parent = mem_cgroup_from_cont(cont->parent);
                 mem->use_hierarchy = parent->use_hierarchy;
@@ -4513,6 +4697,7 @@ static void mem_cgroup_clear_mc(void)
         mc.to = NULL;
         mc.moving_task = NULL;
         spin_unlock(&mc.lock);
+        mem_cgroup_end_move(from);
         memcg_oom_recover(from);
         memcg_oom_recover(to);
         wake_up_all(&mc.waitq);
@@ -4543,6 +4728,7 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                         VM_BUG_ON(mc.moved_charge);
                         VM_BUG_ON(mc.moved_swap);
                         VM_BUG_ON(mc.moving_task);
+                        mem_cgroup_start_move(from);
                         spin_lock(&mc.lock);
                         mc.from = from;
                         mc.to = mem;
diff --git a/mm/swap.c b/mm/swap.c
index 3ce7bc373a52..3f4854205b16 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -378,6 +378,7 @@ void release_pages(struct page **pages, int nr, int cold)
 
         pagevec_free(&pages_to_free);
 }
+EXPORT_SYMBOL(release_pages);
 
 /*
  * The pages which we're about to release may be in the deferred lru-addition