12 files changed, 236 insertions, 268 deletions

diff --git a/mm/Makefile b/mm/Makefile
index 9f117bab5322..a5b0dd93427a 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -32,5 +32,5 @@ obj-$(CONFIG_FS_XIP) += filemap_xip.o
 obj-$(CONFIG_MIGRATION) += migrate.o
 obj-$(CONFIG_SMP) += allocpercpu.o
 obj-$(CONFIG_QUICKLIST) += quicklist.o
-obj-$(CONFIG_CGROUP_MEM_CONT) += memcontrol.o
+obj-$(CONFIG_CGROUP_MEM_RES_CTLR) += memcontrol.o
 
diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
index 7e58322b7134..b0012e27fea8 100644
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -6,6 +6,10 @@
 #include <linux/mm.h>
 #include <linux/module.h>
 
+#ifndef cache_line_size
+#define cache_line_size()       L1_CACHE_BYTES
+#endif
+
 /**
  * percpu_depopulate - depopulate per-cpu data for given cpu
  * @__pdata: per-cpu data to depopulate
@@ -52,6 +56,11 @@ void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu)
         struct percpu_data *pdata = __percpu_disguise(__pdata);
         int node = cpu_to_node(cpu);
 
+        /*
+         * We should make sure each CPU gets private memory.
+         */
+        size = roundup(size, cache_line_size());
+
         BUG_ON(pdata->ptrs[cpu]);
         if (node_online(node))
                 pdata->ptrs[cpu] = kmalloc_node(size, gfp|__GFP_ZERO, node);
@@ -98,7 +107,11 @@ EXPORT_SYMBOL_GPL(__percpu_populate_mask);
  */
 void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
 {
-        void *pdata = kzalloc(nr_cpu_ids * sizeof(void *), gfp);
+        /*
+         * We allocate whole cache lines to avoid false sharing
+         */
+        size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size());
+        void *pdata = kzalloc(sz, gfp);
         void *__pdata = __percpu_disguise(pdata);
 
         if (unlikely(!pdata))
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 89e6286a7f57..dcacc811e70e 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -71,7 +71,25 @@ static void enqueue_huge_page(struct page *page)
         free_huge_pages_node[nid]++;
 }
 
-static struct page *dequeue_huge_page(struct vm_area_struct *vma,
+static struct page *dequeue_huge_page(void)
+{
+        int nid;
+        struct page *page = NULL;
+
+        for (nid = 0; nid < MAX_NUMNODES; ++nid) {
+                if (!list_empty(&hugepage_freelists[nid])) {
+                        page = list_entry(hugepage_freelists[nid].next,
+                                          struct page, lru);
+                        list_del(&page->lru);
+                        free_huge_pages--;
+                        free_huge_pages_node[nid]--;
+                        break;
+                }
+        }
+        return page;
+}
+
+static struct page *dequeue_huge_page_vma(struct vm_area_struct *vma,
                                 unsigned long address)
 {
         int nid;
@@ -296,8 +314,10 @@ static int gather_surplus_pages(int delta)
         int needed, allocated;
 
         needed = (resv_huge_pages + delta) - free_huge_pages;
-        if (needed <= 0)
+        if (needed <= 0) {
+                resv_huge_pages += delta;
                 return 0;
+        }
 
         allocated = 0;
         INIT_LIST_HEAD(&surplus_list);
@@ -335,9 +355,12 @@ retry:
          * The surplus_list now contains _at_least_ the number of extra pages
          * needed to accomodate the reservation.  Add the appropriate number
          * of pages to the hugetlb pool and free the extras back to the buddy
-         * allocator.
+         * allocator.  Commit the entire reservation here to prevent another
+         * process from stealing the pages as they are added to the pool but
+         * before they are reserved.
          */
         needed += allocated;
+        resv_huge_pages += delta;
         ret = 0;
 free:
         list_for_each_entry_safe(page, tmp, &surplus_list, lru) {
@@ -371,6 +394,9 @@ static void return_unused_surplus_pages(unsigned long unused_resv_pages)
         struct page *page;
         unsigned long nr_pages;
 
+        /* Uncommit the reservation */
+        resv_huge_pages -= unused_resv_pages;
+
         nr_pages = min(unused_resv_pages, surplus_huge_pages);
 
         while (nr_pages) {
@@ -402,7 +428,7 @@ static struct page *alloc_huge_page_shared(struct vm_area_struct *vma,
         struct page *page;
 
         spin_lock(&hugetlb_lock);
-        page = dequeue_huge_page(vma, addr);
+        page = dequeue_huge_page_vma(vma, addr);
         spin_unlock(&hugetlb_lock);
         return page ? page : ERR_PTR(-VM_FAULT_OOM);
 }
@@ -417,7 +443,7 @@ static struct page *alloc_huge_page_private(struct vm_area_struct *vma,
 
         spin_lock(&hugetlb_lock);
         if (free_huge_pages > resv_huge_pages)
-                page = dequeue_huge_page(vma, addr);
+                page = dequeue_huge_page_vma(vma, addr);
         spin_unlock(&hugetlb_lock);
         if (!page) {
                 page = alloc_buddy_huge_page(vma, addr);
@@ -570,7 +596,7 @@ static unsigned long set_max_huge_pages(unsigned long count)
         min_count = max(count, min_count);
         try_to_free_low(min_count);
         while (min_count < persistent_huge_pages) {
-                struct page *page = dequeue_huge_page(NULL, 0);
+                struct page *page = dequeue_huge_page();
                 if (!page)
                         break;
                 update_and_free_page(page);
@@ -1205,12 +1231,13 @@ static int hugetlb_acct_memory(long delta)
                 if (gather_surplus_pages(delta) < 0)
                         goto out;
 
-                if (delta > cpuset_mems_nr(free_huge_pages_node))
+                if (delta > cpuset_mems_nr(free_huge_pages_node)) {
+                        return_unused_surplus_pages(delta);
                         goto out;
+                }
         }
 
         ret = 0;
-        resv_huge_pages += delta;
         if (delta < 0)
                 return_unused_surplus_pages((unsigned long) -delta);
 
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 631002d085d1..8b9f6cae938e 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -137,14 +137,21 @@ struct mem_cgroup {
          */
         struct mem_cgroup_stat stat;
 };
+static struct mem_cgroup init_mem_cgroup;
 
 /*
  * We use the lower bit of the page->page_cgroup pointer as a bit spin
- * lock. We need to ensure that page->page_cgroup is atleast two
- * byte aligned (based on comments from Nick Piggin)
+ * lock.  We need to ensure that page->page_cgroup is at least two
+ * byte aligned (based on comments from Nick Piggin).  But since
+ * bit_spin_lock doesn't actually set that lock bit in a non-debug
+ * uniprocessor kernel, we should avoid setting it here too.
  */
 #define PAGE_CGROUP_LOCK_BIT    0x0
-#define PAGE_CGROUP_LOCK                (1 << PAGE_CGROUP_LOCK_BIT)
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+#define PAGE_CGROUP_LOCK        (1 << PAGE_CGROUP_LOCK_BIT)
+#else
+#define PAGE_CGROUP_LOCK        0x0
+#endif
 
 /*
  * A page_cgroup page is associated with every page descriptor. The
@@ -154,37 +161,27 @@ struct page_cgroup {
         struct list_head lru;           /* per cgroup LRU list */
         struct page *page;
         struct mem_cgroup *mem_cgroup;
-        atomic_t ref_cnt;               /* Helpful when pages move b/w  */
-                                        /* mapped and cached states     */
-        int      flags;
+        int ref_cnt;                    /* cached, mapped, migrating */
+        int flags;
 };
 #define PAGE_CGROUP_FLAG_CACHE  (0x1)   /* charged as cache */
 #define PAGE_CGROUP_FLAG_ACTIVE (0x2)   /* page is active in this cgroup */
 
-static inline int page_cgroup_nid(struct page_cgroup *pc)
+static int page_cgroup_nid(struct page_cgroup *pc)
 {
         return page_to_nid(pc->page);
 }
 
-static inline enum zone_type page_cgroup_zid(struct page_cgroup *pc)
+static enum zone_type page_cgroup_zid(struct page_cgroup *pc)
 {
         return page_zonenum(pc->page);
 }
 
-enum {
-        MEM_CGROUP_TYPE_UNSPEC = 0,
-        MEM_CGROUP_TYPE_MAPPED,
-        MEM_CGROUP_TYPE_CACHED,
-        MEM_CGROUP_TYPE_ALL,
-        MEM_CGROUP_TYPE_MAX,
-};
-
 enum charge_type {
         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
         MEM_CGROUP_CHARGE_TYPE_MAPPED,
 };
 
-
 /*
  * Always modified under lru lock. Then, not necessary to preempt_disable()
  */
@@ -193,23 +190,21 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
 {
         int val = (charge)? 1 : -1;
         struct mem_cgroup_stat *stat = &mem->stat;
-        VM_BUG_ON(!irqs_disabled());
 
+        VM_BUG_ON(!irqs_disabled());
         if (flags & PAGE_CGROUP_FLAG_CACHE)
-                __mem_cgroup_stat_add_safe(stat,
-                                        MEM_CGROUP_STAT_CACHE, val);
+                __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_CACHE, val);
         else
                 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
 }
 
-static inline struct mem_cgroup_per_zone *
+static struct mem_cgroup_per_zone *
 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
 {
-        BUG_ON(!mem->info.nodeinfo[nid]);
         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
 }
 
-static inline struct mem_cgroup_per_zone *
+static struct mem_cgroup_per_zone *
 page_cgroup_zoneinfo(struct page_cgroup *pc)
 {
         struct mem_cgroup *mem = pc->mem_cgroup;
@@ -234,18 +229,14 @@ static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
         return total;
 }
 
-static struct mem_cgroup init_mem_cgroup;
-
-static inline
-struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
+static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 {
         return container_of(cgroup_subsys_state(cont,
                                 mem_cgroup_subsys_id), struct mem_cgroup,
                                 css);
 }
 
-static inline
-struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
+static struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 {
         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
                                 struct mem_cgroup, css);
@@ -267,81 +258,33 @@ void mm_free_cgroup(struct mm_struct *mm)
 
 static inline int page_cgroup_locked(struct page *page)
 {
-        return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT,
-                                        &page->page_cgroup);
+        return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 }
 
-void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
+static void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc)
 {
-        int locked;
-
-        /*
-         * While resetting the page_cgroup we might not hold the
-         * page_cgroup lock. free_hot_cold_page() is an example
-         * of such a scenario
-         */
-        if (pc)
-                VM_BUG_ON(!page_cgroup_locked(page));
-        locked = (page->page_cgroup & PAGE_CGROUP_LOCK);
-        page->page_cgroup = ((unsigned long)pc | locked);
+        VM_BUG_ON(!page_cgroup_locked(page));
+        page->page_cgroup = ((unsigned long)pc | PAGE_CGROUP_LOCK);
 }
 
 struct page_cgroup *page_get_page_cgroup(struct page *page)
 {
-        return (struct page_cgroup *)
-                (page->page_cgroup & ~PAGE_CGROUP_LOCK);
+        return (struct page_cgroup *) (page->page_cgroup & ~PAGE_CGROUP_LOCK);
 }
 
-static void __always_inline lock_page_cgroup(struct page *page)
+static void lock_page_cgroup(struct page *page)
 {
         bit_spin_lock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
-        VM_BUG_ON(!page_cgroup_locked(page));
-}
-
-static void __always_inline unlock_page_cgroup(struct page *page)
-{
-        bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 }
 
-/*
- * Tie new page_cgroup to struct page under lock_page_cgroup()
- * This can fail if the page has been tied to a page_cgroup.
- * If success, returns 0.
- */
-static int page_cgroup_assign_new_page_cgroup(struct page *page,
-                                                struct page_cgroup *pc)
+static int try_lock_page_cgroup(struct page *page)
 {
-        int ret = 0;
-
-        lock_page_cgroup(page);
-        if (!page_get_page_cgroup(page))
-                page_assign_page_cgroup(page, pc);
-        else /* A page is tied to other pc. */
-                ret = 1;
-        unlock_page_cgroup(page);
-        return ret;
+        return bit_spin_trylock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 }
 
-/*
- * Clear page->page_cgroup member under lock_page_cgroup().
- * If given "pc" value is different from one page->page_cgroup,
- * page->cgroup is not cleared.
- * Returns a value of page->page_cgroup at lock taken.
- * A can can detect failure of clearing by following
- *  clear_page_cgroup(page, pc) == pc
- */
-
-static struct page_cgroup *clear_page_cgroup(struct page *page,
-                                                struct page_cgroup *pc)
+static void unlock_page_cgroup(struct page *page)
 {
-        struct page_cgroup *ret;
-        /* lock and clear */
-        lock_page_cgroup(page);
-        ret = page_get_page_cgroup(page);
-        if (likely(ret == pc))
-                page_assign_page_cgroup(page, NULL);
-        unlock_page_cgroup(page);
-        return ret;
+        bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 }
 
 static void __mem_cgroup_remove_list(struct page_cgroup *pc)
@@ -399,7 +342,7 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
         int ret;
 
         task_lock(task);
-        ret = task->mm && vm_match_cgroup(task->mm, mem);
+        ret = task->mm && mm_match_cgroup(task->mm, mem);
         task_unlock(task);
         return ret;
 }
@@ -407,18 +350,30 @@ int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
 /*
  * This routine assumes that the appropriate zone's lru lock is already held
  */
-void mem_cgroup_move_lists(struct page_cgroup *pc, bool active)
+void mem_cgroup_move_lists(struct page *page, bool active)
 {
+        struct page_cgroup *pc;
         struct mem_cgroup_per_zone *mz;
         unsigned long flags;
 
-        if (!pc)
+        /*
+         * We cannot lock_page_cgroup while holding zone's lru_lock,
+         * because other holders of lock_page_cgroup can be interrupted
+         * with an attempt to rotate_reclaimable_page.  But we cannot
+         * safely get to page_cgroup without it, so just try_lock it:
+         * mem_cgroup_isolate_pages allows for page left on wrong list.
+         */
+        if (!try_lock_page_cgroup(page))
                 return;
 
-        mz = page_cgroup_zoneinfo(pc);
-        spin_lock_irqsave(&mz->lru_lock, flags);
-        __mem_cgroup_move_lists(pc, active);
-        spin_unlock_irqrestore(&mz->lru_lock, flags);
+        pc = page_get_page_cgroup(page);
+        if (pc) {
+                mz = page_cgroup_zoneinfo(pc);
+                spin_lock_irqsave(&mz->lru_lock, flags);
+                __mem_cgroup_move_lists(pc, active);
+                spin_unlock_irqrestore(&mz->lru_lock, flags);
+        }
+        unlock_page_cgroup(page);
 }
 
 /*
@@ -437,6 +392,7 @@ int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
         rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
         return (int)((rss * 100L) / total);
 }
+
 /*
  * This function is called from vmscan.c. In page reclaiming loop. balance
  * between active and inactive list is calculated. For memory controller
@@ -500,7 +456,6 @@ long mem_cgroup_calc_reclaim_inactive(struct mem_cgroup *mem,
         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
 
         nr_inactive = MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE);
-
         return (nr_inactive >> priority);
 }
 
@@ -586,26 +541,21 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
          * with it
          */
 retry:
-        if (page) {
-                lock_page_cgroup(page);
-                pc = page_get_page_cgroup(page);
-                /*
-                 * The page_cgroup exists and
-                 * the page has already been accounted.
-                 */
-                if (pc) {
-                        if (unlikely(!atomic_inc_not_zero(&pc->ref_cnt))) {
-                                /* this page is under being uncharged ? */
-                                unlock_page_cgroup(page);
-                                cpu_relax();
-                                goto retry;
-                        } else {
-                                unlock_page_cgroup(page);
-                                goto done;
-                        }
-                }
+        lock_page_cgroup(page);
+        pc = page_get_page_cgroup(page);
+        /*
+         * The page_cgroup exists and
+         * the page has already been accounted.
+         */
+        if (pc) {
+                VM_BUG_ON(pc->page != page);
+                VM_BUG_ON(pc->ref_cnt <= 0);
+
+                pc->ref_cnt++;
                 unlock_page_cgroup(page);
+                goto done;
         }
+        unlock_page_cgroup(page);
 
         pc = kzalloc(sizeof(struct page_cgroup), gfp_mask);
         if (pc == NULL)
@@ -623,16 +573,11 @@ retry:
         rcu_read_lock();
         mem = rcu_dereference(mm->mem_cgroup);
         /*
-         * For every charge from the cgroup, increment reference
-         * count
+         * For every charge from the cgroup, increment reference count
          */
         css_get(&mem->css);
         rcu_read_unlock();
 
-        /*
-         * If we created the page_cgroup, we should free it on exceeding
-         * the cgroup limit.
-         */
         while (res_counter_charge(&mem->res, PAGE_SIZE)) {
                 if (!(gfp_mask & __GFP_WAIT))
                         goto out;
@@ -641,12 +586,12 @@ retry:
                         continue;
 
                 /*
-                 * try_to_free_mem_cgroup_pages() might not give us a full
-                 * picture of reclaim. Some pages are reclaimed and might be
-                 * moved to swap cache or just unmapped from the cgroup.
-                 * Check the limit again to see if the reclaim reduced the
-                 * current usage of the cgroup before giving up
-                 */
+                 * try_to_free_mem_cgroup_pages() might not give us a full
+                 * picture of reclaim. Some pages are reclaimed and might be
+                 * moved to swap cache or just unmapped from the cgroup.
+                 * Check the limit again to see if the reclaim reduced the
+                 * current usage of the cgroup before giving up
+                 */
                 if (res_counter_check_under_limit(&mem->res))
                         continue;
 
@@ -657,14 +602,16 @@ retry:
                 congestion_wait(WRITE, HZ/10);
         }
 
-        atomic_set(&pc->ref_cnt, 1);
+        pc->ref_cnt = 1;
         pc->mem_cgroup = mem;
         pc->page = page;
         pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
         if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
                 pc->flags |= PAGE_CGROUP_FLAG_CACHE;
 
-        if (!page || page_cgroup_assign_new_page_cgroup(page, pc)) {
+        lock_page_cgroup(page);
+        if (page_get_page_cgroup(page)) {
+                unlock_page_cgroup(page);
                 /*
                  * Another charge has been added to this page already.
                  * We take lock_page_cgroup(page) again and read
@@ -673,17 +620,16 @@ retry:
                 res_counter_uncharge(&mem->res, PAGE_SIZE);
                 css_put(&mem->css);
                 kfree(pc);
-                if (!page)
-                        goto done;
                 goto retry;
         }
+        page_assign_page_cgroup(page, pc);
 
         mz = page_cgroup_zoneinfo(pc);
         spin_lock_irqsave(&mz->lru_lock, flags);
-        /* Update statistics vector */
         __mem_cgroup_add_list(pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
 
+        unlock_page_cgroup(page);
 done:
         return 0;
 out:
@@ -693,70 +639,61 @@ err:
         return -ENOMEM;
 }
 
-int mem_cgroup_charge(struct page *page, struct mm_struct *mm,
-                        gfp_t gfp_mask)
+int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
 {
         return mem_cgroup_charge_common(page, mm, gfp_mask,
-                        MEM_CGROUP_CHARGE_TYPE_MAPPED);
+                                MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
-/*
- * See if the cached pages should be charged at all?
- */
 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                                 gfp_t gfp_mask)
 {
-        int ret = 0;
         if (!mm)
                 mm = &init_mm;
-
-        ret = mem_cgroup_charge_common(page, mm, gfp_mask,
+        return mem_cgroup_charge_common(page, mm, gfp_mask,
                                 MEM_CGROUP_CHARGE_TYPE_CACHE);
-        return ret;
 }
 
 /*
  * Uncharging is always a welcome operation, we never complain, simply
- * uncharge. This routine should be called with lock_page_cgroup held
+ * uncharge.
  */
-void mem_cgroup_uncharge(struct page_cgroup *pc)
+void mem_cgroup_uncharge_page(struct page *page)
 {
+        struct page_cgroup *pc;
         struct mem_cgroup *mem;
         struct mem_cgroup_per_zone *mz;
-        struct page *page;
         unsigned long flags;
 
         /*
          * Check if our page_cgroup is valid
          */
+        lock_page_cgroup(page);
+        pc = page_get_page_cgroup(page);
         if (!pc)
-                return;
+                goto unlock;
 
-        if (atomic_dec_and_test(&pc->ref_cnt)) {
-                page = pc->page;
+        VM_BUG_ON(pc->page != page);
+        VM_BUG_ON(pc->ref_cnt <= 0);
+
+        if (--(pc->ref_cnt) == 0) {
                 mz = page_cgroup_zoneinfo(pc);
-                /*
-                 * get page->cgroup and clear it under lock.
-                 * force_empty can drop page->cgroup without checking refcnt.
-                 */
+                spin_lock_irqsave(&mz->lru_lock, flags);
+                __mem_cgroup_remove_list(pc);
+                spin_unlock_irqrestore(&mz->lru_lock, flags);
+
+                page_assign_page_cgroup(page, NULL);
                 unlock_page_cgroup(page);
-                if (clear_page_cgroup(page, pc) == pc) {
-                        mem = pc->mem_cgroup;
-                        css_put(&mem->css);
-                        res_counter_uncharge(&mem->res, PAGE_SIZE);
-                        spin_lock_irqsave(&mz->lru_lock, flags);
-                        __mem_cgroup_remove_list(pc);
-                        spin_unlock_irqrestore(&mz->lru_lock, flags);
-                        kfree(pc);
-                }
-                lock_page_cgroup(page);
+
+                mem = pc->mem_cgroup;
+                res_counter_uncharge(&mem->res, PAGE_SIZE);
+                css_put(&mem->css);
+
+                kfree(pc);
+                return;
         }
-}
 
-void mem_cgroup_uncharge_page(struct page *page)
-{
-        lock_page_cgroup(page);
-        mem_cgroup_uncharge(page_get_page_cgroup(page));
+unlock:
         unlock_page_cgroup(page);
 }
 
@@ -764,63 +701,59 @@ void mem_cgroup_uncharge_page(struct page *page)
  * Returns non-zero if a page (under migration) has valid page_cgroup member.
  * Refcnt of page_cgroup is incremented.
  */
-
 int mem_cgroup_prepare_migration(struct page *page)
 {
         struct page_cgroup *pc;
-        int ret = 0;
+
         lock_page_cgroup(page);
         pc = page_get_page_cgroup(page);
-        if (pc && atomic_inc_not_zero(&pc->ref_cnt))
-                ret = 1;
+        if (pc)
+                pc->ref_cnt++;
         unlock_page_cgroup(page);
-        return ret;
+        return pc != NULL;
 }
 
 void mem_cgroup_end_migration(struct page *page)
 {
-        struct page_cgroup *pc;
-
-        lock_page_cgroup(page);
-        pc = page_get_page_cgroup(page);
-        mem_cgroup_uncharge(pc);
-        unlock_page_cgroup(page);
+        mem_cgroup_uncharge_page(page);
 }
+
 /*
- * We know both *page* and *newpage* are now not-on-LRU and Pg_locked.
+ * We know both *page* and *newpage* are now not-on-LRU and PG_locked.
  * And no race with uncharge() routines because page_cgroup for *page*
  * has extra one reference by mem_cgroup_prepare_migration.
  */
-
 void mem_cgroup_page_migration(struct page *page, struct page *newpage)
 {
         struct page_cgroup *pc;
-        struct mem_cgroup *mem;
-        unsigned long flags;
         struct mem_cgroup_per_zone *mz;
-retry:
+        unsigned long flags;
+
+        lock_page_cgroup(page);
         pc = page_get_page_cgroup(page);
-        if (!pc)
+        if (!pc) {
+                unlock_page_cgroup(page);
                 return;
-        mem = pc->mem_cgroup;
+        }
+
         mz = page_cgroup_zoneinfo(pc);
-        if (clear_page_cgroup(page, pc) != pc)
-                goto retry;
         spin_lock_irqsave(&mz->lru_lock, flags);
-
         __mem_cgroup_remove_list(pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
 
+        page_assign_page_cgroup(page, NULL);
+        unlock_page_cgroup(page);
+
         pc->page = newpage;
         lock_page_cgroup(newpage);
         page_assign_page_cgroup(newpage, pc);
-        unlock_page_cgroup(newpage);
 
         mz = page_cgroup_zoneinfo(pc);
         spin_lock_irqsave(&mz->lru_lock, flags);
         __mem_cgroup_add_list(pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
-        return;
+
+        unlock_page_cgroup(newpage);
 }
 
 /*
@@ -829,14 +762,13 @@ retry:
  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  */
 #define FORCE_UNCHARGE_BATCH    (128)
-static void
-mem_cgroup_force_empty_list(struct mem_cgroup *mem,
+static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
                             struct mem_cgroup_per_zone *mz,
                             int active)
 {
         struct page_cgroup *pc;
         struct page *page;
-        int count;
+        int count = FORCE_UNCHARGE_BATCH;
         unsigned long flags;
         struct list_head *list;
 
@@ -845,46 +777,36 @@ mem_cgroup_force_empty_list(struct mem_cgroup *mem,
         else
                 list = &mz->inactive_list;
 
-        if (list_empty(list))
-                return;
-retry:
-        count = FORCE_UNCHARGE_BATCH;
         spin_lock_irqsave(&mz->lru_lock, flags);
-
-        while (--count && !list_empty(list)) {
+        while (!list_empty(list)) {
                 pc = list_entry(list->prev, struct page_cgroup, lru);
                 page = pc->page;
-                /* Avoid race with charge */
-                atomic_set(&pc->ref_cnt, 0);
-                if (clear_page_cgroup(page, pc) == pc) {
-                        css_put(&mem->css);
-                        res_counter_uncharge(&mem->res, PAGE_SIZE);
-                        __mem_cgroup_remove_list(pc);
-                        kfree(pc);
-                } else  /* being uncharged ? ...do relax */
-                        break;
+                get_page(page);
+                spin_unlock_irqrestore(&mz->lru_lock, flags);
+                mem_cgroup_uncharge_page(page);
+                put_page(page);
+                if (--count <= 0) {
+                        count = FORCE_UNCHARGE_BATCH;
+                        cond_resched();
+                }
+                spin_lock_irqsave(&mz->lru_lock, flags);
         }
         spin_unlock_irqrestore(&mz->lru_lock, flags);
-        if (!list_empty(list)) {
-                cond_resched();
-                goto retry;
-        }
-        return;
 }
 
 /*
  * make mem_cgroup's charge to be 0 if there is no task.
  * This enables deleting this mem_cgroup.
  */
-
-int mem_cgroup_force_empty(struct mem_cgroup *mem)
+static int mem_cgroup_force_empty(struct mem_cgroup *mem)
 {
         int ret = -EBUSY;
         int node, zid;
+
         css_get(&mem->css);
         /*
          * page reclaim code (kswapd etc..) will move pages between
-`        * active_list <-> inactive_list while we don't take a lock.
+         * active_list <-> inactive_list while we don't take a lock.
          * So, we have to do loop here until all lists are empty.
          */
         while (mem->res.usage > 0) {
@@ -906,9 +828,7 @@ out:
         return ret;
 }
 
-
-
-int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
+static int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
 {
         *tmp = memparse(buf, &buf);
         if (*buf != '\0')
@@ -945,8 +865,7 @@ static ssize_t mem_force_empty_write(struct cgroup *cont,
                                 size_t nbytes, loff_t *ppos)
 {
         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
-        int ret;
-        ret = mem_cgroup_force_empty(mem);
+        int ret = mem_cgroup_force_empty(mem);
         if (!ret)
                 ret = nbytes;
         return ret;
@@ -955,7 +874,6 @@ static ssize_t mem_force_empty_write(struct cgroup *cont,
 /*
  * Note: This should be removed if cgroup supports write-only file.
  */
-
 static ssize_t mem_force_empty_read(struct cgroup *cont,
                                 struct cftype *cft,
                                 struct file *file, char __user *userbuf,
@@ -964,7 +882,6 @@ static ssize_t mem_force_empty_read(struct cgroup *cont,
         return -EINVAL;
 }
 
-
 static const struct mem_cgroup_stat_desc {
         const char *msg;
         u64 unit;
@@ -1017,8 +934,6 @@ static int mem_control_stat_open(struct inode *unused, struct file *file)
         return single_open(file, mem_control_stat_show, cont);
 }
 
-
-
 static struct cftype mem_cgroup_files[] = {
         {
                 .name = "usage_in_bytes",
@@ -1084,9 +999,6 @@ static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
         kfree(mem->info.nodeinfo[node]);
 }
 
-
-static struct mem_cgroup init_mem_cgroup;
-
 static struct cgroup_subsys_state *
 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 {
@@ -1176,7 +1088,6 @@ static void mem_cgroup_move_task(struct cgroup_subsys *ss,
 
 out:
         mmput(mm);
-        return;
 }
 
 struct cgroup_subsys mem_cgroup_subsys = {
diff --git a/mm/memory.c b/mm/memory.c
index ce3c9e4492d8..0d14d1e58a5f 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -1711,7 +1711,7 @@ unlock:
         }
         return ret;
 oom_free_new:
-        __free_page(new_page);
+        page_cache_release(new_page);
 oom:
         if (old_page)
                 page_cache_release(old_page);
@@ -2093,12 +2093,9 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma,
         unlock_page(page);
 
         if (write_access) {
-                /* XXX: We could OR the do_wp_page code with this one? */
-                if (do_wp_page(mm, vma, address,
-                                page_table, pmd, ptl, pte) & VM_FAULT_OOM) {
-                        mem_cgroup_uncharge_page(page);
-                        ret = VM_FAULT_OOM;
-                }
+                ret |= do_wp_page(mm, vma, address, page_table, pmd, ptl, pte);
+                if (ret & VM_FAULT_ERROR)
+                        ret &= VM_FAULT_ERROR;
                 goto out;
         }
 
@@ -2163,7 +2160,7 @@ release:
         page_cache_release(page);
         goto unlock;
 oom_free_page:
-        __free_page(page);
+        page_cache_release(page);
 oom:
         return VM_FAULT_OOM;
 }
diff --git a/mm/migrate.c b/mm/migrate.c
index a73504ff5ab9..4e0eccca5e26 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -153,11 +153,6 @@ static void remove_migration_pte(struct vm_area_struct *vma,
                 return;
         }
 
-        if (mem_cgroup_charge(new, mm, GFP_KERNEL)) {
-                pte_unmap(ptep);
-                return;
-        }
-
         ptl = pte_lockptr(mm, pmd);
         spin_lock(ptl);
         pte = *ptep;
@@ -169,6 +164,20 @@ static void remove_migration_pte(struct vm_area_struct *vma,
         if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
                 goto out;
 
+        /*
+         * Yes, ignore the return value from a GFP_ATOMIC mem_cgroup_charge.
+         * Failure is not an option here: we're now expected to remove every
+         * migration pte, and will cause crashes otherwise.  Normally this
+         * is not an issue: mem_cgroup_prepare_migration bumped up the old
+         * page_cgroup count for safety, that's now attached to the new page,
+         * so this charge should just be another incrementation of the count,
+         * to keep in balance with rmap.c's mem_cgroup_uncharging.  But if
+         * there's been a force_empty, those reference counts may no longer
+         * be reliable, and this charge can actually fail: oh well, we don't
+         * make the situation any worse by proceeding as if it had succeeded.
+         */
+        mem_cgroup_charge(new, mm, GFP_ATOMIC);
+
         get_page(new);
         pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
         if (is_write_migration_entry(entry))
diff --git a/mm/oom_kill.c b/mm/oom_kill.c
index 4194b9db0104..44b2da11bf43 100644
--- a/mm/oom_kill.c
+++ b/mm/oom_kill.c
@@ -412,7 +412,7 @@ static int oom_kill_process(struct task_struct *p, gfp_t gfp_mask, int order,
         return oom_kill_task(p);
 }
 
-#ifdef CONFIG_CGROUP_MEM_CONT
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
 void mem_cgroup_out_of_memory(struct mem_cgroup *mem, gfp_t gfp_mask)
 {
         unsigned long points = 0;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 8896e874a67d..402a504f1228 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -19,6 +19,7 @@
 #include <linux/swap.h>
 #include <linux/interrupt.h>
 #include <linux/pagemap.h>
+#include <linux/jiffies.h>
 #include <linux/bootmem.h>
 #include <linux/compiler.h>
 #include <linux/kernel.h>
@@ -221,13 +222,19 @@ static inline int bad_range(struct zone *zone, struct page *page)
 
 static void bad_page(struct page *page)
 {
-        printk(KERN_EMERG "Bad page state in process '%s'\n"
-                KERN_EMERG "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d\n"
-                KERN_EMERG "Trying to fix it up, but a reboot is needed\n"
-                KERN_EMERG "Backtrace:\n",
+        void *pc = page_get_page_cgroup(page);
+
+        printk(KERN_EMERG "Bad page state in process '%s'\n" KERN_EMERG
+                "page:%p flags:0x%0*lx mapping:%p mapcount:%d count:%d\n",
                 current->comm, page, (int)(2*sizeof(unsigned long)),
                 (unsigned long)page->flags, page->mapping,
                 page_mapcount(page), page_count(page));
+        if (pc) {
+                printk(KERN_EMERG "cgroup:%p\n", pc);
+                page_reset_bad_cgroup(page);
+        }
+        printk(KERN_EMERG "Trying to fix it up, but a reboot is needed\n"
+                KERN_EMERG "Backtrace:\n");
         dump_stack();
         page->flags &= ~(1 << PG_lru    |
                         1 << PG_private |
@@ -453,6 +460,7 @@ static inline int free_pages_check(struct page *page)
 {
         if (unlikely(page_mapcount(page) |
                 (page->mapping != NULL)  |
+                (page_get_page_cgroup(page) != NULL) |
                 (page_count(page) != 0)  |
                 (page->flags & (
                         1 << PG_lru     |
@@ -602,6 +610,7 @@ static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
 {
         if (unlikely(page_mapcount(page) |
                 (page->mapping != NULL)  |
+                (page_get_page_cgroup(page) != NULL) |
                 (page_count(page) != 0)  |
                 (page->flags & (
                         1 << PG_lru     |
@@ -988,7 +997,6 @@ static void free_hot_cold_page(struct page *page, int cold)
 
         if (!PageHighMem(page))
                 debug_check_no_locks_freed(page_address(page), PAGE_SIZE);
-        VM_BUG_ON(page_get_page_cgroup(page));
         arch_free_page(page, 0);
         kernel_map_pages(page, 1, 0);
 
@@ -1276,7 +1284,7 @@ static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
         if (!zlc)
                 return NULL;
 
-        if (jiffies - zlc->last_full_zap > 1 * HZ) {
+       if (time_after(jiffies, zlc->last_full_zap + HZ)) {
                 bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
                 zlc->last_full_zap = jiffies;
         }
@@ -2527,7 +2535,6 @@ void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
                 set_page_links(page, zone, nid, pfn);
                 init_page_count(page);
                 reset_page_mapcount(page);
-                page_assign_page_cgroup(page, NULL);
                 SetPageReserved(page);
 
                 /*
diff --git a/mm/rmap.c b/mm/rmap.c
index 8fd527c4e2bf..0c9a2df06c39 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -321,7 +321,7 @@ static int page_referenced_anon(struct page *page,
                  * counting on behalf of references from different
                  * cgroups
                  */
-                if (mem_cont && !vm_match_cgroup(vma->vm_mm, mem_cont))
+                if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont))
                         continue;
                 referenced += page_referenced_one(page, vma, &mapcount);
                 if (!mapcount)
@@ -382,7 +382,7 @@ static int page_referenced_file(struct page *page,
                  * counting on behalf of references from different
                  * cgroups
                  */
-                if (mem_cont && !vm_match_cgroup(vma->vm_mm, mem_cont))
+                if (mem_cont && !mm_match_cgroup(vma->vm_mm, mem_cont))
                         continue;
                 if ((vma->vm_flags & (VM_LOCKED|VM_MAYSHARE))
                                   == (VM_LOCKED|VM_MAYSHARE)) {
diff --git a/mm/shmem.c b/mm/shmem.c
index 90b576cbc06e..3372bc579e89 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -1370,14 +1370,17 @@ repeat:
                         shmem_swp_unmap(entry);
                         spin_unlock(&info->lock);
                         unlock_page(swappage);
-                        page_cache_release(swappage);
                         if (error == -ENOMEM) {
                                 /* allow reclaim from this memory cgroup */
-                                error = mem_cgroup_cache_charge(NULL,
+                                error = mem_cgroup_cache_charge(swappage,
                                         current->mm, gfp & ~__GFP_HIGHMEM);
-                                if (error)
+                                if (error) {
+                                        page_cache_release(swappage);
                                         goto failed;
+                                }
+                                mem_cgroup_uncharge_page(swappage);
                         }
+                        page_cache_release(swappage);
                         goto repeat;
                 }
         } else if (sgp == SGP_READ && !filepage) {
diff --git a/mm/swap.c b/mm/swap.c
index 710a20bb9749..d4ec59aa5c46 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -176,7 +176,7 @@ void activate_page(struct page *page)
                 SetPageActive(page);
                 add_page_to_active_list(zone, page);
                 __count_vm_event(PGACTIVATE);
-                mem_cgroup_move_lists(page_get_page_cgroup(page), true);
+                mem_cgroup_move_lists(page, true);
         }
         spin_unlock_irq(&zone->lru_lock);
 }
diff --git a/mm/vmscan.c b/mm/vmscan.c
index a26dabd62fed..45711585684e 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -126,7 +126,7 @@ long vm_total_pages;	/* The total number of pages which the VM controls */
 static LIST_HEAD(shrinker_list);
 static DECLARE_RWSEM(shrinker_rwsem);
 
-#ifdef CONFIG_CGROUP_MEM_CONT
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
 #define scan_global_lru(sc)     (!(sc)->mem_cgroup)
 #else
 #define scan_global_lru(sc)     (1)
@@ -1128,7 +1128,7 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
                 ClearPageActive(page);
 
                 list_move(&page->lru, &zone->inactive_list);
-                mem_cgroup_move_lists(page_get_page_cgroup(page), false);
+                mem_cgroup_move_lists(page, false);
                 pgmoved++;
                 if (!pagevec_add(&pvec, page)) {
                         __mod_zone_page_state(zone, NR_INACTIVE, pgmoved);
@@ -1156,8 +1156,9 @@ static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
                 VM_BUG_ON(PageLRU(page));
                 SetPageLRU(page);
                 VM_BUG_ON(!PageActive(page));
+
                 list_move(&page->lru, &zone->active_list);
-                mem_cgroup_move_lists(page_get_page_cgroup(page), true);
+                mem_cgroup_move_lists(page, true);
                 pgmoved++;
                 if (!pagevec_add(&pvec, page)) {
                         __mod_zone_page_state(zone, NR_ACTIVE, pgmoved);
@@ -1427,7 +1428,7 @@ unsigned long try_to_free_pages(struct zone **zones, int order, gfp_t gfp_mask)
         return do_try_to_free_pages(zones, gfp_mask, &sc);
 }
 
-#ifdef CONFIG_CGROUP_MEM_CONT
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR
 
 unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *mem_cont,
                                                 gfp_t gfp_mask)