1 files changed, 96 insertions, 94 deletions

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 2e0bfc93484b..e46451e1d9b7 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -26,15 +26,18 @@
 #include <linux/backing-dev.h>
 #include <linux/bit_spinlock.h>
 #include <linux/rcupdate.h>
+#include <linux/slab.h>
 #include <linux/swap.h>
 #include <linux/spinlock.h>
 #include <linux/fs.h>
 #include <linux/seq_file.h>
+#include <linux/vmalloc.h>
 
 #include <asm/uaccess.h>
 
 struct cgroup_subsys mem_cgroup_subsys;
 static const int MEM_CGROUP_RECLAIM_RETRIES = 5;
+static struct kmem_cache *page_cgroup_cache;
 
 /*
  * Statistics for memory cgroup.
@@ -45,6 +48,8 @@ enum mem_cgroup_stat_index {
          */
         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
         MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
+        MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
+        MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
 
         MEM_CGROUP_STAT_NSTATS,
 };
@@ -196,6 +201,13 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
                 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_CACHE, val);
         else
                 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val);
+
+        if (charge)
+                __mem_cgroup_stat_add_safe(stat,
+                                MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
+        else
+                __mem_cgroup_stat_add_safe(stat,
+                                MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
 }
 
 static struct mem_cgroup_per_zone *
@@ -236,26 +248,12 @@ static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
                                 css);
 }
 
-static struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
+struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 {
         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
                                 struct mem_cgroup, css);
 }
 
-void mm_init_cgroup(struct mm_struct *mm, struct task_struct *p)
-{
-        struct mem_cgroup *mem;
-
-        mem = mem_cgroup_from_task(p);
-        css_get(&mem->css);
-        mm->mem_cgroup = mem;
-}
-
-void mm_free_cgroup(struct mm_struct *mm)
-{
-        css_put(&mm->mem_cgroup->css);
-}
-
 static inline int page_cgroup_locked(struct page *page)
 {
         return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
@@ -287,10 +285,10 @@ static void unlock_page_cgroup(struct page *page)
         bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup);
 }
 
-static void __mem_cgroup_remove_list(struct page_cgroup *pc)
+static void __mem_cgroup_remove_list(struct mem_cgroup_per_zone *mz,
+                        struct page_cgroup *pc)
 {
         int from = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
-        struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
 
         if (from)
                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_ACTIVE) -= 1;
@@ -301,10 +299,10 @@ static void __mem_cgroup_remove_list(struct page_cgroup *pc)
         list_del_init(&pc->lru);
 }
 
-static void __mem_cgroup_add_list(struct page_cgroup *pc)
+static void __mem_cgroup_add_list(struct mem_cgroup_per_zone *mz,
+                                struct page_cgroup *pc)
 {
         int to = pc->flags & PAGE_CGROUP_FLAG_ACTIVE;
-        struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
 
         if (!to) {
                 MEM_CGROUP_ZSTAT(mz, MEM_CGROUP_ZSTAT_INACTIVE) += 1;
@@ -476,6 +474,7 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
         int zid = zone_idx(z);
         struct mem_cgroup_per_zone *mz;
 
+        BUG_ON(!mem_cont);
         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
         if (active)
                 src = &mz->active_list;
@@ -560,7 +559,7 @@ retry:
         }
         unlock_page_cgroup(page);
 
-        pc = kzalloc(sizeof(struct page_cgroup), gfp_mask);
+        pc = kmem_cache_zalloc(page_cgroup_cache, gfp_mask);
         if (pc == NULL)
                 goto err;
 
@@ -574,7 +573,7 @@ retry:
                 mm = &init_mm;
 
         rcu_read_lock();
-        mem = rcu_dereference(mm->mem_cgroup);
+        mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
         /*
          * For every charge from the cgroup, increment reference count
          */
@@ -602,7 +601,6 @@ retry:
                         mem_cgroup_out_of_memory(mem, gfp_mask);
                         goto out;
                 }
-                congestion_wait(WRITE, HZ/10);
         }
 
         pc->ref_cnt = 1;
@@ -610,7 +608,7 @@ retry:
         pc->page = page;
         pc->flags = PAGE_CGROUP_FLAG_ACTIVE;
         if (ctype == MEM_CGROUP_CHARGE_TYPE_CACHE)
-                pc->flags |= PAGE_CGROUP_FLAG_CACHE;
+                pc->flags = PAGE_CGROUP_FLAG_CACHE;
 
         lock_page_cgroup(page);
         if (page_get_page_cgroup(page)) {
@@ -622,14 +620,14 @@ retry:
                  */
                 res_counter_uncharge(&mem->res, PAGE_SIZE);
                 css_put(&mem->css);
-                kfree(pc);
+                kmem_cache_free(page_cgroup_cache, pc);
                 goto retry;
         }
         page_assign_page_cgroup(page, pc);
 
         mz = page_cgroup_zoneinfo(pc);
         spin_lock_irqsave(&mz->lru_lock, flags);
-        __mem_cgroup_add_list(pc);
+        __mem_cgroup_add_list(mz, pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
 
         unlock_page_cgroup(page);
@@ -637,7 +635,7 @@ done:
         return 0;
 out:
         css_put(&mem->css);
-        kfree(pc);
+        kmem_cache_free(page_cgroup_cache, pc);
 err:
         return -ENOMEM;
 }
@@ -685,7 +683,7 @@ void mem_cgroup_uncharge_page(struct page *page)
         if (--(pc->ref_cnt) == 0) {
                 mz = page_cgroup_zoneinfo(pc);
                 spin_lock_irqsave(&mz->lru_lock, flags);
-                __mem_cgroup_remove_list(pc);
+                __mem_cgroup_remove_list(mz, pc);
                 spin_unlock_irqrestore(&mz->lru_lock, flags);
 
                 page_assign_page_cgroup(page, NULL);
@@ -695,7 +693,7 @@ void mem_cgroup_uncharge_page(struct page *page)
                 res_counter_uncharge(&mem->res, PAGE_SIZE);
                 css_put(&mem->css);
 
-                kfree(pc);
+                kmem_cache_free(page_cgroup_cache, pc);
                 return;
         }
 
@@ -747,7 +745,7 @@ void mem_cgroup_page_migration(struct page *page, struct page *newpage)
 
         mz = page_cgroup_zoneinfo(pc);
         spin_lock_irqsave(&mz->lru_lock, flags);
-        __mem_cgroup_remove_list(pc);
+        __mem_cgroup_remove_list(mz, pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
 
         page_assign_page_cgroup(page, NULL);
@@ -759,7 +757,7 @@ void mem_cgroup_page_migration(struct page *page, struct page *newpage)
 
         mz = page_cgroup_zoneinfo(pc);
         spin_lock_irqsave(&mz->lru_lock, flags);
-        __mem_cgroup_add_list(pc);
+        __mem_cgroup_add_list(mz, pc);
         spin_unlock_irqrestore(&mz->lru_lock, flags);
 
         unlock_page_cgroup(newpage);
@@ -853,13 +851,10 @@ static int mem_cgroup_write_strategy(char *buf, unsigned long long *tmp)
         return 0;
 }
 
-static ssize_t mem_cgroup_read(struct cgroup *cont,
-                        struct cftype *cft, struct file *file,
-                        char __user *userbuf, size_t nbytes, loff_t *ppos)
+static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
 {
-        return res_counter_read(&mem_cgroup_from_cont(cont)->res,
-                                cft->private, userbuf, nbytes, ppos,
-                                NULL);
+        return res_counter_read_u64(&mem_cgroup_from_cont(cont)->res,
+                                    cft->private);
 }
 
 static ssize_t mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
@@ -871,27 +866,25 @@ static ssize_t mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
                                 mem_cgroup_write_strategy);
 }
 
-static ssize_t mem_force_empty_write(struct cgroup *cont,
-                                struct cftype *cft, struct file *file,
-                                const char __user *userbuf,
-                                size_t nbytes, loff_t *ppos)
+static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
 {
-        struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
-        int ret = mem_cgroup_force_empty(mem);
-        if (!ret)
-                ret = nbytes;
-        return ret;
+        struct mem_cgroup *mem;
+
+        mem = mem_cgroup_from_cont(cont);
+        switch (event) {
+        case RES_MAX_USAGE:
+                res_counter_reset_max(&mem->res);
+                break;
+        case RES_FAILCNT:
+                res_counter_reset_failcnt(&mem->res);
+                break;
+        }
+        return 0;
 }
 
-/*
- * Note: This should be removed if cgroup supports write-only file.
- */
-static ssize_t mem_force_empty_read(struct cgroup *cont,
-                                struct cftype *cft,
-                                struct file *file, char __user *userbuf,
-                                size_t nbytes, loff_t *ppos)
+static int mem_force_empty_write(struct cgroup *cont, unsigned int event)
 {
-        return -EINVAL;
+        return mem_cgroup_force_empty(mem_cgroup_from_cont(cont));
 }
 
 static const struct mem_cgroup_stat_desc {
@@ -900,11 +893,13 @@ static const struct mem_cgroup_stat_desc {
 } mem_cgroup_stat_desc[] = {
         [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
         [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
+        [MEM_CGROUP_STAT_PGPGIN_COUNT] = {"pgpgin", 1, },
+        [MEM_CGROUP_STAT_PGPGOUT_COUNT] = {"pgpgout", 1, },
 };
 
-static int mem_control_stat_show(struct seq_file *m, void *arg)
+static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
+                                 struct cgroup_map_cb *cb)
 {
-        struct cgroup *cont = m->private;
         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
         struct mem_cgroup_stat *stat = &mem_cont->stat;
         int i;
@@ -914,8 +909,7 @@ static int mem_control_stat_show(struct seq_file *m, void *arg)
 
                 val = mem_cgroup_read_stat(stat, i);
                 val *= mem_cgroup_stat_desc[i].unit;
-                seq_printf(m, "%s %lld\n", mem_cgroup_stat_desc[i].msg,
-                                (long long)val);
+                cb->fill(cb, mem_cgroup_stat_desc[i].msg, val);
         }
         /* showing # of active pages */
         {
@@ -925,52 +919,43 @@ static int mem_control_stat_show(struct seq_file *m, void *arg)
                                                 MEM_CGROUP_ZSTAT_INACTIVE);
                 active = mem_cgroup_get_all_zonestat(mem_cont,
                                                 MEM_CGROUP_ZSTAT_ACTIVE);
-                seq_printf(m, "active %ld\n", (active) * PAGE_SIZE);
-                seq_printf(m, "inactive %ld\n", (inactive) * PAGE_SIZE);
+                cb->fill(cb, "active", (active) * PAGE_SIZE);
+                cb->fill(cb, "inactive", (inactive) * PAGE_SIZE);
         }
         return 0;
 }
 
-static const struct file_operations mem_control_stat_file_operations = {
-        .read = seq_read,
-        .llseek = seq_lseek,
-        .release = single_release,
-};
-
-static int mem_control_stat_open(struct inode *unused, struct file *file)
-{
-        /* XXX __d_cont */
-        struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;
-
-        file->f_op = &mem_control_stat_file_operations;
-        return single_open(file, mem_control_stat_show, cont);
-}
-
 static struct cftype mem_cgroup_files[] = {
         {
                 .name = "usage_in_bytes",
                 .private = RES_USAGE,
-                .read = mem_cgroup_read,
+                .read_u64 = mem_cgroup_read,
+        },
+        {
+                .name = "max_usage_in_bytes",
+                .private = RES_MAX_USAGE,
+                .trigger = mem_cgroup_reset,
+                .read_u64 = mem_cgroup_read,
         },
         {
                 .name = "limit_in_bytes",
                 .private = RES_LIMIT,
                 .write = mem_cgroup_write,
-                .read = mem_cgroup_read,
+                .read_u64 = mem_cgroup_read,
         },
         {
                 .name = "failcnt",
                 .private = RES_FAILCNT,
-                .read = mem_cgroup_read,
+                .trigger = mem_cgroup_reset,
+                .read_u64 = mem_cgroup_read,
         },
         {
                 .name = "force_empty",
-                .write = mem_force_empty_write,
-                .read = mem_force_empty_read,
+                .trigger = mem_force_empty_write,
         },
         {
                 .name = "stat",
-                .open = mem_control_stat_open,
+                .read_map = mem_control_stat_show,
         },
 };
 
@@ -1010,6 +995,29 @@ static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
         kfree(mem->info.nodeinfo[node]);
 }
 
+static struct mem_cgroup *mem_cgroup_alloc(void)
+{
+        struct mem_cgroup *mem;
+
+        if (sizeof(*mem) < PAGE_SIZE)
+                mem = kmalloc(sizeof(*mem), GFP_KERNEL);
+        else
+                mem = vmalloc(sizeof(*mem));
+
+        if (mem)
+                memset(mem, 0, sizeof(*mem));
+        return mem;
+}
+
+static void mem_cgroup_free(struct mem_cgroup *mem)
+{
+        if (sizeof(*mem) < PAGE_SIZE)
+                kfree(mem);
+        else
+                vfree(mem);
+}
+
+
 static struct cgroup_subsys_state *
 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 {
@@ -1018,17 +1026,15 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 
         if (unlikely((cont->parent) == NULL)) {
                 mem = &init_mem_cgroup;
-                init_mm.mem_cgroup = mem;
-        } else
-                mem = kzalloc(sizeof(struct mem_cgroup), GFP_KERNEL);
-
-        if (mem == NULL)
-                return ERR_PTR(-ENOMEM);
+                page_cgroup_cache = KMEM_CACHE(page_cgroup, SLAB_PANIC);
+        } else {
+                mem = mem_cgroup_alloc();
+                if (!mem)
+                        return ERR_PTR(-ENOMEM);
+        }
 
         res_counter_init(&mem->res);
 
-        memset(&mem->info, 0, sizeof(mem->info));
-
         for_each_node_state(node, N_POSSIBLE)
                 if (alloc_mem_cgroup_per_zone_info(mem, node))
                         goto free_out;
@@ -1038,7 +1044,7 @@ free_out:
         for_each_node_state(node, N_POSSIBLE)
                 free_mem_cgroup_per_zone_info(mem, node);
         if (cont->parent != NULL)
-                kfree(mem);
+                mem_cgroup_free(mem);
         return ERR_PTR(-ENOMEM);
 }
 
@@ -1058,7 +1064,7 @@ static void mem_cgroup_destroy(struct cgroup_subsys *ss,
         for_each_node_state(node, N_POSSIBLE)
                 free_mem_cgroup_per_zone_info(mem, node);
 
-        kfree(mem_cgroup_from_cont(cont));
+        mem_cgroup_free(mem_cgroup_from_cont(cont));
 }
 
 static int mem_cgroup_populate(struct cgroup_subsys *ss,
@@ -1098,10 +1104,6 @@ static void mem_cgroup_move_task(struct cgroup_subsys *ss,
         if (!thread_group_leader(p))
                 goto out;
 
-        css_get(&mem->css);
-        rcu_assign_pointer(mm->mem_cgroup, mem);
-        css_put(&old_mem->css);
-
 out:
         mmput(mm);
 }