/* memcontrol.c - Memory Controller * * Copyright IBM Corporation, 2007 * Author Balbir Singh * * Copyright 2007 OpenVZ SWsoft Inc * Author: Pavel Emelianov * * Memory thresholds * Copyright (C) 2009 Nokia Corporation * Author: Kirill A. Shutemov * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "internal.h" #include #include struct cgroup_subsys mem_cgroup_subsys __read_mostly; #define MEM_CGROUP_RECLAIM_RETRIES 5 struct mem_cgroup *root_mem_cgroup __read_mostly; #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */ int do_swap_account __read_mostly; static int really_do_swap_account __initdata = 1; /* for remember boot option*/ #else #define do_swap_account (0) #endif /* * Per memcg event counter is incremented at every pagein/pageout. This counter * is used for trigger some periodic events. This is straightforward and better * than using jiffies etc. to handle periodic memcg event. * * These values will be used as !((event) & ((1 <<(thresh)) - 1)) */ #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */ #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */ /* * Statistics for memory cgroup. */ enum mem_cgroup_stat_index { /* * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. */ MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */ MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */ MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */ MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */ MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */ MEM_CGROUP_EVENTS, /* incremented at every pagein/pageout */ MEM_CGROUP_STAT_NSTATS, }; struct mem_cgroup_stat_cpu { s64 count[MEM_CGROUP_STAT_NSTATS]; }; /* * per-zone information in memory controller. */ struct mem_cgroup_per_zone { /* * spin_lock to protect the per cgroup LRU */ struct list_head lists[NR_LRU_LISTS]; unsigned long count[NR_LRU_LISTS]; struct zone_reclaim_stat reclaim_stat; struct rb_node tree_node; /* RB tree node */ unsigned long long usage_in_excess;/* Set to the value by which */ /* the soft limit is exceeded*/ bool on_tree; struct mem_cgroup *mem; /* Back pointer, we cannot */ /* use container_of */ }; /* Macro for accessing counter */ #define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)]) struct mem_cgroup_per_node { struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; }; struct mem_cgroup_lru_info { struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; }; /* * Cgroups above their limits are maintained in a RB-Tree, independent of * their hierarchy representation */ struct mem_cgroup_tree_per_zone { struct rb_root rb_root; spinlock_t lock; }; struct mem_cgroup_tree_per_node { struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES]; }; struct mem_cgroup_tree { struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES]; }; static struct mem_cgroup_tree soft_limit_tree __read_mostly; struct mem_cgroup_threshold { struct eventfd_ctx *eventfd; u64 threshold; }; /* For threshold */ struct mem_cgroup_threshold_ary { /* An array index points to threshold just below usage. */ int current_threshold; /* Size of entries[] */ unsigned int size; /* Array of thresholds */ struct mem_cgroup_threshold entries[0]; }; struct mem_cgroup_thresholds { /* Primary thresholds array */ struct mem_cgroup_threshold_ary *primary; /* * Spare threshold array. * This is needed to make mem_cgroup_unregister_event() "never fail". * It must be able to store at least primary->size - 1 entries. */ struct mem_cgroup_threshold_ary *spare; }; /* for OOM */ struct mem_cgroup_eventfd_list { struct list_head list; struct eventfd_ctx *eventfd; }; static void mem_cgroup_threshold(struct mem_cgroup *mem); static void mem_cgroup_oom_notify(struct mem_cgroup *mem); /* * The memory controller data structure. The memory controller controls both * page cache and RSS per cgroup. We would eventually like to provide * statistics based on the statistics developed by Rik Van Riel for clock-pro, * to help the administrator determine what knobs to tune. * * TODO: Add a water mark for the memory controller. Reclaim will begin when * we hit the water mark. May be even add a low water mark, such that * no reclaim occurs from a cgroup at it's low water mark, this is * a feature that will be implemented much later in the future. */ struct mem_cgroup { struct cgroup_subsys_state css; /* * the counter to account for memory usage */ struct res_counter res; /* * the counter to account for mem+swap usage. */ struct res_counter memsw; /* * Per cgroup active and inactive list, similar to the * per zone LRU lists. */ struct mem_cgroup_lru_info info; /* protect against reclaim related member. */ spinlock_t reclaim_param_lock; /* * While reclaiming in a hierarchy, we cache the last child we * reclaimed from. */ int last_scanned_child; /* * Should the accounting and control be hierarchical, per subtree? */ bool use_hierarchy; atomic_t oom_lock; atomic_t refcnt; unsigned int swappiness; /* OOM-Killer disable */ int oom_kill_disable; /* set when res.limit == memsw.limit */ bool memsw_is_minimum; /* protect arrays of thresholds */ struct mutex thresholds_lock; /* thresholds for memory usage. RCU-protected */ struct mem_cgroup_thresholds thresholds; /* thresholds for mem+swap usage. RCU-protected */ struct mem_cgroup_thresholds memsw_thresholds; /* For oom notifier event fd */ struct list_head oom_notify; /* * Should we move charges of a task when a task is moved into this * mem_cgroup ? And what type of charges should we move ? */ unsigned long move_charge_at_immigrate; /* * percpu counter. */ struct mem_cgroup_stat_cpu *stat; }; /* Stuffs for move charges at task migration. */ /* * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a * left-shifted bitmap of these types. */ enum move_type { MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */ MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */ NR_MOVE_TYPE, }; /* "mc" and its members are protected by cgroup_mutex */ static struct move_charge_struct { spinlock_t lock; /* for from, to, moving_task */ struct mem_cgroup *from; struct mem_cgroup *to; unsigned long precharge; unsigned long moved_charge; unsigned long moved_swap; struct task_struct *moving_task; /* a task moving charges */ wait_queue_head_t waitq; /* a waitq for other context */ } mc = { .lock = __SPIN_LOCK_UNLOCKED(mc.lock), .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq), }; static bool move_anon(void) { return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.to->move_charge_at_immigrate); } static bool move_file(void) { return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.to->move_charge_at_immigrate); } /* * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft * limit reclaim to prevent infinite loops, if they ever occur. */ #define MEM_CGROUP_MAX_RECLAIM_LOOPS (100) #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2) enum charge_type { MEM_CGROUP_CHARGE_TYPE_CACHE = 0, MEM_CGROUP_CHARGE_TYPE_MAPPED, MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */ MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */ NR_CHARGE_TYPE, }; /* only for here (for easy reading.) */ #define PCGF_CACHE (1UL << PCG_CACHE) #define PCGF_USED (1UL << PCG_USED) #define PCGF_LOCK (1UL << PCG_LOCK) /* Not used, but added here for completeness */ #define PCGF_ACCT (1UL << PCG_ACCT) /* for encoding cft->private value on file */ #define _MEM (0) #define _MEMSWAP (1) #define _OOM_TYPE (2) #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val)) #define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff) #define MEMFILE_ATTR(val) ((val) & 0xffff) /* Used for OOM nofiier */ #define OOM_CONTROL (0) /* * Reclaim flags for mem_cgroup_hierarchical_reclaim */ #define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0 #define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT) #define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1 #define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT) #define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2 #define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT) static void mem_cgroup_get(struct mem_cgroup *mem); static void mem_cgroup_put(struct mem_cgroup *mem); static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem); static void drain_all_stock_async(void); static struct mem_cgroup_per_zone * mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) { return &mem->info.nodeinfo[nid]->zoneinfo[zid]; } struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem) { return &mem->css; } static struct mem_cgroup_per_zone * page_cgroup_zoneinfo(struct page_cgroup *pc) { struct mem_cgroup *mem = pc->mem_cgroup; int nid = page_cgroup_nid(pc); int zid = page_cgroup_zid(pc); if (!mem) return NULL; return mem_cgroup_zoneinfo(mem, nid, zid); } static struct mem_cgroup_tree_per_zone * soft_limit_tree_node_zone(int nid, int zid) { return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; } static struct mem_cgroup_tree_per_zone * soft_limit_tree_from_page(struct page *page) { int nid = page_to_nid(page); int zid = page_zonenum(page); return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid]; } static void __mem_cgroup_insert_exceeded(struct mem_cgroup *mem, struct mem_cgroup_per_zone *mz, struct mem_cgroup_tree_per_zone *mctz, unsigned long long new_usage_in_excess) { struct rb_node **p = &mctz->rb_root.rb_node; struct rb_node *parent = NULL; struct mem_cgroup_per_zone *mz_node; if (mz->on_tree) return; mz->usage_in_excess = new_usage_in_excess; if (!mz->usage_in_excess) return; while (*p) { parent = *p; mz_node = rb_entry(parent, struct mem_cgroup_per_zone, tree_node); if (mz->usage_in_excess < mz_node->usage_in_excess) p = &(*p)->rb_left; /* * We can't avoid mem cgroups that are over their soft * limit by the same amount */ else if (mz->usage_in_excess >= mz_node->usage_in_excess) p = &(*p)->rb_right; } rb_link_node(&mz->tree_node, parent, p); rb_insert_color(&mz->tree_node, &mctz->rb_root); mz->on_tree = true; } static void __mem_cgroup_remove_exceeded(struct mem_cgroup *mem, struct mem_cgroup_per_zone *mz, struct mem_cgroup_tree_per_zone *mctz) { if (!mz->on_tree) return; rb_erase(&mz->tree_node, &mctz->rb_root); mz->on_tree = false; } static void mem_cgroup_remove_exceeded(struct mem_cgroup *mem, struct mem_cgroup_per_zone *mz, struct mem_cgroup_tree_per_zone *mctz) { spin_lock(&mctz->lock); __mem_cgroup_remove_exceeded(mem, mz, mctz); spin_unlock(&mctz->lock); } static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page) { unsigned long long excess; struct mem_cgroup_per_zone *mz; struct mem_cgroup_tree_per_zone *mctz; int nid = page_to_nid(page); int zid = page_zonenum(page); mctz = soft_limit_tree_from_page(page); /* * Necessary to update all ancestors when hierarchy is used. * because their event counter is not touched. */ for (; mem; mem = parent_mem_cgroup(mem)) { mz = mem_cgroup_zoneinfo(mem, nid, zid); excess = res_counter_soft_limit_excess(&mem->res); /* * We have to update the tree if mz is on RB-tree or * mem is over its softlimit. */ if (excess || mz->on_tree) { spin_lock(&mctz->lock); /* if on-tree, remove it */ if (mz->on_tree) __mem_cgroup_remove_exceeded(mem, mz, mctz); /* * Insert again. mz->usage_in_excess will be updated. * If excess is 0, no tree ops. */ __mem_cgroup_insert_exceeded(mem, mz, mctz, excess); spin_unlock(&mctz->lock); } } } static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem) { int node, zone; struct mem_cgroup_per_zone *mz; struct mem_cgroup_tree_per_zone *mctz; for_each_node_state(node, N_POSSIBLE) { for (zone = 0; zone < MAX_NR_ZONES; zone++) { mz = mem_cgroup_zoneinfo(mem, node, zone); mctz = soft_limit_tree_node_zone(node, zone); mem_cgroup_remove_exceeded(mem, mz, mctz); } } } static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem) { return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT; } static struct mem_cgroup_per_zone * __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) { struct rb_node *rightmost = NULL; struct mem_cgroup_per_zone *mz; retry: mz = NULL; rightmost = rb_last(&mctz->rb_root); if (!rightmost) goto done; /* Nothing to reclaim from */ mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node); /* * Remove the node now but someone else can add it back, * we will to add it back at the end of reclaim to its correct * position in the tree. */ __mem_cgroup_remove_exceeded(mz->mem, mz, mctz); if (!res_counter_soft_limit_excess(&mz->mem->res) || !css_tryget(&mz->mem->css)) goto retry; done: return mz; } static struct mem_cgroup_per_zone * mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz) { struct mem_cgroup_per_zone *mz; spin_lock(&mctz->lock); mz = __mem_cgroup_largest_soft_limit_node(mctz); spin_unlock(&mctz->lock); return mz; } static s64 mem_cgroup_read_stat(struct mem_cgroup *mem, enum mem_cgroup_stat_index idx) { int cpu; s64 val = 0; for_each_possible_cpu(cpu) val += per_cpu(mem->stat->count[idx], cpu); return val; } static s64 mem_cgroup_local_usage(struct mem_cgroup *mem) { s64 ret; ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS); ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE); return ret; } static void mem_cgroup_swap_statistics(struct mem_cgroup *mem, bool charge) { int val = (charge) ? 1 : -1; this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val); } static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, struct page_cgroup *pc, bool charge) { int val = (charge) ? 1 : -1; preempt_disable(); if (PageCgroupCache(pc)) __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val); else __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val); if (charge) __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]); else __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]); __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]); preempt_enable(); } static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem, enum lru_list idx) { int nid, zid; struct mem_cgroup_per_zone *mz; u64 total = 0; for_each_online_node(nid) for (zid = 0; zid < MAX_NR_ZONES; zid++) { mz = mem_cgroup_zoneinfo(mem, nid, zid); total += MEM_CGROUP_ZSTAT(mz, idx); } return total; } static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift) { s64 val; val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]); return !(val & ((1 << event_mask_shift) - 1)); } /* * Check events in order. * */ static void memcg_check_events(struct mem_cgroup *mem, struct page *page) { /* threshold event is triggered in finer grain than soft limit */ if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) { mem_cgroup_threshold(mem); if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH))) mem_cgroup_update_tree(mem, page); } } static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) { return container_of(cgroup_subsys_state(cont, mem_cgroup_subsys_id), struct mem_cgroup, css); } struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) { /* * mm_update_next_owner() may clear mm->owner to NULL * if it races with swapoff, page migration, etc. * So this can be called with p == NULL. */ if (unlikely(!p)) return NULL; return container_of(task_subsys_state(p, mem_cgroup_subsys_id), struct mem_cgroup, css); } static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) { struct mem_cgroup *mem = NULL; if (!mm) return NULL; /* * Because we have no locks, mm->owner's may be being moved to other * cgroup. We use css_tryget() here even if this looks * pessimistic (rather than adding locks here). */ rcu_read_lock(); do { mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); if (unlikely(!mem)) break; } while (!css_tryget(&mem->css)); rcu_read_unlock(); return mem; } /* * Call callback function against all cgroup under hierarchy tree. */ static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data, int (*func)(struct mem_cgroup *, void *)) { int found, ret, nextid; struct cgroup_subsys_state *css; struct mem_cgroup *mem; if (!root->use_hierarchy) return (*func)(root, data); nextid = 1; do { ret = 0; mem = NULL; rcu_read_lock(); css = css_get_next(&mem_cgroup_subsys, nextid, &root->css, &found); if (css && css_tryget(css)) mem = container_of(css, struct mem_cgroup, css); rcu_read_unlock(); if (mem) { ret = (*func)(mem, data); css_put(&mem->css); } nextid = found + 1; } while (!ret && css); return ret; } static inline bool mem_cgroup_is_root(struct mem_cgroup *mem) { return (mem == root_mem_cgroup); } /* * Following LRU functions are allowed to be used without PCG_LOCK. * Operations are called by routine of global LRU independently from memcg. * What we have to take care of here is validness of pc->mem_cgroup. * * Changes to pc->mem_cgroup happens when * 1. charge * 2. moving account * In typical case, "charge" is done before add-to-lru. Exception is SwapCache. * It is added to LRU before charge. * If PCG_USED bit is not set, page_cgroup is not added to this private LRU. * When moving account, the page is not on LRU. It's isolated. */ void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru) { struct page_cgroup *pc; struct mem_cgroup_per_zone *mz; if (mem_cgroup_disabled()) return; pc = lookup_page_cgroup(page); /* can happen while we handle swapcache. */ if (!TestClearPageCgroupAcctLRU(pc)) return; VM_BUG_ON(!pc->mem_cgroup); /* * We don't check PCG_USED bit. It's cleared when the "page" is finally * removed from global LRU. */ mz = page_cgroup_zoneinfo(pc); MEM_CGROUP_ZSTAT(mz, lru) -= 1; if (mem_cgroup_is_root(pc->mem_cgroup)) return; VM_BUG_ON(list_empty(&pc->lru)); list_del_init(&pc->lru); return; } void mem_cgroup_del_lru(struct page *page) { mem_cgroup_del_lru_list(page, page_lru(page)); } void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru) { struct mem_cgroup_per_zone *mz; struct page_cgroup *pc; if (mem_cgroup_disabled()) return; pc = lookup_page_cgroup(page); /* * Used bit is set without atomic ops but after smp_wmb(). * For making pc->mem_cgroup visible, insert smp_rmb() here. */ smp_rmb(); /* unused or root page is not rotated. */ if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup)) return; mz = page_cgroup_zoneinfo(pc); list_move(&pc->lru, &mz->lists[lru]); } void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru) { struct page_cgroup *pc; struct mem_cgroup_per_zone *mz; if (mem_cgroup_disabled()) return; pc = lookup_page_cgroup(page); VM_BUG_ON(PageCgroupAcctLRU(pc)); /* * Used bit is set without atomic ops but after smp_wmb(). * For making pc->mem_cgroup visible, insert smp_rmb() here. */ smp_rmb(); if (!PageCgroupUsed(pc)) return; mz = page_cgroup_zoneinfo(pc); MEM_CGROUP_ZSTAT(mz, lru) += 1; SetPageCgroupAcctLRU(pc); if (mem_cgroup_is_root(pc->mem_cgroup)) return; list_add(&pc->lru, &mz->lists[lru]); } /* * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to * lru because the page may.be reused after it's fully uncharged (because of * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge * it again. This function is only used to charge SwapCache. It's done under * lock_page and expected that zone->lru_lock is never held. */ static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page) { unsigned long flags; struct zone *zone = page_zone(page); struct page_cgroup *pc = lookup_page_cgroup(page); spin_lock_irqsave(&zone->lru_lock, flags); /* * Forget old LRU when this page_cgroup is *not* used. This Used bit * is guarded by lock_page() because the page is SwapCache. */ if (!PageCgroupUsed(pc)) mem_cgroup_del_lru_list(page, page_lru(page)); spin_unlock_irqrestore(&zone->lru_lock, flags); } static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page) { unsigned long flags; struct zone *zone = page_zone(page); struct page_cgroup *pc = lookup_page_cgroup(page); spin_lock_irqsave(&zone->lru_lock, flags); /* link when the page is linked to LRU but page_cgroup isn't */ if (PageLRU(page) && !PageCgroupAcctLRU(pc)) mem_cgroup_add_lru_list(page, page_lru(page)); spin_unlock_irqrestore(&zone->lru_lock, flags); } void mem_cgroup_move_lists(struct page *page, enum lru_list from, enum lru_list to) { if (mem_cgroup_disabled()) return; mem_cgroup_del_lru_list(page, from); mem_cgroup_add_lru_list(page, to); } int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) { int ret; struct mem_cgroup *curr = NULL; struct task_struct *p; p = find_lock_task_mm(task); if (!p) return 0; curr = try_get_mem_cgroup_from_mm(p->mm); task_unlock(p); if (!curr) return 0; /* * We should check use_hierarchy of "mem" not "curr". Because checking * use_hierarchy of "curr" here make this function true if hierarchy is * enabled in "curr" and "curr" is a child of "mem" in *cgroup* * hierarchy(even if use_hierarchy is disabled in "mem"). */ if (mem->use_hierarchy) ret = css_is_ancestor(&curr->css, &mem->css); else ret = (curr == mem); css_put(&curr->css); return ret; } static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages) { unsigned long active; unsigned long inactive; unsigned long gb; unsigned long inactive_ratio; inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON); active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON); gb = (inactive + active) >> (30 - PAGE_SHIFT); if (gb) inactive_ratio = int_sqrt(10 * gb); else inactive_ratio = 1; if (present_pages) { present_pages[0] = inactive; present_pages[1] = active; } return inactive_ratio; } int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg) { unsigned long active; unsigned long inactive; unsigned long present_pages[2]; unsigned long inactive_ratio; inactive_ratio = calc_inactive_ratio(memcg, present_pages); inactive = present_pages[0]; active = present_pages[1]; if (inactive * inactive_ratio < active) return 1; return 0; } int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg) { unsigned long active; unsigned long inactive; inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE); active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE); return (active > inactive); } unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg, struct zone *zone, enum lru_list lru) { int nid = zone_to_nid(zone); int zid = zone_idx(zone); struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); return MEM_CGROUP_ZSTAT(mz, lru); } struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg, struct zone *zone) { int nid = zone_to_nid(zone); int zid = zone_idx(zone); struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid); return &mz->reclaim_stat; } struct zone_reclaim_stat * mem_cgroup_get_reclaim_stat_from_page(struct page *page) { struct page_cgroup *pc; struct mem_cgroup_per_zone *mz; if (mem_cgroup_disabled()) return NULL; pc = lookup_page_cgroup(page); /* * Used bit is set without atomic ops but after smp_wmb(). * For making pc->mem_cgroup visible, insert smp_rmb() here. */ smp_rmb(); if (!PageCgroupUsed(pc)) return NULL; mz = page_cgroup_zoneinfo(pc); if (!mz) return NULL; return &mz->reclaim_stat; } unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, struct list_head *dst, unsigned long *scanned, int order, int mode, struct zone *z, struct mem_cgroup *mem_cont, int active, int file) { unsigned long nr_taken = 0; struct page *page; unsigned long scan; LIST_HEAD(pc_list); struct list_head *src; struct page_cgroup *pc, *tmp; int nid = zone_to_nid(z); int zid = zone_idx(z); struct mem_cgroup_per_zone *mz; int lru = LRU_FILE * file + active; int ret; BUG_ON(!mem_cont); mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); src = &mz->lists[lru]; scan = 0; list_for_each_entry_safe_reverse(pc, tmp, src, lru) { if (scan >= nr_to_scan) break; page = pc->page; if (unlikely(!PageCgroupUsed(pc))) continue; if (unlikely(!PageLRU(page))) continue; scan++; ret = __isolate_lru_page(page, mode, file); switch (ret) { case 0: list_move(&page->lru, dst); mem_cgroup_del_lru(page); nr_taken++; break; case -EBUSY: /* we don't affect global LRU but rotate in our LRU */ mem_cgroup_rotate_lru_list(page, page_lru(page)); break; default: break; } } *scanned = scan; trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken, 0, 0, 0, mode); return nr_taken; } #define mem_cgroup_from_res_counter(counter, member) \ container_of(counter, struct mem_cgroup, member) static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem) { if (do_swap_account) { if (res_counter_check_under_limit(&mem->res) && res_counter_check_under_limit(&mem->memsw)) return true; } else if (res_counter_check_under_limit(&mem->res)) return true; return false; } static unsigned int get_swappiness(struct mem_cgroup *memcg) { struct cgroup *cgrp = memcg->css.cgroup; unsigned int swappiness; /* root ? */ if (cgrp->parent == NULL) return vm_swappiness; spin_lock(&memcg->reclaim_param_lock); swappiness = memcg->swappiness; spin_unlock(&memcg->reclaim_param_lock); return swappiness; } /* A routine for testing mem is not under move_account */ static bool mem_cgroup_under_move(struct mem_cgroup *mem) { struct mem_cgroup *from; struct mem_cgroup *to; bool ret = false; /* * Unlike task_move routines, we access mc.to, mc.from not under * mutual exclusion by cgroup_mutex. Here, we take spinlock instead. */ spin_lock(&mc.lock); from = mc.from; to = mc.to; if (!from) goto unlock; if (from == mem || to == mem || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css)) || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css))) ret = true; unlock: spin_unlock(&mc.lock); return ret; } static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem) { if (mc.moving_task && current != mc.moving_task) { if (mem_cgroup_under_move(mem)) { DEFINE_WAIT(wait); prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE); /* moving charge context might have finished. */ if (mc.moving_task) schedule(); finish_wait(&mc.waitq, &wait); return true; } } return false; } static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data) { int *val = data; (*val)++; return 0; } /** * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode. * @memcg: The memory cgroup that went over limit * @p: Task that is going to be killed * * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is * enabled */ void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) { struct cgroup *task_cgrp; struct cgroup *mem_cgrp; /* * Need a buffer in BSS, can't rely on allocations. The code relies * on the assumption that OOM is serialized for memory controller. * If this assumption is broken, revisit this code. */ static char memcg_name[PATH_MAX]; int ret; if (!memcg || !p) return; rcu_read_lock(); mem_cgrp = memcg->css.cgroup; task_cgrp = task_cgroup(p, mem_cgroup_subsys_id); ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX); if (ret < 0) { /* * Unfortunately, we are unable to convert to a useful name * But we'll still print out the usage information */ rcu_read_unlock(); goto done; } rcu_read_unlock(); printk(KERN_INFO "Task in %s killed", memcg_name); rcu_read_lock(); ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX); if (ret < 0) { rcu_read_unlock(); goto done; } rcu_read_unlock(); /* * Continues from above, so we don't need an KERN_ level */ printk(KERN_CONT " as a result of limit of %s\n", memcg_name); done: printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n", res_counter_read_u64(&memcg->res, RES_USAGE) >> 10, res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10, res_counter_read_u64(&memcg->res, RES_FAILCNT)); printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, " "failcnt %llu\n", res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10, res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10, res_counter_read_u64(&memcg->memsw, RES_FAILCNT)); } /* * This function returns the number of memcg under hierarchy tree. Returns * 1(self count) if no children. */ static int mem_cgroup_count_children(struct mem_cgroup *mem) { int num = 0; mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb); return num; } /* * Return the memory (and swap, if configured) limit for a memcg. */ u64 mem_cgroup_get_limit(struct mem_cgroup *memcg) { u64 limit; u64 memsw; limit = res_counter_read_u64(&memcg->res, RES_LIMIT) + total_swap_pages; memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT); /* * If memsw is finite and limits the amount of swap space available * to this memcg, return that limit. */ return min(limit, memsw); } /* * Visit the first child (need not be the first child as per the ordering * of the cgroup list, since we track last_scanned_child) of @mem and use * that to reclaim free pages from. */ static struct mem_cgroup * mem_cgroup_select_victim(struct mem_cgroup *root_mem) { struct mem_cgroup *ret = NULL; struct cgroup_subsys_state *css; int nextid, found; if (!root_mem->use_hierarchy) { css_get(&root_mem->css); ret = root_mem; } while (!ret) { rcu_read_lock(); nextid = root_mem->last_scanned_child + 1; css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css, &found); if (css && css_tryget(css)) ret = container_of(css, struct mem_cgroup, css); rcu_read_unlock(); /* Updates scanning parameter */ spin_lock(&root_mem->reclaim_param_lock); if (!css) { /* this means start scan from ID:1 */ root_mem->last_scanned_child = 0; } else root_mem->last_scanned_child = found; spin_unlock(&root_mem->reclaim_param_lock); } return ret; } /* * Scan the hierarchy if needed to reclaim memory. We remember the last child * we reclaimed from, so that we don't end up penalizing one child extensively * based on its position in the children list. * * root_mem is the original ancestor that we've been reclaim from. * * We give up and return to the caller when we visit root_mem twice. * (other groups can be removed while we're walking....) * * If shrink==true, for avoiding to free too much, this returns immedieately. */ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem, struct zone *zone, gfp_t gfp_mask, unsigned long reclaim_options) { struct mem_cgroup *victim; int ret, total = 0; int loop = 0; bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP; bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK; bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT; unsigned long excess = mem_cgroup_get_excess(root_mem); /* If memsw_is_minimum==1, swap-out is of-no-use. */ if (root_mem->memsw_is_minimum) noswap = true; while (1) { victim = mem_cgroup_select_victim(root_mem); if (victim == root_mem) { loop++; if (loop >= 1) drain_all_stock_async(); if (loop >= 2) { /* * If we have not been able to reclaim * anything, it might because there are * no reclaimable pages under this hierarchy */ if (!check_soft || !total) { css_put(&victim->css); break; } /* * We want to do more targetted reclaim. * excess >> 2 is not to excessive so as to * reclaim too much, nor too less that we keep * coming back to reclaim from this cgroup */ if (total >= (excess >> 2) || (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) { css_put(&victim->css); break; } } } if (!mem_cgroup_local_usage(victim)) { /* this cgroup's local usage == 0 */ css_put(&victim->css); continue; } /* we use swappiness of local cgroup */ if (check_soft) ret = mem_cgroup_shrink_node_zone(victim, gfp_mask, noswap, get_swappiness(victim), zone); else ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, noswap, get_swappiness(victim)); css_put(&victim->css); /* * At shrinking usage, we can't check we should stop here or * reclaim more. It's depends on callers. last_scanned_child * will work enough for keeping fairness under tree. */ if (shrink) return ret; total += ret; if (check_soft) { if (res_counter_check_under_soft_limit(&root_mem->res)) return total; } else if (mem_cgroup_check_under_limit(root_mem)) return 1 + total; } return total; } static int mem_cgroup_oom_lock_cb(struct mem_cgroup *mem, void *data) { int *val = (int *)data; int x; /* * Logically, we can stop scanning immediately when we find * a memcg is already locked. But condidering unlock ops and * creation/removal of memcg, scan-all is simple operation. */ x = atomic_inc_return(&mem->oom_lock); *val = max(x, *val); return 0; } /* * Check OOM-Killer is already running under our hierarchy. * If someone is running, return false. */ static bool mem_cgroup_oom_lock(struct mem_cgroup *mem) { int lock_count = 0; mem_cgroup_walk_tree(mem, &lock_count, mem_cgroup_oom_lock_cb); if (lock_count == 1) return true; return false; } static int mem_cgroup_oom_unlock_cb(struct mem_cgroup *mem, void *data) { /* * When a new child is created while the hierarchy is under oom, * mem_cgroup_oom_lock() may not be called. We have to use * atomic_add_unless() here. */ atomic_add_unless(&mem->oom_lock, -1, 0); return 0; } static void mem_cgroup_oom_unlock(struct mem_cgroup *mem) { mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_unlock_cb); } static DEFINE_MUTEX(memcg_oom_mutex); static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq); struct oom_wait_info { struct mem_cgroup *mem; wait_queue_t wait; }; static int memcg_oom_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *arg) { struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg; struct oom_wait_info *oom_wait_info; oom_wait_info = container_of(wait, struct oom_wait_info, wait); if (oom_wait_info->mem == wake_mem) goto wakeup; /* if no hierarchy, no match */ if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy) return 0; /* * Both of oom_wait_info->mem and wake_mem are stable under us. * Then we can use css_is_ancestor without taking care of RCU. */ if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) && !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css)) return 0; wakeup: return autoremove_wake_function(wait, mode, sync, arg); } static void memcg_wakeup_oom(struct mem_cgroup *mem) { /* for filtering, pass "mem" as argument. */ __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem); } static void memcg_oom_recover(struct mem_cgroup *mem) { if (mem && atomic_read(&mem->oom_lock)) memcg_wakeup_oom(mem); } /* * try to call OOM killer. returns false if we should exit memory-reclaim loop. */ bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask) { struct oom_wait_info owait; bool locked, need_to_kill; owait.mem = mem; owait.wait.flags = 0; owait.wait.func = memcg_oom_wake_function; owait.wait.private = current; INIT_LIST_HEAD(&owait.wait.task_list); need_to_kill = true; /* At first, try to OOM lock hierarchy under mem.*/ mutex_lock(&memcg_oom_mutex); locked = mem_cgroup_oom_lock(mem); /* * Even if signal_pending(), we can't quit charge() loop without * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL * under OOM is always welcomed, use TASK_KILLABLE here. */ prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE); if (!locked || mem->oom_kill_disable) need_to_kill = false; if (locked) mem_cgroup_oom_notify(mem); mutex_unlock(&memcg_oom_mutex); if (need_to_kill) { finish_wait(&memcg_oom_waitq, &owait.wait); mem_cgroup_out_of_memory(mem, mask); } else { schedule(); finish_wait(&memcg_oom_waitq, &owait.wait); } mutex_lock(&memcg_oom_mutex); mem_cgroup_oom_unlock(mem); memcg_wakeup_oom(mem); mutex_unlock(&memcg_oom_mutex); if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current)) return false; /* Give chance to dying process */ schedule_timeout(1); return true; } /* * Currently used to update mapped file statistics, but the routine can be * generalized to update other statistics as well. */ void mem_cgroup_update_file_mapped(struct page *page, int val) { struct mem_cgroup *mem; struct page_cgroup *pc; pc = lookup_page_cgroup(page); if (unlikely(!pc)) return; lock_page_cgroup(pc); mem = pc->mem_cgroup; if (!mem || !PageCgroupUsed(pc)) goto done; /* * Preemption is already disabled. We can use __this_cpu_xxx */ if (val > 0) { __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); SetPageCgroupFileMapped(pc); } else { __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); ClearPageCgroupFileMapped(pc); } done: unlock_page_cgroup(pc); } /* * size of first charge trial. "32" comes from vmscan.c's magic value. * TODO: maybe necessary to use big numbers in big irons. */ #define CHARGE_SIZE (32 * PAGE_SIZE) struct memcg_stock_pcp { struct mem_cgroup *cached; /* this never be root cgroup */ int charge; struct work_struct work; }; static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock); static atomic_t memcg_drain_count; /* * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed * from local stock and true is returned. If the stock is 0 or charges from a * cgroup which is not current target, returns false. This stock will be * refilled. */ static bool consume_stock(struct mem_cgroup *mem) { struct memcg_stock_pcp *stock; bool ret = true; stock = &get_cpu_var(memcg_stock); if (mem == stock->cached && stock->charge) stock->charge -= PAGE_SIZE; else /* need to call res_counter_charge */ ret = false; put_cpu_var(memcg_stock); return ret; } /* * Returns stocks cached in percpu to res_counter and reset cached information. */ static void drain_stock(struct memcg_stock_pcp *stock) { struct mem_cgroup *old = stock->cached; if (stock->charge) { res_counter_uncharge(&old->res, stock->charge); if (do_swap_account) res_counter_uncharge(&old->memsw, stock->charge); } stock->cached = NULL; stock->charge = 0; } /* * This must be called under preempt disabled or must be called by * a thread which is pinned to local cpu. */ static void drain_local_stock(struct work_struct *dummy) { struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock); drain_stock(stock); } /* * Cache charges(val) which is from res_counter, to local per_cpu area. * This will be consumed by consume_stock() function, later. */ static void refill_stock(struct mem_cgroup *mem, int val) { struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); if (stock->cached != mem) { /* reset if necessary */ drain_stock(stock); stock->cached = mem; } stock->charge += val; put_cpu_var(memcg_stock); } /* * Tries to drain stocked charges in other cpus. This function is asynchronous * and just put a work per cpu for draining localy on each cpu. Caller can * expects some charges will be back to res_counter later but cannot wait for * it. */ static void drain_all_stock_async(void) { int cpu; /* This function is for scheduling "drain" in asynchronous way. * The result of "drain" is not directly handled by callers. Then, * if someone is calling drain, we don't have to call drain more. * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if * there is a race. We just do loose check here. */ if (atomic_read(&memcg_drain_count)) return; /* Notify other cpus that system-wide "drain" is running */ atomic_inc(&memcg_drain_count); get_online_cpus(); for_each_online_cpu(cpu) { struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); schedule_work_on(cpu, &stock->work); } put_online_cpus(); atomic_dec(&memcg_drain_count); /* We don't wait for flush_work */ } /* This is a synchronous drain interface. */ static void drain_all_stock_sync(void) { /* called when force_empty is called */ atomic_inc(&memcg_drain_count); schedule_on_each_cpu(drain_local_stock); atomic_dec(&memcg_drain_count); } static int __cpuinit memcg_stock_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu) { int cpu = (unsigned long)hcpu; struct memcg_stock_pcp *stock; if (action != CPU_DEAD) return NOTIFY_OK; stock = &per_cpu(memcg_stock, cpu); drain_stock(stock); return NOTIFY_OK; } /* See __mem_cgroup_try_charge() for details */ enum { CHARGE_OK, /* success */ CHARGE_RETRY, /* need to retry but retry is not bad */ CHARGE_NOMEM, /* we can't do more. return -ENOMEM */ CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */ CHARGE_OOM_DIE, /* the current is killed because of OOM */ }; static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask, int csize, bool oom_check) { struct mem_cgroup *mem_over_limit; struct res_counter *fail_res; unsigned long flags = 0; int ret; ret = res_counter_charge(&mem->res, csize, &fail_res); if (likely(!ret)) { if (!do_swap_account) return CHARGE_OK; ret = res_counter_charge(&mem->memsw, csize, &fail_res); if (likely(!ret)) return CHARGE_OK; mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw); flags |= MEM_CGROUP_RECLAIM_NOSWAP; } else mem_over_limit = mem_cgroup_from_res_counter(fail_res, res); if (csize > PAGE_SIZE) /* change csize and retry */ return CHARGE_RETRY; if (!(gfp_mask & __GFP_WAIT)) return CHARGE_WOULDBLOCK; ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL, gfp_mask, flags); /* * try_to_free_mem_cgroup_pages() might not give us a full * picture of reclaim. Some pages are reclaimed and might be * moved to swap cache or just unmapped from the cgroup. * Check the limit again to see if the reclaim reduced the * current usage of the cgroup before giving up */ if (ret || mem_cgroup_check_under_limit(mem_over_limit)) return CHARGE_RETRY; /* * At task move, charge accounts can be doubly counted. So, it's * better to wait until the end of task_move if something is going on. */ if (mem_cgroup_wait_acct_move(mem_over_limit)) return CHARGE_RETRY; /* If we don't need to call oom-killer at el, return immediately */ if (!oom_check) return CHARGE_NOMEM; /* check OOM */ if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask)) return CHARGE_OOM_DIE; return CHARGE_RETRY; } /* * Unlike exported interface, "oom" parameter is added. if oom==true, * oom-killer can be invoked. */ static int __mem_cgroup_try_charge(struct mm_struct *mm, gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom) { int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; struct mem_cgroup *mem = NULL; int ret; int csize = CHARGE_SIZE; /* * Unlike gloval-vm's OOM-kill, we're not in memory shortage * in system level. So, allow to go ahead dying process in addition to * MEMDIE process. */ if (unlikely(test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))) goto bypass; /* * We always charge the cgroup the mm_struct belongs to. * The mm_struct's mem_cgroup changes on task migration if the * thread group leader migrates. It's possible that mm is not * set, if so charge the init_mm (happens for pagecache usage). */ if (!*memcg && !mm) goto bypass; again: if (*memcg) { /* css should be a valid one */ mem = *memcg; VM_BUG_ON(css_is_removed(&mem->css)); if (mem_cgroup_is_root(mem)) goto done; if (consume_stock(mem)) goto done; css_get(&mem->css); } else { struct task_struct *p; rcu_read_lock(); p = rcu_dereference(mm->owner); VM_BUG_ON(!p); /* * because we don't have task_lock(), "p" can exit while * we're here. In that case, "mem" can point to root * cgroup but never be NULL. (and task_struct itself is freed * by RCU, cgroup itself is RCU safe.) Then, we have small * risk here to get wrong cgroup. But such kind of mis-account * by race always happens because we don't have cgroup_mutex(). * It's overkill and we allow that small race, here. */ mem = mem_cgroup_from_task(p); VM_BUG_ON(!mem); if (mem_cgroup_is_root(mem)) { rcu_read_unlock(); goto done; } if (consume_stock(mem)) { /* * It seems dagerous to access memcg without css_get(). * But considering how consume_stok works, it's not * necessary. If consume_stock success, some charges * from this memcg are cached on this cpu. So, we * don't need to call css_get()/css_tryget() before * calling consume_stock(). */ rcu_read_unlock(); goto done; } /* after here, we may be blocked. we need to get refcnt */ if (!css_tryget(&mem->css)) { rcu_read_unlock(); goto again; } rcu_read_unlock(); } do { bool oom_check; /* If killed, bypass charge */ if (fatal_signal_pending(current)) { css_put(&mem->css); goto bypass; } oom_check = false; if (oom && !nr_oom_retries) { oom_check = true; nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES; } ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check); switch (ret) { case CHARGE_OK: break; case CHARGE_RETRY: /* not in OOM situation but retry */ csize = PAGE_SIZE; css_put(&mem->css); mem = NULL; goto again; case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */ css_put(&mem->css); goto nomem; case CHARGE_NOMEM: /* OOM routine works */ if (!oom) { css_put(&mem->css); goto nomem; } /* If oom, we never return -ENOMEM */ nr_oom_retries--; break; case CHARGE_OOM_DIE: /* Killed by OOM Killer */ css_put(&mem->css); goto bypass; } } while (ret != CHARGE_OK); if (csize > PAGE_SIZE) refill_stock(mem, csize - PAGE_SIZE); css_put(&mem->css); done: *memcg = mem; return 0; nomem: *memcg = NULL; return -ENOMEM; bypass: *memcg = NULL; return 0; } /* * Somemtimes we have to undo a charge we got by try_charge(). * This function is for that and do uncharge, put css's refcnt. * gotten by try_charge(). */ static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem, unsigned long count) { if (!mem_cgroup_is_root(mem)) { res_counter_uncharge(&mem->res, PAGE_SIZE * count); if (do_swap_account) res_counter_uncharge(&mem->memsw, PAGE_SIZE * count); } } static void mem_cgroup_cancel_charge(struct mem_cgroup *mem) { __mem_cgroup_cancel_charge(mem, 1); } /* * A helper function to get mem_cgroup from ID. must be called under * rcu_read_lock(). The caller must check css_is_removed() or some if * it's concern. (dropping refcnt from swap can be called against removed * memcg.) */ static struct mem_cgroup *mem_cgroup_lookup(unsigned short id) { struct cgroup_subsys_state *css; /* ID 0 is unused ID */ if (!id) return NULL; css = css_lookup(&mem_cgroup_subsys, id); if (!css) return NULL; return container_of(css, struct mem_cgroup, css); } struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) { struct mem_cgroup *mem = NULL; struct page_cgroup *pc; unsigned short id; swp_entry_t ent; VM_BUG_ON(!PageLocked(page)); pc = lookup_page_cgroup(page); lock_page_cgroup(pc); if (PageCgroupUsed(pc)) { mem = pc->mem_cgroup; if (mem && !css_tryget(&mem->css)) mem = NULL; } else if (PageSwapCache(page)) { ent.val = page_private(page); id = lookup_swap_cgroup(ent); rcu_read_lock(); mem = mem_cgroup_lookup(id); if (mem && !css_tryget(&mem->css)) mem = NULL; rcu_read_unlock(); } unlock_page_cgroup(pc); return mem; } /* * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be * USED state. If already USED, uncharge and return. */ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, struct page_cgroup *pc, enum charge_type ctype) { /* try_charge() can return NULL to *memcg, taking care of it. */ if (!mem) return; lock_page_cgroup(pc); if (unlikely(PageCgroupUsed(pc))) { unlock_page_cgroup(pc); mem_cgroup_cancel_charge(mem); return; } pc->mem_cgroup = mem; /* * We access a page_cgroup asynchronously without lock_page_cgroup(). * Especially when a page_cgroup is taken from a page, pc->mem_cgroup * is accessed after testing USED bit. To make pc->mem_cgroup visible * before USED bit, we need memory barrier here. * See mem_cgroup_add_lru_list(), etc. */ smp_wmb(); switch (ctype) { case MEM_CGROUP_CHARGE_TYPE_CACHE: case MEM_CGROUP_CHARGE_TYPE_SHMEM: SetPageCgroupCache(pc); SetPageCgroupUsed(pc); break; case MEM_CGROUP_CHARGE_TYPE_MAPPED: ClearPageCgroupCache(pc); SetPageCgroupUsed(pc); break; default: break; } mem_cgroup_charge_statistics(mem, pc, true); unlock_page_cgroup(pc); /* * "charge_statistics" updated event counter. Then, check it. * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree. * if they exceeds softlimit. */ memcg_check_events(mem, pc->page); } /** * __mem_cgroup_move_account - move account of the page * @pc: page_cgroup of the page. * @from: mem_cgroup which the page is moved from. * @to: mem_cgroup which the page is moved to. @from != @to. * @uncharge: whether we should call uncharge and css_put against @from. * * The caller must confirm following. * - page is not on LRU (isolate_page() is useful.) * - the pc is locked, used, and ->mem_cgroup points to @from. * * This function doesn't do "charge" nor css_get to new cgroup. It should be * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is * true, this function does "uncharge" from old cgroup, but it doesn't if * @uncharge is false, so a caller should do "uncharge". */ static void __mem_cgroup_move_account(struct page_cgroup *pc, struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge) { VM_BUG_ON(from == to); VM_BUG_ON(PageLRU(pc->page)); VM_BUG_ON(!PageCgroupLocked(pc)); VM_BUG_ON(!PageCgroupUsed(pc)); VM_BUG_ON(pc->mem_cgroup != from); if (PageCgroupFileMapped(pc)) { /* Update mapped_file data for mem_cgroup */ preempt_disable(); __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]); preempt_enable(); } mem_cgroup_charge_statistics(from, pc, false); if (uncharge) /* This is not "cancel", but cancel_charge does all we need. */ mem_cgroup_cancel_charge(from); /* caller should have done css_get */ pc->mem_cgroup = to; mem_cgroup_charge_statistics(to, pc, true); /* * We charges against "to" which may not have any tasks. Then, "to" * can be under rmdir(). But in current implementation, caller of * this function is just force_empty() and move charge, so it's * garanteed that "to" is never removed. So, we don't check rmdir * status here. */ } /* * check whether the @pc is valid for moving account and call * __mem_cgroup_move_account() */ static int mem_cgroup_move_account(struct page_cgroup *pc, struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge) { int ret = -EINVAL; lock_page_cgroup(pc); if (PageCgroupUsed(pc) && pc->mem_cgroup == from) { __mem_cgroup_move_account(pc, from, to, uncharge); ret = 0; } unlock_page_cgroup(pc); /* * check events */ memcg_check_events(to, pc->page); memcg_check_events(from, pc->page); return ret; } /* * move charges to its parent. */ static int mem_cgroup_move_parent(struct page_cgroup *pc, struct mem_cgroup *child, gfp_t gfp_mask) { struct page *page = pc->page; struct cgroup *cg = child->css.cgroup; struct cgroup *pcg = cg->parent; struct mem_cgroup *parent; int ret; /* Is ROOT ? */ if (!pcg) return -EINVAL; ret = -EBUSY; if (!get_page_unless_zero(page)) goto out; if (isolate_lru_page(page)) goto put; parent = mem_cgroup_from_cont(pcg); ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false); if (ret || !parent) goto put_back; ret = mem_cgroup_move_account(pc, child, parent, true); if (ret) mem_cgroup_cancel_charge(parent); put_back: putback_lru_page(page); put: put_page(page); out: return ret; } /* * Charge the memory controller for page usage. * Return * 0 if the charge was successful * < 0 if the cgroup is over its limit */ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, gfp_t gfp_mask, enum charge_type ctype) { struct mem_cgroup *mem = NULL; struct page_cgroup *pc; int ret; pc = lookup_page_cgroup(page); /* can happen at boot */ if (unlikely(!pc)) return 0; prefetchw(pc); ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true); if (ret || !mem) return ret; __mem_cgroup_commit_charge(mem, pc, ctype); return 0; } int mem_cgroup_newpage_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) { if (mem_cgroup_disabled()) return 0; if (PageCompound(page)) return 0; /* * If already mapped, we don't have to account. * If page cache, page->mapping has address_space. * But page->mapping may have out-of-use anon_vma pointer, * detecit it by PageAnon() check. newly-mapped-anon's page->mapping * is NULL. */ if (page_mapped(page) || (page->mapping && !PageAnon(page))) return 0; if (unlikely(!mm)) mm = &init_mm; return mem_cgroup_charge_common(page, mm, gfp_mask, MEM_CGROUP_CHARGE_TYPE_MAPPED); } static void __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, enum charge_type ctype); int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask) { int ret; if (mem_cgroup_disabled()) return 0; if (PageCompound(page)) return 0; /* * Corner case handling. This is called from add_to_page_cache() * in usual. But some FS (shmem) precharges this page before calling it * and call add_to_page_cache() with GFP_NOWAIT. * * For GFP_NOWAIT case, the page may be pre-charged before calling * add_to_page_cache(). (See shmem.c) check it here and avoid to call * charge twice. (It works but has to pay a bit larger cost.) * And when the page is SwapCache, it should take swap information * into account. This is under lock_page() now. */ if (!(gfp_mask & __GFP_WAIT)) { struct page_cgroup *pc; pc = lookup_page_cgroup(page); if (!pc) return 0; lock_page_cgroup(pc); if (PageCgroupUsed(pc)) { unlock_page_cgroup(pc); return 0; } unlock_page_cgroup(pc); } if (unlikely(!mm)) mm = &init_mm; if (page_is_file_cache(page)) return mem_cgroup_charge_common(page, mm, gfp_mask, MEM_CGROUP_CHARGE_TYPE_CACHE); /* shmem */ if (PageSwapCache(page)) { struct mem_cgroup *mem = NULL; ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem); if (!ret) __mem_cgroup_commit_charge_swapin(page, mem, MEM_CGROUP_CHARGE_TYPE_SHMEM); } else ret = mem_cgroup_charge_common(page, mm, gfp_mask, MEM_CGROUP_CHARGE_TYPE_SHMEM); return ret; } /* * While swap-in, try_charge -> commit or cancel, the page is locked. * And when try_charge() successfully returns, one refcnt to memcg without * struct page_cgroup is acquired. This refcnt will be consumed by * "commit()" or removed by "cancel()" */ int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page, gfp_t mask, struct mem_cgroup **ptr) { struct mem_cgroup *mem; int ret; if (mem_cgroup_disabled()) return 0; if (!do_swap_account) goto charge_cur_mm; /* * A racing thread's fault, or swapoff, may have already updated * the pte, and even removed page from swap cache: in those cases * do_swap_page()'s pte_same() test will fail; but there's also a * KSM case which does need to charge the page. */ if (!PageSwapCache(page)) goto charge_cur_mm; mem = try_get_mem_cgroup_from_page(page); if (!mem) goto charge_cur_mm; *ptr = mem; ret = __mem_cgroup_try_charge(NULL, mask, ptr, true); css_put(&mem->css); return ret; charge_cur_mm: if (unlikely(!mm)) mm = &init_mm; return __mem_cgroup_try_charge(mm, mask, ptr, true); } static void __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr, enum charge_type ctype) { struct page_cgroup *pc; if (mem_cgroup_disabled()) return; if (!ptr) return; cgroup_exclude_rmdir(&ptr->css); pc = lookup_page_cgroup(page); mem_cgroup_lru_del_before_commit_swapcache(page); __mem_cgroup_commit_charge(ptr, pc, ctype); mem_cgroup_lru_add_after_commit_swapcache(page); /* * Now swap is on-memory. This means this page may be * counted both as mem and swap....double count. * Fix it by uncharging from memsw. Basically, this SwapCache is stable * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page() * may call delete_from_swap_cache() before reach here. */ if (do_swap_account && PageSwapCache(page)) { swp_entry_t ent = {.val = page_private(page)}; unsigned short id; struct mem_cgroup *memcg; id = swap_cgroup_record(ent, 0); rcu_read_lock(); memcg = mem_cgroup_lookup(id); if (memcg) { /* * This recorded memcg can be obsolete one. So, avoid * calling css_tryget */ if (!mem_cgroup_is_root(memcg)) res_counter_uncharge(&memcg->memsw, PAGE_SIZE); mem_cgroup_swap_statistics(memcg, false); mem_cgroup_put(memcg); } rcu_read_unlock(); } /* * At swapin, we may charge account against cgroup which has no tasks. * So, rmdir()->pre_destroy() can be called while we do this charge. * In that case, we need to call pre_destroy() again. check it here. */ cgroup_release_and_wakeup_rmdir(&ptr->css); } void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) { __mem_cgroup_commit_charge_swapin(page, ptr, MEM_CGROUP_CHARGE_TYPE_MAPPED); } void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) { if (mem_cgroup_disabled()) return; if (!mem) return; mem_cgroup_cancel_charge(mem); } static void __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype) { struct memcg_batch_info *batch = NULL; bool uncharge_memsw = true; /* If swapout, usage of swap doesn't decrease */ if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) uncharge_memsw = false; batch = ¤t->memcg_batch; /* * In usual, we do css_get() when we remember memcg pointer. * But in this case, we keep res->usage until end of a series of * uncharges. Then, it's ok to ignore memcg's refcnt. */ if (!batch->memcg) batch->memcg = mem; /* * do_batch > 0 when unmapping pages or inode invalidate/truncate. * In those cases, all pages freed continously can be expected to be in * the same cgroup and we have chance to coalesce uncharges. * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE) * because we want to do uncharge as soon as possible. */ if (!batch->do_batch || test_thread_flag(TIF_MEMDIE)) goto direct_uncharge; /* * In typical case, batch->memcg == mem. This means we can * merge a series of uncharges to an uncharge of res_counter. * If not, we uncharge res_counter ony by one. */ if (batch->memcg != mem) goto direct_uncharge; /* remember freed charge and uncharge it later */ batch->bytes += PAGE_SIZE; if (uncharge_memsw) batch->memsw_bytes += PAGE_SIZE; return; direct_uncharge: res_counter_uncharge(&mem->res, PAGE_SIZE); if (uncharge_memsw) res_counter_uncharge(&mem->memsw, PAGE_SIZE); if (unlikely(batch->memcg != mem)) memcg_oom_recover(mem); return; } /* * uncharge if !page_mapped(page) */ static struct mem_cgroup * __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) { struct page_cgroup *pc; struct mem_cgroup *mem = NULL; if (mem_cgroup_disabled()) return NULL; if (PageSwapCache(page)) return NULL; /* * Check if our page_cgroup is valid */ pc = lookup_page_cgroup(page); if (unlikely(!pc || !PageCgroupUsed(pc))) return NULL; lock_page_cgroup(pc); mem = pc->mem_cgroup; if (!PageCgroupUsed(pc)) goto unlock_out; switch (ctype) { case MEM_CGROUP_CHARGE_TYPE_MAPPED: case MEM_CGROUP_CHARGE_TYPE_DROP: /* See mem_cgroup_prepare_migration() */ if (page_mapped(page) || PageCgroupMigration(pc)) goto unlock_out; break; case MEM_CGROUP_CHARGE_TYPE_SWAPOUT: if (!PageAnon(page)) { /* Shared memory */ if (page->mapping && !page_is_file_cache(page)) goto unlock_out; } else if (page_mapped(page)) /* Anon */ goto unlock_out; break; default: break; } mem_cgroup_charge_statistics(mem, pc, false); ClearPageCgroupUsed(pc); /* * pc->mem_cgroup is not cleared here. It will be accessed when it's * freed from LRU. This is safe because uncharged page is expected not * to be reused (freed soon). Exception is SwapCache, it's handled by * special functions. */ unlock_page_cgroup(pc); /* * even after unlock, we have mem->res.usage here and this memcg * will never be freed. */ memcg_check_events(mem, page); if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) { mem_cgroup_swap_statistics(mem, true); mem_cgroup_get(mem); } if (!mem_cgroup_is_root(mem)) __do_uncharge(mem, ctype); return mem; unlock_out: unlock_page_cgroup(pc); return NULL; } void mem_cgroup_uncharge_page(struct page *page) { /* early check. */ if (page_mapped(page)) return; if (page->mapping && !PageAnon(page)) return; __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED); } void mem_cgroup_uncharge_cache_page(struct page *page) { VM_BUG_ON(page_mapped(page)); VM_BUG_ON(page->mapping); __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE); } /* * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate. * In that cases, pages are freed continuously and we can expect pages * are in the same memcg. All these calls itself limits the number of * pages freed at once, then uncharge_start/end() is called properly. * This may be called prural(2) times in a context, */ void mem_cgroup_uncharge_start(void) { current->memcg_batch.do_batch++; /* We can do nest. */ if (current->memcg_batch.do_batch == 1) { current->memcg_batch.memcg = NULL; current->memcg_batch.bytes = 0; current->memcg_batch.memsw_bytes = 0; } } void mem_cgroup_uncharge_end(void) { struct memcg_batch_info *batch = ¤t->memcg_batch; if (!batch->do_batch) return; batch->do_batch--; if (batch->do_batch) /* If stacked, do nothing. */ return; if (!batch->memcg) return; /* * This "batch->memcg" is valid without any css_get/put etc... * bacause we hide charges behind us. */ if (batch->bytes) res_counter_uncharge(&batch->memcg->res, batch->bytes); if (batch->memsw_bytes) res_counter_uncharge(&batch->memcg->memsw, batch->memsw_byt