From 8a9f3ccd24741b50200c3f33d62534c7271f3dfc Mon Sep 17 00:00:00 2001 From: Balbir Singh Date: Thu, 7 Feb 2008 00:13:53 -0800 Subject: Memory controller: memory accounting Add the accounting hooks. The accounting is carried out for RSS and Page Cache (unmapped) pages. There is now a common limit and accounting for both. The RSS accounting is accounted at page_add_*_rmap() and page_remove_rmap() time. Page cache is accounted at add_to_page_cache(), __delete_from_page_cache(). Swap cache is also accounted for. Each page's page_cgroup is protected with the last bit of the page_cgroup pointer, this makes handling of race conditions involving simultaneous mappings of a page easier. A reference count is kept in the page_cgroup to deal with cases where a page might be unmapped from the RSS of all tasks, but still lives in the page cache. Credits go to Vaidyanathan Srinivasan for helping with reference counting work of the page cgroup. Almost all of the page cache accounting code has help from Vaidyanathan Srinivasan. [hugh@veritas.com: fix swapoff breakage] [akpm@linux-foundation.org: fix locking] Signed-off-by: Vaidyanathan Srinivasan Signed-off-by: Balbir Singh Cc: Pavel Emelianov Cc: Paul Menage Cc: Peter Zijlstra Cc: "Eric W. Biederman" Cc: Nick Piggin Cc: Kirill Korotaev Cc: Herbert Poetzl Cc: David Rientjes Cc: Signed-off-by: Hugh Dickins Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- mm/memcontrol.c | 166 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 163 insertions(+), 3 deletions(-) (limited to 'mm/memcontrol.c') diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 4d4805eb37c7..ebca767292dc 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include struct cgroup_subsys mem_cgroup_subsys; @@ -31,7 +34,9 @@ struct cgroup_subsys mem_cgroup_subsys; * to help the administrator determine what knobs to tune. * * TODO: Add a water mark for the memory controller. Reclaim will begin when - * we hit the water mark. + * we hit the water mark. May be even add a low water mark, such that + * no reclaim occurs from a cgroup at it's low water mark, this is + * a feature that will be implemented much later in the future. */ struct mem_cgroup { struct cgroup_subsys_state css; @@ -48,6 +53,14 @@ struct mem_cgroup { struct list_head inactive_list; }; +/* + * We use the lower bit of the page->page_cgroup pointer as a bit spin + * lock. We need to ensure that page->page_cgroup is atleast two + * byte aligned (based on comments from Nick Piggin) + */ +#define PAGE_CGROUP_LOCK_BIT 0x0 +#define PAGE_CGROUP_LOCK (1 << PAGE_CGROUP_LOCK_BIT) + /* * A page_cgroup page is associated with every page descriptor. The * page_cgroup helps us identify information about the cgroup @@ -56,6 +69,8 @@ struct page_cgroup { struct list_head lru; /* per cgroup LRU list */ struct page *page; struct mem_cgroup *mem_cgroup; + atomic_t ref_cnt; /* Helpful when pages move b/w */ + /* mapped and cached states */ }; @@ -88,14 +103,157 @@ void mm_free_cgroup(struct mm_struct *mm) css_put(&mm->mem_cgroup->css); } +static inline int page_cgroup_locked(struct page *page) +{ + return bit_spin_is_locked(PAGE_CGROUP_LOCK_BIT, + &page->page_cgroup); +} + void page_assign_page_cgroup(struct page *page, struct page_cgroup *pc) { - page->page_cgroup = (unsigned long)pc; + int locked; + + /* + * While resetting the page_cgroup we might not hold the + * page_cgroup lock. free_hot_cold_page() is an example + * of such a scenario + */ + if (pc) + VM_BUG_ON(!page_cgroup_locked(page)); + locked = (page->page_cgroup & PAGE_CGROUP_LOCK); + page->page_cgroup = ((unsigned long)pc | locked); } struct page_cgroup *page_get_page_cgroup(struct page *page) { - return page->page_cgroup; + return (struct page_cgroup *) + (page->page_cgroup & ~PAGE_CGROUP_LOCK); +} + +void __always_inline lock_page_cgroup(struct page *page) +{ + bit_spin_lock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup); + VM_BUG_ON(!page_cgroup_locked(page)); +} + +void __always_inline unlock_page_cgroup(struct page *page) +{ + bit_spin_unlock(PAGE_CGROUP_LOCK_BIT, &page->page_cgroup); +} + +/* + * Charge the memory controller for page usage. + * Return + * 0 if the charge was successful + * < 0 if the cgroup is over its limit + */ +int mem_cgroup_charge(struct page *page, struct mm_struct *mm) +{ + struct mem_cgroup *mem; + struct page_cgroup *pc, *race_pc; + + /* + * Should page_cgroup's go to their own slab? + * One could optimize the performance of the charging routine + * by saving a bit in the page_flags and using it as a lock + * to see if the cgroup page already has a page_cgroup associated + * with it + */ + lock_page_cgroup(page); + pc = page_get_page_cgroup(page); + /* + * The page_cgroup exists and the page has already been accounted + */ + if (pc) { + atomic_inc(&pc->ref_cnt); + goto done; + } + + unlock_page_cgroup(page); + + pc = kzalloc(sizeof(struct page_cgroup), GFP_KERNEL); + if (pc == NULL) + goto err; + + rcu_read_lock(); + /* + * We always charge the cgroup the mm_struct belongs to + * the mm_struct's mem_cgroup changes on task migration if the + * thread group leader migrates. It's possible that mm is not + * set, if so charge the init_mm (happens for pagecache usage). + */ + if (!mm) + mm = &init_mm; + + mem = rcu_dereference(mm->mem_cgroup); + /* + * For every charge from the cgroup, increment reference + * count + */ + css_get(&mem->css); + rcu_read_unlock(); + + /* + * If we created the page_cgroup, we should free it on exceeding + * the cgroup limit. + */ + if (res_counter_charge(&mem->res, 1)) { + css_put(&mem->css); + goto free_pc; + } + + lock_page_cgroup(page); + /* + * Check if somebody else beat us to allocating the page_cgroup + */ + race_pc = page_get_page_cgroup(page); + if (race_pc) { + kfree(pc); + pc = race_pc; + atomic_inc(&pc->ref_cnt); + res_counter_uncharge(&mem->res, 1); + css_put(&mem->css); + goto done; + } + + atomic_set(&pc->ref_cnt, 1); + pc->mem_cgroup = mem; + pc->page = page; + page_assign_page_cgroup(page, pc); + +done: + unlock_page_cgroup(page); + return 0; +free_pc: + kfree(pc); + return -ENOMEM; +err: + unlock_page_cgroup(page); + return -ENOMEM; +} + +/* + * Uncharging is always a welcome operation, we never complain, simply + * uncharge. + */ +void mem_cgroup_uncharge(struct page_cgroup *pc) +{ + struct mem_cgroup *mem; + struct page *page; + + if (!pc) + return; + + if (atomic_dec_and_test(&pc->ref_cnt)) { + page = pc->page; + lock_page_cgroup(page); + mem = pc->mem_cgroup; + css_put(&mem->css); + page_assign_page_cgroup(page, NULL); + unlock_page_cgroup(page); + res_counter_uncharge(&mem->res, 1); + kfree(pc); + } } static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft, @@ -150,6 +308,8 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) return NULL; res_counter_init(&mem->res); + INIT_LIST_HEAD(&mem->active_list); + INIT_LIST_HEAD(&mem->inactive_list); return &mem->css; } -- cgit v1.2.2