aboutsummaryrefslogtreecommitdiffstats
path: root/include
diff options
context:
space:
mode:
authorJohannes Weiner <jweiner@redhat.com>2012-01-12 20:18:15 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2012-01-12 23:13:05 -0500
commit925b7673cce39116ce61e7a06683a4a0dad1e72a (patch)
tree66c134db836e531e196ee3dfc23c124ff74ac827 /include
parent6290df545814990ca2663baf6e894669132d5f73 (diff)
mm: make per-memcg LRU lists exclusive
Now that all code that operated on global per-zone LRU lists is converted to operate on per-memory cgroup LRU lists instead, there is no reason to keep the double-LRU scheme around any longer. The pc->lru member is removed and page->lru is linked directly to the per-memory cgroup LRU lists, which removes two pointers from a descriptor that exists for every page frame in the system. Signed-off-by: Johannes Weiner <jweiner@redhat.com> Signed-off-by: Hugh Dickins <hughd@google.com> Signed-off-by: Ying Han <yinghan@google.com> Reviewed-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Reviewed-by: Michal Hocko <mhocko@suse.cz> Reviewed-by: Kirill A. Shutemov <kirill@shutemov.name> Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp> Cc: Balbir Singh <bsingharora@gmail.com> Cc: Greg Thelen <gthelen@google.com> Cc: Michel Lespinasse <walken@google.com> Cc: Rik van Riel <riel@redhat.com> Cc: Minchan Kim <minchan.kim@gmail.com> Cc: Christoph Hellwig <hch@infradead.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include')
-rw-r--r--include/linux/memcontrol.h51
-rw-r--r--include/linux/mm_inline.h21
-rw-r--r--include/linux/page_cgroup.h1
3 files changed, 30 insertions, 43 deletions
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 3b99dce85293..e2f8e7caf04b 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -32,14 +32,6 @@ enum mem_cgroup_page_stat_item {
32 MEMCG_NR_FILE_MAPPED, /* # of pages charged as file rss */ 32 MEMCG_NR_FILE_MAPPED, /* # of pages charged as file rss */
33}; 33};
34 34
35extern unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
36 struct list_head *dst,
37 unsigned long *scanned, int order,
38 isolate_mode_t mode,
39 struct zone *z,
40 struct mem_cgroup *mem_cont,
41 int active, int file);
42
43struct mem_cgroup_reclaim_cookie { 35struct mem_cgroup_reclaim_cookie {
44 struct zone *zone; 36 struct zone *zone;
45 int priority; 37 int priority;
@@ -69,13 +61,14 @@ extern void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *ptr);
69 61
70extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, 62extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
71 gfp_t gfp_mask); 63 gfp_t gfp_mask);
72extern void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru); 64
73extern void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru); 65struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *);
74extern void mem_cgroup_rotate_reclaimable_page(struct page *page); 66struct lruvec *mem_cgroup_lru_add_list(struct zone *, struct page *,
75extern void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru); 67 enum lru_list);
76extern void mem_cgroup_del_lru(struct page *page); 68void mem_cgroup_lru_del_list(struct page *, enum lru_list);
77extern void mem_cgroup_move_lists(struct page *page, 69void mem_cgroup_lru_del(struct page *);
78 enum lru_list from, enum lru_list to); 70struct lruvec *mem_cgroup_lru_move_lists(struct zone *, struct page *,
71 enum lru_list, enum lru_list);
79 72
80/* For coalescing uncharge for reducing memcg' overhead*/ 73/* For coalescing uncharge for reducing memcg' overhead*/
81extern void mem_cgroup_uncharge_start(void); 74extern void mem_cgroup_uncharge_start(void);
@@ -223,33 +216,33 @@ static inline void mem_cgroup_uncharge_cache_page(struct page *page)
223{ 216{
224} 217}
225 218
226static inline void mem_cgroup_add_lru_list(struct page *page, int lru) 219static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
227{ 220 struct mem_cgroup *memcg)
228}
229
230static inline void mem_cgroup_del_lru_list(struct page *page, int lru)
231{ 221{
232 return ; 222 return &zone->lruvec;
233} 223}
234 224
235static inline void mem_cgroup_rotate_reclaimable_page(struct page *page) 225static inline struct lruvec *mem_cgroup_lru_add_list(struct zone *zone,
226 struct page *page,
227 enum lru_list lru)
236{ 228{
237 return ; 229 return &zone->lruvec;
238} 230}
239 231
240static inline void mem_cgroup_rotate_lru_list(struct page *page, int lru) 232static inline void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru)
241{ 233{
242 return ;
243} 234}
244 235
245static inline void mem_cgroup_del_lru(struct page *page) 236static inline void mem_cgroup_lru_del(struct page *page)
246{ 237{
247 return ;
248} 238}
249 239
250static inline void 240static inline struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone,
251mem_cgroup_move_lists(struct page *page, enum lru_list from, enum lru_list to) 241 struct page *page,
242 enum lru_list from,
243 enum lru_list to)
252{ 244{
245 return &zone->lruvec;
253} 246}
254 247
255static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) 248static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index e6a7ffe16d31..4e3478e71926 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -22,26 +22,21 @@ static inline int page_is_file_cache(struct page *page)
22} 22}
23 23
24static inline void 24static inline void
25__add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list l,
26 struct list_head *head)
27{
28 list_add(&page->lru, head);
29 __mod_zone_page_state(zone, NR_LRU_BASE + l, hpage_nr_pages(page));
30 mem_cgroup_add_lru_list(page, l);
31}
32
33static inline void
34add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list l) 25add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list l)
35{ 26{
36 __add_page_to_lru_list(zone, page, l, &zone->lruvec.lists[l]); 27 struct lruvec *lruvec;
28
29 lruvec = mem_cgroup_lru_add_list(zone, page, l);
30 list_add(&page->lru, &lruvec->lists[l]);
31 __mod_zone_page_state(zone, NR_LRU_BASE + l, hpage_nr_pages(page));
37} 32}
38 33
39static inline void 34static inline void
40del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list l) 35del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list l)
41{ 36{
37 mem_cgroup_lru_del_list(page, l);
42 list_del(&page->lru); 38 list_del(&page->lru);
43 __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page)); 39 __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page));
44 mem_cgroup_del_lru_list(page, l);
45} 40}
46 41
47/** 42/**
@@ -64,7 +59,6 @@ del_page_from_lru(struct zone *zone, struct page *page)
64{ 59{
65 enum lru_list l; 60 enum lru_list l;
66 61
67 list_del(&page->lru);
68 if (PageUnevictable(page)) { 62 if (PageUnevictable(page)) {
69 __ClearPageUnevictable(page); 63 __ClearPageUnevictable(page);
70 l = LRU_UNEVICTABLE; 64 l = LRU_UNEVICTABLE;
@@ -75,8 +69,9 @@ del_page_from_lru(struct zone *zone, struct page *page)
75 l += LRU_ACTIVE; 69 l += LRU_ACTIVE;
76 } 70 }
77 } 71 }
72 mem_cgroup_lru_del_list(page, l);
73 list_del(&page->lru);
78 __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page)); 74 __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page));
79 mem_cgroup_del_lru_list(page, l);
80} 75}
81 76
82/** 77/**
diff --git a/include/linux/page_cgroup.h b/include/linux/page_cgroup.h
index 961ecc7d30bc..5bae7535c202 100644
--- a/include/linux/page_cgroup.h
+++ b/include/linux/page_cgroup.h
@@ -31,7 +31,6 @@ enum {
31struct page_cgroup { 31struct page_cgroup {
32 unsigned long flags; 32 unsigned long flags;
33 struct mem_cgroup *mem_cgroup; 33 struct mem_cgroup *mem_cgroup;
34 struct list_head lru; /* per cgroup LRU list */
35}; 34};
36 35
37void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat); 36void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat);