diff options
author | Johannes Weiner <jweiner@redhat.com> | 2012-01-12 20:18:15 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2012-01-12 23:13:05 -0500 |
commit | 925b7673cce39116ce61e7a06683a4a0dad1e72a (patch) | |
tree | 66c134db836e531e196ee3dfc23c124ff74ac827 /include | |
parent | 6290df545814990ca2663baf6e894669132d5f73 (diff) |
mm: make per-memcg LRU lists exclusive
Now that all code that operated on global per-zone LRU lists is
converted to operate on per-memory cgroup LRU lists instead, there is no
reason to keep the double-LRU scheme around any longer.
The pc->lru member is removed and page->lru is linked directly to the
per-memory cgroup LRU lists, which removes two pointers from a
descriptor that exists for every page frame in the system.
Signed-off-by: Johannes Weiner <jweiner@redhat.com>
Signed-off-by: Hugh Dickins <hughd@google.com>
Signed-off-by: Ying Han <yinghan@google.com>
Reviewed-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Reviewed-by: Michal Hocko <mhocko@suse.cz>
Reviewed-by: Kirill A. Shutemov <kirill@shutemov.name>
Cc: Daisuke Nishimura <nishimura@mxp.nes.nec.co.jp>
Cc: Balbir Singh <bsingharora@gmail.com>
Cc: Greg Thelen <gthelen@google.com>
Cc: Michel Lespinasse <walken@google.com>
Cc: Rik van Riel <riel@redhat.com>
Cc: Minchan Kim <minchan.kim@gmail.com>
Cc: Christoph Hellwig <hch@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'include')
-rw-r--r-- | include/linux/memcontrol.h | 51 | ||||
-rw-r--r-- | include/linux/mm_inline.h | 21 | ||||
-rw-r--r-- | include/linux/page_cgroup.h | 1 |
3 files changed, 30 insertions, 43 deletions
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 3b99dce85293..e2f8e7caf04b 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h | |||
@@ -32,14 +32,6 @@ enum mem_cgroup_page_stat_item { | |||
32 | MEMCG_NR_FILE_MAPPED, /* # of pages charged as file rss */ | 32 | MEMCG_NR_FILE_MAPPED, /* # of pages charged as file rss */ |
33 | }; | 33 | }; |
34 | 34 | ||
35 | extern unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, | ||
36 | struct list_head *dst, | ||
37 | unsigned long *scanned, int order, | ||
38 | isolate_mode_t mode, | ||
39 | struct zone *z, | ||
40 | struct mem_cgroup *mem_cont, | ||
41 | int active, int file); | ||
42 | |||
43 | struct mem_cgroup_reclaim_cookie { | 35 | struct mem_cgroup_reclaim_cookie { |
44 | struct zone *zone; | 36 | struct zone *zone; |
45 | int priority; | 37 | int priority; |
@@ -69,13 +61,14 @@ extern void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *ptr); | |||
69 | 61 | ||
70 | extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, | 62 | extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, |
71 | gfp_t gfp_mask); | 63 | gfp_t gfp_mask); |
72 | extern void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru); | 64 | |
73 | extern void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru); | 65 | struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *); |
74 | extern void mem_cgroup_rotate_reclaimable_page(struct page *page); | 66 | struct lruvec *mem_cgroup_lru_add_list(struct zone *, struct page *, |
75 | extern void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru); | 67 | enum lru_list); |
76 | extern void mem_cgroup_del_lru(struct page *page); | 68 | void mem_cgroup_lru_del_list(struct page *, enum lru_list); |
77 | extern void mem_cgroup_move_lists(struct page *page, | 69 | void mem_cgroup_lru_del(struct page *); |
78 | enum lru_list from, enum lru_list to); | 70 | struct lruvec *mem_cgroup_lru_move_lists(struct zone *, struct page *, |
71 | enum lru_list, enum lru_list); | ||
79 | 72 | ||
80 | /* For coalescing uncharge for reducing memcg' overhead*/ | 73 | /* For coalescing uncharge for reducing memcg' overhead*/ |
81 | extern void mem_cgroup_uncharge_start(void); | 74 | extern void mem_cgroup_uncharge_start(void); |
@@ -223,33 +216,33 @@ static inline void mem_cgroup_uncharge_cache_page(struct page *page) | |||
223 | { | 216 | { |
224 | } | 217 | } |
225 | 218 | ||
226 | static inline void mem_cgroup_add_lru_list(struct page *page, int lru) | 219 | static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone, |
227 | { | 220 | struct mem_cgroup *memcg) |
228 | } | ||
229 | |||
230 | static inline void mem_cgroup_del_lru_list(struct page *page, int lru) | ||
231 | { | 221 | { |
232 | return ; | 222 | return &zone->lruvec; |
233 | } | 223 | } |
234 | 224 | ||
235 | static inline void mem_cgroup_rotate_reclaimable_page(struct page *page) | 225 | static inline struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, |
226 | struct page *page, | ||
227 | enum lru_list lru) | ||
236 | { | 228 | { |
237 | return ; | 229 | return &zone->lruvec; |
238 | } | 230 | } |
239 | 231 | ||
240 | static inline void mem_cgroup_rotate_lru_list(struct page *page, int lru) | 232 | static inline void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru) |
241 | { | 233 | { |
242 | return ; | ||
243 | } | 234 | } |
244 | 235 | ||
245 | static inline void mem_cgroup_del_lru(struct page *page) | 236 | static inline void mem_cgroup_lru_del(struct page *page) |
246 | { | 237 | { |
247 | return ; | ||
248 | } | 238 | } |
249 | 239 | ||
250 | static inline void | 240 | static inline struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone, |
251 | mem_cgroup_move_lists(struct page *page, enum lru_list from, enum lru_list to) | 241 | struct page *page, |
242 | enum lru_list from, | ||
243 | enum lru_list to) | ||
252 | { | 244 | { |
245 | return &zone->lruvec; | ||
253 | } | 246 | } |
254 | 247 | ||
255 | static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) | 248 | static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) |
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index e6a7ffe16d31..4e3478e71926 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h | |||
@@ -22,26 +22,21 @@ static inline int page_is_file_cache(struct page *page) | |||
22 | } | 22 | } |
23 | 23 | ||
24 | static inline void | 24 | static inline void |
25 | __add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list l, | ||
26 | struct list_head *head) | ||
27 | { | ||
28 | list_add(&page->lru, head); | ||
29 | __mod_zone_page_state(zone, NR_LRU_BASE + l, hpage_nr_pages(page)); | ||
30 | mem_cgroup_add_lru_list(page, l); | ||
31 | } | ||
32 | |||
33 | static inline void | ||
34 | add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list l) | 25 | add_page_to_lru_list(struct zone *zone, struct page *page, enum lru_list l) |
35 | { | 26 | { |
36 | __add_page_to_lru_list(zone, page, l, &zone->lruvec.lists[l]); | 27 | struct lruvec *lruvec; |
28 | |||
29 | lruvec = mem_cgroup_lru_add_list(zone, page, l); | ||
30 | list_add(&page->lru, &lruvec->lists[l]); | ||
31 | __mod_zone_page_state(zone, NR_LRU_BASE + l, hpage_nr_pages(page)); | ||
37 | } | 32 | } |
38 | 33 | ||
39 | static inline void | 34 | static inline void |
40 | del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list l) | 35 | del_page_from_lru_list(struct zone *zone, struct page *page, enum lru_list l) |
41 | { | 36 | { |
37 | mem_cgroup_lru_del_list(page, l); | ||
42 | list_del(&page->lru); | 38 | list_del(&page->lru); |
43 | __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page)); | 39 | __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page)); |
44 | mem_cgroup_del_lru_list(page, l); | ||
45 | } | 40 | } |
46 | 41 | ||
47 | /** | 42 | /** |
@@ -64,7 +59,6 @@ del_page_from_lru(struct zone *zone, struct page *page) | |||
64 | { | 59 | { |
65 | enum lru_list l; | 60 | enum lru_list l; |
66 | 61 | ||
67 | list_del(&page->lru); | ||
68 | if (PageUnevictable(page)) { | 62 | if (PageUnevictable(page)) { |
69 | __ClearPageUnevictable(page); | 63 | __ClearPageUnevictable(page); |
70 | l = LRU_UNEVICTABLE; | 64 | l = LRU_UNEVICTABLE; |
@@ -75,8 +69,9 @@ del_page_from_lru(struct zone *zone, struct page *page) | |||
75 | l += LRU_ACTIVE; | 69 | l += LRU_ACTIVE; |
76 | } | 70 | } |
77 | } | 71 | } |
72 | mem_cgroup_lru_del_list(page, l); | ||
73 | list_del(&page->lru); | ||
78 | __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page)); | 74 | __mod_zone_page_state(zone, NR_LRU_BASE + l, -hpage_nr_pages(page)); |
79 | mem_cgroup_del_lru_list(page, l); | ||
80 | } | 75 | } |
81 | 76 | ||
82 | /** | 77 | /** |
diff --git a/include/linux/page_cgroup.h b/include/linux/page_cgroup.h index 961ecc7d30bc..5bae7535c202 100644 --- a/include/linux/page_cgroup.h +++ b/include/linux/page_cgroup.h | |||
@@ -31,7 +31,6 @@ enum { | |||
31 | struct page_cgroup { | 31 | struct page_cgroup { |
32 | unsigned long flags; | 32 | unsigned long flags; |
33 | struct mem_cgroup *mem_cgroup; | 33 | struct mem_cgroup *mem_cgroup; |
34 | struct list_head lru; /* per cgroup LRU list */ | ||
35 | }; | 34 | }; |
36 | 35 | ||
37 | void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat); | 36 | void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat); |