aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/rmap.h50
-rw-r--r--mm/ksm.c4
-rw-r--r--mm/migrate.c4
-rw-r--r--mm/rmap.c6
4 files changed, 24 insertions, 40 deletions
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 567d43f29a10..77216742c178 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -26,11 +26,17 @@
26 */ 26 */
27struct anon_vma { 27struct anon_vma {
28 spinlock_t lock; /* Serialize access to vma list */ 28 spinlock_t lock; /* Serialize access to vma list */
29#ifdef CONFIG_KSM 29#if defined(CONFIG_KSM) || defined(CONFIG_MIGRATION)
30 atomic_t ksm_refcount; 30
31#endif 31 /*
32#ifdef CONFIG_MIGRATION 32 * The external_refcount is taken by either KSM or page migration
33 atomic_t migrate_refcount; 33 * to take a reference to an anon_vma when there is no
34 * guarantee that the vma of page tables will exist for
35 * the duration of the operation. A caller that takes
36 * the reference is responsible for clearing up the
37 * anon_vma if they are the last user on release
38 */
39 atomic_t external_refcount;
34#endif 40#endif
35 /* 41 /*
36 * NOTE: the LSB of the head.next is set by 42 * NOTE: the LSB of the head.next is set by
@@ -64,46 +70,26 @@ struct anon_vma_chain {
64}; 70};
65 71
66#ifdef CONFIG_MMU 72#ifdef CONFIG_MMU
67#ifdef CONFIG_KSM 73#if defined(CONFIG_KSM) || defined(CONFIG_MIGRATION)
68static inline void ksm_refcount_init(struct anon_vma *anon_vma) 74static inline void anonvma_external_refcount_init(struct anon_vma *anon_vma)
69{ 75{
70 atomic_set(&anon_vma->ksm_refcount, 0); 76 atomic_set(&anon_vma->external_refcount, 0);
71} 77}
72 78
73static inline int ksm_refcount(struct anon_vma *anon_vma) 79static inline int anonvma_external_refcount(struct anon_vma *anon_vma)
74{ 80{
75 return atomic_read(&anon_vma->ksm_refcount); 81 return atomic_read(&anon_vma->external_refcount);
76} 82}
77#else 83#else
78static inline void ksm_refcount_init(struct anon_vma *anon_vma) 84static inline void anonvma_external_refcount_init(struct anon_vma *anon_vma)
79{ 85{
80} 86}
81 87
82static inline int ksm_refcount(struct anon_vma *anon_vma) 88static inline int anonvma_external_refcount(struct anon_vma *anon_vma)
83{ 89{
84 return 0; 90 return 0;
85} 91}
86#endif /* CONFIG_KSM */ 92#endif /* CONFIG_KSM */
87#ifdef CONFIG_MIGRATION
88static inline void migrate_refcount_init(struct anon_vma *anon_vma)
89{
90 atomic_set(&anon_vma->migrate_refcount, 0);
91}
92
93static inline int migrate_refcount(struct anon_vma *anon_vma)
94{
95 return atomic_read(&anon_vma->migrate_refcount);
96}
97#else
98static inline void migrate_refcount_init(struct anon_vma *anon_vma)
99{
100}
101
102static inline int migrate_refcount(struct anon_vma *anon_vma)
103{
104 return 0;
105}
106#endif /* CONFIG_MIGRATE */
107 93
108static inline struct anon_vma *page_anon_vma(struct page *page) 94static inline struct anon_vma *page_anon_vma(struct page *page)
109{ 95{
diff --git a/mm/ksm.c b/mm/ksm.c
index 956880f2ff49..6c3e99b4ae7c 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -318,14 +318,14 @@ static void hold_anon_vma(struct rmap_item *rmap_item,
318 struct anon_vma *anon_vma) 318 struct anon_vma *anon_vma)
319{ 319{
320 rmap_item->anon_vma = anon_vma; 320 rmap_item->anon_vma = anon_vma;
321 atomic_inc(&anon_vma->ksm_refcount); 321 atomic_inc(&anon_vma->external_refcount);
322} 322}
323 323
324static void drop_anon_vma(struct rmap_item *rmap_item) 324static void drop_anon_vma(struct rmap_item *rmap_item)
325{ 325{
326 struct anon_vma *anon_vma = rmap_item->anon_vma; 326 struct anon_vma *anon_vma = rmap_item->anon_vma;
327 327
328 if (atomic_dec_and_lock(&anon_vma->ksm_refcount, &anon_vma->lock)) { 328 if (atomic_dec_and_lock(&anon_vma->external_refcount, &anon_vma->lock)) {
329 int empty = list_empty(&anon_vma->head); 329 int empty = list_empty(&anon_vma->head);
330 spin_unlock(&anon_vma->lock); 330 spin_unlock(&anon_vma->lock);
331 if (empty) 331 if (empty)
diff --git a/mm/migrate.c b/mm/migrate.c
index b768a1d4fa43..42a3d24d1107 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -601,7 +601,7 @@ static int unmap_and_move(new_page_t get_new_page, unsigned long private,
601 rcu_read_lock(); 601 rcu_read_lock();
602 rcu_locked = 1; 602 rcu_locked = 1;
603 anon_vma = page_anon_vma(page); 603 anon_vma = page_anon_vma(page);
604 atomic_inc(&anon_vma->migrate_refcount); 604 atomic_inc(&anon_vma->external_refcount);
605 } 605 }
606 606
607 /* 607 /*
@@ -643,7 +643,7 @@ skip_unmap:
643rcu_unlock: 643rcu_unlock:
644 644
645 /* Drop an anon_vma reference if we took one */ 645 /* Drop an anon_vma reference if we took one */
646 if (anon_vma && atomic_dec_and_lock(&anon_vma->migrate_refcount, &anon_vma->lock)) { 646 if (anon_vma && atomic_dec_and_lock(&anon_vma->external_refcount, &anon_vma->lock)) {
647 int empty = list_empty(&anon_vma->head); 647 int empty = list_empty(&anon_vma->head);
648 spin_unlock(&anon_vma->lock); 648 spin_unlock(&anon_vma->lock);
649 if (empty) 649 if (empty)
diff --git a/mm/rmap.c b/mm/rmap.c
index f522cb008646..b5c320f7d0a5 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -250,8 +250,7 @@ static void anon_vma_unlink(struct anon_vma_chain *anon_vma_chain)
250 list_del(&anon_vma_chain->same_anon_vma); 250 list_del(&anon_vma_chain->same_anon_vma);
251 251
252 /* We must garbage collect the anon_vma if it's empty */ 252 /* We must garbage collect the anon_vma if it's empty */
253 empty = list_empty(&anon_vma->head) && !ksm_refcount(anon_vma) && 253 empty = list_empty(&anon_vma->head) && !anonvma_external_refcount(anon_vma);
254 !migrate_refcount(anon_vma);
255 spin_unlock(&anon_vma->lock); 254 spin_unlock(&anon_vma->lock);
256 255
257 if (empty) 256 if (empty)
@@ -275,8 +274,7 @@ static void anon_vma_ctor(void *data)
275 struct anon_vma *anon_vma = data; 274 struct anon_vma *anon_vma = data;
276 275
277 spin_lock_init(&anon_vma->lock); 276 spin_lock_init(&anon_vma->lock);
278 ksm_refcount_init(anon_vma); 277 anonvma_external_refcount_init(anon_vma);
279 migrate_refcount_init(anon_vma);
280 INIT_LIST_HEAD(&anon_vma->head); 278 INIT_LIST_HEAD(&anon_vma->head);
281} 279}
282 280