aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/huge_mm.h8
-rw-r--r--include/linux/mmu_notifier.h2
-rw-r--r--include/linux/rmap.h14
-rw-r--r--mm/huge_memory.c4
-rw-r--r--mm/mmap.c10
-rw-r--r--mm/rmap.c8
6 files changed, 21 insertions, 25 deletions
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 8847c8c29791..48c32ebf65a7 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -92,12 +92,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd);
92#define wait_split_huge_page(__anon_vma, __pmd) \ 92#define wait_split_huge_page(__anon_vma, __pmd) \
93 do { \ 93 do { \
94 pmd_t *____pmd = (__pmd); \ 94 pmd_t *____pmd = (__pmd); \
95 spin_unlock_wait(&(__anon_vma)->root->lock); \ 95 anon_vma_lock(__anon_vma); \
96 /* \ 96 anon_vma_unlock(__anon_vma); \
97 * spin_unlock_wait() is just a loop in C and so the \
98 * CPU can reorder anything around it. \
99 */ \
100 smp_mb(); \
101 BUG_ON(pmd_trans_splitting(*____pmd) || \ 97 BUG_ON(pmd_trans_splitting(*____pmd) || \
102 pmd_trans_huge(*____pmd)); \ 98 pmd_trans_huge(*____pmd)); \
103 } while (0) 99 } while (0)
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index a877dfc243eb..1d1b1e13f79f 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -150,7 +150,7 @@ struct mmu_notifier_ops {
150 * Therefore notifier chains can only be traversed when either 150 * Therefore notifier chains can only be traversed when either
151 * 151 *
152 * 1. mmap_sem is held. 152 * 1. mmap_sem is held.
153 * 2. One of the reverse map locks is held (i_mmap_mutex or anon_vma->lock). 153 * 2. One of the reverse map locks is held (i_mmap_mutex or anon_vma->mutex).
154 * 3. No other concurrent thread can access the list (release) 154 * 3. No other concurrent thread can access the list (release)
155 */ 155 */
156struct mmu_notifier { 156struct mmu_notifier {
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index 590c291a8cd9..2148b122779b 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -7,7 +7,7 @@
7#include <linux/list.h> 7#include <linux/list.h>
8#include <linux/slab.h> 8#include <linux/slab.h>
9#include <linux/mm.h> 9#include <linux/mm.h>
10#include <linux/spinlock.h> 10#include <linux/mutex.h>
11#include <linux/memcontrol.h> 11#include <linux/memcontrol.h>
12 12
13/* 13/*
@@ -26,7 +26,7 @@
26 */ 26 */
27struct anon_vma { 27struct anon_vma {
28 struct anon_vma *root; /* Root of this anon_vma tree */ 28 struct anon_vma *root; /* Root of this anon_vma tree */
29 spinlock_t lock; /* Serialize access to vma list */ 29 struct mutex mutex; /* Serialize access to vma list */
30 /* 30 /*
31 * The refcount is taken on an anon_vma when there is no 31 * The refcount is taken on an anon_vma when there is no
32 * guarantee that the vma of page tables will exist for 32 * guarantee that the vma of page tables will exist for
@@ -64,7 +64,7 @@ struct anon_vma_chain {
64 struct vm_area_struct *vma; 64 struct vm_area_struct *vma;
65 struct anon_vma *anon_vma; 65 struct anon_vma *anon_vma;
66 struct list_head same_vma; /* locked by mmap_sem & page_table_lock */ 66 struct list_head same_vma; /* locked by mmap_sem & page_table_lock */
67 struct list_head same_anon_vma; /* locked by anon_vma->lock */ 67 struct list_head same_anon_vma; /* locked by anon_vma->mutex */
68}; 68};
69 69
70#ifdef CONFIG_MMU 70#ifdef CONFIG_MMU
@@ -93,24 +93,24 @@ static inline void vma_lock_anon_vma(struct vm_area_struct *vma)
93{ 93{
94 struct anon_vma *anon_vma = vma->anon_vma; 94 struct anon_vma *anon_vma = vma->anon_vma;
95 if (anon_vma) 95 if (anon_vma)
96 spin_lock(&anon_vma->root->lock); 96 mutex_lock(&anon_vma->root->mutex);
97} 97}
98 98
99static inline void vma_unlock_anon_vma(struct vm_area_struct *vma) 99static inline void vma_unlock_anon_vma(struct vm_area_struct *vma)
100{ 100{
101 struct anon_vma *anon_vma = vma->anon_vma; 101 struct anon_vma *anon_vma = vma->anon_vma;
102 if (anon_vma) 102 if (anon_vma)
103 spin_unlock(&anon_vma->root->lock); 103 mutex_unlock(&anon_vma->root->mutex);
104} 104}
105 105
106static inline void anon_vma_lock(struct anon_vma *anon_vma) 106static inline void anon_vma_lock(struct anon_vma *anon_vma)
107{ 107{
108 spin_lock(&anon_vma->root->lock); 108 mutex_lock(&anon_vma->root->mutex);
109} 109}
110 110
111static inline void anon_vma_unlock(struct anon_vma *anon_vma) 111static inline void anon_vma_unlock(struct anon_vma *anon_vma)
112{ 112{
113 spin_unlock(&anon_vma->root->lock); 113 mutex_unlock(&anon_vma->root->mutex);
114} 114}
115 115
116/* 116/*
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 83326ad66d9b..90eef404ec2e 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1139,7 +1139,7 @@ static int __split_huge_page_splitting(struct page *page,
1139 * We can't temporarily set the pmd to null in order 1139 * We can't temporarily set the pmd to null in order
1140 * to split it, the pmd must remain marked huge at all 1140 * to split it, the pmd must remain marked huge at all
1141 * times or the VM won't take the pmd_trans_huge paths 1141 * times or the VM won't take the pmd_trans_huge paths
1142 * and it won't wait on the anon_vma->root->lock to 1142 * and it won't wait on the anon_vma->root->mutex to
1143 * serialize against split_huge_page*. 1143 * serialize against split_huge_page*.
1144 */ 1144 */
1145 pmdp_splitting_flush_notify(vma, address, pmd); 1145 pmdp_splitting_flush_notify(vma, address, pmd);
@@ -1333,7 +1333,7 @@ static int __split_huge_page_map(struct page *page,
1333 return ret; 1333 return ret;
1334} 1334}
1335 1335
1336/* must be called with anon_vma->root->lock hold */ 1336/* must be called with anon_vma->root->mutex hold */
1337static void __split_huge_page(struct page *page, 1337static void __split_huge_page(struct page *page,
1338 struct anon_vma *anon_vma) 1338 struct anon_vma *anon_vma)
1339{ 1339{
diff --git a/mm/mmap.c b/mm/mmap.c
index 26efbfca0b20..ac2631b7477f 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2502,15 +2502,15 @@ static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
2502 * The LSB of head.next can't change from under us 2502 * The LSB of head.next can't change from under us
2503 * because we hold the mm_all_locks_mutex. 2503 * because we hold the mm_all_locks_mutex.
2504 */ 2504 */
2505 spin_lock_nest_lock(&anon_vma->root->lock, &mm->mmap_sem); 2505 mutex_lock_nest_lock(&anon_vma->root->mutex, &mm->mmap_sem);
2506 /* 2506 /*
2507 * We can safely modify head.next after taking the 2507 * We can safely modify head.next after taking the
2508 * anon_vma->root->lock. If some other vma in this mm shares 2508 * anon_vma->root->mutex. If some other vma in this mm shares
2509 * the same anon_vma we won't take it again. 2509 * the same anon_vma we won't take it again.
2510 * 2510 *
2511 * No need of atomic instructions here, head.next 2511 * No need of atomic instructions here, head.next
2512 * can't change from under us thanks to the 2512 * can't change from under us thanks to the
2513 * anon_vma->root->lock. 2513 * anon_vma->root->mutex.
2514 */ 2514 */
2515 if (__test_and_set_bit(0, (unsigned long *) 2515 if (__test_and_set_bit(0, (unsigned long *)
2516 &anon_vma->root->head.next)) 2516 &anon_vma->root->head.next))
@@ -2559,7 +2559,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
2559 * vma in this mm is backed by the same anon_vma or address_space. 2559 * vma in this mm is backed by the same anon_vma or address_space.
2560 * 2560 *
2561 * We can take all the locks in random order because the VM code 2561 * We can take all the locks in random order because the VM code
2562 * taking i_mmap_mutex or anon_vma->lock outside the mmap_sem never 2562 * taking i_mmap_mutex or anon_vma->mutex outside the mmap_sem never
2563 * takes more than one of them in a row. Secondly we're protected 2563 * takes more than one of them in a row. Secondly we're protected
2564 * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex. 2564 * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex.
2565 * 2565 *
@@ -2615,7 +2615,7 @@ static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
2615 * 2615 *
2616 * No need of atomic instructions here, head.next 2616 * No need of atomic instructions here, head.next
2617 * can't change from under us until we release the 2617 * can't change from under us until we release the
2618 * anon_vma->root->lock. 2618 * anon_vma->root->mutex.
2619 */ 2619 */
2620 if (!__test_and_clear_bit(0, (unsigned long *) 2620 if (!__test_and_clear_bit(0, (unsigned long *)
2621 &anon_vma->root->head.next)) 2621 &anon_vma->root->head.next))
diff --git a/mm/rmap.c b/mm/rmap.c
index d271845d7d15..ce29d405d093 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -25,7 +25,7 @@
25 * mm->mmap_sem 25 * mm->mmap_sem
26 * page->flags PG_locked (lock_page) 26 * page->flags PG_locked (lock_page)
27 * mapping->i_mmap_mutex 27 * mapping->i_mmap_mutex
28 * anon_vma->lock 28 * anon_vma->mutex
29 * mm->page_table_lock or pte_lock 29 * mm->page_table_lock or pte_lock
30 * zone->lru_lock (in mark_page_accessed, isolate_lru_page) 30 * zone->lru_lock (in mark_page_accessed, isolate_lru_page)
31 * swap_lock (in swap_duplicate, swap_info_get) 31 * swap_lock (in swap_duplicate, swap_info_get)
@@ -40,7 +40,7 @@
40 * 40 *
41 * (code doesn't rely on that order so it could be switched around) 41 * (code doesn't rely on that order so it could be switched around)
42 * ->tasklist_lock 42 * ->tasklist_lock
43 * anon_vma->lock (memory_failure, collect_procs_anon) 43 * anon_vma->mutex (memory_failure, collect_procs_anon)
44 * pte map lock 44 * pte map lock
45 */ 45 */
46 46
@@ -307,7 +307,7 @@ static void anon_vma_ctor(void *data)
307{ 307{
308 struct anon_vma *anon_vma = data; 308 struct anon_vma *anon_vma = data;
309 309
310 spin_lock_init(&anon_vma->lock); 310 mutex_init(&anon_vma->mutex);
311 atomic_set(&anon_vma->refcount, 0); 311 atomic_set(&anon_vma->refcount, 0);
312 INIT_LIST_HEAD(&anon_vma->head); 312 INIT_LIST_HEAD(&anon_vma->head);
313} 313}
@@ -1143,7 +1143,7 @@ out_mlock:
1143 /* 1143 /*
1144 * We need mmap_sem locking, Otherwise VM_LOCKED check makes 1144 * We need mmap_sem locking, Otherwise VM_LOCKED check makes
1145 * unstable result and race. Plus, We can't wait here because 1145 * unstable result and race. Plus, We can't wait here because
1146 * we now hold anon_vma->lock or mapping->i_mmap_mutex. 1146 * we now hold anon_vma->mutex or mapping->i_mmap_mutex.
1147 * if trylock failed, the page remain in evictable lru and later 1147 * if trylock failed, the page remain in evictable lru and later
1148 * vmscan could retry to move the page to unevictable lru if the 1148 * vmscan could retry to move the page to unevictable lru if the
1149 * page is actually mlocked. 1149 * page is actually mlocked.