diff options
-rw-r--r-- | include/linux/huge_mm.h | 8 | ||||
-rw-r--r-- | include/linux/mmu_notifier.h | 2 | ||||
-rw-r--r-- | include/linux/rmap.h | 14 | ||||
-rw-r--r-- | mm/huge_memory.c | 4 | ||||
-rw-r--r-- | mm/mmap.c | 10 | ||||
-rw-r--r-- | mm/rmap.c | 8 |
6 files changed, 21 insertions, 25 deletions
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 8847c8c29791..48c32ebf65a7 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h | |||
@@ -92,12 +92,8 @@ extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd); | |||
92 | #define wait_split_huge_page(__anon_vma, __pmd) \ | 92 | #define wait_split_huge_page(__anon_vma, __pmd) \ |
93 | do { \ | 93 | do { \ |
94 | pmd_t *____pmd = (__pmd); \ | 94 | pmd_t *____pmd = (__pmd); \ |
95 | spin_unlock_wait(&(__anon_vma)->root->lock); \ | 95 | anon_vma_lock(__anon_vma); \ |
96 | /* \ | 96 | anon_vma_unlock(__anon_vma); \ |
97 | * spin_unlock_wait() is just a loop in C and so the \ | ||
98 | * CPU can reorder anything around it. \ | ||
99 | */ \ | ||
100 | smp_mb(); \ | ||
101 | BUG_ON(pmd_trans_splitting(*____pmd) || \ | 97 | BUG_ON(pmd_trans_splitting(*____pmd) || \ |
102 | pmd_trans_huge(*____pmd)); \ | 98 | pmd_trans_huge(*____pmd)); \ |
103 | } while (0) | 99 | } while (0) |
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index a877dfc243eb..1d1b1e13f79f 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h | |||
@@ -150,7 +150,7 @@ struct mmu_notifier_ops { | |||
150 | * Therefore notifier chains can only be traversed when either | 150 | * Therefore notifier chains can only be traversed when either |
151 | * | 151 | * |
152 | * 1. mmap_sem is held. | 152 | * 1. mmap_sem is held. |
153 | * 2. One of the reverse map locks is held (i_mmap_mutex or anon_vma->lock). | 153 | * 2. One of the reverse map locks is held (i_mmap_mutex or anon_vma->mutex). |
154 | * 3. No other concurrent thread can access the list (release) | 154 | * 3. No other concurrent thread can access the list (release) |
155 | */ | 155 | */ |
156 | struct mmu_notifier { | 156 | struct mmu_notifier { |
diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 590c291a8cd9..2148b122779b 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h | |||
@@ -7,7 +7,7 @@ | |||
7 | #include <linux/list.h> | 7 | #include <linux/list.h> |
8 | #include <linux/slab.h> | 8 | #include <linux/slab.h> |
9 | #include <linux/mm.h> | 9 | #include <linux/mm.h> |
10 | #include <linux/spinlock.h> | 10 | #include <linux/mutex.h> |
11 | #include <linux/memcontrol.h> | 11 | #include <linux/memcontrol.h> |
12 | 12 | ||
13 | /* | 13 | /* |
@@ -26,7 +26,7 @@ | |||
26 | */ | 26 | */ |
27 | struct anon_vma { | 27 | struct anon_vma { |
28 | struct anon_vma *root; /* Root of this anon_vma tree */ | 28 | struct anon_vma *root; /* Root of this anon_vma tree */ |
29 | spinlock_t lock; /* Serialize access to vma list */ | 29 | struct mutex mutex; /* Serialize access to vma list */ |
30 | /* | 30 | /* |
31 | * The refcount is taken on an anon_vma when there is no | 31 | * The refcount is taken on an anon_vma when there is no |
32 | * guarantee that the vma of page tables will exist for | 32 | * guarantee that the vma of page tables will exist for |
@@ -64,7 +64,7 @@ struct anon_vma_chain { | |||
64 | struct vm_area_struct *vma; | 64 | struct vm_area_struct *vma; |
65 | struct anon_vma *anon_vma; | 65 | struct anon_vma *anon_vma; |
66 | struct list_head same_vma; /* locked by mmap_sem & page_table_lock */ | 66 | struct list_head same_vma; /* locked by mmap_sem & page_table_lock */ |
67 | struct list_head same_anon_vma; /* locked by anon_vma->lock */ | 67 | struct list_head same_anon_vma; /* locked by anon_vma->mutex */ |
68 | }; | 68 | }; |
69 | 69 | ||
70 | #ifdef CONFIG_MMU | 70 | #ifdef CONFIG_MMU |
@@ -93,24 +93,24 @@ static inline void vma_lock_anon_vma(struct vm_area_struct *vma) | |||
93 | { | 93 | { |
94 | struct anon_vma *anon_vma = vma->anon_vma; | 94 | struct anon_vma *anon_vma = vma->anon_vma; |
95 | if (anon_vma) | 95 | if (anon_vma) |
96 | spin_lock(&anon_vma->root->lock); | 96 | mutex_lock(&anon_vma->root->mutex); |
97 | } | 97 | } |
98 | 98 | ||
99 | static inline void vma_unlock_anon_vma(struct vm_area_struct *vma) | 99 | static inline void vma_unlock_anon_vma(struct vm_area_struct *vma) |
100 | { | 100 | { |
101 | struct anon_vma *anon_vma = vma->anon_vma; | 101 | struct anon_vma *anon_vma = vma->anon_vma; |
102 | if (anon_vma) | 102 | if (anon_vma) |
103 | spin_unlock(&anon_vma->root->lock); | 103 | mutex_unlock(&anon_vma->root->mutex); |
104 | } | 104 | } |
105 | 105 | ||
106 | static inline void anon_vma_lock(struct anon_vma *anon_vma) | 106 | static inline void anon_vma_lock(struct anon_vma *anon_vma) |
107 | { | 107 | { |
108 | spin_lock(&anon_vma->root->lock); | 108 | mutex_lock(&anon_vma->root->mutex); |
109 | } | 109 | } |
110 | 110 | ||
111 | static inline void anon_vma_unlock(struct anon_vma *anon_vma) | 111 | static inline void anon_vma_unlock(struct anon_vma *anon_vma) |
112 | { | 112 | { |
113 | spin_unlock(&anon_vma->root->lock); | 113 | mutex_unlock(&anon_vma->root->mutex); |
114 | } | 114 | } |
115 | 115 | ||
116 | /* | 116 | /* |
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 83326ad66d9b..90eef404ec2e 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -1139,7 +1139,7 @@ static int __split_huge_page_splitting(struct page *page, | |||
1139 | * We can't temporarily set the pmd to null in order | 1139 | * We can't temporarily set the pmd to null in order |
1140 | * to split it, the pmd must remain marked huge at all | 1140 | * to split it, the pmd must remain marked huge at all |
1141 | * times or the VM won't take the pmd_trans_huge paths | 1141 | * times or the VM won't take the pmd_trans_huge paths |
1142 | * and it won't wait on the anon_vma->root->lock to | 1142 | * and it won't wait on the anon_vma->root->mutex to |
1143 | * serialize against split_huge_page*. | 1143 | * serialize against split_huge_page*. |
1144 | */ | 1144 | */ |
1145 | pmdp_splitting_flush_notify(vma, address, pmd); | 1145 | pmdp_splitting_flush_notify(vma, address, pmd); |
@@ -1333,7 +1333,7 @@ static int __split_huge_page_map(struct page *page, | |||
1333 | return ret; | 1333 | return ret; |
1334 | } | 1334 | } |
1335 | 1335 | ||
1336 | /* must be called with anon_vma->root->lock hold */ | 1336 | /* must be called with anon_vma->root->mutex hold */ |
1337 | static void __split_huge_page(struct page *page, | 1337 | static void __split_huge_page(struct page *page, |
1338 | struct anon_vma *anon_vma) | 1338 | struct anon_vma *anon_vma) |
1339 | { | 1339 | { |
@@ -2502,15 +2502,15 @@ static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma) | |||
2502 | * The LSB of head.next can't change from under us | 2502 | * The LSB of head.next can't change from under us |
2503 | * because we hold the mm_all_locks_mutex. | 2503 | * because we hold the mm_all_locks_mutex. |
2504 | */ | 2504 | */ |
2505 | spin_lock_nest_lock(&anon_vma->root->lock, &mm->mmap_sem); | 2505 | mutex_lock_nest_lock(&anon_vma->root->mutex, &mm->mmap_sem); |
2506 | /* | 2506 | /* |
2507 | * We can safely modify head.next after taking the | 2507 | * We can safely modify head.next after taking the |
2508 | * anon_vma->root->lock. If some other vma in this mm shares | 2508 | * anon_vma->root->mutex. If some other vma in this mm shares |
2509 | * the same anon_vma we won't take it again. | 2509 | * the same anon_vma we won't take it again. |
2510 | * | 2510 | * |
2511 | * No need of atomic instructions here, head.next | 2511 | * No need of atomic instructions here, head.next |
2512 | * can't change from under us thanks to the | 2512 | * can't change from under us thanks to the |
2513 | * anon_vma->root->lock. | 2513 | * anon_vma->root->mutex. |
2514 | */ | 2514 | */ |
2515 | if (__test_and_set_bit(0, (unsigned long *) | 2515 | if (__test_and_set_bit(0, (unsigned long *) |
2516 | &anon_vma->root->head.next)) | 2516 | &anon_vma->root->head.next)) |
@@ -2559,7 +2559,7 @@ static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping) | |||
2559 | * vma in this mm is backed by the same anon_vma or address_space. | 2559 | * vma in this mm is backed by the same anon_vma or address_space. |
2560 | * | 2560 | * |
2561 | * We can take all the locks in random order because the VM code | 2561 | * We can take all the locks in random order because the VM code |
2562 | * taking i_mmap_mutex or anon_vma->lock outside the mmap_sem never | 2562 | * taking i_mmap_mutex or anon_vma->mutex outside the mmap_sem never |
2563 | * takes more than one of them in a row. Secondly we're protected | 2563 | * takes more than one of them in a row. Secondly we're protected |
2564 | * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex. | 2564 | * against a concurrent mm_take_all_locks() by the mm_all_locks_mutex. |
2565 | * | 2565 | * |
@@ -2615,7 +2615,7 @@ static void vm_unlock_anon_vma(struct anon_vma *anon_vma) | |||
2615 | * | 2615 | * |
2616 | * No need of atomic instructions here, head.next | 2616 | * No need of atomic instructions here, head.next |
2617 | * can't change from under us until we release the | 2617 | * can't change from under us until we release the |
2618 | * anon_vma->root->lock. | 2618 | * anon_vma->root->mutex. |
2619 | */ | 2619 | */ |
2620 | if (!__test_and_clear_bit(0, (unsigned long *) | 2620 | if (!__test_and_clear_bit(0, (unsigned long *) |
2621 | &anon_vma->root->head.next)) | 2621 | &anon_vma->root->head.next)) |
@@ -25,7 +25,7 @@ | |||
25 | * mm->mmap_sem | 25 | * mm->mmap_sem |
26 | * page->flags PG_locked (lock_page) | 26 | * page->flags PG_locked (lock_page) |
27 | * mapping->i_mmap_mutex | 27 | * mapping->i_mmap_mutex |
28 | * anon_vma->lock | 28 | * anon_vma->mutex |
29 | * mm->page_table_lock or pte_lock | 29 | * mm->page_table_lock or pte_lock |
30 | * zone->lru_lock (in mark_page_accessed, isolate_lru_page) | 30 | * zone->lru_lock (in mark_page_accessed, isolate_lru_page) |
31 | * swap_lock (in swap_duplicate, swap_info_get) | 31 | * swap_lock (in swap_duplicate, swap_info_get) |
@@ -40,7 +40,7 @@ | |||
40 | * | 40 | * |
41 | * (code doesn't rely on that order so it could be switched around) | 41 | * (code doesn't rely on that order so it could be switched around) |
42 | * ->tasklist_lock | 42 | * ->tasklist_lock |
43 | * anon_vma->lock (memory_failure, collect_procs_anon) | 43 | * anon_vma->mutex (memory_failure, collect_procs_anon) |
44 | * pte map lock | 44 | * pte map lock |
45 | */ | 45 | */ |
46 | 46 | ||
@@ -307,7 +307,7 @@ static void anon_vma_ctor(void *data) | |||
307 | { | 307 | { |
308 | struct anon_vma *anon_vma = data; | 308 | struct anon_vma *anon_vma = data; |
309 | 309 | ||
310 | spin_lock_init(&anon_vma->lock); | 310 | mutex_init(&anon_vma->mutex); |
311 | atomic_set(&anon_vma->refcount, 0); | 311 | atomic_set(&anon_vma->refcount, 0); |
312 | INIT_LIST_HEAD(&anon_vma->head); | 312 | INIT_LIST_HEAD(&anon_vma->head); |
313 | } | 313 | } |
@@ -1143,7 +1143,7 @@ out_mlock: | |||
1143 | /* | 1143 | /* |
1144 | * We need mmap_sem locking, Otherwise VM_LOCKED check makes | 1144 | * We need mmap_sem locking, Otherwise VM_LOCKED check makes |
1145 | * unstable result and race. Plus, We can't wait here because | 1145 | * unstable result and race. Plus, We can't wait here because |
1146 | * we now hold anon_vma->lock or mapping->i_mmap_mutex. | 1146 | * we now hold anon_vma->mutex or mapping->i_mmap_mutex. |
1147 | * if trylock failed, the page remain in evictable lru and later | 1147 | * if trylock failed, the page remain in evictable lru and later |
1148 | * vmscan could retry to move the page to unevictable lru if the | 1148 | * vmscan could retry to move the page to unevictable lru if the |
1149 | * page is actually mlocked. | 1149 | * page is actually mlocked. |