diff options
author | Matthew Wilcox <mawilcox@microsoft.com> | 2018-04-10 19:36:56 -0400 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2018-04-11 13:28:39 -0400 |
commit | b93b016313b3ba8003c3b8bb71f569af91f19fc7 (patch) | |
tree | ad4be96414189dcdf8c972f351ba430996e9fdff | |
parent | f6bb2a2c0b81c47282ddb7883f92e65a063c27dd (diff) |
page cache: use xa_lock
Remove the address_space ->tree_lock and use the xa_lock newly added to
the radix_tree_root. Rename the address_space ->page_tree to ->i_pages,
since we don't really care that it's a tree.
[willy@infradead.org: fix nds32, fs/dax.c]
Link: http://lkml.kernel.org/r/20180406145415.GB20605@bombadil.infradead.orgLink: http://lkml.kernel.org/r/20180313132639.17387-9-willy@infradead.org
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Acked-by: Jeff Layton <jlayton@redhat.com>
Cc: Darrick J. Wong <darrick.wong@oracle.com>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
39 files changed, 345 insertions, 366 deletions
diff --git a/Documentation/cgroup-v1/memory.txt b/Documentation/cgroup-v1/memory.txt index a4af2e124e24..3682e99234c2 100644 --- a/Documentation/cgroup-v1/memory.txt +++ b/Documentation/cgroup-v1/memory.txt | |||
@@ -262,7 +262,7 @@ When oom event notifier is registered, event will be delivered. | |||
262 | 2.6 Locking | 262 | 2.6 Locking |
263 | 263 | ||
264 | lock_page_cgroup()/unlock_page_cgroup() should not be called under | 264 | lock_page_cgroup()/unlock_page_cgroup() should not be called under |
265 | mapping->tree_lock. | 265 | the i_pages lock. |
266 | 266 | ||
267 | Other lock order is following: | 267 | Other lock order is following: |
268 | PG_locked. | 268 | PG_locked. |
diff --git a/Documentation/vm/page_migration b/Documentation/vm/page_migration index 0478ae2ad44a..496868072e24 100644 --- a/Documentation/vm/page_migration +++ b/Documentation/vm/page_migration | |||
@@ -90,7 +90,7 @@ Steps: | |||
90 | 90 | ||
91 | 1. Lock the page to be migrated | 91 | 1. Lock the page to be migrated |
92 | 92 | ||
93 | 2. Insure that writeback is complete. | 93 | 2. Ensure that writeback is complete. |
94 | 94 | ||
95 | 3. Lock the new page that we want to move to. It is locked so that accesses to | 95 | 3. Lock the new page that we want to move to. It is locked so that accesses to |
96 | this (not yet uptodate) page immediately lock while the move is in progress. | 96 | this (not yet uptodate) page immediately lock while the move is in progress. |
@@ -100,8 +100,8 @@ Steps: | |||
100 | mapcount is not zero then we do not migrate the page. All user space | 100 | mapcount is not zero then we do not migrate the page. All user space |
101 | processes that attempt to access the page will now wait on the page lock. | 101 | processes that attempt to access the page will now wait on the page lock. |
102 | 102 | ||
103 | 5. The radix tree lock is taken. This will cause all processes trying | 103 | 5. The i_pages lock is taken. This will cause all processes trying |
104 | to access the page via the mapping to block on the radix tree spinlock. | 104 | to access the page via the mapping to block on the spinlock. |
105 | 105 | ||
106 | 6. The refcount of the page is examined and we back out if references remain | 106 | 6. The refcount of the page is examined and we back out if references remain |
107 | otherwise we know that we are the only one referencing this page. | 107 | otherwise we know that we are the only one referencing this page. |
@@ -114,12 +114,12 @@ Steps: | |||
114 | 114 | ||
115 | 9. The radix tree is changed to point to the new page. | 115 | 9. The radix tree is changed to point to the new page. |
116 | 116 | ||
117 | 10. The reference count of the old page is dropped because the radix tree | 117 | 10. The reference count of the old page is dropped because the address space |
118 | reference is gone. A reference to the new page is established because | 118 | reference is gone. A reference to the new page is established because |
119 | the new page is referenced to by the radix tree. | 119 | the new page is referenced by the address space. |
120 | 120 | ||
121 | 11. The radix tree lock is dropped. With that lookups in the mapping | 121 | 11. The i_pages lock is dropped. With that lookups in the mapping |
122 | become possible again. Processes will move from spinning on the tree_lock | 122 | become possible again. Processes will move from spinning on the lock |
123 | to sleeping on the locked new page. | 123 | to sleeping on the locked new page. |
124 | 124 | ||
125 | 12. The page contents are copied to the new page. | 125 | 12. The page contents are copied to the new page. |
diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 74504b154256..869080bedb89 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h | |||
@@ -318,10 +318,8 @@ static inline void flush_anon_page(struct vm_area_struct *vma, | |||
318 | #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE | 318 | #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE |
319 | extern void flush_kernel_dcache_page(struct page *); | 319 | extern void flush_kernel_dcache_page(struct page *); |
320 | 320 | ||
321 | #define flush_dcache_mmap_lock(mapping) \ | 321 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) |
322 | spin_lock_irq(&(mapping)->tree_lock) | 322 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) |
323 | #define flush_dcache_mmap_unlock(mapping) \ | ||
324 | spin_unlock_irq(&(mapping)->tree_lock) | ||
325 | 323 | ||
326 | #define flush_icache_user_range(vma,page,addr,len) \ | 324 | #define flush_icache_user_range(vma,page,addr,len) \ |
327 | flush_dcache_page(page) | 325 | flush_dcache_page(page) |
diff --git a/arch/nds32/include/asm/cacheflush.h b/arch/nds32/include/asm/cacheflush.h index 7b9b20a381cb..1240f148ec0f 100644 --- a/arch/nds32/include/asm/cacheflush.h +++ b/arch/nds32/include/asm/cacheflush.h | |||
@@ -34,8 +34,8 @@ void flush_anon_page(struct vm_area_struct *vma, | |||
34 | void flush_kernel_dcache_page(struct page *page); | 34 | void flush_kernel_dcache_page(struct page *page); |
35 | void flush_icache_range(unsigned long start, unsigned long end); | 35 | void flush_icache_range(unsigned long start, unsigned long end); |
36 | void flush_icache_page(struct vm_area_struct *vma, struct page *page); | 36 | void flush_icache_page(struct vm_area_struct *vma, struct page *page); |
37 | #define flush_dcache_mmap_lock(mapping) spin_lock_irq(&(mapping)->tree_lock) | 37 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&(mapping)->i_pages) |
38 | #define flush_dcache_mmap_unlock(mapping) spin_unlock_irq(&(mapping)->tree_lock) | 38 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&(mapping)->i_pages) |
39 | 39 | ||
40 | #else | 40 | #else |
41 | #include <asm-generic/cacheflush.h> | 41 | #include <asm-generic/cacheflush.h> |
diff --git a/arch/nios2/include/asm/cacheflush.h b/arch/nios2/include/asm/cacheflush.h index 55e383c173f7..18eb9f69f806 100644 --- a/arch/nios2/include/asm/cacheflush.h +++ b/arch/nios2/include/asm/cacheflush.h | |||
@@ -46,9 +46,7 @@ extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page, | |||
46 | extern void flush_dcache_range(unsigned long start, unsigned long end); | 46 | extern void flush_dcache_range(unsigned long start, unsigned long end); |
47 | extern void invalidate_dcache_range(unsigned long start, unsigned long end); | 47 | extern void invalidate_dcache_range(unsigned long start, unsigned long end); |
48 | 48 | ||
49 | #define flush_dcache_mmap_lock(mapping) \ | 49 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) |
50 | spin_lock_irq(&(mapping)->tree_lock) | 50 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) |
51 | #define flush_dcache_mmap_unlock(mapping) \ | ||
52 | spin_unlock_irq(&(mapping)->tree_lock) | ||
53 | 51 | ||
54 | #endif /* _ASM_NIOS2_CACHEFLUSH_H */ | 52 | #endif /* _ASM_NIOS2_CACHEFLUSH_H */ |
diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index bd5ce31936f5..0c83644bfa5c 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h | |||
@@ -55,10 +55,8 @@ void invalidate_kernel_vmap_range(void *vaddr, int size); | |||
55 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 | 55 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 |
56 | extern void flush_dcache_page(struct page *page); | 56 | extern void flush_dcache_page(struct page *page); |
57 | 57 | ||
58 | #define flush_dcache_mmap_lock(mapping) \ | 58 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) |
59 | spin_lock_irq(&(mapping)->tree_lock) | 59 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) |
60 | #define flush_dcache_mmap_unlock(mapping) \ | ||
61 | spin_unlock_irq(&(mapping)->tree_lock) | ||
62 | 60 | ||
63 | #define flush_icache_page(vma,page) do { \ | 61 | #define flush_icache_page(vma,page) do { \ |
64 | flush_kernel_dcache_page(page); \ | 62 | flush_kernel_dcache_page(page); \ |
diff --git a/drivers/staging/lustre/lustre/llite/glimpse.c b/drivers/staging/lustre/lustre/llite/glimpse.c index c43ac574274c..3075358f3f08 100644 --- a/drivers/staging/lustre/lustre/llite/glimpse.c +++ b/drivers/staging/lustre/lustre/llite/glimpse.c | |||
@@ -69,7 +69,7 @@ blkcnt_t dirty_cnt(struct inode *inode) | |||
69 | void *results[1]; | 69 | void *results[1]; |
70 | 70 | ||
71 | if (inode->i_mapping) | 71 | if (inode->i_mapping) |
72 | cnt += radix_tree_gang_lookup_tag(&inode->i_mapping->page_tree, | 72 | cnt += radix_tree_gang_lookup_tag(&inode->i_mapping->i_pages, |
73 | results, 0, 1, | 73 | results, 0, 1, |
74 | PAGECACHE_TAG_DIRTY); | 74 | PAGECACHE_TAG_DIRTY); |
75 | if (cnt == 0 && atomic_read(&vob->vob_mmap_cnt) > 0) | 75 | if (cnt == 0 && atomic_read(&vob->vob_mmap_cnt) > 0) |
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_request.c b/drivers/staging/lustre/lustre/mdc/mdc_request.c index 3b1c8e5a3053..8ee7b4d273b2 100644 --- a/drivers/staging/lustre/lustre/mdc/mdc_request.c +++ b/drivers/staging/lustre/lustre/mdc/mdc_request.c | |||
@@ -934,14 +934,14 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, | |||
934 | struct page *page; | 934 | struct page *page; |
935 | int found; | 935 | int found; |
936 | 936 | ||
937 | spin_lock_irq(&mapping->tree_lock); | 937 | xa_lock_irq(&mapping->i_pages); |
938 | found = radix_tree_gang_lookup(&mapping->page_tree, | 938 | found = radix_tree_gang_lookup(&mapping->i_pages, |
939 | (void **)&page, offset, 1); | 939 | (void **)&page, offset, 1); |
940 | if (found > 0 && !radix_tree_exceptional_entry(page)) { | 940 | if (found > 0 && !radix_tree_exceptional_entry(page)) { |
941 | struct lu_dirpage *dp; | 941 | struct lu_dirpage *dp; |
942 | 942 | ||
943 | get_page(page); | 943 | get_page(page); |
944 | spin_unlock_irq(&mapping->tree_lock); | 944 | xa_unlock_irq(&mapping->i_pages); |
945 | /* | 945 | /* |
946 | * In contrast to find_lock_page() we are sure that directory | 946 | * In contrast to find_lock_page() we are sure that directory |
947 | * page cannot be truncated (while DLM lock is held) and, | 947 | * page cannot be truncated (while DLM lock is held) and, |
@@ -989,7 +989,7 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, | |||
989 | page = ERR_PTR(-EIO); | 989 | page = ERR_PTR(-EIO); |
990 | } | 990 | } |
991 | } else { | 991 | } else { |
992 | spin_unlock_irq(&mapping->tree_lock); | 992 | xa_unlock_irq(&mapping->i_pages); |
993 | page = NULL; | 993 | page = NULL; |
994 | } | 994 | } |
995 | return page; | 995 | return page; |
diff --git a/fs/afs/write.c b/fs/afs/write.c index 9370e2feb999..dbc3c0b0142d 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c | |||
@@ -570,10 +570,11 @@ static int afs_writepages_region(struct address_space *mapping, | |||
570 | 570 | ||
571 | _debug("wback %lx", page->index); | 571 | _debug("wback %lx", page->index); |
572 | 572 | ||
573 | /* at this point we hold neither mapping->tree_lock nor lock on | 573 | /* |
574 | * the page itself: the page may be truncated or invalidated | 574 | * at this point we hold neither the i_pages lock nor the |
575 | * (changing page->mapping to NULL), or even swizzled back from | 575 | * page lock: the page may be truncated or invalidated |
576 | * swapper_space to tmpfs file mapping | 576 | * (changing page->mapping to NULL), or even swizzled |
577 | * back from swapper_space to tmpfs file mapping | ||
577 | */ | 578 | */ |
578 | ret = lock_page_killable(page); | 579 | ret = lock_page_killable(page); |
579 | if (ret < 0) { | 580 | if (ret < 0) { |
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 562c3e633403..578181cd96b5 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c | |||
@@ -458,7 +458,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, | |||
458 | break; | 458 | break; |
459 | 459 | ||
460 | rcu_read_lock(); | 460 | rcu_read_lock(); |
461 | page = radix_tree_lookup(&mapping->page_tree, pg_index); | 461 | page = radix_tree_lookup(&mapping->i_pages, pg_index); |
462 | rcu_read_unlock(); | 462 | rcu_read_unlock(); |
463 | if (page && !radix_tree_exceptional_entry(page)) { | 463 | if (page && !radix_tree_exceptional_entry(page)) { |
464 | misses++; | 464 | misses++; |
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 47a8fe9d22e8..cf87976e389d 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c | |||
@@ -3963,11 +3963,11 @@ retry: | |||
3963 | 3963 | ||
3964 | done_index = page->index; | 3964 | done_index = page->index; |
3965 | /* | 3965 | /* |
3966 | * At this point we hold neither mapping->tree_lock nor | 3966 | * At this point we hold neither the i_pages lock nor |
3967 | * lock on the page itself: the page may be truncated or | 3967 | * the page lock: the page may be truncated or |
3968 | * invalidated (changing page->mapping to NULL), or even | 3968 | * invalidated (changing page->mapping to NULL), |
3969 | * swizzled back from swapper_space to tmpfs file | 3969 | * or even swizzled back from swapper_space to |
3970 | * mapping | 3970 | * tmpfs file mapping |
3971 | */ | 3971 | */ |
3972 | if (!trylock_page(page)) { | 3972 | if (!trylock_page(page)) { |
3973 | flush_write_bio(epd); | 3973 | flush_write_bio(epd); |
@@ -5174,13 +5174,13 @@ void clear_extent_buffer_dirty(struct extent_buffer *eb) | |||
5174 | WARN_ON(!PagePrivate(page)); | 5174 | WARN_ON(!PagePrivate(page)); |
5175 | 5175 | ||
5176 | clear_page_dirty_for_io(page); | 5176 | clear_page_dirty_for_io(page); |
5177 | spin_lock_irq(&page->mapping->tree_lock); | 5177 | xa_lock_irq(&page->mapping->i_pages); |
5178 | if (!PageDirty(page)) { | 5178 | if (!PageDirty(page)) { |
5179 | radix_tree_tag_clear(&page->mapping->page_tree, | 5179 | radix_tree_tag_clear(&page->mapping->i_pages, |
5180 | page_index(page), | 5180 | page_index(page), |
5181 | PAGECACHE_TAG_DIRTY); | 5181 | PAGECACHE_TAG_DIRTY); |
5182 | } | 5182 | } |
5183 | spin_unlock_irq(&page->mapping->tree_lock); | 5183 | xa_unlock_irq(&page->mapping->i_pages); |
5184 | ClearPageError(page); | 5184 | ClearPageError(page); |
5185 | unlock_page(page); | 5185 | unlock_page(page); |
5186 | } | 5186 | } |
diff --git a/fs/buffer.c b/fs/buffer.c index 64b1e2065b6b..f3491074b035 100644 --- a/fs/buffer.c +++ b/fs/buffer.c | |||
@@ -185,10 +185,9 @@ EXPORT_SYMBOL(end_buffer_write_sync); | |||
185 | * we get exclusion from try_to_free_buffers with the blockdev mapping's | 185 | * we get exclusion from try_to_free_buffers with the blockdev mapping's |
186 | * private_lock. | 186 | * private_lock. |
187 | * | 187 | * |
188 | * Hack idea: for the blockdev mapping, i_bufferlist_lock contention | 188 | * Hack idea: for the blockdev mapping, private_lock contention |
189 | * may be quite high. This code could TryLock the page, and if that | 189 | * may be quite high. This code could TryLock the page, and if that |
190 | * succeeds, there is no need to take private_lock. (But if | 190 | * succeeds, there is no need to take private_lock. |
191 | * private_lock is contended then so is mapping->tree_lock). | ||
192 | */ | 191 | */ |
193 | static struct buffer_head * | 192 | static struct buffer_head * |
194 | __find_get_block_slow(struct block_device *bdev, sector_t block) | 193 | __find_get_block_slow(struct block_device *bdev, sector_t block) |
@@ -599,14 +598,14 @@ void __set_page_dirty(struct page *page, struct address_space *mapping, | |||
599 | { | 598 | { |
600 | unsigned long flags; | 599 | unsigned long flags; |
601 | 600 | ||
602 | spin_lock_irqsave(&mapping->tree_lock, flags); | 601 | xa_lock_irqsave(&mapping->i_pages, flags); |
603 | if (page->mapping) { /* Race with truncate? */ | 602 | if (page->mapping) { /* Race with truncate? */ |
604 | WARN_ON_ONCE(warn && !PageUptodate(page)); | 603 | WARN_ON_ONCE(warn && !PageUptodate(page)); |
605 | account_page_dirtied(page, mapping); | 604 | account_page_dirtied(page, mapping); |
606 | radix_tree_tag_set(&mapping->page_tree, | 605 | radix_tree_tag_set(&mapping->i_pages, |
607 | page_index(page), PAGECACHE_TAG_DIRTY); | 606 | page_index(page), PAGECACHE_TAG_DIRTY); |
608 | } | 607 | } |
609 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 608 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
610 | } | 609 | } |
611 | EXPORT_SYMBOL_GPL(__set_page_dirty); | 610 | EXPORT_SYMBOL_GPL(__set_page_dirty); |
612 | 611 | ||
@@ -1096,7 +1095,7 @@ __getblk_slow(struct block_device *bdev, sector_t block, | |||
1096 | * inode list. | 1095 | * inode list. |
1097 | * | 1096 | * |
1098 | * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, | 1097 | * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, |
1099 | * mapping->tree_lock and mapping->host->i_lock. | 1098 | * i_pages lock and mapping->host->i_lock. |
1100 | */ | 1099 | */ |
1101 | void mark_buffer_dirty(struct buffer_head *bh) | 1100 | void mark_buffer_dirty(struct buffer_head *bh) |
1102 | { | 1101 | { |
diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 7cee97b93a61..4bcd4e838b47 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c | |||
@@ -1987,11 +1987,10 @@ wdata_prepare_pages(struct cifs_writedata *wdata, unsigned int found_pages, | |||
1987 | for (i = 0; i < found_pages; i++) { | 1987 | for (i = 0; i < found_pages; i++) { |
1988 | page = wdata->pages[i]; | 1988 | page = wdata->pages[i]; |
1989 | /* | 1989 | /* |
1990 | * At this point we hold neither mapping->tree_lock nor | 1990 | * At this point we hold neither the i_pages lock nor the |
1991 | * lock on the page itself: the page may be truncated or | 1991 | * page lock: the page may be truncated or invalidated |
1992 | * invalidated (changing page->mapping to NULL), or even | 1992 | * (changing page->mapping to NULL), or even swizzled |
1993 | * swizzled back from swapper_space to tmpfs file | 1993 | * back from swapper_space to tmpfs file mapping |
1994 | * mapping | ||
1995 | */ | 1994 | */ |
1996 | 1995 | ||
1997 | if (nr_pages == 0) | 1996 | if (nr_pages == 0) |
@@ -158,11 +158,9 @@ static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mo | |||
158 | } | 158 | } |
159 | 159 | ||
160 | /* | 160 | /* |
161 | * We do not necessarily hold the mapping->tree_lock when we call this | 161 | * @entry may no longer be the entry at the index in the mapping. |
162 | * function so it is possible that 'entry' is no longer a valid item in the | 162 | * The important information it's conveying is whether the entry at |
163 | * radix tree. This is okay because all we really need to do is to find the | 163 | * this index used to be a PMD entry. |
164 | * correct waitqueue where tasks might be waiting for that old 'entry' and | ||
165 | * wake them. | ||
166 | */ | 164 | */ |
167 | static void dax_wake_mapping_entry_waiter(struct address_space *mapping, | 165 | static void dax_wake_mapping_entry_waiter(struct address_space *mapping, |
168 | pgoff_t index, void *entry, bool wake_all) | 166 | pgoff_t index, void *entry, bool wake_all) |
@@ -174,7 +172,7 @@ static void dax_wake_mapping_entry_waiter(struct address_space *mapping, | |||
174 | 172 | ||
175 | /* | 173 | /* |
176 | * Checking for locked entry and prepare_to_wait_exclusive() happens | 174 | * Checking for locked entry and prepare_to_wait_exclusive() happens |
177 | * under mapping->tree_lock, ditto for entry handling in our callers. | 175 | * under the i_pages lock, ditto for entry handling in our callers. |
178 | * So at this point all tasks that could have seen our entry locked | 176 | * So at this point all tasks that could have seen our entry locked |
179 | * must be in the waitqueue and the following check will see them. | 177 | * must be in the waitqueue and the following check will see them. |
180 | */ | 178 | */ |
@@ -183,41 +181,39 @@ static void dax_wake_mapping_entry_waiter(struct address_space *mapping, | |||
183 | } | 181 | } |
184 | 182 | ||
185 | /* | 183 | /* |
186 | * Check whether the given slot is locked. The function must be called with | 184 | * Check whether the given slot is locked. Must be called with the i_pages |
187 | * mapping->tree_lock held | 185 | * lock held. |
188 | */ | 186 | */ |
189 | static inline int slot_locked(struct address_space *mapping, void **slot) | 187 | static inline int slot_locked(struct address_space *mapping, void **slot) |
190 | { | 188 | { |
191 | unsigned long entry = (unsigned long) | 189 | unsigned long entry = (unsigned long) |
192 | radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 190 | radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); |
193 | return entry & RADIX_DAX_ENTRY_LOCK; | 191 | return entry & RADIX_DAX_ENTRY_LOCK; |
194 | } | 192 | } |
195 | 193 | ||
196 | /* | 194 | /* |
197 | * Mark the given slot is locked. The function must be called with | 195 | * Mark the given slot as locked. Must be called with the i_pages lock held. |
198 | * mapping->tree_lock held | ||
199 | */ | 196 | */ |
200 | static inline void *lock_slot(struct address_space *mapping, void **slot) | 197 | static inline void *lock_slot(struct address_space *mapping, void **slot) |
201 | { | 198 | { |
202 | unsigned long entry = (unsigned long) | 199 | unsigned long entry = (unsigned long) |
203 | radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 200 | radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); |
204 | 201 | ||
205 | entry |= RADIX_DAX_ENTRY_LOCK; | 202 | entry |= RADIX_DAX_ENTRY_LOCK; |
206 | radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); | 203 | radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); |
207 | return (void *)entry; | 204 | return (void *)entry; |
208 | } | 205 | } |
209 | 206 | ||
210 | /* | 207 | /* |
211 | * Mark the given slot is unlocked. The function must be called with | 208 | * Mark the given slot as unlocked. Must be called with the i_pages lock held. |
212 | * mapping->tree_lock held | ||
213 | */ | 209 | */ |
214 | static inline void *unlock_slot(struct address_space *mapping, void **slot) | 210 | static inline void *unlock_slot(struct address_space *mapping, void **slot) |
215 | { | 211 | { |
216 | unsigned long entry = (unsigned long) | 212 | unsigned long entry = (unsigned long) |
217 | radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 213 | radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); |
218 | 214 | ||
219 | entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; | 215 | entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; |
220 | radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); | 216 | radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); |
221 | return (void *)entry; | 217 | return (void *)entry; |
222 | } | 218 | } |
223 | 219 | ||
@@ -228,7 +224,7 @@ static inline void *unlock_slot(struct address_space *mapping, void **slot) | |||
228 | * put_locked_mapping_entry() when he locked the entry and now wants to | 224 | * put_locked_mapping_entry() when he locked the entry and now wants to |
229 | * unlock it. | 225 | * unlock it. |
230 | * | 226 | * |
231 | * The function must be called with mapping->tree_lock held. | 227 | * Must be called with the i_pages lock held. |
232 | */ | 228 | */ |
233 | static void *get_unlocked_mapping_entry(struct address_space *mapping, | 229 | static void *get_unlocked_mapping_entry(struct address_space *mapping, |
234 | pgoff_t index, void ***slotp) | 230 | pgoff_t index, void ***slotp) |
@@ -241,7 +237,7 @@ static void *get_unlocked_mapping_entry(struct address_space *mapping, | |||
241 | ewait.wait.func = wake_exceptional_entry_func; | 237 | ewait.wait.func = wake_exceptional_entry_func; |
242 | 238 | ||
243 | for (;;) { | 239 | for (;;) { |
244 | entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, | 240 | entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, |
245 | &slot); | 241 | &slot); |
246 | if (!entry || | 242 | if (!entry || |
247 | WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || | 243 | WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || |
@@ -254,10 +250,10 @@ static void *get_unlocked_mapping_entry(struct address_space *mapping, | |||
254 | wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); | 250 | wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); |
255 | prepare_to_wait_exclusive(wq, &ewait.wait, | 251 | prepare_to_wait_exclusive(wq, &ewait.wait, |
256 | TASK_UNINTERRUPTIBLE); | 252 | TASK_UNINTERRUPTIBLE); |
257 | spin_unlock_irq(&mapping->tree_lock); | 253 | xa_unlock_irq(&mapping->i_pages); |
258 | schedule(); | 254 | schedule(); |
259 | finish_wait(wq, &ewait.wait); | 255 | finish_wait(wq, &ewait.wait); |
260 | spin_lock_irq(&mapping->tree_lock); | 256 | xa_lock_irq(&mapping->i_pages); |
261 | } | 257 | } |
262 | } | 258 | } |
263 | 259 | ||
@@ -266,15 +262,15 @@ static void dax_unlock_mapping_entry(struct address_space *mapping, | |||
266 | { | 262 | { |
267 | void *entry, **slot; | 263 | void *entry, **slot; |
268 | 264 | ||
269 | spin_lock_irq(&mapping->tree_lock); | 265 | xa_lock_irq(&mapping->i_pages); |
270 | entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); | 266 | entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot); |
271 | if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || | 267 | if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || |
272 | !slot_locked(mapping, slot))) { | 268 | !slot_locked(mapping, slot))) { |
273 | spin_unlock_irq(&mapping->tree_lock); | 269 | xa_unlock_irq(&mapping->i_pages); |
274 | return; | 270 | return; |
275 | } | 271 | } |
276 | unlock_slot(mapping, slot); | 272 | unlock_slot(mapping, slot); |
277 | spin_unlock_irq(&mapping->tree_lock); | 273 | xa_unlock_irq(&mapping->i_pages); |
278 | dax_wake_mapping_entry_waiter(mapping, index, entry, false); | 274 | dax_wake_mapping_entry_waiter(mapping, index, entry, false); |
279 | } | 275 | } |
280 | 276 | ||
@@ -388,7 +384,7 @@ static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index, | |||
388 | void *entry, **slot; | 384 | void *entry, **slot; |
389 | 385 | ||
390 | restart: | 386 | restart: |
391 | spin_lock_irq(&mapping->tree_lock); | 387 | xa_lock_irq(&mapping->i_pages); |
392 | entry = get_unlocked_mapping_entry(mapping, index, &slot); | 388 | entry = get_unlocked_mapping_entry(mapping, index, &slot); |
393 | 389 | ||
394 | if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { | 390 | if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { |
@@ -420,12 +416,12 @@ restart: | |||
420 | if (pmd_downgrade) { | 416 | if (pmd_downgrade) { |
421 | /* | 417 | /* |
422 | * Make sure 'entry' remains valid while we drop | 418 | * Make sure 'entry' remains valid while we drop |
423 | * mapping->tree_lock. | 419 | * the i_pages lock. |
424 | */ | 420 | */ |
425 | entry = lock_slot(mapping, slot); | 421 | entry = lock_slot(mapping, slot); |
426 | } | 422 | } |
427 | 423 | ||
428 | spin_unlock_irq(&mapping->tree_lock); | 424 | xa_unlock_irq(&mapping->i_pages); |
429 | /* | 425 | /* |
430 | * Besides huge zero pages the only other thing that gets | 426 | * Besides huge zero pages the only other thing that gets |
431 | * downgraded are empty entries which don't need to be | 427 | * downgraded are empty entries which don't need to be |
@@ -442,27 +438,27 @@ restart: | |||
442 | put_locked_mapping_entry(mapping, index); | 438 | put_locked_mapping_entry(mapping, index); |
443 | return ERR_PTR(err); | 439 | return ERR_PTR(err); |
444 | } | 440 | } |
445 | spin_lock_irq(&mapping->tree_lock); | 441 | xa_lock_irq(&mapping->i_pages); |
446 | 442 | ||
447 | if (!entry) { | 443 | if (!entry) { |
448 | /* | 444 | /* |
449 | * We needed to drop the page_tree lock while calling | 445 | * We needed to drop the i_pages lock while calling |
450 | * radix_tree_preload() and we didn't have an entry to | 446 | * radix_tree_preload() and we didn't have an entry to |
451 | * lock. See if another thread inserted an entry at | 447 | * lock. See if another thread inserted an entry at |
452 | * our index during this time. | 448 | * our index during this time. |
453 | */ | 449 | */ |
454 | entry = __radix_tree_lookup(&mapping->page_tree, index, | 450 | entry = __radix_tree_lookup(&mapping->i_pages, index, |
455 | NULL, &slot); | 451 | NULL, &slot); |
456 | if (entry) { | 452 | if (entry) { |
457 | radix_tree_preload_end(); | 453 | radix_tree_preload_end(); |
458 | spin_unlock_irq(&mapping->tree_lock); | 454 | xa_unlock_irq(&mapping->i_pages); |
459 | goto restart; | 455 | goto restart; |
460 | } | 456 | } |
461 | } | 457 | } |
462 | 458 | ||
463 | if (pmd_downgrade) { | 459 | if (pmd_downgrade) { |
464 | dax_disassociate_entry(entry, mapping, false); | 460 | dax_disassociate_entry(entry, mapping, false); |
465 | radix_tree_delete(&mapping->page_tree, index); | 461 | radix_tree_delete(&mapping->i_pages, index); |
466 | mapping->nrexceptional--; | 462 | mapping->nrexceptional--; |
467 | dax_wake_mapping_entry_waiter(mapping, index, entry, | 463 | dax_wake_mapping_entry_waiter(mapping, index, entry, |
468 | true); | 464 | true); |
@@ -470,11 +466,11 @@ restart: | |||
470 | 466 | ||
471 | entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); | 467 | entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); |
472 | 468 | ||
473 | err = __radix_tree_insert(&mapping->page_tree, index, | 469 | err = __radix_tree_insert(&mapping->i_pages, index, |
474 | dax_radix_order(entry), entry); | 470 | dax_radix_order(entry), entry); |
475 | radix_tree_preload_end(); | 471 | radix_tree_preload_end(); |
476 | if (err) { | 472 | if (err) { |
477 | spin_unlock_irq(&mapping->tree_lock); | 473 | xa_unlock_irq(&mapping->i_pages); |
478 | /* | 474 | /* |
479 | * Our insertion of a DAX entry failed, most likely | 475 | * Our insertion of a DAX entry failed, most likely |
480 | * because we were inserting a PMD entry and it | 476 | * because we were inserting a PMD entry and it |
@@ -487,12 +483,12 @@ restart: | |||
487 | } | 483 | } |
488 | /* Good, we have inserted empty locked entry into the tree. */ | 484 | /* Good, we have inserted empty locked entry into the tree. */ |
489 | mapping->nrexceptional++; | 485 | mapping->nrexceptional++; |
490 | spin_unlock_irq(&mapping->tree_lock); | 486 | xa_unlock_irq(&mapping->i_pages); |
491 | return entry; | 487 | return entry; |
492 | } | 488 | } |
493 | entry = lock_slot(mapping, slot); | 489 | entry = lock_slot(mapping, slot); |
494 | out_unlock: | 490 | out_unlock: |
495 | spin_unlock_irq(&mapping->tree_lock); | 491 | xa_unlock_irq(&mapping->i_pages); |
496 | return entry; | 492 | return entry; |
497 | } | 493 | } |
498 | 494 | ||
@@ -501,23 +497,23 @@ static int __dax_invalidate_mapping_entry(struct address_space *mapping, | |||
501 | { | 497 | { |
502 | int ret = 0; | 498 | int ret = 0; |
503 | void *entry; | 499 | void *entry; |
504 | struct radix_tree_root *page_tree = &mapping->page_tree; | 500 | struct radix_tree_root *pages = &mapping->i_pages; |
505 | 501 | ||
506 | spin_lock_irq(&mapping->tree_lock); | 502 | xa_lock_irq(pages); |
507 | entry = get_unlocked_mapping_entry(mapping, index, NULL); | 503 | entry = get_unlocked_mapping_entry(mapping, index, NULL); |
508 | if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) | 504 | if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) |
509 | goto out; | 505 | goto out; |
510 | if (!trunc && | 506 | if (!trunc && |
511 | (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) || | 507 | (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) || |
512 | radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))) | 508 | radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE))) |
513 | goto out; | 509 | goto out; |
514 | dax_disassociate_entry(entry, mapping, trunc); | 510 | dax_disassociate_entry(entry, mapping, trunc); |
515 | radix_tree_delete(page_tree, index); | 511 | radix_tree_delete(pages, index); |
516 | mapping->nrexceptional--; | 512 | mapping->nrexceptional--; |
517 | ret = 1; | 513 | ret = 1; |
518 | out: | 514 | out: |
519 | put_unlocked_mapping_entry(mapping, index, entry); | 515 | put_unlocked_mapping_entry(mapping, index, entry); |
520 | spin_unlock_irq(&mapping->tree_lock); | 516 | xa_unlock_irq(pages); |
521 | return ret; | 517 | return ret; |
522 | } | 518 | } |
523 | /* | 519 | /* |
@@ -587,7 +583,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
587 | void *entry, pfn_t pfn_t, | 583 | void *entry, pfn_t pfn_t, |
588 | unsigned long flags, bool dirty) | 584 | unsigned long flags, bool dirty) |
589 | { | 585 | { |
590 | struct radix_tree_root *page_tree = &mapping->page_tree; | 586 | struct radix_tree_root *pages = &mapping->i_pages; |
591 | unsigned long pfn = pfn_t_to_pfn(pfn_t); | 587 | unsigned long pfn = pfn_t_to_pfn(pfn_t); |
592 | pgoff_t index = vmf->pgoff; | 588 | pgoff_t index = vmf->pgoff; |
593 | void *new_entry; | 589 | void *new_entry; |
@@ -604,7 +600,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
604 | unmap_mapping_pages(mapping, vmf->pgoff, 1, false); | 600 | unmap_mapping_pages(mapping, vmf->pgoff, 1, false); |
605 | } | 601 | } |
606 | 602 | ||
607 | spin_lock_irq(&mapping->tree_lock); | 603 | xa_lock_irq(pages); |
608 | new_entry = dax_radix_locked_entry(pfn, flags); | 604 | new_entry = dax_radix_locked_entry(pfn, flags); |
609 | if (dax_entry_size(entry) != dax_entry_size(new_entry)) { | 605 | if (dax_entry_size(entry) != dax_entry_size(new_entry)) { |
610 | dax_disassociate_entry(entry, mapping, false); | 606 | dax_disassociate_entry(entry, mapping, false); |
@@ -624,17 +620,17 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
624 | void **slot; | 620 | void **slot; |
625 | void *ret; | 621 | void *ret; |
626 | 622 | ||
627 | ret = __radix_tree_lookup(page_tree, index, &node, &slot); | 623 | ret = __radix_tree_lookup(pages, index, &node, &slot); |
628 | WARN_ON_ONCE(ret != entry); | 624 | WARN_ON_ONCE(ret != entry); |
629 | __radix_tree_replace(page_tree, node, slot, | 625 | __radix_tree_replace(pages, node, slot, |
630 | new_entry, NULL); | 626 | new_entry, NULL); |
631 | entry = new_entry; | 627 | entry = new_entry; |
632 | } | 628 | } |
633 | 629 | ||
634 | if (dirty) | 630 | if (dirty) |
635 | radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY); | 631 | radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY); |
636 | 632 | ||
637 | spin_unlock_irq(&mapping->tree_lock); | 633 | xa_unlock_irq(pages); |
638 | return entry; | 634 | return entry; |
639 | } | 635 | } |
640 | 636 | ||
@@ -723,7 +719,7 @@ unlock_pte: | |||
723 | static int dax_writeback_one(struct dax_device *dax_dev, | 719 | static int dax_writeback_one(struct dax_device *dax_dev, |
724 | struct address_space *mapping, pgoff_t index, void *entry) | 720 | struct address_space *mapping, pgoff_t index, void *entry) |
725 | { | 721 | { |
726 | struct radix_tree_root *page_tree = &mapping->page_tree; | 722 | struct radix_tree_root *pages = &mapping->i_pages; |
727 | void *entry2, **slot; | 723 | void *entry2, **slot; |
728 | unsigned long pfn; | 724 | unsigned long pfn; |
729 | long ret = 0; | 725 | long ret = 0; |
@@ -736,7 +732,7 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
736 | if (WARN_ON(!radix_tree_exceptional_entry(entry))) | 732 | if (WARN_ON(!radix_tree_exceptional_entry(entry))) |
737 | return -EIO; | 733 | return -EIO; |
738 | 734 | ||
739 | spin_lock_irq(&mapping->tree_lock); | 735 | xa_lock_irq(pages); |
740 | entry2 = get_unlocked_mapping_entry(mapping, index, &slot); | 736 | entry2 = get_unlocked_mapping_entry(mapping, index, &slot); |
741 | /* Entry got punched out / reallocated? */ | 737 | /* Entry got punched out / reallocated? */ |
742 | if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) | 738 | if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) |
@@ -755,7 +751,7 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
755 | } | 751 | } |
756 | 752 | ||
757 | /* Another fsync thread may have already written back this entry */ | 753 | /* Another fsync thread may have already written back this entry */ |
758 | if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) | 754 | if (!radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE)) |
759 | goto put_unlocked; | 755 | goto put_unlocked; |
760 | /* Lock the entry to serialize with page faults */ | 756 | /* Lock the entry to serialize with page faults */ |
761 | entry = lock_slot(mapping, slot); | 757 | entry = lock_slot(mapping, slot); |
@@ -763,11 +759,11 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
763 | * We can clear the tag now but we have to be careful so that concurrent | 759 | * We can clear the tag now but we have to be careful so that concurrent |
764 | * dax_writeback_one() calls for the same index cannot finish before we | 760 | * dax_writeback_one() calls for the same index cannot finish before we |
765 | * actually flush the caches. This is achieved as the calls will look | 761 | * actually flush the caches. This is achieved as the calls will look |
766 | * at the entry only under tree_lock and once they do that they will | 762 | * at the entry only under the i_pages lock and once they do that |
767 | * see the entry locked and wait for it to unlock. | 763 | * they will see the entry locked and wait for it to unlock. |
768 | */ | 764 | */ |
769 | radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE); | 765 | radix_tree_tag_clear(pages, index, PAGECACHE_TAG_TOWRITE); |
770 | spin_unlock_irq(&mapping->tree_lock); | 766 | xa_unlock_irq(pages); |
771 | 767 | ||
772 | /* | 768 | /* |
773 | * Even if dax_writeback_mapping_range() was given a wbc->range_start | 769 | * Even if dax_writeback_mapping_range() was given a wbc->range_start |
@@ -787,16 +783,16 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
787 | * the pfn mappings are writeprotected and fault waits for mapping | 783 | * the pfn mappings are writeprotected and fault waits for mapping |
788 | * entry lock. | 784 | * entry lock. |
789 | */ | 785 | */ |
790 | spin_lock_irq(&mapping->tree_lock); | 786 | xa_lock_irq(pages); |
791 | radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY); | 787 | radix_tree_tag_clear(pages, index, PAGECACHE_TAG_DIRTY); |
792 | spin_unlock_irq(&mapping->tree_lock); | 788 | xa_unlock_irq(pages); |
793 | trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); | 789 | trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); |
794 | put_locked_mapping_entry(mapping, index); | 790 | put_locked_mapping_entry(mapping, index); |
795 | return ret; | 791 | return ret; |
796 | 792 | ||
797 | put_unlocked: | 793 | put_unlocked: |
798 | put_unlocked_mapping_entry(mapping, index, entry2); | 794 | put_unlocked_mapping_entry(mapping, index, entry2); |
799 | spin_unlock_irq(&mapping->tree_lock); | 795 | xa_unlock_irq(pages); |
800 | return ret; | 796 | return ret; |
801 | } | 797 | } |
802 | 798 | ||
@@ -1566,21 +1562,21 @@ static int dax_insert_pfn_mkwrite(struct vm_fault *vmf, | |||
1566 | pgoff_t index = vmf->pgoff; | 1562 | pgoff_t index = vmf->pgoff; |
1567 | int vmf_ret, error; | 1563 | int vmf_ret, error; |
1568 | 1564 | ||
1569 | spin_lock_irq(&mapping->tree_lock); | 1565 | xa_lock_irq(&mapping->i_pages); |
1570 | entry = get_unlocked_mapping_entry(mapping, index, &slot); | 1566 | entry = get_unlocked_mapping_entry(mapping, index, &slot); |
1571 | /* Did we race with someone splitting entry or so? */ | 1567 | /* Did we race with someone splitting entry or so? */ |
1572 | if (!entry || | 1568 | if (!entry || |
1573 | (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || | 1569 | (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || |
1574 | (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { | 1570 | (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { |
1575 | put_unlocked_mapping_entry(mapping, index, entry); | 1571 | put_unlocked_mapping_entry(mapping, index, entry); |
1576 | spin_unlock_irq(&mapping->tree_lock); | 1572 | xa_unlock_irq(&mapping->i_pages); |
1577 | trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, | 1573 | trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, |
1578 | VM_FAULT_NOPAGE); | 1574 | VM_FAULT_NOPAGE); |
1579 | return VM_FAULT_NOPAGE; | 1575 | return VM_FAULT_NOPAGE; |
1580 | } | 1576 | } |
1581 | radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY); | 1577 | radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY); |
1582 | entry = lock_slot(mapping, slot); | 1578 | entry = lock_slot(mapping, slot); |
1583 | spin_unlock_irq(&mapping->tree_lock); | 1579 | xa_unlock_irq(&mapping->i_pages); |
1584 | switch (pe_size) { | 1580 | switch (pe_size) { |
1585 | case PE_SIZE_PTE: | 1581 | case PE_SIZE_PTE: |
1586 | error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn); | 1582 | error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn); |
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index db50686f5096..02237d4d91f5 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c | |||
@@ -2424,12 +2424,12 @@ void f2fs_set_page_dirty_nobuffers(struct page *page) | |||
2424 | SetPageDirty(page); | 2424 | SetPageDirty(page); |
2425 | spin_unlock(&mapping->private_lock); | 2425 | spin_unlock(&mapping->private_lock); |
2426 | 2426 | ||
2427 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2427 | xa_lock_irqsave(&mapping->i_pages, flags); |
2428 | WARN_ON_ONCE(!PageUptodate(page)); | 2428 | WARN_ON_ONCE(!PageUptodate(page)); |
2429 | account_page_dirtied(page, mapping); | 2429 | account_page_dirtied(page, mapping); |
2430 | radix_tree_tag_set(&mapping->page_tree, | 2430 | radix_tree_tag_set(&mapping->i_pages, |
2431 | page_index(page), PAGECACHE_TAG_DIRTY); | 2431 | page_index(page), PAGECACHE_TAG_DIRTY); |
2432 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2432 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
2433 | unlock_page_memcg(page); | 2433 | unlock_page_memcg(page); |
2434 | 2434 | ||
2435 | __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); | 2435 | __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); |
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index fe661274ff10..8c9c2f31b253 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c | |||
@@ -732,10 +732,10 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, | |||
732 | 732 | ||
733 | if (bit_pos == NR_DENTRY_IN_BLOCK && | 733 | if (bit_pos == NR_DENTRY_IN_BLOCK && |
734 | !truncate_hole(dir, page->index, page->index + 1)) { | 734 | !truncate_hole(dir, page->index, page->index + 1)) { |
735 | spin_lock_irqsave(&mapping->tree_lock, flags); | 735 | xa_lock_irqsave(&mapping->i_pages, flags); |
736 | radix_tree_tag_clear(&mapping->page_tree, page_index(page), | 736 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
737 | PAGECACHE_TAG_DIRTY); | 737 | PAGECACHE_TAG_DIRTY); |
738 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 738 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
739 | 739 | ||
740 | clear_page_dirty_for_io(page); | 740 | clear_page_dirty_for_io(page); |
741 | ClearPagePrivate(page); | 741 | ClearPagePrivate(page); |
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 3b77d6421218..265da200daa8 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c | |||
@@ -226,10 +226,10 @@ int f2fs_write_inline_data(struct inode *inode, struct page *page) | |||
226 | kunmap_atomic(src_addr); | 226 | kunmap_atomic(src_addr); |
227 | set_page_dirty(dn.inode_page); | 227 | set_page_dirty(dn.inode_page); |
228 | 228 | ||
229 | spin_lock_irqsave(&mapping->tree_lock, flags); | 229 | xa_lock_irqsave(&mapping->i_pages, flags); |
230 | radix_tree_tag_clear(&mapping->page_tree, page_index(page), | 230 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
231 | PAGECACHE_TAG_DIRTY); | 231 | PAGECACHE_TAG_DIRTY); |
232 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 232 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
233 | 233 | ||
234 | set_inode_flag(inode, FI_APPEND_WRITE); | 234 | set_inode_flag(inode, FI_APPEND_WRITE); |
235 | set_inode_flag(inode, FI_DATA_EXIST); | 235 | set_inode_flag(inode, FI_DATA_EXIST); |
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 9a99243054ba..f202398e20ea 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c | |||
@@ -91,11 +91,11 @@ static void clear_node_page_dirty(struct page *page) | |||
91 | unsigned int long flags; | 91 | unsigned int long flags; |
92 | 92 | ||
93 | if (PageDirty(page)) { | 93 | if (PageDirty(page)) { |
94 | spin_lock_irqsave(&mapping->tree_lock, flags); | 94 | xa_lock_irqsave(&mapping->i_pages, flags); |
95 | radix_tree_tag_clear(&mapping->page_tree, | 95 | radix_tree_tag_clear(&mapping->i_pages, |
96 | page_index(page), | 96 | page_index(page), |
97 | PAGECACHE_TAG_DIRTY); | 97 | PAGECACHE_TAG_DIRTY); |
98 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 98 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
99 | 99 | ||
100 | clear_page_dirty_for_io(page); | 100 | clear_page_dirty_for_io(page); |
101 | dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); | 101 | dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); |
@@ -1161,7 +1161,7 @@ void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) | |||
1161 | f2fs_bug_on(sbi, check_nid_range(sbi, nid)); | 1161 | f2fs_bug_on(sbi, check_nid_range(sbi, nid)); |
1162 | 1162 | ||
1163 | rcu_read_lock(); | 1163 | rcu_read_lock(); |
1164 | apage = radix_tree_lookup(&NODE_MAPPING(sbi)->page_tree, nid); | 1164 | apage = radix_tree_lookup(&NODE_MAPPING(sbi)->i_pages, nid); |
1165 | rcu_read_unlock(); | 1165 | rcu_read_unlock(); |
1166 | if (apage) | 1166 | if (apage) |
1167 | return; | 1167 | return; |
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 1280f915079b..4b12ba70a895 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c | |||
@@ -347,9 +347,9 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) | |||
347 | * By the time control reaches here, RCU grace period has passed | 347 | * By the time control reaches here, RCU grace period has passed |
348 | * since I_WB_SWITCH assertion and all wb stat update transactions | 348 | * since I_WB_SWITCH assertion and all wb stat update transactions |
349 | * between unlocked_inode_to_wb_begin/end() are guaranteed to be | 349 | * between unlocked_inode_to_wb_begin/end() are guaranteed to be |
350 | * synchronizing against mapping->tree_lock. | 350 | * synchronizing against the i_pages lock. |
351 | * | 351 | * |
352 | * Grabbing old_wb->list_lock, inode->i_lock and mapping->tree_lock | 352 | * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock |
353 | * gives us exclusion against all wb related operations on @inode | 353 | * gives us exclusion against all wb related operations on @inode |
354 | * including IO list manipulations and stat updates. | 354 | * including IO list manipulations and stat updates. |
355 | */ | 355 | */ |
@@ -361,7 +361,7 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) | |||
361 | spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); | 361 | spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); |
362 | } | 362 | } |
363 | spin_lock(&inode->i_lock); | 363 | spin_lock(&inode->i_lock); |
364 | spin_lock_irq(&mapping->tree_lock); | 364 | xa_lock_irq(&mapping->i_pages); |
365 | 365 | ||
366 | /* | 366 | /* |
367 | * Once I_FREEING is visible under i_lock, the eviction path owns | 367 | * Once I_FREEING is visible under i_lock, the eviction path owns |
@@ -373,22 +373,22 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) | |||
373 | /* | 373 | /* |
374 | * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points | 374 | * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points |
375 | * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to | 375 | * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to |
376 | * pages actually under underwriteback. | 376 | * pages actually under writeback. |
377 | */ | 377 | */ |
378 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, | 378 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0, |
379 | PAGECACHE_TAG_DIRTY) { | 379 | PAGECACHE_TAG_DIRTY) { |
380 | struct page *page = radix_tree_deref_slot_protected(slot, | 380 | struct page *page = radix_tree_deref_slot_protected(slot, |
381 | &mapping->tree_lock); | 381 | &mapping->i_pages.xa_lock); |
382 | if (likely(page) && PageDirty(page)) { | 382 | if (likely(page) && PageDirty(page)) { |
383 | dec_wb_stat(old_wb, WB_RECLAIMABLE); | 383 | dec_wb_stat(old_wb, WB_RECLAIMABLE); |
384 | inc_wb_stat(new_wb, WB_RECLAIMABLE); | 384 | inc_wb_stat(new_wb, WB_RECLAIMABLE); |
385 | } | 385 | } |
386 | } | 386 | } |
387 | 387 | ||
388 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, | 388 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0, |
389 | PAGECACHE_TAG_WRITEBACK) { | 389 | PAGECACHE_TAG_WRITEBACK) { |
390 | struct page *page = radix_tree_deref_slot_protected(slot, | 390 | struct page *page = radix_tree_deref_slot_protected(slot, |
391 | &mapping->tree_lock); | 391 | &mapping->i_pages.xa_lock); |
392 | if (likely(page)) { | 392 | if (likely(page)) { |
393 | WARN_ON_ONCE(!PageWriteback(page)); | 393 | WARN_ON_ONCE(!PageWriteback(page)); |
394 | dec_wb_stat(old_wb, WB_WRITEBACK); | 394 | dec_wb_stat(old_wb, WB_WRITEBACK); |
@@ -430,7 +430,7 @@ skip_switch: | |||
430 | */ | 430 | */ |
431 | smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); | 431 | smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); |
432 | 432 | ||
433 | spin_unlock_irq(&mapping->tree_lock); | 433 | xa_unlock_irq(&mapping->i_pages); |
434 | spin_unlock(&inode->i_lock); | 434 | spin_unlock(&inode->i_lock); |
435 | spin_unlock(&new_wb->list_lock); | 435 | spin_unlock(&new_wb->list_lock); |
436 | spin_unlock(&old_wb->list_lock); | 436 | spin_unlock(&old_wb->list_lock); |
@@ -506,8 +506,8 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) | |||
506 | 506 | ||
507 | /* | 507 | /* |
508 | * In addition to synchronizing among switchers, I_WB_SWITCH tells | 508 | * In addition to synchronizing among switchers, I_WB_SWITCH tells |
509 | * the RCU protected stat update paths to grab the mapping's | 509 | * the RCU protected stat update paths to grab the i_page |
510 | * tree_lock so that stat transfer can synchronize against them. | 510 | * lock so that stat transfer can synchronize against them. |
511 | * Let's continue after I_WB_SWITCH is guaranteed to be visible. | 511 | * Let's continue after I_WB_SWITCH is guaranteed to be visible. |
512 | */ | 512 | */ |
513 | call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn); | 513 | call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn); |
diff --git a/fs/inode.c b/fs/inode.c index b153aeaa61ea..13ceb98c3bd3 100644 --- a/fs/inode.c +++ b/fs/inode.c | |||
@@ -348,8 +348,7 @@ EXPORT_SYMBOL(inc_nlink); | |||
348 | 348 | ||
349 | static void __address_space_init_once(struct address_space *mapping) | 349 | static void __address_space_init_once(struct address_space *mapping) |
350 | { | 350 | { |
351 | INIT_RADIX_TREE(&mapping->page_tree, GFP_ATOMIC | __GFP_ACCOUNT); | 351 | INIT_RADIX_TREE(&mapping->i_pages, GFP_ATOMIC | __GFP_ACCOUNT); |
352 | spin_lock_init(&mapping->tree_lock); | ||
353 | init_rwsem(&mapping->i_mmap_rwsem); | 352 | init_rwsem(&mapping->i_mmap_rwsem); |
354 | INIT_LIST_HEAD(&mapping->private_list); | 353 | INIT_LIST_HEAD(&mapping->private_list); |
355 | spin_lock_init(&mapping->private_lock); | 354 | spin_lock_init(&mapping->private_lock); |
@@ -504,14 +503,14 @@ EXPORT_SYMBOL(__remove_inode_hash); | |||
504 | void clear_inode(struct inode *inode) | 503 | void clear_inode(struct inode *inode) |
505 | { | 504 | { |
506 | /* | 505 | /* |
507 | * We have to cycle tree_lock here because reclaim can be still in the | 506 | * We have to cycle the i_pages lock here because reclaim can be in the |
508 | * process of removing the last page (in __delete_from_page_cache()) | 507 | * process of removing the last page (in __delete_from_page_cache()) |
509 | * and we must not free mapping under it. | 508 | * and we must not free the mapping under it. |
510 | */ | 509 | */ |
511 | spin_lock_irq(&inode->i_data.tree_lock); | 510 | xa_lock_irq(&inode->i_data.i_pages); |
512 | BUG_ON(inode->i_data.nrpages); | 511 | BUG_ON(inode->i_data.nrpages); |
513 | BUG_ON(inode->i_data.nrexceptional); | 512 | BUG_ON(inode->i_data.nrexceptional); |
514 | spin_unlock_irq(&inode->i_data.tree_lock); | 513 | xa_unlock_irq(&inode->i_data.i_pages); |
515 | BUG_ON(!list_empty(&inode->i_data.private_list)); | 514 | BUG_ON(!list_empty(&inode->i_data.private_list)); |
516 | BUG_ON(!(inode->i_state & I_FREEING)); | 515 | BUG_ON(!(inode->i_state & I_FREEING)); |
517 | BUG_ON(inode->i_state & I_CLEAR); | 516 | BUG_ON(inode->i_state & I_CLEAR); |
diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index c21e0b4454a6..dec98cab729d 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c | |||
@@ -193,9 +193,9 @@ retry: | |||
193 | (unsigned long long)oldkey, | 193 | (unsigned long long)oldkey, |
194 | (unsigned long long)newkey); | 194 | (unsigned long long)newkey); |
195 | 195 | ||
196 | spin_lock_irq(&btnc->tree_lock); | 196 | xa_lock_irq(&btnc->i_pages); |
197 | err = radix_tree_insert(&btnc->page_tree, newkey, obh->b_page); | 197 | err = radix_tree_insert(&btnc->i_pages, newkey, obh->b_page); |
198 | spin_unlock_irq(&btnc->tree_lock); | 198 | xa_unlock_irq(&btnc->i_pages); |
199 | /* | 199 | /* |
200 | * Note: page->index will not change to newkey until | 200 | * Note: page->index will not change to newkey until |
201 | * nilfs_btnode_commit_change_key() will be called. | 201 | * nilfs_btnode_commit_change_key() will be called. |
@@ -251,11 +251,11 @@ void nilfs_btnode_commit_change_key(struct address_space *btnc, | |||
251 | (unsigned long long)newkey); | 251 | (unsigned long long)newkey); |
252 | mark_buffer_dirty(obh); | 252 | mark_buffer_dirty(obh); |
253 | 253 | ||
254 | spin_lock_irq(&btnc->tree_lock); | 254 | xa_lock_irq(&btnc->i_pages); |
255 | radix_tree_delete(&btnc->page_tree, oldkey); | 255 | radix_tree_delete(&btnc->i_pages, oldkey); |
256 | radix_tree_tag_set(&btnc->page_tree, newkey, | 256 | radix_tree_tag_set(&btnc->i_pages, newkey, |
257 | PAGECACHE_TAG_DIRTY); | 257 | PAGECACHE_TAG_DIRTY); |
258 | spin_unlock_irq(&btnc->tree_lock); | 258 | xa_unlock_irq(&btnc->i_pages); |
259 | 259 | ||
260 | opage->index = obh->b_blocknr = newkey; | 260 | opage->index = obh->b_blocknr = newkey; |
261 | unlock_page(opage); | 261 | unlock_page(opage); |
@@ -283,9 +283,9 @@ void nilfs_btnode_abort_change_key(struct address_space *btnc, | |||
283 | return; | 283 | return; |
284 | 284 | ||
285 | if (nbh == NULL) { /* blocksize == pagesize */ | 285 | if (nbh == NULL) { /* blocksize == pagesize */ |
286 | spin_lock_irq(&btnc->tree_lock); | 286 | xa_lock_irq(&btnc->i_pages); |
287 | radix_tree_delete(&btnc->page_tree, newkey); | 287 | radix_tree_delete(&btnc->i_pages, newkey); |
288 | spin_unlock_irq(&btnc->tree_lock); | 288 | xa_unlock_irq(&btnc->i_pages); |
289 | unlock_page(ctxt->bh->b_page); | 289 | unlock_page(ctxt->bh->b_page); |
290 | } else | 290 | } else |
291 | brelse(nbh); | 291 | brelse(nbh); |
diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 68241512d7c1..4cb850a6f1c2 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c | |||
@@ -331,15 +331,15 @@ repeat: | |||
331 | struct page *page2; | 331 | struct page *page2; |
332 | 332 | ||
333 | /* move the page to the destination cache */ | 333 | /* move the page to the destination cache */ |
334 | spin_lock_irq(&smap->tree_lock); | 334 | xa_lock_irq(&smap->i_pages); |
335 | page2 = radix_tree_delete(&smap->page_tree, offset); | 335 | page2 = radix_tree_delete(&smap->i_pages, offset); |
336 | WARN_ON(page2 != page); | 336 | WARN_ON(page2 != page); |
337 | 337 | ||
338 | smap->nrpages--; | 338 | smap->nrpages--; |
339 | spin_unlock_irq(&smap->tree_lock); | 339 | xa_unlock_irq(&smap->i_pages); |
340 | 340 | ||
341 | spin_lock_irq(&dmap->tree_lock); | 341 | xa_lock_irq(&dmap->i_pages); |
342 | err = radix_tree_insert(&dmap->page_tree, offset, page); | 342 | err = radix_tree_insert(&dmap->i_pages, offset, page); |
343 | if (unlikely(err < 0)) { | 343 | if (unlikely(err < 0)) { |
344 | WARN_ON(err == -EEXIST); | 344 | WARN_ON(err == -EEXIST); |
345 | page->mapping = NULL; | 345 | page->mapping = NULL; |
@@ -348,11 +348,11 @@ repeat: | |||
348 | page->mapping = dmap; | 348 | page->mapping = dmap; |
349 | dmap->nrpages++; | 349 | dmap->nrpages++; |
350 | if (PageDirty(page)) | 350 | if (PageDirty(page)) |
351 | radix_tree_tag_set(&dmap->page_tree, | 351 | radix_tree_tag_set(&dmap->i_pages, |
352 | offset, | 352 | offset, |
353 | PAGECACHE_TAG_DIRTY); | 353 | PAGECACHE_TAG_DIRTY); |
354 | } | 354 | } |
355 | spin_unlock_irq(&dmap->tree_lock); | 355 | xa_unlock_irq(&dmap->i_pages); |
356 | } | 356 | } |
357 | unlock_page(page); | 357 | unlock_page(page); |
358 | } | 358 | } |
@@ -474,15 +474,15 @@ int __nilfs_clear_page_dirty(struct page *page) | |||
474 | struct address_space *mapping = page->mapping; | 474 | struct address_space *mapping = page->mapping; |
475 | 475 | ||
476 | if (mapping) { | 476 | if (mapping) { |
477 | spin_lock_irq(&mapping->tree_lock); | 477 | xa_lock_irq(&mapping->i_pages); |
478 | if (test_bit(PG_dirty, &page->flags)) { | 478 | if (test_bit(PG_dirty, &page->flags)) { |
479 | radix_tree_tag_clear(&mapping->page_tree, | 479 | radix_tree_tag_clear(&mapping->i_pages, |
480 | page_index(page), | 480 | page_index(page), |
481 | PAGECACHE_TAG_DIRTY); | 481 | PAGECACHE_TAG_DIRTY); |
482 | spin_unlock_irq(&mapping->tree_lock); | 482 | xa_unlock_irq(&mapping->i_pages); |
483 | return clear_page_dirty_for_io(page); | 483 | return clear_page_dirty_for_io(page); |
484 | } | 484 | } |
485 | spin_unlock_irq(&mapping->tree_lock); | 485 | xa_unlock_irq(&mapping->i_pages); |
486 | return 0; | 486 | return 0; |
487 | } | 487 | } |
488 | return TestClearPageDirty(page); | 488 | return TestClearPageDirty(page); |
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index e6cbb915ee56..09da0f124699 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h | |||
@@ -329,7 +329,7 @@ static inline bool inode_to_wb_is_valid(struct inode *inode) | |||
329 | * @inode: inode of interest | 329 | * @inode: inode of interest |
330 | * | 330 | * |
331 | * Returns the wb @inode is currently associated with. The caller must be | 331 | * Returns the wb @inode is currently associated with. The caller must be |
332 | * holding either @inode->i_lock, @inode->i_mapping->tree_lock, or the | 332 | * holding either @inode->i_lock, the i_pages lock, or the |
333 | * associated wb's list_lock. | 333 | * associated wb's list_lock. |
334 | */ | 334 | */ |
335 | static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) | 335 | static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) |
@@ -337,7 +337,7 @@ static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) | |||
337 | #ifdef CONFIG_LOCKDEP | 337 | #ifdef CONFIG_LOCKDEP |
338 | WARN_ON_ONCE(debug_locks && | 338 | WARN_ON_ONCE(debug_locks && |
339 | (!lockdep_is_held(&inode->i_lock) && | 339 | (!lockdep_is_held(&inode->i_lock) && |
340 | !lockdep_is_held(&inode->i_mapping->tree_lock) && | 340 | !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) && |
341 | !lockdep_is_held(&inode->i_wb->list_lock))); | 341 | !lockdep_is_held(&inode->i_wb->list_lock))); |
342 | #endif | 342 | #endif |
343 | return inode->i_wb; | 343 | return inode->i_wb; |
@@ -349,7 +349,7 @@ static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) | |||
349 | * @lockedp: temp bool output param, to be passed to the end function | 349 | * @lockedp: temp bool output param, to be passed to the end function |
350 | * | 350 | * |
351 | * The caller wants to access the wb associated with @inode but isn't | 351 | * The caller wants to access the wb associated with @inode but isn't |
352 | * holding inode->i_lock, mapping->tree_lock or wb->list_lock. This | 352 | * holding inode->i_lock, the i_pages lock or wb->list_lock. This |
353 | * function determines the wb associated with @inode and ensures that the | 353 | * function determines the wb associated with @inode and ensures that the |
354 | * association doesn't change until the transaction is finished with | 354 | * association doesn't change until the transaction is finished with |
355 | * unlocked_inode_to_wb_end(). | 355 | * unlocked_inode_to_wb_end(). |
@@ -370,11 +370,11 @@ unlocked_inode_to_wb_begin(struct inode *inode, bool *lockedp) | |||
370 | *lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH; | 370 | *lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH; |
371 | 371 | ||
372 | if (unlikely(*lockedp)) | 372 | if (unlikely(*lockedp)) |
373 | spin_lock_irq(&inode->i_mapping->tree_lock); | 373 | xa_lock_irq(&inode->i_mapping->i_pages); |
374 | 374 | ||
375 | /* | 375 | /* |
376 | * Protected by either !I_WB_SWITCH + rcu_read_lock() or tree_lock. | 376 | * Protected by either !I_WB_SWITCH + rcu_read_lock() or the i_pages |
377 | * inode_to_wb() will bark. Deref directly. | 377 | * lock. inode_to_wb() will bark. Deref directly. |
378 | */ | 378 | */ |
379 | return inode->i_wb; | 379 | return inode->i_wb; |
380 | } | 380 | } |
@@ -387,7 +387,7 @@ unlocked_inode_to_wb_begin(struct inode *inode, bool *lockedp) | |||
387 | static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked) | 387 | static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked) |
388 | { | 388 | { |
389 | if (unlikely(locked)) | 389 | if (unlikely(locked)) |
390 | spin_unlock_irq(&inode->i_mapping->tree_lock); | 390 | xa_unlock_irq(&inode->i_mapping->i_pages); |
391 | 391 | ||
392 | rcu_read_unlock(); | 392 | rcu_read_unlock(); |
393 | } | 393 | } |
diff --git a/include/linux/fs.h b/include/linux/fs.h index 2aa02cad94d4..92efaf1f8977 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h | |||
@@ -13,6 +13,7 @@ | |||
13 | #include <linux/list_lru.h> | 13 | #include <linux/list_lru.h> |
14 | #include <linux/llist.h> | 14 | #include <linux/llist.h> |
15 | #include <linux/radix-tree.h> | 15 | #include <linux/radix-tree.h> |
16 | #include <linux/xarray.h> | ||
16 | #include <linux/rbtree.h> | 17 | #include <linux/rbtree.h> |
17 | #include <linux/init.h> | 18 | #include <linux/init.h> |
18 | #include <linux/pid.h> | 19 | #include <linux/pid.h> |
@@ -390,12 +391,11 @@ int pagecache_write_end(struct file *, struct address_space *mapping, | |||
390 | 391 | ||
391 | struct address_space { | 392 | struct address_space { |
392 | struct inode *host; /* owner: inode, block_device */ | 393 | struct inode *host; /* owner: inode, block_device */ |
393 | struct radix_tree_root page_tree; /* radix tree of all pages */ | 394 | struct radix_tree_root i_pages; /* cached pages */ |
394 | spinlock_t tree_lock; /* and lock protecting it */ | ||
395 | atomic_t i_mmap_writable;/* count VM_SHARED mappings */ | 395 | atomic_t i_mmap_writable;/* count VM_SHARED mappings */ |
396 | struct rb_root_cached i_mmap; /* tree of private and shared mappings */ | 396 | struct rb_root_cached i_mmap; /* tree of private and shared mappings */ |
397 | struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */ | 397 | struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */ |
398 | /* Protected by tree_lock together with the radix tree */ | 398 | /* Protected by the i_pages lock */ |
399 | unsigned long nrpages; /* number of total pages */ | 399 | unsigned long nrpages; /* number of total pages */ |
400 | /* number of shadow or DAX exceptional entries */ | 400 | /* number of shadow or DAX exceptional entries */ |
401 | unsigned long nrexceptional; | 401 | unsigned long nrexceptional; |
@@ -1989,7 +1989,7 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) | |||
1989 | * | 1989 | * |
1990 | * I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to | 1990 | * I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to |
1991 | * synchronize competing switching instances and to tell | 1991 | * synchronize competing switching instances and to tell |
1992 | * wb stat updates to grab mapping->tree_lock. See | 1992 | * wb stat updates to grab the i_pages lock. See |
1993 | * inode_switch_wb_work_fn() for details. | 1993 | * inode_switch_wb_work_fn() for details. |
1994 | * | 1994 | * |
1995 | * I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper | 1995 | * I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper |
diff --git a/include/linux/mm.h b/include/linux/mm.h index f13bc25f7a9f..1ac1f06a4be6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
@@ -747,7 +747,7 @@ int finish_mkwrite_fault(struct vm_fault *vmf); | |||
747 | * refcount. The each user mapping also has a reference to the page. | 747 | * refcount. The each user mapping also has a reference to the page. |
748 | * | 748 | * |
749 | * The pagecache pages are stored in a per-mapping radix tree, which is | 749 | * The pagecache pages are stored in a per-mapping radix tree, which is |
750 | * rooted at mapping->page_tree, and indexed by offset. | 750 | * rooted at mapping->i_pages, and indexed by offset. |
751 | * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space | 751 | * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space |
752 | * lists, we instead now tag pages as dirty/writeback in the radix tree. | 752 | * lists, we instead now tag pages as dirty/writeback in the radix tree. |
753 | * | 753 | * |
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 34ce3ebf97d5..b1bd2186e6d2 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h | |||
@@ -144,7 +144,7 @@ void release_pages(struct page **pages, int nr); | |||
144 | * 3. check the page is still in pagecache (if no, goto 1) | 144 | * 3. check the page is still in pagecache (if no, goto 1) |
145 | * | 145 | * |
146 | * Remove-side that cares about stability of _refcount (eg. reclaim) has the | 146 | * Remove-side that cares about stability of _refcount (eg. reclaim) has the |
147 | * following (with tree_lock held for write): | 147 | * following (with the i_pages lock held): |
148 | * A. atomically check refcount is correct and set it to 0 (atomic_cmpxchg) | 148 | * A. atomically check refcount is correct and set it to 0 (atomic_cmpxchg) |
149 | * B. remove page from pagecache | 149 | * B. remove page from pagecache |
150 | * C. free the page | 150 | * C. free the page |
@@ -157,7 +157,7 @@ void release_pages(struct page **pages, int nr); | |||
157 | * | 157 | * |
158 | * It is possible that between 1 and 2, the page is removed then the exact same | 158 | * It is possible that between 1 and 2, the page is removed then the exact same |
159 | * page is inserted into the same position in pagecache. That's OK: the | 159 | * page is inserted into the same position in pagecache. That's OK: the |
160 | * old find_get_page using tree_lock could equally have run before or after | 160 | * old find_get_page using a lock could equally have run before or after |
161 | * such a re-insertion, depending on order that locks are granted. | 161 | * such a re-insertion, depending on order that locks are granted. |
162 | * | 162 | * |
163 | * Lookups racing against pagecache insertion isn't a big problem: either 1 | 163 | * Lookups racing against pagecache insertion isn't a big problem: either 1 |
diff --git a/mm/filemap.c b/mm/filemap.c index 693f62212a59..ab77e19ab09c 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -66,7 +66,7 @@ | |||
66 | * ->i_mmap_rwsem (truncate_pagecache) | 66 | * ->i_mmap_rwsem (truncate_pagecache) |
67 | * ->private_lock (__free_pte->__set_page_dirty_buffers) | 67 | * ->private_lock (__free_pte->__set_page_dirty_buffers) |
68 | * ->swap_lock (exclusive_swap_page, others) | 68 | * ->swap_lock (exclusive_swap_page, others) |
69 | * ->mapping->tree_lock | 69 | * ->i_pages lock |
70 | * | 70 | * |
71 | * ->i_mutex | 71 | * ->i_mutex |
72 | * ->i_mmap_rwsem (truncate->unmap_mapping_range) | 72 | * ->i_mmap_rwsem (truncate->unmap_mapping_range) |
@@ -74,7 +74,7 @@ | |||
74 | * ->mmap_sem | 74 | * ->mmap_sem |
75 | * ->i_mmap_rwsem | 75 | * ->i_mmap_rwsem |
76 | * ->page_table_lock or pte_lock (various, mainly in memory.c) | 76 | * ->page_table_lock or pte_lock (various, mainly in memory.c) |
77 | * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) | 77 | * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) |
78 | * | 78 | * |
79 | * ->mmap_sem | 79 | * ->mmap_sem |
80 | * ->lock_page (access_process_vm) | 80 | * ->lock_page (access_process_vm) |
@@ -84,7 +84,7 @@ | |||
84 | * | 84 | * |
85 | * bdi->wb.list_lock | 85 | * bdi->wb.list_lock |
86 | * sb_lock (fs/fs-writeback.c) | 86 | * sb_lock (fs/fs-writeback.c) |
87 | * ->mapping->tree_lock (__sync_single_inode) | 87 | * ->i_pages lock (__sync_single_inode) |
88 | * | 88 | * |
89 | * ->i_mmap_rwsem | 89 | * ->i_mmap_rwsem |
90 | * ->anon_vma.lock (vma_adjust) | 90 | * ->anon_vma.lock (vma_adjust) |
@@ -95,11 +95,11 @@ | |||
95 | * ->page_table_lock or pte_lock | 95 | * ->page_table_lock or pte_lock |
96 | * ->swap_lock (try_to_unmap_one) | 96 | * ->swap_lock (try_to_unmap_one) |
97 | * ->private_lock (try_to_unmap_one) | 97 | * ->private_lock (try_to_unmap_one) |
98 | * ->tree_lock (try_to_unmap_one) | 98 | * ->i_pages lock (try_to_unmap_one) |
99 | * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) | 99 | * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) |
100 | * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) | 100 | * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) |
101 | * ->private_lock (page_remove_rmap->set_page_dirty) | 101 | * ->private_lock (page_remove_rmap->set_page_dirty) |
102 | * ->tree_lock (page_remove_rmap->set_page_dirty) | 102 | * ->i_pages lock (page_remove_rmap->set_page_dirty) |
103 | * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) | 103 | * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) |
104 | * ->inode->i_lock (page_remove_rmap->set_page_dirty) | 104 | * ->inode->i_lock (page_remove_rmap->set_page_dirty) |
105 | * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) | 105 | * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) |
@@ -118,14 +118,15 @@ static int page_cache_tree_insert(struct address_space *mapping, | |||
118 | void **slot; | 118 | void **slot; |
119 | int error; | 119 | int error; |
120 | 120 | ||
121 | error = __radix_tree_create(&mapping->page_tree, page->index, 0, | 121 | error = __radix_tree_create(&mapping->i_pages, page->index, 0, |
122 | &node, &slot); | 122 | &node, &slot); |
123 | if (error) | 123 | if (error) |
124 | return error; | 124 | return error; |
125 | if (*slot) { | 125 | if (*slot) { |
126 | void *p; | 126 | void *p; |
127 | 127 | ||
128 | p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 128 | p = radix_tree_deref_slot_protected(slot, |
129 | &mapping->i_pages.xa_lock); | ||
129 | if (!radix_tree_exceptional_entry(p)) | 130 | if (!radix_tree_exceptional_entry(p)) |
130 | return -EEXIST; | 131 | return -EEXIST; |
131 | 132 | ||
@@ -133,7 +134,7 @@ static int page_cache_tree_insert(struct address_space *mapping, | |||
133 | if (shadowp) | 134 | if (shadowp) |
134 | *shadowp = p; | 135 | *shadowp = p; |
135 | } | 136 | } |
136 | __radix_tree_replace(&mapping->page_tree, node, slot, page, | 137 | __radix_tree_replace(&mapping->i_pages, node, slot, page, |
137 | workingset_lookup_update(mapping)); | 138 | workingset_lookup_update(mapping)); |
138 | mapping->nrpages++; | 139 | mapping->nrpages++; |
139 | return 0; | 140 | return 0; |
@@ -155,13 +156,13 @@ static void page_cache_tree_delete(struct address_space *mapping, | |||
155 | struct radix_tree_node *node; | 156 | struct radix_tree_node *node; |
156 | void **slot; | 157 | void **slot; |
157 | 158 | ||
158 | __radix_tree_lookup(&mapping->page_tree, page->index + i, | 159 | __radix_tree_lookup(&mapping->i_pages, page->index + i, |
159 | &node, &slot); | 160 | &node, &slot); |
160 | 161 | ||
161 | VM_BUG_ON_PAGE(!node && nr != 1, page); | 162 | VM_BUG_ON_PAGE(!node && nr != 1, page); |
162 | 163 | ||
163 | radix_tree_clear_tags(&mapping->page_tree, node, slot); | 164 | radix_tree_clear_tags(&mapping->i_pages, node, slot); |
164 | __radix_tree_replace(&mapping->page_tree, node, slot, shadow, | 165 | __radix_tree_replace(&mapping->i_pages, node, slot, shadow, |
165 | workingset_lookup_update(mapping)); | 166 | workingset_lookup_update(mapping)); |
166 | } | 167 | } |
167 | 168 | ||
@@ -253,7 +254,7 @@ static void unaccount_page_cache_page(struct address_space *mapping, | |||
253 | /* | 254 | /* |
254 | * Delete a page from the page cache and free it. Caller has to make | 255 | * Delete a page from the page cache and free it. Caller has to make |
255 | * sure the page is locked and that nobody else uses it - or that usage | 256 | * sure the page is locked and that nobody else uses it - or that usage |
256 | * is safe. The caller must hold the mapping's tree_lock. | 257 | * is safe. The caller must hold the i_pages lock. |
257 | */ | 258 | */ |
258 | void __delete_from_page_cache(struct page *page, void *shadow) | 259 | void __delete_from_page_cache(struct page *page, void *shadow) |
259 | { | 260 | { |
@@ -296,9 +297,9 @@ void delete_from_page_cache(struct page *page) | |||
296 | unsigned long flags; | 297 | unsigned long flags; |
297 | 298 | ||
298 | BUG_ON(!PageLocked(page)); | 299 | BUG_ON(!PageLocked(page)); |
299 | spin_lock_irqsave(&mapping->tree_lock, flags); | 300 | xa_lock_irqsave(&mapping->i_pages, flags); |
300 | __delete_from_page_cache(page, NULL); | 301 | __delete_from_page_cache(page, NULL); |
301 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 302 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
302 | 303 | ||
303 | page_cache_free_page(mapping, page); | 304 | page_cache_free_page(mapping, page); |
304 | } | 305 | } |
@@ -309,14 +310,14 @@ EXPORT_SYMBOL(delete_from_page_cache); | |||
309 | * @mapping: the mapping to which pages belong | 310 | * @mapping: the mapping to which pages belong |
310 | * @pvec: pagevec with pages to delete | 311 | * @pvec: pagevec with pages to delete |
311 | * | 312 | * |
312 | * The function walks over mapping->page_tree and removes pages passed in @pvec | 313 | * The function walks over mapping->i_pages and removes pages passed in @pvec |
313 | * from the radix tree. The function expects @pvec to be sorted by page index. | 314 | * from the mapping. The function expects @pvec to be sorted by page index. |
314 | * It tolerates holes in @pvec (radix tree entries at those indices are not | 315 | * It tolerates holes in @pvec (mapping entries at those indices are not |
315 | * modified). The function expects only THP head pages to be present in the | 316 | * modified). The function expects only THP head pages to be present in the |
316 | * @pvec and takes care to delete all corresponding tail pages from the radix | 317 | * @pvec and takes care to delete all corresponding tail pages from the |
317 | * tree as well. | 318 | * mapping as well. |
318 | * | 319 | * |
319 | * The function expects mapping->tree_lock to be held. | 320 | * The function expects the i_pages lock to be held. |
320 | */ | 321 | */ |
321 | static void | 322 | static void |
322 | page_cache_tree_delete_batch(struct address_space *mapping, | 323 | page_cache_tree_delete_batch(struct address_space *mapping, |
@@ -330,11 +331,11 @@ page_cache_tree_delete_batch(struct address_space *mapping, | |||
330 | pgoff_t start; | 331 | pgoff_t start; |
331 | 332 | ||
332 | start = pvec->pages[0]->index; | 333 | start = pvec->pages[0]->index; |
333 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 334 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
334 | if (i >= pagevec_count(pvec) && !tail_pages) | 335 | if (i >= pagevec_count(pvec) && !tail_pages) |
335 | break; | 336 | break; |
336 | page = radix_tree_deref_slot_protected(slot, | 337 | page = radix_tree_deref_slot_protected(slot, |
337 | &mapping->tree_lock); | 338 | &mapping->i_pages.xa_lock); |
338 | if (radix_tree_exceptional_entry(page)) | 339 | if (radix_tree_exceptional_entry(page)) |
339 | continue; | 340 | continue; |
340 | if (!tail_pages) { | 341 | if (!tail_pages) { |
@@ -357,8 +358,8 @@ page_cache_tree_delete_batch(struct address_space *mapping, | |||
357 | } else { | 358 | } else { |
358 | tail_pages--; | 359 | tail_pages--; |
359 | } | 360 | } |
360 | radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); | 361 | radix_tree_clear_tags(&mapping->i_pages, iter.node, slot); |
361 | __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, | 362 | __radix_tree_replace(&mapping->i_pages, iter.node, slot, NULL, |
362 | workingset_lookup_update(mapping)); | 363 | workingset_lookup_update(mapping)); |
363 | total_pages++; | 364 | total_pages++; |
364 | } | 365 | } |
@@ -374,14 +375,14 @@ void delete_from_page_cache_batch(struct address_space *mapping, | |||
374 | if (!pagevec_count(pvec)) | 375 | if (!pagevec_count(pvec)) |
375 | return; | 376 | return; |
376 | 377 | ||
377 | spin_lock_irqsave(&mapping->tree_lock, flags); | 378 | xa_lock_irqsave(&mapping->i_pages, flags); |
378 | for (i = 0; i < pagevec_count(pvec); i++) { | 379 | for (i = 0; i < pagevec_count(pvec); i++) { |
379 | trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); | 380 | trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); |
380 | 381 | ||
381 | unaccount_page_cache_page(mapping, pvec->pages[i]); | 382 | unaccount_page_cache_page(mapping, pvec->pages[i]); |
382 | } | 383 | } |
383 | page_cache_tree_delete_batch(mapping, pvec); | 384 | page_cache_tree_delete_batch(mapping, pvec); |
384 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 385 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
385 | 386 | ||
386 | for (i = 0; i < pagevec_count(pvec); i++) | 387 | for (i = 0; i < pagevec_count(pvec); i++) |
387 | page_cache_free_page(mapping, pvec->pages[i]); | 388 | page_cache_free_page(mapping, pvec->pages[i]); |
@@ -798,7 +799,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) | |||
798 | new->mapping = mapping; | 799 | new->mapping = mapping; |
799 | new->index = offset; | 800 | new->index = offset; |
800 | 801 | ||
801 | spin_lock_irqsave(&mapping->tree_lock, flags); | 802 | xa_lock_irqsave(&mapping->i_pages, flags); |
802 | __delete_from_page_cache(old, NULL); | 803 | __delete_from_page_cache(old, NULL); |
803 | error = page_cache_tree_insert(mapping, new, NULL); | 804 | error = page_cache_tree_insert(mapping, new, NULL); |
804 | BUG_ON(error); | 805 | BUG_ON(error); |
@@ -810,7 +811,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) | |||
810 | __inc_node_page_state(new, NR_FILE_PAGES); | 811 | __inc_node_page_state(new, NR_FILE_PAGES); |
811 | if (PageSwapBacked(new)) | 812 | if (PageSwapBacked(new)) |
812 | __inc_node_page_state(new, NR_SHMEM); | 813 | __inc_node_page_state(new, NR_SHMEM); |
813 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 814 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
814 | mem_cgroup_migrate(old, new); | 815 | mem_cgroup_migrate(old, new); |
815 | radix_tree_preload_end(); | 816 | radix_tree_preload_end(); |
816 | if (freepage) | 817 | if (freepage) |
@@ -852,7 +853,7 @@ static int __add_to_page_cache_locked(struct page *page, | |||
852 | page->mapping = mapping; | 853 | page->mapping = mapping; |
853 | page->index = offset; | 854 | page->index = offset; |
854 | 855 | ||
855 | spin_lock_irq(&mapping->tree_lock); | 856 | xa_lock_irq(&mapping->i_pages); |
856 | error = page_cache_tree_insert(mapping, page, shadowp); | 857 | error = page_cache_tree_insert(mapping, page, shadowp); |
857 | radix_tree_preload_end(); | 858 | radix_tree_preload_end(); |
858 | if (unlikely(error)) | 859 | if (unlikely(error)) |
@@ -861,7 +862,7 @@ static int __add_to_page_cache_locked(struct page *page, | |||
861 | /* hugetlb pages do not participate in page cache accounting. */ | 862 | /* hugetlb pages do not participate in page cache accounting. */ |
862 | if (!huge) | 863 | if (!huge) |
863 | __inc_node_page_state(page, NR_FILE_PAGES); | 864 | __inc_node_page_state(page, NR_FILE_PAGES); |
864 | spin_unlock_irq(&mapping->tree_lock); | 865 | xa_unlock_irq(&mapping->i_pages); |
865 | if (!huge) | 866 | if (!huge) |
866 | mem_cgroup_commit_charge(page, memcg, false, false); | 867 | mem_cgroup_commit_charge(page, memcg, false, false); |
867 | trace_mm_filemap_add_to_page_cache(page); | 868 | trace_mm_filemap_add_to_page_cache(page); |
@@ -869,7 +870,7 @@ static int __add_to_page_cache_locked(struct page *page, | |||
869 | err_insert: | 870 | err_insert: |
870 | page->mapping = NULL; | 871 | page->mapping = NULL; |
871 | /* Leave page->index set: truncation relies upon it */ | 872 | /* Leave page->index set: truncation relies upon it */ |
872 | spin_unlock_irq(&mapping->tree_lock); | 873 | xa_unlock_irq(&mapping->i_pages); |
873 | if (!huge) | 874 | if (!huge) |
874 | mem_cgroup_cancel_charge(page, memcg, false); | 875 | mem_cgroup_cancel_charge(page, memcg, false); |
875 | put_page(page); | 876 | put_page(page); |
@@ -1353,7 +1354,7 @@ pgoff_t page_cache_next_hole(struct address_space *mapping, | |||
1353 | for (i = 0; i < max_scan; i++) { | 1354 | for (i = 0; i < max_scan; i++) { |
1354 | struct page *page; | 1355 | struct page *page; |
1355 | 1356 | ||
1356 | page = radix_tree_lookup(&mapping->page_tree, index); | 1357 | page = radix_tree_lookup(&mapping->i_pages, index); |
1357 | if (!page || radix_tree_exceptional_entry(page)) | 1358 | if (!page || radix_tree_exceptional_entry(page)) |
1358 | break; | 1359 | break; |
1359 | index++; | 1360 | index++; |
@@ -1394,7 +1395,7 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping, | |||
1394 | for (i = 0; i < max_scan; i++) { | 1395 | for (i = 0; i < max_scan; i++) { |
1395 | struct page *page; | 1396 | struct page *page; |
1396 | 1397 | ||
1397 | page = radix_tree_lookup(&mapping->page_tree, index); | 1398 | page = radix_tree_lookup(&mapping->i_pages, index); |
1398 | if (!page || radix_tree_exceptional_entry(page)) | 1399 | if (!page || radix_tree_exceptional_entry(page)) |
1399 | break; | 1400 | break; |
1400 | index--; | 1401 | index--; |
@@ -1427,7 +1428,7 @@ struct page *find_get_entry(struct address_space *mapping, pgoff_t offset) | |||
1427 | rcu_read_lock(); | 1428 | rcu_read_lock(); |
1428 | repeat: | 1429 | repeat: |
1429 | page = NULL; | 1430 | page = NULL; |
1430 | pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); | 1431 | pagep = radix_tree_lookup_slot(&mapping->i_pages, offset); |
1431 | if (pagep) { | 1432 | if (pagep) { |
1432 | page = radix_tree_deref_slot(pagep); | 1433 | page = radix_tree_deref_slot(pagep); |
1433 | if (unlikely(!page)) | 1434 | if (unlikely(!page)) |
@@ -1633,7 +1634,7 @@ unsigned find_get_entries(struct address_space *mapping, | |||
1633 | return 0; | 1634 | return 0; |
1634 | 1635 | ||
1635 | rcu_read_lock(); | 1636 | rcu_read_lock(); |
1636 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 1637 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
1637 | struct page *head, *page; | 1638 | struct page *head, *page; |
1638 | repeat: | 1639 | repeat: |
1639 | page = radix_tree_deref_slot(slot); | 1640 | page = radix_tree_deref_slot(slot); |
@@ -1710,7 +1711,7 @@ unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, | |||
1710 | return 0; | 1711 | return 0; |
1711 | 1712 | ||
1712 | rcu_read_lock(); | 1713 | rcu_read_lock(); |
1713 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) { | 1714 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, *start) { |
1714 | struct page *head, *page; | 1715 | struct page *head, *page; |
1715 | 1716 | ||
1716 | if (iter.index > end) | 1717 | if (iter.index > end) |
@@ -1795,7 +1796,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, | |||
1795 | return 0; | 1796 | return 0; |
1796 | 1797 | ||
1797 | rcu_read_lock(); | 1798 | rcu_read_lock(); |
1798 | radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) { | 1799 | radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) { |
1799 | struct page *head, *page; | 1800 | struct page *head, *page; |
1800 | repeat: | 1801 | repeat: |
1801 | page = radix_tree_deref_slot(slot); | 1802 | page = radix_tree_deref_slot(slot); |
@@ -1875,8 +1876,7 @@ unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, | |||
1875 | return 0; | 1876 | return 0; |
1876 | 1877 | ||
1877 | rcu_read_lock(); | 1878 | rcu_read_lock(); |
1878 | radix_tree_for_each_tagged(slot, &mapping->page_tree, | 1879 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, *index, tag) { |
1879 | &iter, *index, tag) { | ||
1880 | struct page *head, *page; | 1880 | struct page *head, *page; |
1881 | 1881 | ||
1882 | if (iter.index > end) | 1882 | if (iter.index > end) |
@@ -1969,8 +1969,7 @@ unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start, | |||
1969 | return 0; | 1969 | return 0; |
1970 | 1970 | ||
1971 | rcu_read_lock(); | 1971 | rcu_read_lock(); |
1972 | radix_tree_for_each_tagged(slot, &mapping->page_tree, | 1972 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, tag) { |
1973 | &iter, start, tag) { | ||
1974 | struct page *head, *page; | 1973 | struct page *head, *page; |
1975 | repeat: | 1974 | repeat: |
1976 | page = radix_tree_deref_slot(slot); | 1975 | page = radix_tree_deref_slot(slot); |
@@ -2624,8 +2623,7 @@ void filemap_map_pages(struct vm_fault *vmf, | |||
2624 | struct page *head, *page; | 2623 | struct page *head, *page; |
2625 | 2624 | ||
2626 | rcu_read_lock(); | 2625 | rcu_read_lock(); |
2627 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, | 2626 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start_pgoff) { |
2628 | start_pgoff) { | ||
2629 | if (iter.index > end_pgoff) | 2627 | if (iter.index > end_pgoff) |
2630 | break; | 2628 | break; |
2631 | repeat: | 2629 | repeat: |
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 3f3267af4e3b..14ed6ee5e02f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
@@ -2450,7 +2450,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, | |||
2450 | } else { | 2450 | } else { |
2451 | /* Additional pin to radix tree */ | 2451 | /* Additional pin to radix tree */ |
2452 | page_ref_add(head, 2); | 2452 | page_ref_add(head, 2); |
2453 | spin_unlock(&head->mapping->tree_lock); | 2453 | xa_unlock(&head->mapping->i_pages); |
2454 | } | 2454 | } |
2455 | 2455 | ||
2456 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); | 2456 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); |
@@ -2658,15 +2658,15 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) | |||
2658 | if (mapping) { | 2658 | if (mapping) { |
2659 | void **pslot; | 2659 | void **pslot; |
2660 | 2660 | ||
2661 | spin_lock(&mapping->tree_lock); | 2661 | xa_lock(&mapping->i_pages); |
2662 | pslot = radix_tree_lookup_slot(&mapping->page_tree, | 2662 | pslot = radix_tree_lookup_slot(&mapping->i_pages, |
2663 | page_index(head)); | 2663 | page_index(head)); |
2664 | /* | 2664 | /* |
2665 | * Check if the head page is present in radix tree. | 2665 | * Check if the head page is present in radix tree. |
2666 | * We assume all tail are present too, if head is there. | 2666 | * We assume all tail are present too, if head is there. |
2667 | */ | 2667 | */ |
2668 | if (radix_tree_deref_slot_protected(pslot, | 2668 | if (radix_tree_deref_slot_protected(pslot, |
2669 | &mapping->tree_lock) != head) | 2669 | &mapping->i_pages.xa_lock) != head) |
2670 | goto fail; | 2670 | goto fail; |
2671 | } | 2671 | } |
2672 | 2672 | ||
@@ -2700,7 +2700,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) | |||
2700 | } | 2700 | } |
2701 | spin_unlock(&pgdata->split_queue_lock); | 2701 | spin_unlock(&pgdata->split_queue_lock); |
2702 | fail: if (mapping) | 2702 | fail: if (mapping) |
2703 | spin_unlock(&mapping->tree_lock); | 2703 | xa_unlock(&mapping->i_pages); |
2704 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); | 2704 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); |
2705 | unfreeze_page(head); | 2705 | unfreeze_page(head); |
2706 | ret = -EBUSY; | 2706 | ret = -EBUSY; |
diff --git a/mm/khugepaged.c b/mm/khugepaged.c index eb32d0707c80..d7b2a4bf8671 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c | |||
@@ -1344,8 +1344,8 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1344 | */ | 1344 | */ |
1345 | 1345 | ||
1346 | index = start; | 1346 | index = start; |
1347 | spin_lock_irq(&mapping->tree_lock); | 1347 | xa_lock_irq(&mapping->i_pages); |
1348 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 1348 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
1349 | int n = min(iter.index, end) - index; | 1349 | int n = min(iter.index, end) - index; |
1350 | 1350 | ||
1351 | /* | 1351 | /* |
@@ -1358,7 +1358,7 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1358 | } | 1358 | } |
1359 | nr_none += n; | 1359 | nr_none += n; |
1360 | for (; index < min(iter.index, end); index++) { | 1360 | for (; index < min(iter.index, end); index++) { |
1361 | radix_tree_insert(&mapping->page_tree, index, | 1361 | radix_tree_insert(&mapping->i_pages, index, |
1362 | new_page + (index % HPAGE_PMD_NR)); | 1362 | new_page + (index % HPAGE_PMD_NR)); |
1363 | } | 1363 | } |
1364 | 1364 | ||
@@ -1367,16 +1367,16 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1367 | break; | 1367 | break; |
1368 | 1368 | ||
1369 | page = radix_tree_deref_slot_protected(slot, | 1369 | page = radix_tree_deref_slot_protected(slot, |
1370 | &mapping->tree_lock); | 1370 | &mapping->i_pages.xa_lock); |
1371 | if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { | 1371 | if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { |
1372 | spin_unlock_irq(&mapping->tree_lock); | 1372 | xa_unlock_irq(&mapping->i_pages); |
1373 | /* swap in or instantiate fallocated page */ | 1373 | /* swap in or instantiate fallocated page */ |
1374 | if (shmem_getpage(mapping->host, index, &page, | 1374 | if (shmem_getpage(mapping->host, index, &page, |
1375 | SGP_NOHUGE)) { | 1375 | SGP_NOHUGE)) { |
1376 | result = SCAN_FAIL; | 1376 | result = SCAN_FAIL; |
1377 | goto tree_unlocked; | 1377 | goto tree_unlocked; |
1378 | } | 1378 | } |
1379 | spin_lock_irq(&mapping->tree_lock); | 1379 | xa_lock_irq(&mapping->i_pages); |
1380 | } else if (trylock_page(page)) { | 1380 | } else if (trylock_page(page)) { |
1381 | get_page(page); | 1381 | get_page(page); |
1382 | } else { | 1382 | } else { |
@@ -1385,7 +1385,7 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1385 | } | 1385 | } |
1386 | 1386 | ||
1387 | /* | 1387 | /* |
1388 | * The page must be locked, so we can drop the tree_lock | 1388 | * The page must be locked, so we can drop the i_pages lock |
1389 | * without racing with truncate. | 1389 | * without racing with truncate. |
1390 | */ | 1390 | */ |
1391 | VM_BUG_ON_PAGE(!PageLocked(page), page); | 1391 | VM_BUG_ON_PAGE(!PageLocked(page), page); |
@@ -1396,7 +1396,7 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1396 | result = SCAN_TRUNCATED; | 1396 | result = SCAN_TRUNCATED; |
1397 | goto out_unlock; | 1397 | goto out_unlock; |
1398 | } | 1398 | } |
1399 | spin_unlock_irq(&mapping->tree_lock); | 1399 | xa_unlock_irq(&mapping->i_pages); |
1400 | 1400 | ||
1401 | if (isolate_lru_page(page)) { | 1401 | if (isolate_lru_page(page)) { |
1402 | result = SCAN_DEL_PAGE_LRU; | 1402 | result = SCAN_DEL_PAGE_LRU; |
@@ -1406,11 +1406,11 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1406 | if (page_mapped(page)) | 1406 | if (page_mapped(page)) |
1407 | unmap_mapping_pages(mapping, index, 1, false); | 1407 | unmap_mapping_pages(mapping, index, 1, false); |
1408 | 1408 | ||
1409 | spin_lock_irq(&mapping->tree_lock); | 1409 | xa_lock_irq(&mapping->i_pages); |
1410 | 1410 | ||
1411 | slot = radix_tree_lookup_slot(&mapping->page_tree, index); | 1411 | slot = radix_tree_lookup_slot(&mapping->i_pages, index); |
1412 | VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, | 1412 | VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, |
1413 | &mapping->tree_lock), page); | 1413 | &mapping->i_pages.xa_lock), page); |
1414 | VM_BUG_ON_PAGE(page_mapped(page), page); | 1414 | VM_BUG_ON_PAGE(page_mapped(page), page); |
1415 | 1415 | ||
1416 | /* | 1416 | /* |
@@ -1431,14 +1431,14 @@ static void collapse_shmem(struct mm_struct *mm, | |||
1431 | list_add_tail(&page->lru, &pagelist); | 1431 | list_add_tail(&page->lru, &pagelist); |
1432 | 1432 | ||
1433 | /* Finally, replace with the new page. */ | 1433 | /* Finally, replace with the new page. */ |
1434 | radix_tree_replace_slot(&mapping->page_tree, slot, | 1434 | radix_tree_replace_slot(&mapping->i_pages, slot, |
1435 | new_page + (index % HPAGE_PMD_NR)); | 1435 | new_page + (index % HPAGE_PMD_NR)); |
1436 | 1436 | ||
1437 | slot = radix_tree_iter_resume(slot, &iter); | 1437 | slot = radix_tree_iter_resume(slot, &iter); |
1438 | index++; | 1438 | index++; |
1439 | continue; | 1439 | continue; |
1440 | out_lru: | 1440 | out_lru: |
1441 | spin_unlock_irq(&mapping->tree_lock); | 1441 | xa_unlock_irq(&mapping->i_pages); |
1442 | putback_lru_page(page); | 1442 | putback_lru_page(page); |
1443 | out_isolate_failed: | 1443 | out_isolate_failed: |
1444 | unlock_page(page); | 1444 | unlock_page(page); |
@@ -1464,14 +1464,14 @@ out_unlock: | |||
1464 | } | 1464 | } |
1465 | 1465 | ||
1466 | for (; index < end; index++) { | 1466 | for (; index < end; index++) { |
1467 | radix_tree_insert(&mapping->page_tree, index, | 1467 | radix_tree_insert(&mapping->i_pages, index, |
1468 | new_page + (index % HPAGE_PMD_NR)); | 1468 | new_page + (index % HPAGE_PMD_NR)); |
1469 | } | 1469 | } |
1470 | nr_none += n; | 1470 | nr_none += n; |
1471 | } | 1471 | } |
1472 | 1472 | ||
1473 | tree_locked: | 1473 | tree_locked: |
1474 | spin_unlock_irq(&mapping->tree_lock); | 1474 | xa_unlock_irq(&mapping->i_pages); |
1475 | tree_unlocked: | 1475 | tree_unlocked: |
1476 | 1476 | ||
1477 | if (result == SCAN_SUCCEED) { | 1477 | if (result == SCAN_SUCCEED) { |
@@ -1520,9 +1520,8 @@ tree_unlocked: | |||
1520 | } else { | 1520 | } else { |
1521 | /* Something went wrong: rollback changes to the radix-tree */ | 1521 | /* Something went wrong: rollback changes to the radix-tree */ |
1522 | shmem_uncharge(mapping->host, nr_none); | 1522 | shmem_uncharge(mapping->host, nr_none); |
1523 | spin_lock_irq(&mapping->tree_lock); | 1523 | xa_lock_irq(&mapping->i_pages); |
1524 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, | 1524 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
1525 | start) { | ||
1526 | if (iter.index >= end) | 1525 | if (iter.index >= end) |
1527 | break; | 1526 | break; |
1528 | page = list_first_entry_or_null(&pagelist, | 1527 | page = list_first_entry_or_null(&pagelist, |
@@ -1532,8 +1531,7 @@ tree_unlocked: | |||
1532 | break; | 1531 | break; |
1533 | nr_none--; | 1532 | nr_none--; |
1534 | /* Put holes back where they were */ | 1533 | /* Put holes back where they were */ |
1535 | radix_tree_delete(&mapping->page_tree, | 1534 | radix_tree_delete(&mapping->i_pages, iter.index); |
1536 | iter.index); | ||
1537 | continue; | 1535 | continue; |
1538 | } | 1536 | } |
1539 | 1537 | ||
@@ -1542,16 +1540,15 @@ tree_unlocked: | |||
1542 | /* Unfreeze the page. */ | 1540 | /* Unfreeze the page. */ |
1543 | list_del(&page->lru); | 1541 | list_del(&page->lru); |
1544 | page_ref_unfreeze(page, 2); | 1542 | page_ref_unfreeze(page, 2); |
1545 | radix_tree_replace_slot(&mapping->page_tree, | 1543 | radix_tree_replace_slot(&mapping->i_pages, slot, page); |
1546 | slot, page); | ||
1547 | slot = radix_tree_iter_resume(slot, &iter); | 1544 | slot = radix_tree_iter_resume(slot, &iter); |
1548 | spin_unlock_irq(&mapping->tree_lock); | 1545 | xa_unlock_irq(&mapping->i_pages); |
1549 | putback_lru_page(page); | 1546 | putback_lru_page(page); |
1550 | unlock_page(page); | 1547 | unlock_page(page); |
1551 | spin_lock_irq(&mapping->tree_lock); | 1548 | xa_lock_irq(&mapping->i_pages); |
1552 | } | 1549 | } |
1553 | VM_BUG_ON(nr_none); | 1550 | VM_BUG_ON(nr_none); |
1554 | spin_unlock_irq(&mapping->tree_lock); | 1551 | xa_unlock_irq(&mapping->i_pages); |
1555 | 1552 | ||
1556 | /* Unfreeze new_page, caller would take care about freeing it */ | 1553 | /* Unfreeze new_page, caller would take care about freeing it */ |
1557 | page_ref_unfreeze(new_page, 1); | 1554 | page_ref_unfreeze(new_page, 1); |
@@ -1579,7 +1576,7 @@ static void khugepaged_scan_shmem(struct mm_struct *mm, | |||
1579 | swap = 0; | 1576 | swap = 0; |
1580 | memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); | 1577 | memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); |
1581 | rcu_read_lock(); | 1578 | rcu_read_lock(); |
1582 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 1579 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
1583 | if (iter.index >= start + HPAGE_PMD_NR) | 1580 | if (iter.index >= start + HPAGE_PMD_NR) |
1584 | break; | 1581 | break; |
1585 | 1582 | ||
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7978c6faae06..e074f7c637aa 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c | |||
@@ -5974,9 +5974,9 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) | |||
5974 | 5974 | ||
5975 | /* | 5975 | /* |
5976 | * Interrupts should be disabled here because the caller holds the | 5976 | * Interrupts should be disabled here because the caller holds the |
5977 | * mapping->tree_lock lock which is taken with interrupts-off. It is | 5977 | * i_pages lock which is taken with interrupts-off. It is |
5978 | * important here to have the interrupts disabled because it is the | 5978 | * important here to have the interrupts disabled because it is the |
5979 | * only synchronisation we have for udpating the per-CPU variables. | 5979 | * only synchronisation we have for updating the per-CPU variables. |
5980 | */ | 5980 | */ |
5981 | VM_BUG_ON(!irqs_disabled()); | 5981 | VM_BUG_ON(!irqs_disabled()); |
5982 | mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), | 5982 | mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), |
diff --git a/mm/migrate.c b/mm/migrate.c index 51b55f2d2db5..f65dd69e1fd1 100644 --- a/mm/migrate.c +++ b/mm/migrate.c | |||
@@ -467,20 +467,21 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
467 | oldzone = page_zone(page); | 467 | oldzone = page_zone(page); |
468 | newzone = page_zone(newpage); | 468 | newzone = page_zone(newpage); |
469 | 469 | ||
470 | spin_lock_irq(&mapping->tree_lock); | 470 | xa_lock_irq(&mapping->i_pages); |
471 | 471 | ||
472 | pslot = radix_tree_lookup_slot(&mapping->page_tree, | 472 | pslot = radix_tree_lookup_slot(&mapping->i_pages, |
473 | page_index(page)); | 473 | page_index(page)); |
474 | 474 | ||
475 | expected_count += 1 + page_has_private(page); | 475 | expected_count += 1 + page_has_private(page); |
476 | if (page_count(page) != expected_count || | 476 | if (page_count(page) != expected_count || |
477 | radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { | 477 | radix_tree_deref_slot_protected(pslot, |
478 | spin_unlock_irq(&mapping->tree_lock); | 478 | &mapping->i_pages.xa_lock) != page) { |
479 | xa_unlock_irq(&mapping->i_pages); | ||
479 | return -EAGAIN; | 480 | return -EAGAIN; |
480 | } | 481 | } |
481 | 482 | ||
482 | if (!page_ref_freeze(page, expected_count)) { | 483 | if (!page_ref_freeze(page, expected_count)) { |
483 | spin_unlock_irq(&mapping->tree_lock); | 484 | xa_unlock_irq(&mapping->i_pages); |
484 | return -EAGAIN; | 485 | return -EAGAIN; |
485 | } | 486 | } |
486 | 487 | ||
@@ -494,7 +495,7 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
494 | if (mode == MIGRATE_ASYNC && head && | 495 | if (mode == MIGRATE_ASYNC && head && |
495 | !buffer_migrate_lock_buffers(head, mode)) { | 496 | !buffer_migrate_lock_buffers(head, mode)) { |
496 | page_ref_unfreeze(page, expected_count); | 497 | page_ref_unfreeze(page, expected_count); |
497 | spin_unlock_irq(&mapping->tree_lock); | 498 | xa_unlock_irq(&mapping->i_pages); |
498 | return -EAGAIN; | 499 | return -EAGAIN; |
499 | } | 500 | } |
500 | 501 | ||
@@ -522,7 +523,7 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
522 | SetPageDirty(newpage); | 523 | SetPageDirty(newpage); |
523 | } | 524 | } |
524 | 525 | ||
525 | radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); | 526 | radix_tree_replace_slot(&mapping->i_pages, pslot, newpage); |
526 | 527 | ||
527 | /* | 528 | /* |
528 | * Drop cache reference from old page by unfreezing | 529 | * Drop cache reference from old page by unfreezing |
@@ -531,7 +532,7 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
531 | */ | 532 | */ |
532 | page_ref_unfreeze(page, expected_count - 1); | 533 | page_ref_unfreeze(page, expected_count - 1); |
533 | 534 | ||
534 | spin_unlock(&mapping->tree_lock); | 535 | xa_unlock(&mapping->i_pages); |
535 | /* Leave irq disabled to prevent preemption while updating stats */ | 536 | /* Leave irq disabled to prevent preemption while updating stats */ |
536 | 537 | ||
537 | /* | 538 | /* |
@@ -574,20 +575,19 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, | |||
574 | int expected_count; | 575 | int expected_count; |
575 | void **pslot; | 576 | void **pslot; |
576 | 577 | ||
577 | spin_lock_irq(&mapping->tree_lock); | 578 | xa_lock_irq(&mapping->i_pages); |
578 | 579 | ||
579 | pslot = radix_tree_lookup_slot(&mapping->page_tree, | 580 | pslot = radix_tree_lookup_slot(&mapping->i_pages, page_index(page)); |
580 | page_index(page)); | ||
581 | 581 | ||
582 | expected_count = 2 + page_has_private(page); | 582 | expected_count = 2 + page_has_private(page); |
583 | if (page_count(page) != expected_count || | 583 | if (page_count(page) != expected_count || |
584 | radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { | 584 | radix_tree_deref_slot_protected(pslot, &mapping->i_pages.xa_lock) != page) { |
585 | spin_unlock_irq(&mapping->tree_lock); | 585 | xa_unlock_irq(&mapping->i_pages); |
586 | return -EAGAIN; | 586 | return -EAGAIN; |
587 | } | 587 | } |
588 | 588 | ||
589 | if (!page_ref_freeze(page, expected_count)) { | 589 | if (!page_ref_freeze(page, expected_count)) { |
590 | spin_unlock_irq(&mapping->tree_lock); | 590 | xa_unlock_irq(&mapping->i_pages); |
591 | return -EAGAIN; | 591 | return -EAGAIN; |
592 | } | 592 | } |
593 | 593 | ||
@@ -596,11 +596,11 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, | |||
596 | 596 | ||
597 | get_page(newpage); | 597 | get_page(newpage); |
598 | 598 | ||
599 | radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); | 599 | radix_tree_replace_slot(&mapping->i_pages, pslot, newpage); |
600 | 600 | ||
601 | page_ref_unfreeze(page, expected_count - 1); | 601 | page_ref_unfreeze(page, expected_count - 1); |
602 | 602 | ||
603 | spin_unlock_irq(&mapping->tree_lock); | 603 | xa_unlock_irq(&mapping->i_pages); |
604 | 604 | ||
605 | return MIGRATEPAGE_SUCCESS; | 605 | return MIGRATEPAGE_SUCCESS; |
606 | } | 606 | } |
diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 586f31261c83..5c1a3279e63f 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c | |||
@@ -2099,7 +2099,8 @@ void __init page_writeback_init(void) | |||
2099 | * so that it can tag pages faster than a dirtying process can create them). | 2099 | * so that it can tag pages faster than a dirtying process can create them). |
2100 | */ | 2100 | */ |
2101 | /* | 2101 | /* |
2102 | * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency. | 2102 | * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce the i_pages lock |
2103 | * latency. | ||
2103 | */ | 2104 | */ |
2104 | void tag_pages_for_writeback(struct address_space *mapping, | 2105 | void tag_pages_for_writeback(struct address_space *mapping, |
2105 | pgoff_t start, pgoff_t end) | 2106 | pgoff_t start, pgoff_t end) |
@@ -2109,22 +2110,22 @@ void tag_pages_for_writeback(struct address_space *mapping, | |||
2109 | struct radix_tree_iter iter; | 2110 | struct radix_tree_iter iter; |
2110 | void **slot; | 2111 | void **slot; |
2111 | 2112 | ||
2112 | spin_lock_irq(&mapping->tree_lock); | 2113 | xa_lock_irq(&mapping->i_pages); |
2113 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start, | 2114 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, |
2114 | PAGECACHE_TAG_DIRTY) { | 2115 | PAGECACHE_TAG_DIRTY) { |
2115 | if (iter.index > end) | 2116 | if (iter.index > end) |
2116 | break; | 2117 | break; |
2117 | radix_tree_iter_tag_set(&mapping->page_tree, &iter, | 2118 | radix_tree_iter_tag_set(&mapping->i_pages, &iter, |
2118 | PAGECACHE_TAG_TOWRITE); | 2119 | PAGECACHE_TAG_TOWRITE); |
2119 | tagged++; | 2120 | tagged++; |
2120 | if ((tagged % WRITEBACK_TAG_BATCH) != 0) | 2121 | if ((tagged % WRITEBACK_TAG_BATCH) != 0) |
2121 | continue; | 2122 | continue; |
2122 | slot = radix_tree_iter_resume(slot, &iter); | 2123 | slot = radix_tree_iter_resume(slot, &iter); |
2123 | spin_unlock_irq(&mapping->tree_lock); | 2124 | xa_unlock_irq(&mapping->i_pages); |
2124 | cond_resched(); | 2125 | cond_resched(); |
2125 | spin_lock_irq(&mapping->tree_lock); | 2126 | xa_lock_irq(&mapping->i_pages); |
2126 | } | 2127 | } |
2127 | spin_unlock_irq(&mapping->tree_lock); | 2128 | xa_unlock_irq(&mapping->i_pages); |
2128 | } | 2129 | } |
2129 | EXPORT_SYMBOL(tag_pages_for_writeback); | 2130 | EXPORT_SYMBOL(tag_pages_for_writeback); |
2130 | 2131 | ||
@@ -2467,13 +2468,13 @@ int __set_page_dirty_nobuffers(struct page *page) | |||
2467 | return 1; | 2468 | return 1; |
2468 | } | 2469 | } |
2469 | 2470 | ||
2470 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2471 | xa_lock_irqsave(&mapping->i_pages, flags); |
2471 | BUG_ON(page_mapping(page) != mapping); | 2472 | BUG_ON(page_mapping(page) != mapping); |
2472 | WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); | 2473 | WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); |
2473 | account_page_dirtied(page, mapping); | 2474 | account_page_dirtied(page, mapping); |
2474 | radix_tree_tag_set(&mapping->page_tree, page_index(page), | 2475 | radix_tree_tag_set(&mapping->i_pages, page_index(page), |
2475 | PAGECACHE_TAG_DIRTY); | 2476 | PAGECACHE_TAG_DIRTY); |
2476 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2477 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
2477 | unlock_page_memcg(page); | 2478 | unlock_page_memcg(page); |
2478 | 2479 | ||
2479 | if (mapping->host) { | 2480 | if (mapping->host) { |
@@ -2718,11 +2719,10 @@ int test_clear_page_writeback(struct page *page) | |||
2718 | struct backing_dev_info *bdi = inode_to_bdi(inode); | 2719 | struct backing_dev_info *bdi = inode_to_bdi(inode); |
2719 | unsigned long flags; | 2720 | unsigned long flags; |
2720 | 2721 | ||
2721 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2722 | xa_lock_irqsave(&mapping->i_pages, flags); |
2722 | ret = TestClearPageWriteback(page); | 2723 | ret = TestClearPageWriteback(page); |
2723 | if (ret) { | 2724 | if (ret) { |
2724 | radix_tree_tag_clear(&mapping->page_tree, | 2725 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
2725 | page_index(page), | ||
2726 | PAGECACHE_TAG_WRITEBACK); | 2726 | PAGECACHE_TAG_WRITEBACK); |
2727 | if (bdi_cap_account_writeback(bdi)) { | 2727 | if (bdi_cap_account_writeback(bdi)) { |
2728 | struct bdi_writeback *wb = inode_to_wb(inode); | 2728 | struct bdi_writeback *wb = inode_to_wb(inode); |
@@ -2736,7 +2736,7 @@ int test_clear_page_writeback(struct page *page) | |||
2736 | PAGECACHE_TAG_WRITEBACK)) | 2736 | PAGECACHE_TAG_WRITEBACK)) |
2737 | sb_clear_inode_writeback(mapping->host); | 2737 | sb_clear_inode_writeback(mapping->host); |
2738 | 2738 | ||
2739 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2739 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
2740 | } else { | 2740 | } else { |
2741 | ret = TestClearPageWriteback(page); | 2741 | ret = TestClearPageWriteback(page); |
2742 | } | 2742 | } |
@@ -2766,7 +2766,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) | |||
2766 | struct backing_dev_info *bdi = inode_to_bdi(inode); | 2766 | struct backing_dev_info *bdi = inode_to_bdi(inode); |
2767 | unsigned long flags; | 2767 | unsigned long flags; |
2768 | 2768 | ||
2769 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2769 | xa_lock_irqsave(&mapping->i_pages, flags); |
2770 | ret = TestSetPageWriteback(page); | 2770 | ret = TestSetPageWriteback(page); |
2771 | if (!ret) { | 2771 | if (!ret) { |
2772 | bool on_wblist; | 2772 | bool on_wblist; |
@@ -2774,8 +2774,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) | |||
2774 | on_wblist = mapping_tagged(mapping, | 2774 | on_wblist = mapping_tagged(mapping, |
2775 | PAGECACHE_TAG_WRITEBACK); | 2775 | PAGECACHE_TAG_WRITEBACK); |
2776 | 2776 | ||
2777 | radix_tree_tag_set(&mapping->page_tree, | 2777 | radix_tree_tag_set(&mapping->i_pages, page_index(page), |
2778 | page_index(page), | ||
2779 | PAGECACHE_TAG_WRITEBACK); | 2778 | PAGECACHE_TAG_WRITEBACK); |
2780 | if (bdi_cap_account_writeback(bdi)) | 2779 | if (bdi_cap_account_writeback(bdi)) |
2781 | inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); | 2780 | inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); |
@@ -2789,14 +2788,12 @@ int __test_set_page_writeback(struct page *page, bool keep_write) | |||
2789 | sb_mark_inode_writeback(mapping->host); | 2788 | sb_mark_inode_writeback(mapping->host); |
2790 | } | 2789 | } |
2791 | if (!PageDirty(page)) | 2790 | if (!PageDirty(page)) |
2792 | radix_tree_tag_clear(&mapping->page_tree, | 2791 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
2793 | page_index(page), | ||
2794 | PAGECACHE_TAG_DIRTY); | 2792 | PAGECACHE_TAG_DIRTY); |
2795 | if (!keep_write) | 2793 | if (!keep_write) |
2796 | radix_tree_tag_clear(&mapping->page_tree, | 2794 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
2797 | page_index(page), | ||
2798 | PAGECACHE_TAG_TOWRITE); | 2795 | PAGECACHE_TAG_TOWRITE); |
2799 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2796 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
2800 | } else { | 2797 | } else { |
2801 | ret = TestSetPageWriteback(page); | 2798 | ret = TestSetPageWriteback(page); |
2802 | } | 2799 | } |
@@ -2816,7 +2813,7 @@ EXPORT_SYMBOL(__test_set_page_writeback); | |||
2816 | */ | 2813 | */ |
2817 | int mapping_tagged(struct address_space *mapping, int tag) | 2814 | int mapping_tagged(struct address_space *mapping, int tag) |
2818 | { | 2815 | { |
2819 | return radix_tree_tagged(&mapping->page_tree, tag); | 2816 | return radix_tree_tagged(&mapping->i_pages, tag); |
2820 | } | 2817 | } |
2821 | EXPORT_SYMBOL(mapping_tagged); | 2818 | EXPORT_SYMBOL(mapping_tagged); |
2822 | 2819 | ||
diff --git a/mm/readahead.c b/mm/readahead.c index 4d57b4644f98..539bbb6c1fad 100644 --- a/mm/readahead.c +++ b/mm/readahead.c | |||
@@ -175,7 +175,7 @@ int __do_page_cache_readahead(struct address_space *mapping, struct file *filp, | |||
175 | break; | 175 | break; |
176 | 176 | ||
177 | rcu_read_lock(); | 177 | rcu_read_lock(); |
178 | page = radix_tree_lookup(&mapping->page_tree, page_offset); | 178 | page = radix_tree_lookup(&mapping->i_pages, page_offset); |
179 | rcu_read_unlock(); | 179 | rcu_read_unlock(); |
180 | if (page && !radix_tree_exceptional_entry(page)) | 180 | if (page && !radix_tree_exceptional_entry(page)) |
181 | continue; | 181 | continue; |
@@ -32,11 +32,11 @@ | |||
32 | * mmlist_lock (in mmput, drain_mmlist and others) | 32 | * mmlist_lock (in mmput, drain_mmlist and others) |
33 | * mapping->private_lock (in __set_page_dirty_buffers) | 33 | * mapping->private_lock (in __set_page_dirty_buffers) |
34 | * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) | 34 | * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) |
35 | * mapping->tree_lock (widely used) | 35 | * i_pages lock (widely used) |
36 | * inode->i_lock (in set_page_dirty's __mark_inode_dirty) | 36 | * inode->i_lock (in set_page_dirty's __mark_inode_dirty) |
37 | * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) | 37 | * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) |
38 | * sb_lock (within inode_lock in fs/fs-writeback.c) | 38 | * sb_lock (within inode_lock in fs/fs-writeback.c) |
39 | * mapping->tree_lock (widely used, in set_page_dirty, | 39 | * i_pages lock (widely used, in set_page_dirty, |
40 | * in arch-dependent flush_dcache_mmap_lock, | 40 | * in arch-dependent flush_dcache_mmap_lock, |
41 | * within bdi.wb->list_lock in __sync_single_inode) | 41 | * within bdi.wb->list_lock in __sync_single_inode) |
42 | * | 42 | * |
diff --git a/mm/shmem.c b/mm/shmem.c index 4424fc0c33aa..9d6c7e595415 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
@@ -332,12 +332,12 @@ static int shmem_radix_tree_replace(struct address_space *mapping, | |||
332 | 332 | ||
333 | VM_BUG_ON(!expected); | 333 | VM_BUG_ON(!expected); |
334 | VM_BUG_ON(!replacement); | 334 | VM_BUG_ON(!replacement); |
335 | item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); | 335 | item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot); |
336 | if (!item) | 336 | if (!item) |
337 | return -ENOENT; | 337 | return -ENOENT; |
338 | if (item != expected) | 338 | if (item != expected) |
339 | return -ENOENT; | 339 | return -ENOENT; |
340 | __radix_tree_replace(&mapping->page_tree, node, pslot, | 340 | __radix_tree_replace(&mapping->i_pages, node, pslot, |
341 | replacement, NULL); | 341 | replacement, NULL); |
342 | return 0; | 342 | return 0; |
343 | } | 343 | } |
@@ -355,7 +355,7 @@ static bool shmem_confirm_swap(struct address_space *mapping, | |||
355 | void *item; | 355 | void *item; |
356 | 356 | ||
357 | rcu_read_lock(); | 357 | rcu_read_lock(); |
358 | item = radix_tree_lookup(&mapping->page_tree, index); | 358 | item = radix_tree_lookup(&mapping->i_pages, index); |
359 | rcu_read_unlock(); | 359 | rcu_read_unlock(); |
360 | return item == swp_to_radix_entry(swap); | 360 | return item == swp_to_radix_entry(swap); |
361 | } | 361 | } |
@@ -590,14 +590,14 @@ static int shmem_add_to_page_cache(struct page *page, | |||
590 | page->mapping = mapping; | 590 | page->mapping = mapping; |
591 | page->index = index; | 591 | page->index = index; |
592 | 592 | ||
593 | spin_lock_irq(&mapping->tree_lock); | 593 | xa_lock_irq(&mapping->i_pages); |
594 | if (PageTransHuge(page)) { | 594 | if (PageTransHuge(page)) { |
595 | void __rcu **results; | 595 | void __rcu **results; |
596 | pgoff_t idx; | 596 | pgoff_t idx; |
597 | int i; | 597 | int i; |
598 | 598 | ||
599 | error = 0; | 599 | error = 0; |
600 | if (radix_tree_gang_lookup_slot(&mapping->page_tree, | 600 | if (radix_tree_gang_lookup_slot(&mapping->i_pages, |
601 | &results, &idx, index, 1) && | 601 | &results, &idx, index, 1) && |
602 | idx < index + HPAGE_PMD_NR) { | 602 | idx < index + HPAGE_PMD_NR) { |
603 | error = -EEXIST; | 603 | error = -EEXIST; |
@@ -605,14 +605,14 @@ static int shmem_add_to_page_cache(struct page *page, | |||
605 | 605 | ||
606 | if (!error) { | 606 | if (!error) { |
607 | for (i = 0; i < HPAGE_PMD_NR; i++) { | 607 | for (i = 0; i < HPAGE_PMD_NR; i++) { |
608 | error = radix_tree_insert(&mapping->page_tree, | 608 | error = radix_tree_insert(&mapping->i_pages, |
609 | index + i, page + i); | 609 | index + i, page + i); |
610 | VM_BUG_ON(error); | 610 | VM_BUG_ON(error); |
611 | } | 611 | } |
612 | count_vm_event(THP_FILE_ALLOC); | 612 | count_vm_event(THP_FILE_ALLOC); |
613 | } | 613 | } |
614 | } else if (!expected) { | 614 | } else if (!expected) { |
615 | error = radix_tree_insert(&mapping->page_tree, index, page); | 615 | error = radix_tree_insert(&mapping->i_pages, index, page); |
616 | } else { | 616 | } else { |
617 | error = shmem_radix_tree_replace(mapping, index, expected, | 617 | error = shmem_radix_tree_replace(mapping, index, expected, |
618 | page); | 618 | page); |
@@ -624,10 +624,10 @@ static int shmem_add_to_page_cache(struct page *page, | |||
624 | __inc_node_page_state(page, NR_SHMEM_THPS); | 624 | __inc_node_page_state(page, NR_SHMEM_THPS); |
625 | __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); | 625 | __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); |
626 | __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); | 626 | __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); |
627 | spin_unlock_irq(&mapping->tree_lock); | 627 | xa_unlock_irq(&mapping->i_pages); |
628 | } else { | 628 | } else { |
629 | page->mapping = NULL; | 629 | page->mapping = NULL; |
630 | spin_unlock_irq(&mapping->tree_lock); | 630 | xa_unlock_irq(&mapping->i_pages); |
631 | page_ref_sub(page, nr); | 631 | page_ref_sub(page, nr); |
632 | } | 632 | } |
633 | return error; | 633 | return error; |
@@ -643,13 +643,13 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap) | |||
643 | 643 | ||
644 | VM_BUG_ON_PAGE(PageCompound(page), page); | 644 | VM_BUG_ON_PAGE(PageCompound(page), page); |
645 | 645 | ||
646 | spin_lock_irq(&mapping->tree_lock); | 646 | xa_lock_irq(&mapping->i_pages); |
647 | error = shmem_radix_tree_replace(mapping, page->index, page, radswap); | 647 | error = shmem_radix_tree_replace(mapping, page->index, page, radswap); |
648 | page->mapping = NULL; | 648 | page->mapping = NULL; |
649 | mapping->nrpages--; | 649 | mapping->nrpages--; |
650 | __dec_node_page_state(page, NR_FILE_PAGES); | 650 | __dec_node_page_state(page, NR_FILE_PAGES); |
651 | __dec_node_page_state(page, NR_SHMEM); | 651 | __dec_node_page_state(page, NR_SHMEM); |
652 | spin_unlock_irq(&mapping->tree_lock); | 652 | xa_unlock_irq(&mapping->i_pages); |
653 | put_page(page); | 653 | put_page(page); |
654 | BUG_ON(error); | 654 | BUG_ON(error); |
655 | } | 655 | } |
@@ -662,9 +662,9 @@ static int shmem_free_swap(struct address_space *mapping, | |||
662 | { | 662 | { |
663 | void *old; | 663 | void *old; |
664 | 664 | ||
665 | spin_lock_irq(&mapping->tree_lock); | 665 | xa_lock_irq(&mapping->i_pages); |
666 | old = radix_tree_delete_item(&mapping->page_tree, index, radswap); | 666 | old = radix_tree_delete_item(&mapping->i_pages, index, radswap); |
667 | spin_unlock_irq(&mapping->tree_lock); | 667 | xa_unlock_irq(&mapping->i_pages); |
668 | if (old != radswap) | 668 | if (old != radswap) |
669 | return -ENOENT; | 669 | return -ENOENT; |
670 | free_swap_and_cache(radix_to_swp_entry(radswap)); | 670 | free_swap_and_cache(radix_to_swp_entry(radswap)); |
@@ -675,7 +675,7 @@ static int shmem_free_swap(struct address_space *mapping, | |||
675 | * Determine (in bytes) how many of the shmem object's pages mapped by the | 675 | * Determine (in bytes) how many of the shmem object's pages mapped by the |
676 | * given offsets are swapped out. | 676 | * given offsets are swapped out. |
677 | * | 677 | * |
678 | * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, | 678 | * This is safe to call without i_mutex or the i_pages lock thanks to RCU, |
679 | * as long as the inode doesn't go away and racy results are not a problem. | 679 | * as long as the inode doesn't go away and racy results are not a problem. |
680 | */ | 680 | */ |
681 | unsigned long shmem_partial_swap_usage(struct address_space *mapping, | 681 | unsigned long shmem_partial_swap_usage(struct address_space *mapping, |
@@ -688,7 +688,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, | |||
688 | 688 | ||
689 | rcu_read_lock(); | 689 | rcu_read_lock(); |
690 | 690 | ||
691 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 691 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
692 | if (iter.index >= end) | 692 | if (iter.index >= end) |
693 | break; | 693 | break; |
694 | 694 | ||
@@ -717,7 +717,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, | |||
717 | * Determine (in bytes) how many of the shmem object's pages mapped by the | 717 | * Determine (in bytes) how many of the shmem object's pages mapped by the |
718 | * given vma is swapped out. | 718 | * given vma is swapped out. |
719 | * | 719 | * |
720 | * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, | 720 | * This is safe to call without i_mutex or the i_pages lock thanks to RCU, |
721 | * as long as the inode doesn't go away and racy results are not a problem. | 721 | * as long as the inode doesn't go away and racy results are not a problem. |
722 | */ | 722 | */ |
723 | unsigned long shmem_swap_usage(struct vm_area_struct *vma) | 723 | unsigned long shmem_swap_usage(struct vm_area_struct *vma) |
@@ -1132,7 +1132,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, | |||
1132 | int error = 0; | 1132 | int error = 0; |
1133 | 1133 | ||
1134 | radswap = swp_to_radix_entry(swap); | 1134 | radswap = swp_to_radix_entry(swap); |
1135 | index = find_swap_entry(&mapping->page_tree, radswap); | 1135 | index = find_swap_entry(&mapping->i_pages, radswap); |
1136 | if (index == -1) | 1136 | if (index == -1) |
1137 | return -EAGAIN; /* tell shmem_unuse we found nothing */ | 1137 | return -EAGAIN; /* tell shmem_unuse we found nothing */ |
1138 | 1138 | ||
@@ -1448,7 +1448,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, | |||
1448 | 1448 | ||
1449 | hindex = round_down(index, HPAGE_PMD_NR); | 1449 | hindex = round_down(index, HPAGE_PMD_NR); |
1450 | rcu_read_lock(); | 1450 | rcu_read_lock(); |
1451 | if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, | 1451 | if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx, |
1452 | hindex, 1) && idx < hindex + HPAGE_PMD_NR) { | 1452 | hindex, 1) && idx < hindex + HPAGE_PMD_NR) { |
1453 | rcu_read_unlock(); | 1453 | rcu_read_unlock(); |
1454 | return NULL; | 1454 | return NULL; |
@@ -1561,14 +1561,14 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp, | |||
1561 | * Our caller will very soon move newpage out of swapcache, but it's | 1561 | * Our caller will very soon move newpage out of swapcache, but it's |
1562 | * a nice clean interface for us to replace oldpage by newpage there. | 1562 | * a nice clean interface for us to replace oldpage by newpage there. |
1563 | */ | 1563 | */ |
1564 | spin_lock_irq(&swap_mapping->tree_lock); | 1564 | xa_lock_irq(&swap_mapping->i_pages); |
1565 | error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, | 1565 | error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, |
1566 | newpage); | 1566 | newpage); |
1567 | if (!error) { | 1567 | if (!error) { |
1568 | __inc_node_page_state(newpage, NR_FILE_PAGES); | 1568 | __inc_node_page_state(newpage, NR_FILE_PAGES); |
1569 | __dec_node_page_state(oldpage, NR_FILE_PAGES); | 1569 | __dec_node_page_state(oldpage, NR_FILE_PAGES); |
1570 | } | 1570 | } |
1571 | spin_unlock_irq(&swap_mapping->tree_lock); | 1571 | xa_unlock_irq(&swap_mapping->i_pages); |
1572 | 1572 | ||
1573 | if (unlikely(error)) { | 1573 | if (unlikely(error)) { |
1574 | /* | 1574 | /* |
@@ -2634,7 +2634,7 @@ static void shmem_tag_pins(struct address_space *mapping) | |||
2634 | start = 0; | 2634 | start = 0; |
2635 | rcu_read_lock(); | 2635 | rcu_read_lock(); |
2636 | 2636 | ||
2637 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 2637 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
2638 | page = radix_tree_deref_slot(slot); | 2638 | page = radix_tree_deref_slot(slot); |
2639 | if (!page || radix_tree_exception(page)) { | 2639 | if (!page || radix_tree_exception(page)) { |
2640 | if (radix_tree_deref_retry(page)) { | 2640 | if (radix_tree_deref_retry(page)) { |
@@ -2642,10 +2642,10 @@ static void shmem_tag_pins(struct address_space *mapping) | |||
2642 | continue; | 2642 | continue; |
2643 | } | 2643 | } |
2644 | } else if (page_count(page) - page_mapcount(page) > 1) { | 2644 | } else if (page_count(page) - page_mapcount(page) > 1) { |
2645 | spin_lock_irq(&mapping->tree_lock); | 2645 | xa_lock_irq(&mapping->i_pages); |
2646 | radix_tree_tag_set(&mapping->page_tree, iter.index, | 2646 | radix_tree_tag_set(&mapping->i_pages, iter.index, |
2647 | SHMEM_TAG_PINNED); | 2647 | SHMEM_TAG_PINNED); |
2648 | spin_unlock_irq(&mapping->tree_lock); | 2648 | xa_unlock_irq(&mapping->i_pages); |
2649 | } | 2649 | } |
2650 | 2650 | ||
2651 | if (need_resched()) { | 2651 | if (need_resched()) { |
@@ -2677,7 +2677,7 @@ static int shmem_wait_for_pins(struct address_space *mapping) | |||
2677 | 2677 | ||
2678 | error = 0; | 2678 | error = 0; |
2679 | for (scan = 0; scan <= LAST_SCAN; scan++) { | 2679 | for (scan = 0; scan <= LAST_SCAN; scan++) { |
2680 | if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) | 2680 | if (!radix_tree_tagged(&mapping->i_pages, SHMEM_TAG_PINNED)) |
2681 | break; | 2681 | break; |
2682 | 2682 | ||
2683 | if (!scan) | 2683 | if (!scan) |
@@ -2687,7 +2687,7 @@ static int shmem_wait_for_pins(struct address_space *mapping) | |||
2687 | 2687 | ||
2688 | start = 0; | 2688 | start = 0; |
2689 | rcu_read_lock(); | 2689 | rcu_read_lock(); |
2690 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, | 2690 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, |
2691 | start, SHMEM_TAG_PINNED) { | 2691 | start, SHMEM_TAG_PINNED) { |
2692 | 2692 | ||
2693 | page = radix_tree_deref_slot(slot); | 2693 | page = radix_tree_deref_slot(slot); |
@@ -2713,10 +2713,10 @@ static int shmem_wait_for_pins(struct address_space *mapping) | |||
2713 | error = -EBUSY; | 2713 | error = -EBUSY; |
2714 | } | 2714 | } |
2715 | 2715 | ||
2716 | spin_lock_irq(&mapping->tree_lock); | 2716 | xa_lock_irq(&mapping->i_pages); |
2717 | radix_tree_tag_clear(&mapping->page_tree, | 2717 | radix_tree_tag_clear(&mapping->i_pages, |
2718 | iter.index, SHMEM_TAG_PINNED); | 2718 | iter.index, SHMEM_TAG_PINNED); |
2719 | spin_unlock_irq(&mapping->tree_lock); | 2719 | xa_unlock_irq(&mapping->i_pages); |
2720 | continue_resched: | 2720 | continue_resched: |
2721 | if (need_resched()) { | 2721 | if (need_resched()) { |
2722 | slot = radix_tree_iter_resume(slot, &iter); | 2722 | slot = radix_tree_iter_resume(slot, &iter); |
diff --git a/mm/swap_state.c b/mm/swap_state.c index f233dccd3b1b..07f9aa2340c3 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c | |||
@@ -124,10 +124,10 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) | |||
124 | SetPageSwapCache(page); | 124 | SetPageSwapCache(page); |
125 | 125 | ||
126 | address_space = swap_address_space(entry); | 126 | address_space = swap_address_space(entry); |
127 | spin_lock_irq(&address_space->tree_lock); | 127 | xa_lock_irq(&address_space->i_pages); |
128 | for (i = 0; i < nr; i++) { | 128 | for (i = 0; i < nr; i++) { |
129 | set_page_private(page + i, entry.val + i); | 129 | set_page_private(page + i, entry.val + i); |
130 | error = radix_tree_insert(&address_space->page_tree, | 130 | error = radix_tree_insert(&address_space->i_pages, |
131 | idx + i, page + i); | 131 | idx + i, page + i); |
132 | if (unlikely(error)) | 132 | if (unlikely(error)) |
133 | break; | 133 | break; |
@@ -145,13 +145,13 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) | |||
145 | VM_BUG_ON(error == -EEXIST); | 145 | VM_BUG_ON(error == -EEXIST); |
146 | set_page_private(page + i, 0UL); | 146 | set_page_private(page + i, 0UL); |
147 | while (i--) { | 147 | while (i--) { |
148 | radix_tree_delete(&address_space->page_tree, idx + i); | 148 | radix_tree_delete(&address_space->i_pages, idx + i); |
149 | set_page_private(page + i, 0UL); | 149 | set_page_private(page + i, 0UL); |
150 | } | 150 | } |
151 | ClearPageSwapCache(page); | 151 | ClearPageSwapCache(page); |
152 | page_ref_sub(page, nr); | 152 | page_ref_sub(page, nr); |
153 | } | 153 | } |
154 | spin_unlock_irq(&address_space->tree_lock); | 154 | xa_unlock_irq(&address_space->i_pages); |
155 | 155 | ||
156 | return error; | 156 | return error; |
157 | } | 157 | } |
@@ -188,7 +188,7 @@ void __delete_from_swap_cache(struct page *page) | |||
188 | address_space = swap_address_space(entry); | 188 | address_space = swap_address_space(entry); |
189 | idx = swp_offset(entry); | 189 | idx = swp_offset(entry); |
190 | for (i = 0; i < nr; i++) { | 190 | for (i = 0; i < nr; i++) { |
191 | radix_tree_delete(&address_space->page_tree, idx + i); | 191 | radix_tree_delete(&address_space->i_pages, idx + i); |
192 | set_page_private(page + i, 0); | 192 | set_page_private(page + i, 0); |
193 | } | 193 | } |
194 | ClearPageSwapCache(page); | 194 | ClearPageSwapCache(page); |
@@ -272,9 +272,9 @@ void delete_from_swap_cache(struct page *page) | |||
272 | entry.val = page_private(page); | 272 | entry.val = page_private(page); |
273 | 273 | ||
274 | address_space = swap_address_space(entry); | 274 | address_space = swap_address_space(entry); |
275 | spin_lock_irq(&address_space->tree_lock); | 275 | xa_lock_irq(&address_space->i_pages); |
276 | __delete_from_swap_cache(page); | 276 | __delete_from_swap_cache(page); |
277 | spin_unlock_irq(&address_space->tree_lock); | 277 | xa_unlock_irq(&address_space->i_pages); |
278 | 278 | ||
279 | put_swap_page(page, entry); | 279 | put_swap_page(page, entry); |
280 | page_ref_sub(page, hpage_nr_pages(page)); | 280 | page_ref_sub(page, hpage_nr_pages(page)); |
@@ -628,12 +628,11 @@ int init_swap_address_space(unsigned int type, unsigned long nr_pages) | |||
628 | return -ENOMEM; | 628 | return -ENOMEM; |
629 | for (i = 0; i < nr; i++) { | 629 | for (i = 0; i < nr; i++) { |
630 | space = spaces + i; | 630 | space = spaces + i; |
631 | INIT_RADIX_TREE(&space->page_tree, GFP_ATOMIC|__GFP_NOWARN); | 631 | INIT_RADIX_TREE(&space->i_pages, GFP_ATOMIC|__GFP_NOWARN); |
632 | atomic_set(&space->i_mmap_writable, 0); | 632 | atomic_set(&space->i_mmap_writable, 0); |
633 | space->a_ops = &swap_aops; | 633 | space->a_ops = &swap_aops; |
634 | /* swap cache doesn't use writeback related tags */ | 634 | /* swap cache doesn't use writeback related tags */ |
635 | mapping_set_no_writeback_tags(space); | 635 | mapping_set_no_writeback_tags(space); |
636 | spin_lock_init(&space->tree_lock); | ||
637 | } | 636 | } |
638 | nr_swapper_spaces[type] = nr; | 637 | nr_swapper_spaces[type] = nr; |
639 | rcu_assign_pointer(swapper_spaces[type], spaces); | 638 | rcu_assign_pointer(swapper_spaces[type], spaces); |
diff --git a/mm/truncate.c b/mm/truncate.c index c34e2fd4f583..1d2fb2dca96f 100644 --- a/mm/truncate.c +++ b/mm/truncate.c | |||
@@ -36,11 +36,11 @@ static inline void __clear_shadow_entry(struct address_space *mapping, | |||
36 | struct radix_tree_node *node; | 36 | struct radix_tree_node *node; |
37 | void **slot; | 37 | void **slot; |
38 | 38 | ||
39 | if (!__radix_tree_lookup(&mapping->page_tree, index, &node, &slot)) | 39 | if (!__radix_tree_lookup(&mapping->i_pages, index, &node, &slot)) |
40 | return; | 40 | return; |
41 | if (*slot != entry) | 41 | if (*slot != entry) |
42 | return; | 42 | return; |
43 | __radix_tree_replace(&mapping->page_tree, node, slot, NULL, | 43 | __radix_tree_replace(&mapping->i_pages, node, slot, NULL, |
44 | workingset_update_node); | 44 | workingset_update_node); |
45 | mapping->nrexceptional--; | 45 | mapping->nrexceptional--; |
46 | } | 46 | } |
@@ -48,9 +48,9 @@ static inline void __clear_shadow_entry(struct address_space *mapping, | |||
48 | static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, | 48 | static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, |
49 | void *entry) | 49 | void *entry) |
50 | { | 50 | { |
51 | spin_lock_irq(&mapping->tree_lock); | 51 | xa_lock_irq(&mapping->i_pages); |
52 | __clear_shadow_entry(mapping, index, entry); | 52 | __clear_shadow_entry(mapping, index, entry); |
53 | spin_unlock_irq(&mapping->tree_lock); | 53 | xa_unlock_irq(&mapping->i_pages); |
54 | } | 54 | } |
55 | 55 | ||
56 | /* | 56 | /* |
@@ -79,7 +79,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, | |||
79 | dax = dax_mapping(mapping); | 79 | dax = dax_mapping(mapping); |
80 | lock = !dax && indices[j] < end; | 80 | lock = !dax && indices[j] < end; |
81 | if (lock) | 81 | if (lock) |
82 | spin_lock_irq(&mapping->tree_lock); | 82 | xa_lock_irq(&mapping->i_pages); |
83 | 83 | ||
84 | for (i = j; i < pagevec_count(pvec); i++) { | 84 | for (i = j; i < pagevec_count(pvec); i++) { |
85 | struct page *page = pvec->pages[i]; | 85 | struct page *page = pvec->pages[i]; |
@@ -102,7 +102,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, | |||
102 | } | 102 | } |
103 | 103 | ||
104 | if (lock) | 104 | if (lock) |
105 | spin_unlock_irq(&mapping->tree_lock); | 105 | xa_unlock_irq(&mapping->i_pages); |
106 | pvec->nr = j; | 106 | pvec->nr = j; |
107 | } | 107 | } |
108 | 108 | ||
@@ -518,8 +518,8 @@ void truncate_inode_pages_final(struct address_space *mapping) | |||
518 | * modification that does not see AS_EXITING is | 518 | * modification that does not see AS_EXITING is |
519 | * completed before starting the final truncate. | 519 | * completed before starting the final truncate. |
520 | */ | 520 | */ |
521 | spin_lock_irq(&mapping->tree_lock); | 521 | xa_lock_irq(&mapping->i_pages); |
522 | spin_unlock_irq(&mapping->tree_lock); | 522 | xa_unlock_irq(&mapping->i_pages); |
523 | 523 | ||
524 | truncate_inode_pages(mapping, 0); | 524 | truncate_inode_pages(mapping, 0); |
525 | } | 525 | } |
@@ -627,13 +627,13 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page) | |||
627 | if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) | 627 | if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) |
628 | return 0; | 628 | return 0; |
629 | 629 | ||
630 | spin_lock_irqsave(&mapping->tree_lock, flags); | 630 | xa_lock_irqsave(&mapping->i_pages, flags); |
631 | if (PageDirty(page)) | 631 | if (PageDirty(page)) |
632 | goto failed; | 632 | goto failed; |
633 | 633 | ||
634 | BUG_ON(page_has_private(page)); | 634 | BUG_ON(page_has_private(page)); |
635 | __delete_from_page_cache(page, NULL); | 635 | __delete_from_page_cache(page, NULL); |
636 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 636 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
637 | 637 | ||
638 | if (mapping->a_ops->freepage) | 638 | if (mapping->a_ops->freepage) |
639 | mapping->a_ops->freepage(page); | 639 | mapping->a_ops->freepage(page); |
@@ -641,7 +641,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page) | |||
641 | put_page(page); /* pagecache ref */ | 641 | put_page(page); /* pagecache ref */ |
642 | return 1; | 642 | return 1; |
643 | failed: | 643 | failed: |
644 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 644 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
645 | return 0; | 645 | return 0; |
646 | } | 646 | } |
647 | 647 | ||
diff --git a/mm/vmscan.c b/mm/vmscan.c index 671597ce1ea0..8b920ce3ae02 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
@@ -693,7 +693,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
693 | BUG_ON(!PageLocked(page)); | 693 | BUG_ON(!PageLocked(page)); |
694 | BUG_ON(mapping != page_mapping(page)); | 694 | BUG_ON(mapping != page_mapping(page)); |
695 | 695 | ||
696 | spin_lock_irqsave(&mapping->tree_lock, flags); | 696 | xa_lock_irqsave(&mapping->i_pages, flags); |
697 | /* | 697 | /* |
698 | * The non racy check for a busy page. | 698 | * The non racy check for a busy page. |
699 | * | 699 | * |
@@ -717,7 +717,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
717 | * load is not satisfied before that of page->_refcount. | 717 | * load is not satisfied before that of page->_refcount. |
718 | * | 718 | * |
719 | * Note that if SetPageDirty is always performed via set_page_dirty, | 719 | * Note that if SetPageDirty is always performed via set_page_dirty, |
720 | * and thus under tree_lock, then this ordering is not required. | 720 | * and thus under the i_pages lock, then this ordering is not required. |
721 | */ | 721 | */ |
722 | if (unlikely(PageTransHuge(page)) && PageSwapCache(page)) | 722 | if (unlikely(PageTransHuge(page)) && PageSwapCache(page)) |
723 | refcount = 1 + HPAGE_PMD_NR; | 723 | refcount = 1 + HPAGE_PMD_NR; |
@@ -735,7 +735,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
735 | swp_entry_t swap = { .val = page_private(page) }; | 735 | swp_entry_t swap = { .val = page_private(page) }; |
736 | mem_cgroup_swapout(page, swap); | 736 | mem_cgroup_swapout(page, swap); |
737 | __delete_from_swap_cache(page); | 737 | __delete_from_swap_cache(page); |
738 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 738 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
739 | put_swap_page(page, swap); | 739 | put_swap_page(page, swap); |
740 | } else { | 740 | } else { |
741 | void (*freepage)(struct page *); | 741 | void (*freepage)(struct page *); |
@@ -756,13 +756,13 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
756 | * only page cache pages found in these are zero pages | 756 | * only page cache pages found in these are zero pages |
757 | * covering holes, and because we don't want to mix DAX | 757 | * covering holes, and because we don't want to mix DAX |
758 | * exceptional entries and shadow exceptional entries in the | 758 | * exceptional entries and shadow exceptional entries in the |
759 | * same page_tree. | 759 | * same address_space. |
760 | */ | 760 | */ |
761 | if (reclaimed && page_is_file_cache(page) && | 761 | if (reclaimed && page_is_file_cache(page) && |
762 | !mapping_exiting(mapping) && !dax_mapping(mapping)) | 762 | !mapping_exiting(mapping) && !dax_mapping(mapping)) |
763 | shadow = workingset_eviction(mapping, page); | 763 | shadow = workingset_eviction(mapping, page); |
764 | __delete_from_page_cache(page, shadow); | 764 | __delete_from_page_cache(page, shadow); |
765 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 765 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
766 | 766 | ||
767 | if (freepage != NULL) | 767 | if (freepage != NULL) |
768 | freepage(page); | 768 | freepage(page); |
@@ -771,7 +771,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
771 | return 1; | 771 | return 1; |
772 | 772 | ||
773 | cannot_free: | 773 | cannot_free: |
774 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 774 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
775 | return 0; | 775 | return 0; |
776 | } | 776 | } |
777 | 777 | ||
diff --git a/mm/workingset.c b/mm/workingset.c index b7d616a3bbbe..40ee02c83978 100644 --- a/mm/workingset.c +++ b/mm/workingset.c | |||
@@ -202,7 +202,7 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, | |||
202 | * @mapping: address space the page was backing | 202 | * @mapping: address space the page was backing |
203 | * @page: the page being evicted | 203 | * @page: the page being evicted |
204 | * | 204 | * |
205 | * Returns a shadow entry to be stored in @mapping->page_tree in place | 205 | * Returns a shadow entry to be stored in @mapping->i_pages in place |
206 | * of the evicted @page so that a later refault can be detected. | 206 | * of the evicted @page so that a later refault can be detected. |
207 | */ | 207 | */ |
208 | void *workingset_eviction(struct address_space *mapping, struct page *page) | 208 | void *workingset_eviction(struct address_space *mapping, struct page *page) |
@@ -348,7 +348,7 @@ void workingset_update_node(struct radix_tree_node *node) | |||
348 | * | 348 | * |
349 | * Avoid acquiring the list_lru lock when the nodes are | 349 | * Avoid acquiring the list_lru lock when the nodes are |
350 | * already where they should be. The list_empty() test is safe | 350 | * already where they should be. The list_empty() test is safe |
351 | * as node->private_list is protected by &mapping->tree_lock. | 351 | * as node->private_list is protected by the i_pages lock. |
352 | */ | 352 | */ |
353 | if (node->count && node->count == node->exceptional) { | 353 | if (node->count && node->count == node->exceptional) { |
354 | if (list_empty(&node->private_list)) | 354 | if (list_empty(&node->private_list)) |
@@ -366,7 +366,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, | |||
366 | unsigned long nodes; | 366 | unsigned long nodes; |
367 | unsigned long cache; | 367 | unsigned long cache; |
368 | 368 | ||
369 | /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ | 369 | /* list_lru lock nests inside the IRQ-safe i_pages lock */ |
370 | local_irq_disable(); | 370 | local_irq_disable(); |
371 | nodes = list_lru_shrink_count(&shadow_nodes, sc); | 371 | nodes = list_lru_shrink_count(&shadow_nodes, sc); |
372 | local_irq_enable(); | 372 | local_irq_enable(); |
@@ -419,21 +419,21 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, | |||
419 | 419 | ||
420 | /* | 420 | /* |
421 | * Page cache insertions and deletions synchroneously maintain | 421 | * Page cache insertions and deletions synchroneously maintain |
422 | * the shadow node LRU under the mapping->tree_lock and the | 422 | * the shadow node LRU under the i_pages lock and the |
423 | * lru_lock. Because the page cache tree is emptied before | 423 | * lru_lock. Because the page cache tree is emptied before |
424 | * the inode can be destroyed, holding the lru_lock pins any | 424 | * the inode can be destroyed, holding the lru_lock pins any |
425 | * address_space that has radix tree nodes on the LRU. | 425 | * address_space that has radix tree nodes on the LRU. |
426 | * | 426 | * |
427 | * We can then safely transition to the mapping->tree_lock to | 427 | * We can then safely transition to the i_pages lock to |
428 | * pin only the address_space of the particular node we want | 428 | * pin only the address_space of the particular node we want |
429 | * to reclaim, take the node off-LRU, and drop the lru_lock. | 429 | * to reclaim, take the node off-LRU, and drop the lru_lock. |
430 | */ | 430 | */ |
431 | 431 | ||
432 | node = container_of(item, struct radix_tree_node, private_list); | 432 | node = container_of(item, struct radix_tree_node, private_list); |
433 | mapping = container_of(node->root, struct address_space, page_tree); | 433 | mapping = container_of(node->root, struct address_space, i_pages); |
434 | 434 | ||
435 | /* Coming from the list, invert the lock order */ | 435 | /* Coming from the list, invert the lock order */ |
436 | if (!spin_trylock(&mapping->tree_lock)) { | 436 | if (!xa_trylock(&mapping->i_pages)) { |
437 | spin_unlock(lru_lock); | 437 | spin_unlock(lru_lock); |
438 | ret = LRU_RETRY; | 438 | ret = LRU_RETRY; |
439 | goto out; | 439 | goto out; |
@@ -468,11 +468,11 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, | |||
468 | if (WARN_ON_ONCE(node->exceptional)) | 468 | if (WARN_ON_ONCE(node->exceptional)) |
469 | goto out_invalid; | 469 | goto out_invalid; |
470 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); | 470 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); |
471 | __radix_tree_delete_node(&mapping->page_tree, node, | 471 | __radix_tree_delete_node(&mapping->i_pages, node, |
472 | workingset_lookup_update(mapping)); | 472 | workingset_lookup_update(mapping)); |
473 | 473 | ||
474 | out_invalid: | 474 | out_invalid: |
475 | spin_unlock(&mapping->tree_lock); | 475 | xa_unlock(&mapping->i_pages); |
476 | ret = LRU_REMOVED_RETRY; | 476 | ret = LRU_REMOVED_RETRY; |
477 | out: | 477 | out: |
478 | local_irq_enable(); | 478 | local_irq_enable(); |
@@ -487,7 +487,7 @@ static unsigned long scan_shadow_nodes(struct shrinker *shrinker, | |||
487 | { | 487 | { |
488 | unsigned long ret; | 488 | unsigned long ret; |
489 | 489 | ||
490 | /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ | 490 | /* list_lru lock nests inside the IRQ-safe i_pages lock */ |
491 | local_irq_disable(); | 491 | local_irq_disable(); |
492 | ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); | 492 | ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); |
493 | local_irq_enable(); | 493 | local_irq_enable(); |
@@ -503,7 +503,7 @@ static struct shrinker workingset_shadow_shrinker = { | |||
503 | 503 | ||
504 | /* | 504 | /* |
505 | * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe | 505 | * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe |
506 | * mapping->tree_lock. | 506 | * i_pages lock. |
507 | */ | 507 | */ |
508 | static struct lock_class_key shadow_nodes_key; | 508 | static struct lock_class_key shadow_nodes_key; |
509 | 509 | ||