diff options
39 files changed, 345 insertions, 366 deletions
diff --git a/Documentation/cgroup-v1/memory.txt b/Documentation/cgroup-v1/memory.txt index a4af2e124e24..3682e99234c2 100644 --- a/Documentation/cgroup-v1/memory.txt +++ b/Documentation/cgroup-v1/memory.txt | |||
| @@ -262,7 +262,7 @@ When oom event notifier is registered, event will be delivered. | |||
| 262 | 2.6 Locking | 262 | 2.6 Locking |
| 263 | 263 | ||
| 264 | lock_page_cgroup()/unlock_page_cgroup() should not be called under | 264 | lock_page_cgroup()/unlock_page_cgroup() should not be called under |
| 265 | mapping->tree_lock. | 265 | the i_pages lock. |
| 266 | 266 | ||
| 267 | Other lock order is following: | 267 | Other lock order is following: |
| 268 | PG_locked. | 268 | PG_locked. |
diff --git a/Documentation/vm/page_migration b/Documentation/vm/page_migration index 0478ae2ad44a..496868072e24 100644 --- a/Documentation/vm/page_migration +++ b/Documentation/vm/page_migration | |||
| @@ -90,7 +90,7 @@ Steps: | |||
| 90 | 90 | ||
| 91 | 1. Lock the page to be migrated | 91 | 1. Lock the page to be migrated |
| 92 | 92 | ||
| 93 | 2. Insure that writeback is complete. | 93 | 2. Ensure that writeback is complete. |
| 94 | 94 | ||
| 95 | 3. Lock the new page that we want to move to. It is locked so that accesses to | 95 | 3. Lock the new page that we want to move to. It is locked so that accesses to |
| 96 | this (not yet uptodate) page immediately lock while the move is in progress. | 96 | this (not yet uptodate) page immediately lock while the move is in progress. |
| @@ -100,8 +100,8 @@ Steps: | |||
| 100 | mapcount is not zero then we do not migrate the page. All user space | 100 | mapcount is not zero then we do not migrate the page. All user space |
| 101 | processes that attempt to access the page will now wait on the page lock. | 101 | processes that attempt to access the page will now wait on the page lock. |
| 102 | 102 | ||
| 103 | 5. The radix tree lock is taken. This will cause all processes trying | 103 | 5. The i_pages lock is taken. This will cause all processes trying |
| 104 | to access the page via the mapping to block on the radix tree spinlock. | 104 | to access the page via the mapping to block on the spinlock. |
| 105 | 105 | ||
| 106 | 6. The refcount of the page is examined and we back out if references remain | 106 | 6. The refcount of the page is examined and we back out if references remain |
| 107 | otherwise we know that we are the only one referencing this page. | 107 | otherwise we know that we are the only one referencing this page. |
| @@ -114,12 +114,12 @@ Steps: | |||
| 114 | 114 | ||
| 115 | 9. The radix tree is changed to point to the new page. | 115 | 9. The radix tree is changed to point to the new page. |
| 116 | 116 | ||
| 117 | 10. The reference count of the old page is dropped because the radix tree | 117 | 10. The reference count of the old page is dropped because the address space |
| 118 | reference is gone. A reference to the new page is established because | 118 | reference is gone. A reference to the new page is established because |
| 119 | the new page is referenced to by the radix tree. | 119 | the new page is referenced by the address space. |
| 120 | 120 | ||
| 121 | 11. The radix tree lock is dropped. With that lookups in the mapping | 121 | 11. The i_pages lock is dropped. With that lookups in the mapping |
| 122 | become possible again. Processes will move from spinning on the tree_lock | 122 | become possible again. Processes will move from spinning on the lock |
| 123 | to sleeping on the locked new page. | 123 | to sleeping on the locked new page. |
| 124 | 124 | ||
| 125 | 12. The page contents are copied to the new page. | 125 | 12. The page contents are copied to the new page. |
diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 74504b154256..869080bedb89 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h | |||
| @@ -318,10 +318,8 @@ static inline void flush_anon_page(struct vm_area_struct *vma, | |||
| 318 | #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE | 318 | #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE |
| 319 | extern void flush_kernel_dcache_page(struct page *); | 319 | extern void flush_kernel_dcache_page(struct page *); |
| 320 | 320 | ||
| 321 | #define flush_dcache_mmap_lock(mapping) \ | 321 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) |
| 322 | spin_lock_irq(&(mapping)->tree_lock) | 322 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) |
| 323 | #define flush_dcache_mmap_unlock(mapping) \ | ||
| 324 | spin_unlock_irq(&(mapping)->tree_lock) | ||
| 325 | 323 | ||
| 326 | #define flush_icache_user_range(vma,page,addr,len) \ | 324 | #define flush_icache_user_range(vma,page,addr,len) \ |
| 327 | flush_dcache_page(page) | 325 | flush_dcache_page(page) |
diff --git a/arch/nds32/include/asm/cacheflush.h b/arch/nds32/include/asm/cacheflush.h index 7b9b20a381cb..1240f148ec0f 100644 --- a/arch/nds32/include/asm/cacheflush.h +++ b/arch/nds32/include/asm/cacheflush.h | |||
| @@ -34,8 +34,8 @@ void flush_anon_page(struct vm_area_struct *vma, | |||
| 34 | void flush_kernel_dcache_page(struct page *page); | 34 | void flush_kernel_dcache_page(struct page *page); |
| 35 | void flush_icache_range(unsigned long start, unsigned long end); | 35 | void flush_icache_range(unsigned long start, unsigned long end); |
| 36 | void flush_icache_page(struct vm_area_struct *vma, struct page *page); | 36 | void flush_icache_page(struct vm_area_struct *vma, struct page *page); |
| 37 | #define flush_dcache_mmap_lock(mapping) spin_lock_irq(&(mapping)->tree_lock) | 37 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&(mapping)->i_pages) |
| 38 | #define flush_dcache_mmap_unlock(mapping) spin_unlock_irq(&(mapping)->tree_lock) | 38 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&(mapping)->i_pages) |
| 39 | 39 | ||
| 40 | #else | 40 | #else |
| 41 | #include <asm-generic/cacheflush.h> | 41 | #include <asm-generic/cacheflush.h> |
diff --git a/arch/nios2/include/asm/cacheflush.h b/arch/nios2/include/asm/cacheflush.h index 55e383c173f7..18eb9f69f806 100644 --- a/arch/nios2/include/asm/cacheflush.h +++ b/arch/nios2/include/asm/cacheflush.h | |||
| @@ -46,9 +46,7 @@ extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page, | |||
| 46 | extern void flush_dcache_range(unsigned long start, unsigned long end); | 46 | extern void flush_dcache_range(unsigned long start, unsigned long end); |
| 47 | extern void invalidate_dcache_range(unsigned long start, unsigned long end); | 47 | extern void invalidate_dcache_range(unsigned long start, unsigned long end); |
| 48 | 48 | ||
| 49 | #define flush_dcache_mmap_lock(mapping) \ | 49 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) |
| 50 | spin_lock_irq(&(mapping)->tree_lock) | 50 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) |
| 51 | #define flush_dcache_mmap_unlock(mapping) \ | ||
| 52 | spin_unlock_irq(&(mapping)->tree_lock) | ||
| 53 | 51 | ||
| 54 | #endif /* _ASM_NIOS2_CACHEFLUSH_H */ | 52 | #endif /* _ASM_NIOS2_CACHEFLUSH_H */ |
diff --git a/arch/parisc/include/asm/cacheflush.h b/arch/parisc/include/asm/cacheflush.h index bd5ce31936f5..0c83644bfa5c 100644 --- a/arch/parisc/include/asm/cacheflush.h +++ b/arch/parisc/include/asm/cacheflush.h | |||
| @@ -55,10 +55,8 @@ void invalidate_kernel_vmap_range(void *vaddr, int size); | |||
| 55 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 | 55 | #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 |
| 56 | extern void flush_dcache_page(struct page *page); | 56 | extern void flush_dcache_page(struct page *page); |
| 57 | 57 | ||
| 58 | #define flush_dcache_mmap_lock(mapping) \ | 58 | #define flush_dcache_mmap_lock(mapping) xa_lock_irq(&mapping->i_pages) |
| 59 | spin_lock_irq(&(mapping)->tree_lock) | 59 | #define flush_dcache_mmap_unlock(mapping) xa_unlock_irq(&mapping->i_pages) |
| 60 | #define flush_dcache_mmap_unlock(mapping) \ | ||
| 61 | spin_unlock_irq(&(mapping)->tree_lock) | ||
| 62 | 60 | ||
| 63 | #define flush_icache_page(vma,page) do { \ | 61 | #define flush_icache_page(vma,page) do { \ |
| 64 | flush_kernel_dcache_page(page); \ | 62 | flush_kernel_dcache_page(page); \ |
diff --git a/drivers/staging/lustre/lustre/llite/glimpse.c b/drivers/staging/lustre/lustre/llite/glimpse.c index c43ac574274c..3075358f3f08 100644 --- a/drivers/staging/lustre/lustre/llite/glimpse.c +++ b/drivers/staging/lustre/lustre/llite/glimpse.c | |||
| @@ -69,7 +69,7 @@ blkcnt_t dirty_cnt(struct inode *inode) | |||
| 69 | void *results[1]; | 69 | void *results[1]; |
| 70 | 70 | ||
| 71 | if (inode->i_mapping) | 71 | if (inode->i_mapping) |
| 72 | cnt += radix_tree_gang_lookup_tag(&inode->i_mapping->page_tree, | 72 | cnt += radix_tree_gang_lookup_tag(&inode->i_mapping->i_pages, |
| 73 | results, 0, 1, | 73 | results, 0, 1, |
| 74 | PAGECACHE_TAG_DIRTY); | 74 | PAGECACHE_TAG_DIRTY); |
| 75 | if (cnt == 0 && atomic_read(&vob->vob_mmap_cnt) > 0) | 75 | if (cnt == 0 && atomic_read(&vob->vob_mmap_cnt) > 0) |
diff --git a/drivers/staging/lustre/lustre/mdc/mdc_request.c b/drivers/staging/lustre/lustre/mdc/mdc_request.c index 3b1c8e5a3053..8ee7b4d273b2 100644 --- a/drivers/staging/lustre/lustre/mdc/mdc_request.c +++ b/drivers/staging/lustre/lustre/mdc/mdc_request.c | |||
| @@ -934,14 +934,14 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, | |||
| 934 | struct page *page; | 934 | struct page *page; |
| 935 | int found; | 935 | int found; |
| 936 | 936 | ||
| 937 | spin_lock_irq(&mapping->tree_lock); | 937 | xa_lock_irq(&mapping->i_pages); |
| 938 | found = radix_tree_gang_lookup(&mapping->page_tree, | 938 | found = radix_tree_gang_lookup(&mapping->i_pages, |
| 939 | (void **)&page, offset, 1); | 939 | (void **)&page, offset, 1); |
| 940 | if (found > 0 && !radix_tree_exceptional_entry(page)) { | 940 | if (found > 0 && !radix_tree_exceptional_entry(page)) { |
| 941 | struct lu_dirpage *dp; | 941 | struct lu_dirpage *dp; |
| 942 | 942 | ||
| 943 | get_page(page); | 943 | get_page(page); |
| 944 | spin_unlock_irq(&mapping->tree_lock); | 944 | xa_unlock_irq(&mapping->i_pages); |
| 945 | /* | 945 | /* |
| 946 | * In contrast to find_lock_page() we are sure that directory | 946 | * In contrast to find_lock_page() we are sure that directory |
| 947 | * page cannot be truncated (while DLM lock is held) and, | 947 | * page cannot be truncated (while DLM lock is held) and, |
| @@ -989,7 +989,7 @@ static struct page *mdc_page_locate(struct address_space *mapping, __u64 *hash, | |||
| 989 | page = ERR_PTR(-EIO); | 989 | page = ERR_PTR(-EIO); |
| 990 | } | 990 | } |
| 991 | } else { | 991 | } else { |
| 992 | spin_unlock_irq(&mapping->tree_lock); | 992 | xa_unlock_irq(&mapping->i_pages); |
| 993 | page = NULL; | 993 | page = NULL; |
| 994 | } | 994 | } |
| 995 | return page; | 995 | return page; |
diff --git a/fs/afs/write.c b/fs/afs/write.c index 9370e2feb999..dbc3c0b0142d 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c | |||
| @@ -570,10 +570,11 @@ static int afs_writepages_region(struct address_space *mapping, | |||
| 570 | 570 | ||
| 571 | _debug("wback %lx", page->index); | 571 | _debug("wback %lx", page->index); |
| 572 | 572 | ||
| 573 | /* at this point we hold neither mapping->tree_lock nor lock on | 573 | /* |
| 574 | * the page itself: the page may be truncated or invalidated | 574 | * at this point we hold neither the i_pages lock nor the |
| 575 | * (changing page->mapping to NULL), or even swizzled back from | 575 | * page lock: the page may be truncated or invalidated |
| 576 | * swapper_space to tmpfs file mapping | 576 | * (changing page->mapping to NULL), or even swizzled |
| 577 | * back from swapper_space to tmpfs file mapping | ||
| 577 | */ | 578 | */ |
| 578 | ret = lock_page_killable(page); | 579 | ret = lock_page_killable(page); |
| 579 | if (ret < 0) { | 580 | if (ret < 0) { |
diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 562c3e633403..578181cd96b5 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c | |||
| @@ -458,7 +458,7 @@ static noinline int add_ra_bio_pages(struct inode *inode, | |||
| 458 | break; | 458 | break; |
| 459 | 459 | ||
| 460 | rcu_read_lock(); | 460 | rcu_read_lock(); |
| 461 | page = radix_tree_lookup(&mapping->page_tree, pg_index); | 461 | page = radix_tree_lookup(&mapping->i_pages, pg_index); |
| 462 | rcu_read_unlock(); | 462 | rcu_read_unlock(); |
| 463 | if (page && !radix_tree_exceptional_entry(page)) { | 463 | if (page && !radix_tree_exceptional_entry(page)) { |
| 464 | misses++; | 464 | misses++; |
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index 47a8fe9d22e8..cf87976e389d 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c | |||
| @@ -3963,11 +3963,11 @@ retry: | |||
| 3963 | 3963 | ||
| 3964 | done_index = page->index; | 3964 | done_index = page->index; |
| 3965 | /* | 3965 | /* |
| 3966 | * At this point we hold neither mapping->tree_lock nor | 3966 | * At this point we hold neither the i_pages lock nor |
| 3967 | * lock on the page itself: the page may be truncated or | 3967 | * the page lock: the page may be truncated or |
| 3968 | * invalidated (changing page->mapping to NULL), or even | 3968 | * invalidated (changing page->mapping to NULL), |
| 3969 | * swizzled back from swapper_space to tmpfs file | 3969 | * or even swizzled back from swapper_space to |
| 3970 | * mapping | 3970 | * tmpfs file mapping |
| 3971 | */ | 3971 | */ |
| 3972 | if (!trylock_page(page)) { | 3972 | if (!trylock_page(page)) { |
| 3973 | flush_write_bio(epd); | 3973 | flush_write_bio(epd); |
| @@ -5174,13 +5174,13 @@ void clear_extent_buffer_dirty(struct extent_buffer *eb) | |||
| 5174 | WARN_ON(!PagePrivate(page)); | 5174 | WARN_ON(!PagePrivate(page)); |
| 5175 | 5175 | ||
| 5176 | clear_page_dirty_for_io(page); | 5176 | clear_page_dirty_for_io(page); |
| 5177 | spin_lock_irq(&page->mapping->tree_lock); | 5177 | xa_lock_irq(&page->mapping->i_pages); |
| 5178 | if (!PageDirty(page)) { | 5178 | if (!PageDirty(page)) { |
| 5179 | radix_tree_tag_clear(&page->mapping->page_tree, | 5179 | radix_tree_tag_clear(&page->mapping->i_pages, |
| 5180 | page_index(page), | 5180 | page_index(page), |
| 5181 | PAGECACHE_TAG_DIRTY); | 5181 | PAGECACHE_TAG_DIRTY); |
| 5182 | } | 5182 | } |
| 5183 | spin_unlock_irq(&page->mapping->tree_lock); | 5183 | xa_unlock_irq(&page->mapping->i_pages); |
| 5184 | ClearPageError(page); | 5184 | ClearPageError(page); |
| 5185 | unlock_page(page); | 5185 | unlock_page(page); |
| 5186 | } | 5186 | } |
diff --git a/fs/buffer.c b/fs/buffer.c index 64b1e2065b6b..f3491074b035 100644 --- a/fs/buffer.c +++ b/fs/buffer.c | |||
| @@ -185,10 +185,9 @@ EXPORT_SYMBOL(end_buffer_write_sync); | |||
| 185 | * we get exclusion from try_to_free_buffers with the blockdev mapping's | 185 | * we get exclusion from try_to_free_buffers with the blockdev mapping's |
| 186 | * private_lock. | 186 | * private_lock. |
| 187 | * | 187 | * |
| 188 | * Hack idea: for the blockdev mapping, i_bufferlist_lock contention | 188 | * Hack idea: for the blockdev mapping, private_lock contention |
| 189 | * may be quite high. This code could TryLock the page, and if that | 189 | * may be quite high. This code could TryLock the page, and if that |
| 190 | * succeeds, there is no need to take private_lock. (But if | 190 | * succeeds, there is no need to take private_lock. |
| 191 | * private_lock is contended then so is mapping->tree_lock). | ||
| 192 | */ | 191 | */ |
| 193 | static struct buffer_head * | 192 | static struct buffer_head * |
| 194 | __find_get_block_slow(struct block_device *bdev, sector_t block) | 193 | __find_get_block_slow(struct block_device *bdev, sector_t block) |
| @@ -599,14 +598,14 @@ void __set_page_dirty(struct page *page, struct address_space *mapping, | |||
| 599 | { | 598 | { |
| 600 | unsigned long flags; | 599 | unsigned long flags; |
| 601 | 600 | ||
| 602 | spin_lock_irqsave(&mapping->tree_lock, flags); | 601 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 603 | if (page->mapping) { /* Race with truncate? */ | 602 | if (page->mapping) { /* Race with truncate? */ |
| 604 | WARN_ON_ONCE(warn && !PageUptodate(page)); | 603 | WARN_ON_ONCE(warn && !PageUptodate(page)); |
| 605 | account_page_dirtied(page, mapping); | 604 | account_page_dirtied(page, mapping); |
| 606 | radix_tree_tag_set(&mapping->page_tree, | 605 | radix_tree_tag_set(&mapping->i_pages, |
| 607 | page_index(page), PAGECACHE_TAG_DIRTY); | 606 | page_index(page), PAGECACHE_TAG_DIRTY); |
| 608 | } | 607 | } |
| 609 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 608 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 610 | } | 609 | } |
| 611 | EXPORT_SYMBOL_GPL(__set_page_dirty); | 610 | EXPORT_SYMBOL_GPL(__set_page_dirty); |
| 612 | 611 | ||
| @@ -1096,7 +1095,7 @@ __getblk_slow(struct block_device *bdev, sector_t block, | |||
| 1096 | * inode list. | 1095 | * inode list. |
| 1097 | * | 1096 | * |
| 1098 | * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, | 1097 | * mark_buffer_dirty() is atomic. It takes bh->b_page->mapping->private_lock, |
| 1099 | * mapping->tree_lock and mapping->host->i_lock. | 1098 | * i_pages lock and mapping->host->i_lock. |
| 1100 | */ | 1099 | */ |
| 1101 | void mark_buffer_dirty(struct buffer_head *bh) | 1100 | void mark_buffer_dirty(struct buffer_head *bh) |
| 1102 | { | 1101 | { |
diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 7cee97b93a61..4bcd4e838b47 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c | |||
| @@ -1987,11 +1987,10 @@ wdata_prepare_pages(struct cifs_writedata *wdata, unsigned int found_pages, | |||
| 1987 | for (i = 0; i < found_pages; i++) { | 1987 | for (i = 0; i < found_pages; i++) { |
| 1988 | page = wdata->pages[i]; | 1988 | page = wdata->pages[i]; |
| 1989 | /* | 1989 | /* |
| 1990 | * At this point we hold neither mapping->tree_lock nor | 1990 | * At this point we hold neither the i_pages lock nor the |
| 1991 | * lock on the page itself: the page may be truncated or | 1991 | * page lock: the page may be truncated or invalidated |
| 1992 | * invalidated (changing page->mapping to NULL), or even | 1992 | * (changing page->mapping to NULL), or even swizzled |
| 1993 | * swizzled back from swapper_space to tmpfs file | 1993 | * back from swapper_space to tmpfs file mapping |
| 1994 | * mapping | ||
| 1995 | */ | 1994 | */ |
| 1996 | 1995 | ||
| 1997 | if (nr_pages == 0) | 1996 | if (nr_pages == 0) |
| @@ -158,11 +158,9 @@ static int wake_exceptional_entry_func(wait_queue_entry_t *wait, unsigned int mo | |||
| 158 | } | 158 | } |
| 159 | 159 | ||
| 160 | /* | 160 | /* |
| 161 | * We do not necessarily hold the mapping->tree_lock when we call this | 161 | * @entry may no longer be the entry at the index in the mapping. |
| 162 | * function so it is possible that 'entry' is no longer a valid item in the | 162 | * The important information it's conveying is whether the entry at |
| 163 | * radix tree. This is okay because all we really need to do is to find the | 163 | * this index used to be a PMD entry. |
| 164 | * correct waitqueue where tasks might be waiting for that old 'entry' and | ||
| 165 | * wake them. | ||
| 166 | */ | 164 | */ |
| 167 | static void dax_wake_mapping_entry_waiter(struct address_space *mapping, | 165 | static void dax_wake_mapping_entry_waiter(struct address_space *mapping, |
| 168 | pgoff_t index, void *entry, bool wake_all) | 166 | pgoff_t index, void *entry, bool wake_all) |
| @@ -174,7 +172,7 @@ static void dax_wake_mapping_entry_waiter(struct address_space *mapping, | |||
| 174 | 172 | ||
| 175 | /* | 173 | /* |
| 176 | * Checking for locked entry and prepare_to_wait_exclusive() happens | 174 | * Checking for locked entry and prepare_to_wait_exclusive() happens |
| 177 | * under mapping->tree_lock, ditto for entry handling in our callers. | 175 | * under the i_pages lock, ditto for entry handling in our callers. |
| 178 | * So at this point all tasks that could have seen our entry locked | 176 | * So at this point all tasks that could have seen our entry locked |
| 179 | * must be in the waitqueue and the following check will see them. | 177 | * must be in the waitqueue and the following check will see them. |
| 180 | */ | 178 | */ |
| @@ -183,41 +181,39 @@ static void dax_wake_mapping_entry_waiter(struct address_space *mapping, | |||
| 183 | } | 181 | } |
| 184 | 182 | ||
| 185 | /* | 183 | /* |
| 186 | * Check whether the given slot is locked. The function must be called with | 184 | * Check whether the given slot is locked. Must be called with the i_pages |
| 187 | * mapping->tree_lock held | 185 | * lock held. |
| 188 | */ | 186 | */ |
| 189 | static inline int slot_locked(struct address_space *mapping, void **slot) | 187 | static inline int slot_locked(struct address_space *mapping, void **slot) |
| 190 | { | 188 | { |
| 191 | unsigned long entry = (unsigned long) | 189 | unsigned long entry = (unsigned long) |
| 192 | radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 190 | radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); |
| 193 | return entry & RADIX_DAX_ENTRY_LOCK; | 191 | return entry & RADIX_DAX_ENTRY_LOCK; |
| 194 | } | 192 | } |
| 195 | 193 | ||
| 196 | /* | 194 | /* |
| 197 | * Mark the given slot is locked. The function must be called with | 195 | * Mark the given slot as locked. Must be called with the i_pages lock held. |
| 198 | * mapping->tree_lock held | ||
| 199 | */ | 196 | */ |
| 200 | static inline void *lock_slot(struct address_space *mapping, void **slot) | 197 | static inline void *lock_slot(struct address_space *mapping, void **slot) |
| 201 | { | 198 | { |
| 202 | unsigned long entry = (unsigned long) | 199 | unsigned long entry = (unsigned long) |
| 203 | radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 200 | radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); |
| 204 | 201 | ||
| 205 | entry |= RADIX_DAX_ENTRY_LOCK; | 202 | entry |= RADIX_DAX_ENTRY_LOCK; |
| 206 | radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); | 203 | radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); |
| 207 | return (void *)entry; | 204 | return (void *)entry; |
| 208 | } | 205 | } |
| 209 | 206 | ||
| 210 | /* | 207 | /* |
| 211 | * Mark the given slot is unlocked. The function must be called with | 208 | * Mark the given slot as unlocked. Must be called with the i_pages lock held. |
| 212 | * mapping->tree_lock held | ||
| 213 | */ | 209 | */ |
| 214 | static inline void *unlock_slot(struct address_space *mapping, void **slot) | 210 | static inline void *unlock_slot(struct address_space *mapping, void **slot) |
| 215 | { | 211 | { |
| 216 | unsigned long entry = (unsigned long) | 212 | unsigned long entry = (unsigned long) |
| 217 | radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 213 | radix_tree_deref_slot_protected(slot, &mapping->i_pages.xa_lock); |
| 218 | 214 | ||
| 219 | entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; | 215 | entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK; |
| 220 | radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry); | 216 | radix_tree_replace_slot(&mapping->i_pages, slot, (void *)entry); |
| 221 | return (void *)entry; | 217 | return (void *)entry; |
| 222 | } | 218 | } |
| 223 | 219 | ||
| @@ -228,7 +224,7 @@ static inline void *unlock_slot(struct address_space *mapping, void **slot) | |||
| 228 | * put_locked_mapping_entry() when he locked the entry and now wants to | 224 | * put_locked_mapping_entry() when he locked the entry and now wants to |
| 229 | * unlock it. | 225 | * unlock it. |
| 230 | * | 226 | * |
| 231 | * The function must be called with mapping->tree_lock held. | 227 | * Must be called with the i_pages lock held. |
| 232 | */ | 228 | */ |
| 233 | static void *get_unlocked_mapping_entry(struct address_space *mapping, | 229 | static void *get_unlocked_mapping_entry(struct address_space *mapping, |
| 234 | pgoff_t index, void ***slotp) | 230 | pgoff_t index, void ***slotp) |
| @@ -241,7 +237,7 @@ static void *get_unlocked_mapping_entry(struct address_space *mapping, | |||
| 241 | ewait.wait.func = wake_exceptional_entry_func; | 237 | ewait.wait.func = wake_exceptional_entry_func; |
| 242 | 238 | ||
| 243 | for (;;) { | 239 | for (;;) { |
| 244 | entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, | 240 | entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, |
| 245 | &slot); | 241 | &slot); |
| 246 | if (!entry || | 242 | if (!entry || |
| 247 | WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || | 243 | WARN_ON_ONCE(!radix_tree_exceptional_entry(entry)) || |
| @@ -254,10 +250,10 @@ static void *get_unlocked_mapping_entry(struct address_space *mapping, | |||
| 254 | wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); | 250 | wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key); |
| 255 | prepare_to_wait_exclusive(wq, &ewait.wait, | 251 | prepare_to_wait_exclusive(wq, &ewait.wait, |
| 256 | TASK_UNINTERRUPTIBLE); | 252 | TASK_UNINTERRUPTIBLE); |
| 257 | spin_unlock_irq(&mapping->tree_lock); | 253 | xa_unlock_irq(&mapping->i_pages); |
| 258 | schedule(); | 254 | schedule(); |
| 259 | finish_wait(wq, &ewait.wait); | 255 | finish_wait(wq, &ewait.wait); |
| 260 | spin_lock_irq(&mapping->tree_lock); | 256 | xa_lock_irq(&mapping->i_pages); |
| 261 | } | 257 | } |
| 262 | } | 258 | } |
| 263 | 259 | ||
| @@ -266,15 +262,15 @@ static void dax_unlock_mapping_entry(struct address_space *mapping, | |||
| 266 | { | 262 | { |
| 267 | void *entry, **slot; | 263 | void *entry, **slot; |
| 268 | 264 | ||
| 269 | spin_lock_irq(&mapping->tree_lock); | 265 | xa_lock_irq(&mapping->i_pages); |
| 270 | entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot); | 266 | entry = __radix_tree_lookup(&mapping->i_pages, index, NULL, &slot); |
| 271 | if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || | 267 | if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) || |
| 272 | !slot_locked(mapping, slot))) { | 268 | !slot_locked(mapping, slot))) { |
| 273 | spin_unlock_irq(&mapping->tree_lock); | 269 | xa_unlock_irq(&mapping->i_pages); |
| 274 | return; | 270 | return; |
| 275 | } | 271 | } |
| 276 | unlock_slot(mapping, slot); | 272 | unlock_slot(mapping, slot); |
| 277 | spin_unlock_irq(&mapping->tree_lock); | 273 | xa_unlock_irq(&mapping->i_pages); |
| 278 | dax_wake_mapping_entry_waiter(mapping, index, entry, false); | 274 | dax_wake_mapping_entry_waiter(mapping, index, entry, false); |
| 279 | } | 275 | } |
| 280 | 276 | ||
| @@ -388,7 +384,7 @@ static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index, | |||
| 388 | void *entry, **slot; | 384 | void *entry, **slot; |
| 389 | 385 | ||
| 390 | restart: | 386 | restart: |
| 391 | spin_lock_irq(&mapping->tree_lock); | 387 | xa_lock_irq(&mapping->i_pages); |
| 392 | entry = get_unlocked_mapping_entry(mapping, index, &slot); | 388 | entry = get_unlocked_mapping_entry(mapping, index, &slot); |
| 393 | 389 | ||
| 394 | if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { | 390 | if (WARN_ON_ONCE(entry && !radix_tree_exceptional_entry(entry))) { |
| @@ -420,12 +416,12 @@ restart: | |||
| 420 | if (pmd_downgrade) { | 416 | if (pmd_downgrade) { |
| 421 | /* | 417 | /* |
| 422 | * Make sure 'entry' remains valid while we drop | 418 | * Make sure 'entry' remains valid while we drop |
| 423 | * mapping->tree_lock. | 419 | * the i_pages lock. |
| 424 | */ | 420 | */ |
| 425 | entry = lock_slot(mapping, slot); | 421 | entry = lock_slot(mapping, slot); |
| 426 | } | 422 | } |
| 427 | 423 | ||
| 428 | spin_unlock_irq(&mapping->tree_lock); | 424 | xa_unlock_irq(&mapping->i_pages); |
| 429 | /* | 425 | /* |
| 430 | * Besides huge zero pages the only other thing that gets | 426 | * Besides huge zero pages the only other thing that gets |
| 431 | * downgraded are empty entries which don't need to be | 427 | * downgraded are empty entries which don't need to be |
| @@ -442,27 +438,27 @@ restart: | |||
| 442 | put_locked_mapping_entry(mapping, index); | 438 | put_locked_mapping_entry(mapping, index); |
| 443 | return ERR_PTR(err); | 439 | return ERR_PTR(err); |
| 444 | } | 440 | } |
| 445 | spin_lock_irq(&mapping->tree_lock); | 441 | xa_lock_irq(&mapping->i_pages); |
| 446 | 442 | ||
| 447 | if (!entry) { | 443 | if (!entry) { |
| 448 | /* | 444 | /* |
| 449 | * We needed to drop the page_tree lock while calling | 445 | * We needed to drop the i_pages lock while calling |
| 450 | * radix_tree_preload() and we didn't have an entry to | 446 | * radix_tree_preload() and we didn't have an entry to |
| 451 | * lock. See if another thread inserted an entry at | 447 | * lock. See if another thread inserted an entry at |
| 452 | * our index during this time. | 448 | * our index during this time. |
| 453 | */ | 449 | */ |
| 454 | entry = __radix_tree_lookup(&mapping->page_tree, index, | 450 | entry = __radix_tree_lookup(&mapping->i_pages, index, |
| 455 | NULL, &slot); | 451 | NULL, &slot); |
| 456 | if (entry) { | 452 | if (entry) { |
| 457 | radix_tree_preload_end(); | 453 | radix_tree_preload_end(); |
| 458 | spin_unlock_irq(&mapping->tree_lock); | 454 | xa_unlock_irq(&mapping->i_pages); |
| 459 | goto restart; | 455 | goto restart; |
| 460 | } | 456 | } |
| 461 | } | 457 | } |
| 462 | 458 | ||
| 463 | if (pmd_downgrade) { | 459 | if (pmd_downgrade) { |
| 464 | dax_disassociate_entry(entry, mapping, false); | 460 | dax_disassociate_entry(entry, mapping, false); |
| 465 | radix_tree_delete(&mapping->page_tree, index); | 461 | radix_tree_delete(&mapping->i_pages, index); |
| 466 | mapping->nrexceptional--; | 462 | mapping->nrexceptional--; |
| 467 | dax_wake_mapping_entry_waiter(mapping, index, entry, | 463 | dax_wake_mapping_entry_waiter(mapping, index, entry, |
| 468 | true); | 464 | true); |
| @@ -470,11 +466,11 @@ restart: | |||
| 470 | 466 | ||
| 471 | entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); | 467 | entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY); |
| 472 | 468 | ||
| 473 | err = __radix_tree_insert(&mapping->page_tree, index, | 469 | err = __radix_tree_insert(&mapping->i_pages, index, |
| 474 | dax_radix_order(entry), entry); | 470 | dax_radix_order(entry), entry); |
| 475 | radix_tree_preload_end(); | 471 | radix_tree_preload_end(); |
| 476 | if (err) { | 472 | if (err) { |
| 477 | spin_unlock_irq(&mapping->tree_lock); | 473 | xa_unlock_irq(&mapping->i_pages); |
| 478 | /* | 474 | /* |
| 479 | * Our insertion of a DAX entry failed, most likely | 475 | * Our insertion of a DAX entry failed, most likely |
| 480 | * because we were inserting a PMD entry and it | 476 | * because we were inserting a PMD entry and it |
| @@ -487,12 +483,12 @@ restart: | |||
| 487 | } | 483 | } |
| 488 | /* Good, we have inserted empty locked entry into the tree. */ | 484 | /* Good, we have inserted empty locked entry into the tree. */ |
| 489 | mapping->nrexceptional++; | 485 | mapping->nrexceptional++; |
| 490 | spin_unlock_irq(&mapping->tree_lock); | 486 | xa_unlock_irq(&mapping->i_pages); |
| 491 | return entry; | 487 | return entry; |
| 492 | } | 488 | } |
| 493 | entry = lock_slot(mapping, slot); | 489 | entry = lock_slot(mapping, slot); |
| 494 | out_unlock: | 490 | out_unlock: |
| 495 | spin_unlock_irq(&mapping->tree_lock); | 491 | xa_unlock_irq(&mapping->i_pages); |
| 496 | return entry; | 492 | return entry; |
| 497 | } | 493 | } |
| 498 | 494 | ||
| @@ -501,23 +497,23 @@ static int __dax_invalidate_mapping_entry(struct address_space *mapping, | |||
| 501 | { | 497 | { |
| 502 | int ret = 0; | 498 | int ret = 0; |
| 503 | void *entry; | 499 | void *entry; |
| 504 | struct radix_tree_root *page_tree = &mapping->page_tree; | 500 | struct radix_tree_root *pages = &mapping->i_pages; |
| 505 | 501 | ||
| 506 | spin_lock_irq(&mapping->tree_lock); | 502 | xa_lock_irq(pages); |
| 507 | entry = get_unlocked_mapping_entry(mapping, index, NULL); | 503 | entry = get_unlocked_mapping_entry(mapping, index, NULL); |
| 508 | if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) | 504 | if (!entry || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry))) |
| 509 | goto out; | 505 | goto out; |
| 510 | if (!trunc && | 506 | if (!trunc && |
| 511 | (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) || | 507 | (radix_tree_tag_get(pages, index, PAGECACHE_TAG_DIRTY) || |
| 512 | radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))) | 508 | radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE))) |
| 513 | goto out; | 509 | goto out; |
| 514 | dax_disassociate_entry(entry, mapping, trunc); | 510 | dax_disassociate_entry(entry, mapping, trunc); |
| 515 | radix_tree_delete(page_tree, index); | 511 | radix_tree_delete(pages, index); |
| 516 | mapping->nrexceptional--; | 512 | mapping->nrexceptional--; |
| 517 | ret = 1; | 513 | ret = 1; |
| 518 | out: | 514 | out: |
| 519 | put_unlocked_mapping_entry(mapping, index, entry); | 515 | put_unlocked_mapping_entry(mapping, index, entry); |
| 520 | spin_unlock_irq(&mapping->tree_lock); | 516 | xa_unlock_irq(pages); |
| 521 | return ret; | 517 | return ret; |
| 522 | } | 518 | } |
| 523 | /* | 519 | /* |
| @@ -587,7 +583,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
| 587 | void *entry, pfn_t pfn_t, | 583 | void *entry, pfn_t pfn_t, |
| 588 | unsigned long flags, bool dirty) | 584 | unsigned long flags, bool dirty) |
| 589 | { | 585 | { |
| 590 | struct radix_tree_root *page_tree = &mapping->page_tree; | 586 | struct radix_tree_root *pages = &mapping->i_pages; |
| 591 | unsigned long pfn = pfn_t_to_pfn(pfn_t); | 587 | unsigned long pfn = pfn_t_to_pfn(pfn_t); |
| 592 | pgoff_t index = vmf->pgoff; | 588 | pgoff_t index = vmf->pgoff; |
| 593 | void *new_entry; | 589 | void *new_entry; |
| @@ -604,7 +600,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
| 604 | unmap_mapping_pages(mapping, vmf->pgoff, 1, false); | 600 | unmap_mapping_pages(mapping, vmf->pgoff, 1, false); |
| 605 | } | 601 | } |
| 606 | 602 | ||
| 607 | spin_lock_irq(&mapping->tree_lock); | 603 | xa_lock_irq(pages); |
| 608 | new_entry = dax_radix_locked_entry(pfn, flags); | 604 | new_entry = dax_radix_locked_entry(pfn, flags); |
| 609 | if (dax_entry_size(entry) != dax_entry_size(new_entry)) { | 605 | if (dax_entry_size(entry) != dax_entry_size(new_entry)) { |
| 610 | dax_disassociate_entry(entry, mapping, false); | 606 | dax_disassociate_entry(entry, mapping, false); |
| @@ -624,17 +620,17 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
| 624 | void **slot; | 620 | void **slot; |
| 625 | void *ret; | 621 | void *ret; |
| 626 | 622 | ||
| 627 | ret = __radix_tree_lookup(page_tree, index, &node, &slot); | 623 | ret = __radix_tree_lookup(pages, index, &node, &slot); |
| 628 | WARN_ON_ONCE(ret != entry); | 624 | WARN_ON_ONCE(ret != entry); |
| 629 | __radix_tree_replace(page_tree, node, slot, | 625 | __radix_tree_replace(pages, node, slot, |
| 630 | new_entry, NULL); | 626 | new_entry, NULL); |
| 631 | entry = new_entry; | 627 | entry = new_entry; |
| 632 | } | 628 | } |
| 633 | 629 | ||
| 634 | if (dirty) | 630 | if (dirty) |
| 635 | radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY); | 631 | radix_tree_tag_set(pages, index, PAGECACHE_TAG_DIRTY); |
| 636 | 632 | ||
| 637 | spin_unlock_irq(&mapping->tree_lock); | 633 | xa_unlock_irq(pages); |
| 638 | return entry; | 634 | return entry; |
| 639 | } | 635 | } |
| 640 | 636 | ||
| @@ -723,7 +719,7 @@ unlock_pte: | |||
| 723 | static int dax_writeback_one(struct dax_device *dax_dev, | 719 | static int dax_writeback_one(struct dax_device *dax_dev, |
| 724 | struct address_space *mapping, pgoff_t index, void *entry) | 720 | struct address_space *mapping, pgoff_t index, void *entry) |
| 725 | { | 721 | { |
| 726 | struct radix_tree_root *page_tree = &mapping->page_tree; | 722 | struct radix_tree_root *pages = &mapping->i_pages; |
| 727 | void *entry2, **slot; | 723 | void *entry2, **slot; |
| 728 | unsigned long pfn; | 724 | unsigned long pfn; |
| 729 | long ret = 0; | 725 | long ret = 0; |
| @@ -736,7 +732,7 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
| 736 | if (WARN_ON(!radix_tree_exceptional_entry(entry))) | 732 | if (WARN_ON(!radix_tree_exceptional_entry(entry))) |
| 737 | return -EIO; | 733 | return -EIO; |
| 738 | 734 | ||
| 739 | spin_lock_irq(&mapping->tree_lock); | 735 | xa_lock_irq(pages); |
| 740 | entry2 = get_unlocked_mapping_entry(mapping, index, &slot); | 736 | entry2 = get_unlocked_mapping_entry(mapping, index, &slot); |
| 741 | /* Entry got punched out / reallocated? */ | 737 | /* Entry got punched out / reallocated? */ |
| 742 | if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) | 738 | if (!entry2 || WARN_ON_ONCE(!radix_tree_exceptional_entry(entry2))) |
| @@ -755,7 +751,7 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
| 755 | } | 751 | } |
| 756 | 752 | ||
| 757 | /* Another fsync thread may have already written back this entry */ | 753 | /* Another fsync thread may have already written back this entry */ |
| 758 | if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)) | 754 | if (!radix_tree_tag_get(pages, index, PAGECACHE_TAG_TOWRITE)) |
| 759 | goto put_unlocked; | 755 | goto put_unlocked; |
| 760 | /* Lock the entry to serialize with page faults */ | 756 | /* Lock the entry to serialize with page faults */ |
| 761 | entry = lock_slot(mapping, slot); | 757 | entry = lock_slot(mapping, slot); |
| @@ -763,11 +759,11 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
| 763 | * We can clear the tag now but we have to be careful so that concurrent | 759 | * We can clear the tag now but we have to be careful so that concurrent |
| 764 | * dax_writeback_one() calls for the same index cannot finish before we | 760 | * dax_writeback_one() calls for the same index cannot finish before we |
| 765 | * actually flush the caches. This is achieved as the calls will look | 761 | * actually flush the caches. This is achieved as the calls will look |
| 766 | * at the entry only under tree_lock and once they do that they will | 762 | * at the entry only under the i_pages lock and once they do that |
| 767 | * see the entry locked and wait for it to unlock. | 763 | * they will see the entry locked and wait for it to unlock. |
| 768 | */ | 764 | */ |
| 769 | radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE); | 765 | radix_tree_tag_clear(pages, index, PAGECACHE_TAG_TOWRITE); |
| 770 | spin_unlock_irq(&mapping->tree_lock); | 766 | xa_unlock_irq(pages); |
| 771 | 767 | ||
| 772 | /* | 768 | /* |
| 773 | * Even if dax_writeback_mapping_range() was given a wbc->range_start | 769 | * Even if dax_writeback_mapping_range() was given a wbc->range_start |
| @@ -787,16 +783,16 @@ static int dax_writeback_one(struct dax_device *dax_dev, | |||
| 787 | * the pfn mappings are writeprotected and fault waits for mapping | 783 | * the pfn mappings are writeprotected and fault waits for mapping |
| 788 | * entry lock. | 784 | * entry lock. |
| 789 | */ | 785 | */ |
| 790 | spin_lock_irq(&mapping->tree_lock); | 786 | xa_lock_irq(pages); |
| 791 | radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY); | 787 | radix_tree_tag_clear(pages, index, PAGECACHE_TAG_DIRTY); |
| 792 | spin_unlock_irq(&mapping->tree_lock); | 788 | xa_unlock_irq(pages); |
| 793 | trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); | 789 | trace_dax_writeback_one(mapping->host, index, size >> PAGE_SHIFT); |
| 794 | put_locked_mapping_entry(mapping, index); | 790 | put_locked_mapping_entry(mapping, index); |
| 795 | return ret; | 791 | return ret; |
| 796 | 792 | ||
| 797 | put_unlocked: | 793 | put_unlocked: |
| 798 | put_unlocked_mapping_entry(mapping, index, entry2); | 794 | put_unlocked_mapping_entry(mapping, index, entry2); |
| 799 | spin_unlock_irq(&mapping->tree_lock); | 795 | xa_unlock_irq(pages); |
| 800 | return ret; | 796 | return ret; |
| 801 | } | 797 | } |
| 802 | 798 | ||
| @@ -1566,21 +1562,21 @@ static int dax_insert_pfn_mkwrite(struct vm_fault *vmf, | |||
| 1566 | pgoff_t index = vmf->pgoff; | 1562 | pgoff_t index = vmf->pgoff; |
| 1567 | int vmf_ret, error; | 1563 | int vmf_ret, error; |
| 1568 | 1564 | ||
| 1569 | spin_lock_irq(&mapping->tree_lock); | 1565 | xa_lock_irq(&mapping->i_pages); |
| 1570 | entry = get_unlocked_mapping_entry(mapping, index, &slot); | 1566 | entry = get_unlocked_mapping_entry(mapping, index, &slot); |
| 1571 | /* Did we race with someone splitting entry or so? */ | 1567 | /* Did we race with someone splitting entry or so? */ |
| 1572 | if (!entry || | 1568 | if (!entry || |
| 1573 | (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || | 1569 | (pe_size == PE_SIZE_PTE && !dax_is_pte_entry(entry)) || |
| 1574 | (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { | 1570 | (pe_size == PE_SIZE_PMD && !dax_is_pmd_entry(entry))) { |
| 1575 | put_unlocked_mapping_entry(mapping, index, entry); | 1571 | put_unlocked_mapping_entry(mapping, index, entry); |
| 1576 | spin_unlock_irq(&mapping->tree_lock); | 1572 | xa_unlock_irq(&mapping->i_pages); |
| 1577 | trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, | 1573 | trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf, |
| 1578 | VM_FAULT_NOPAGE); | 1574 | VM_FAULT_NOPAGE); |
| 1579 | return VM_FAULT_NOPAGE; | 1575 | return VM_FAULT_NOPAGE; |
| 1580 | } | 1576 | } |
| 1581 | radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY); | 1577 | radix_tree_tag_set(&mapping->i_pages, index, PAGECACHE_TAG_DIRTY); |
| 1582 | entry = lock_slot(mapping, slot); | 1578 | entry = lock_slot(mapping, slot); |
| 1583 | spin_unlock_irq(&mapping->tree_lock); | 1579 | xa_unlock_irq(&mapping->i_pages); |
| 1584 | switch (pe_size) { | 1580 | switch (pe_size) { |
| 1585 | case PE_SIZE_PTE: | 1581 | case PE_SIZE_PTE: |
| 1586 | error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn); | 1582 | error = vm_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn); |
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c index db50686f5096..02237d4d91f5 100644 --- a/fs/f2fs/data.c +++ b/fs/f2fs/data.c | |||
| @@ -2424,12 +2424,12 @@ void f2fs_set_page_dirty_nobuffers(struct page *page) | |||
| 2424 | SetPageDirty(page); | 2424 | SetPageDirty(page); |
| 2425 | spin_unlock(&mapping->private_lock); | 2425 | spin_unlock(&mapping->private_lock); |
| 2426 | 2426 | ||
| 2427 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2427 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 2428 | WARN_ON_ONCE(!PageUptodate(page)); | 2428 | WARN_ON_ONCE(!PageUptodate(page)); |
| 2429 | account_page_dirtied(page, mapping); | 2429 | account_page_dirtied(page, mapping); |
| 2430 | radix_tree_tag_set(&mapping->page_tree, | 2430 | radix_tree_tag_set(&mapping->i_pages, |
| 2431 | page_index(page), PAGECACHE_TAG_DIRTY); | 2431 | page_index(page), PAGECACHE_TAG_DIRTY); |
| 2432 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2432 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 2433 | unlock_page_memcg(page); | 2433 | unlock_page_memcg(page); |
| 2434 | 2434 | ||
| 2435 | __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); | 2435 | __mark_inode_dirty(mapping->host, I_DIRTY_PAGES); |
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c index fe661274ff10..8c9c2f31b253 100644 --- a/fs/f2fs/dir.c +++ b/fs/f2fs/dir.c | |||
| @@ -732,10 +732,10 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, | |||
| 732 | 732 | ||
| 733 | if (bit_pos == NR_DENTRY_IN_BLOCK && | 733 | if (bit_pos == NR_DENTRY_IN_BLOCK && |
| 734 | !truncate_hole(dir, page->index, page->index + 1)) { | 734 | !truncate_hole(dir, page->index, page->index + 1)) { |
| 735 | spin_lock_irqsave(&mapping->tree_lock, flags); | 735 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 736 | radix_tree_tag_clear(&mapping->page_tree, page_index(page), | 736 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
| 737 | PAGECACHE_TAG_DIRTY); | 737 | PAGECACHE_TAG_DIRTY); |
| 738 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 738 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 739 | 739 | ||
| 740 | clear_page_dirty_for_io(page); | 740 | clear_page_dirty_for_io(page); |
| 741 | ClearPagePrivate(page); | 741 | ClearPagePrivate(page); |
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c index 3b77d6421218..265da200daa8 100644 --- a/fs/f2fs/inline.c +++ b/fs/f2fs/inline.c | |||
| @@ -226,10 +226,10 @@ int f2fs_write_inline_data(struct inode *inode, struct page *page) | |||
| 226 | kunmap_atomic(src_addr); | 226 | kunmap_atomic(src_addr); |
| 227 | set_page_dirty(dn.inode_page); | 227 | set_page_dirty(dn.inode_page); |
| 228 | 228 | ||
| 229 | spin_lock_irqsave(&mapping->tree_lock, flags); | 229 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 230 | radix_tree_tag_clear(&mapping->page_tree, page_index(page), | 230 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
| 231 | PAGECACHE_TAG_DIRTY); | 231 | PAGECACHE_TAG_DIRTY); |
| 232 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 232 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 233 | 233 | ||
| 234 | set_inode_flag(inode, FI_APPEND_WRITE); | 234 | set_inode_flag(inode, FI_APPEND_WRITE); |
| 235 | set_inode_flag(inode, FI_DATA_EXIST); | 235 | set_inode_flag(inode, FI_DATA_EXIST); |
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c index 9a99243054ba..f202398e20ea 100644 --- a/fs/f2fs/node.c +++ b/fs/f2fs/node.c | |||
| @@ -91,11 +91,11 @@ static void clear_node_page_dirty(struct page *page) | |||
| 91 | unsigned int long flags; | 91 | unsigned int long flags; |
| 92 | 92 | ||
| 93 | if (PageDirty(page)) { | 93 | if (PageDirty(page)) { |
| 94 | spin_lock_irqsave(&mapping->tree_lock, flags); | 94 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 95 | radix_tree_tag_clear(&mapping->page_tree, | 95 | radix_tree_tag_clear(&mapping->i_pages, |
| 96 | page_index(page), | 96 | page_index(page), |
| 97 | PAGECACHE_TAG_DIRTY); | 97 | PAGECACHE_TAG_DIRTY); |
| 98 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 98 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 99 | 99 | ||
| 100 | clear_page_dirty_for_io(page); | 100 | clear_page_dirty_for_io(page); |
| 101 | dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); | 101 | dec_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_NODES); |
| @@ -1161,7 +1161,7 @@ void ra_node_page(struct f2fs_sb_info *sbi, nid_t nid) | |||
| 1161 | f2fs_bug_on(sbi, check_nid_range(sbi, nid)); | 1161 | f2fs_bug_on(sbi, check_nid_range(sbi, nid)); |
| 1162 | 1162 | ||
| 1163 | rcu_read_lock(); | 1163 | rcu_read_lock(); |
| 1164 | apage = radix_tree_lookup(&NODE_MAPPING(sbi)->page_tree, nid); | 1164 | apage = radix_tree_lookup(&NODE_MAPPING(sbi)->i_pages, nid); |
| 1165 | rcu_read_unlock(); | 1165 | rcu_read_unlock(); |
| 1166 | if (apage) | 1166 | if (apage) |
| 1167 | return; | 1167 | return; |
diff --git a/fs/fs-writeback.c b/fs/fs-writeback.c index 1280f915079b..4b12ba70a895 100644 --- a/fs/fs-writeback.c +++ b/fs/fs-writeback.c | |||
| @@ -347,9 +347,9 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) | |||
| 347 | * By the time control reaches here, RCU grace period has passed | 347 | * By the time control reaches here, RCU grace period has passed |
| 348 | * since I_WB_SWITCH assertion and all wb stat update transactions | 348 | * since I_WB_SWITCH assertion and all wb stat update transactions |
| 349 | * between unlocked_inode_to_wb_begin/end() are guaranteed to be | 349 | * between unlocked_inode_to_wb_begin/end() are guaranteed to be |
| 350 | * synchronizing against mapping->tree_lock. | 350 | * synchronizing against the i_pages lock. |
| 351 | * | 351 | * |
| 352 | * Grabbing old_wb->list_lock, inode->i_lock and mapping->tree_lock | 352 | * Grabbing old_wb->list_lock, inode->i_lock and the i_pages lock |
| 353 | * gives us exclusion against all wb related operations on @inode | 353 | * gives us exclusion against all wb related operations on @inode |
| 354 | * including IO list manipulations and stat updates. | 354 | * including IO list manipulations and stat updates. |
| 355 | */ | 355 | */ |
| @@ -361,7 +361,7 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) | |||
| 361 | spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); | 361 | spin_lock_nested(&old_wb->list_lock, SINGLE_DEPTH_NESTING); |
| 362 | } | 362 | } |
| 363 | spin_lock(&inode->i_lock); | 363 | spin_lock(&inode->i_lock); |
| 364 | spin_lock_irq(&mapping->tree_lock); | 364 | xa_lock_irq(&mapping->i_pages); |
| 365 | 365 | ||
| 366 | /* | 366 | /* |
| 367 | * Once I_FREEING is visible under i_lock, the eviction path owns | 367 | * Once I_FREEING is visible under i_lock, the eviction path owns |
| @@ -373,22 +373,22 @@ static void inode_switch_wbs_work_fn(struct work_struct *work) | |||
| 373 | /* | 373 | /* |
| 374 | * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points | 374 | * Count and transfer stats. Note that PAGECACHE_TAG_DIRTY points |
| 375 | * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to | 375 | * to possibly dirty pages while PAGECACHE_TAG_WRITEBACK points to |
| 376 | * pages actually under underwriteback. | 376 | * pages actually under writeback. |
| 377 | */ | 377 | */ |
| 378 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, | 378 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0, |
| 379 | PAGECACHE_TAG_DIRTY) { | 379 | PAGECACHE_TAG_DIRTY) { |
| 380 | struct page *page = radix_tree_deref_slot_protected(slot, | 380 | struct page *page = radix_tree_deref_slot_protected(slot, |
| 381 | &mapping->tree_lock); | 381 | &mapping->i_pages.xa_lock); |
| 382 | if (likely(page) && PageDirty(page)) { | 382 | if (likely(page) && PageDirty(page)) { |
| 383 | dec_wb_stat(old_wb, WB_RECLAIMABLE); | 383 | dec_wb_stat(old_wb, WB_RECLAIMABLE); |
| 384 | inc_wb_stat(new_wb, WB_RECLAIMABLE); | 384 | inc_wb_stat(new_wb, WB_RECLAIMABLE); |
| 385 | } | 385 | } |
| 386 | } | 386 | } |
| 387 | 387 | ||
| 388 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 0, | 388 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, 0, |
| 389 | PAGECACHE_TAG_WRITEBACK) { | 389 | PAGECACHE_TAG_WRITEBACK) { |
| 390 | struct page *page = radix_tree_deref_slot_protected(slot, | 390 | struct page *page = radix_tree_deref_slot_protected(slot, |
| 391 | &mapping->tree_lock); | 391 | &mapping->i_pages.xa_lock); |
| 392 | if (likely(page)) { | 392 | if (likely(page)) { |
| 393 | WARN_ON_ONCE(!PageWriteback(page)); | 393 | WARN_ON_ONCE(!PageWriteback(page)); |
| 394 | dec_wb_stat(old_wb, WB_WRITEBACK); | 394 | dec_wb_stat(old_wb, WB_WRITEBACK); |
| @@ -430,7 +430,7 @@ skip_switch: | |||
| 430 | */ | 430 | */ |
| 431 | smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); | 431 | smp_store_release(&inode->i_state, inode->i_state & ~I_WB_SWITCH); |
| 432 | 432 | ||
| 433 | spin_unlock_irq(&mapping->tree_lock); | 433 | xa_unlock_irq(&mapping->i_pages); |
| 434 | spin_unlock(&inode->i_lock); | 434 | spin_unlock(&inode->i_lock); |
| 435 | spin_unlock(&new_wb->list_lock); | 435 | spin_unlock(&new_wb->list_lock); |
| 436 | spin_unlock(&old_wb->list_lock); | 436 | spin_unlock(&old_wb->list_lock); |
| @@ -506,8 +506,8 @@ static void inode_switch_wbs(struct inode *inode, int new_wb_id) | |||
| 506 | 506 | ||
| 507 | /* | 507 | /* |
| 508 | * In addition to synchronizing among switchers, I_WB_SWITCH tells | 508 | * In addition to synchronizing among switchers, I_WB_SWITCH tells |
| 509 | * the RCU protected stat update paths to grab the mapping's | 509 | * the RCU protected stat update paths to grab the i_page |
| 510 | * tree_lock so that stat transfer can synchronize against them. | 510 | * lock so that stat transfer can synchronize against them. |
| 511 | * Let's continue after I_WB_SWITCH is guaranteed to be visible. | 511 | * Let's continue after I_WB_SWITCH is guaranteed to be visible. |
| 512 | */ | 512 | */ |
| 513 | call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn); | 513 | call_rcu(&isw->rcu_head, inode_switch_wbs_rcu_fn); |
diff --git a/fs/inode.c b/fs/inode.c index b153aeaa61ea..13ceb98c3bd3 100644 --- a/fs/inode.c +++ b/fs/inode.c | |||
| @@ -348,8 +348,7 @@ EXPORT_SYMBOL(inc_nlink); | |||
| 348 | 348 | ||
| 349 | static void __address_space_init_once(struct address_space *mapping) | 349 | static void __address_space_init_once(struct address_space *mapping) |
| 350 | { | 350 | { |
| 351 | INIT_RADIX_TREE(&mapping->page_tree, GFP_ATOMIC | __GFP_ACCOUNT); | 351 | INIT_RADIX_TREE(&mapping->i_pages, GFP_ATOMIC | __GFP_ACCOUNT); |
| 352 | spin_lock_init(&mapping->tree_lock); | ||
| 353 | init_rwsem(&mapping->i_mmap_rwsem); | 352 | init_rwsem(&mapping->i_mmap_rwsem); |
| 354 | INIT_LIST_HEAD(&mapping->private_list); | 353 | INIT_LIST_HEAD(&mapping->private_list); |
| 355 | spin_lock_init(&mapping->private_lock); | 354 | spin_lock_init(&mapping->private_lock); |
| @@ -504,14 +503,14 @@ EXPORT_SYMBOL(__remove_inode_hash); | |||
| 504 | void clear_inode(struct inode *inode) | 503 | void clear_inode(struct inode *inode) |
| 505 | { | 504 | { |
| 506 | /* | 505 | /* |
| 507 | * We have to cycle tree_lock here because reclaim can be still in the | 506 | * We have to cycle the i_pages lock here because reclaim can be in the |
| 508 | * process of removing the last page (in __delete_from_page_cache()) | 507 | * process of removing the last page (in __delete_from_page_cache()) |
| 509 | * and we must not free mapping under it. | 508 | * and we must not free the mapping under it. |
| 510 | */ | 509 | */ |
| 511 | spin_lock_irq(&inode->i_data.tree_lock); | 510 | xa_lock_irq(&inode->i_data.i_pages); |
| 512 | BUG_ON(inode->i_data.nrpages); | 511 | BUG_ON(inode->i_data.nrpages); |
| 513 | BUG_ON(inode->i_data.nrexceptional); | 512 | BUG_ON(inode->i_data.nrexceptional); |
| 514 | spin_unlock_irq(&inode->i_data.tree_lock); | 513 | xa_unlock_irq(&inode->i_data.i_pages); |
| 515 | BUG_ON(!list_empty(&inode->i_data.private_list)); | 514 | BUG_ON(!list_empty(&inode->i_data.private_list)); |
| 516 | BUG_ON(!(inode->i_state & I_FREEING)); | 515 | BUG_ON(!(inode->i_state & I_FREEING)); |
| 517 | BUG_ON(inode->i_state & I_CLEAR); | 516 | BUG_ON(inode->i_state & I_CLEAR); |
diff --git a/fs/nilfs2/btnode.c b/fs/nilfs2/btnode.c index c21e0b4454a6..dec98cab729d 100644 --- a/fs/nilfs2/btnode.c +++ b/fs/nilfs2/btnode.c | |||
| @@ -193,9 +193,9 @@ retry: | |||
| 193 | (unsigned long long)oldkey, | 193 | (unsigned long long)oldkey, |
| 194 | (unsigned long long)newkey); | 194 | (unsigned long long)newkey); |
| 195 | 195 | ||
| 196 | spin_lock_irq(&btnc->tree_lock); | 196 | xa_lock_irq(&btnc->i_pages); |
| 197 | err = radix_tree_insert(&btnc->page_tree, newkey, obh->b_page); | 197 | err = radix_tree_insert(&btnc->i_pages, newkey, obh->b_page); |
| 198 | spin_unlock_irq(&btnc->tree_lock); | 198 | xa_unlock_irq(&btnc->i_pages); |
| 199 | /* | 199 | /* |
| 200 | * Note: page->index will not change to newkey until | 200 | * Note: page->index will not change to newkey until |
| 201 | * nilfs_btnode_commit_change_key() will be called. | 201 | * nilfs_btnode_commit_change_key() will be called. |
| @@ -251,11 +251,11 @@ void nilfs_btnode_commit_change_key(struct address_space *btnc, | |||
| 251 | (unsigned long long)newkey); | 251 | (unsigned long long)newkey); |
| 252 | mark_buffer_dirty(obh); | 252 | mark_buffer_dirty(obh); |
| 253 | 253 | ||
| 254 | spin_lock_irq(&btnc->tree_lock); | 254 | xa_lock_irq(&btnc->i_pages); |
| 255 | radix_tree_delete(&btnc->page_tree, oldkey); | 255 | radix_tree_delete(&btnc->i_pages, oldkey); |
| 256 | radix_tree_tag_set(&btnc->page_tree, newkey, | 256 | radix_tree_tag_set(&btnc->i_pages, newkey, |
| 257 | PAGECACHE_TAG_DIRTY); | 257 | PAGECACHE_TAG_DIRTY); |
| 258 | spin_unlock_irq(&btnc->tree_lock); | 258 | xa_unlock_irq(&btnc->i_pages); |
| 259 | 259 | ||
| 260 | opage->index = obh->b_blocknr = newkey; | 260 | opage->index = obh->b_blocknr = newkey; |
| 261 | unlock_page(opage); | 261 | unlock_page(opage); |
| @@ -283,9 +283,9 @@ void nilfs_btnode_abort_change_key(struct address_space *btnc, | |||
| 283 | return; | 283 | return; |
| 284 | 284 | ||
| 285 | if (nbh == NULL) { /* blocksize == pagesize */ | 285 | if (nbh == NULL) { /* blocksize == pagesize */ |
| 286 | spin_lock_irq(&btnc->tree_lock); | 286 | xa_lock_irq(&btnc->i_pages); |
| 287 | radix_tree_delete(&btnc->page_tree, newkey); | 287 | radix_tree_delete(&btnc->i_pages, newkey); |
| 288 | spin_unlock_irq(&btnc->tree_lock); | 288 | xa_unlock_irq(&btnc->i_pages); |
| 289 | unlock_page(ctxt->bh->b_page); | 289 | unlock_page(ctxt->bh->b_page); |
| 290 | } else | 290 | } else |
| 291 | brelse(nbh); | 291 | brelse(nbh); |
diff --git a/fs/nilfs2/page.c b/fs/nilfs2/page.c index 68241512d7c1..4cb850a6f1c2 100644 --- a/fs/nilfs2/page.c +++ b/fs/nilfs2/page.c | |||
| @@ -331,15 +331,15 @@ repeat: | |||
| 331 | struct page *page2; | 331 | struct page *page2; |
| 332 | 332 | ||
| 333 | /* move the page to the destination cache */ | 333 | /* move the page to the destination cache */ |
| 334 | spin_lock_irq(&smap->tree_lock); | 334 | xa_lock_irq(&smap->i_pages); |
| 335 | page2 = radix_tree_delete(&smap->page_tree, offset); | 335 | page2 = radix_tree_delete(&smap->i_pages, offset); |
| 336 | WARN_ON(page2 != page); | 336 | WARN_ON(page2 != page); |
| 337 | 337 | ||
| 338 | smap->nrpages--; | 338 | smap->nrpages--; |
| 339 | spin_unlock_irq(&smap->tree_lock); | 339 | xa_unlock_irq(&smap->i_pages); |
| 340 | 340 | ||
| 341 | spin_lock_irq(&dmap->tree_lock); | 341 | xa_lock_irq(&dmap->i_pages); |
| 342 | err = radix_tree_insert(&dmap->page_tree, offset, page); | 342 | err = radix_tree_insert(&dmap->i_pages, offset, page); |
| 343 | if (unlikely(err < 0)) { | 343 | if (unlikely(err < 0)) { |
| 344 | WARN_ON(err == -EEXIST); | 344 | WARN_ON(err == -EEXIST); |
| 345 | page->mapping = NULL; | 345 | page->mapping = NULL; |
| @@ -348,11 +348,11 @@ repeat: | |||
| 348 | page->mapping = dmap; | 348 | page->mapping = dmap; |
| 349 | dmap->nrpages++; | 349 | dmap->nrpages++; |
| 350 | if (PageDirty(page)) | 350 | if (PageDirty(page)) |
| 351 | radix_tree_tag_set(&dmap->page_tree, | 351 | radix_tree_tag_set(&dmap->i_pages, |
| 352 | offset, | 352 | offset, |
| 353 | PAGECACHE_TAG_DIRTY); | 353 | PAGECACHE_TAG_DIRTY); |
| 354 | } | 354 | } |
| 355 | spin_unlock_irq(&dmap->tree_lock); | 355 | xa_unlock_irq(&dmap->i_pages); |
| 356 | } | 356 | } |
| 357 | unlock_page(page); | 357 | unlock_page(page); |
| 358 | } | 358 | } |
| @@ -474,15 +474,15 @@ int __nilfs_clear_page_dirty(struct page *page) | |||
| 474 | struct address_space *mapping = page->mapping; | 474 | struct address_space *mapping = page->mapping; |
| 475 | 475 | ||
| 476 | if (mapping) { | 476 | if (mapping) { |
| 477 | spin_lock_irq(&mapping->tree_lock); | 477 | xa_lock_irq(&mapping->i_pages); |
| 478 | if (test_bit(PG_dirty, &page->flags)) { | 478 | if (test_bit(PG_dirty, &page->flags)) { |
| 479 | radix_tree_tag_clear(&mapping->page_tree, | 479 | radix_tree_tag_clear(&mapping->i_pages, |
| 480 | page_index(page), | 480 | page_index(page), |
| 481 | PAGECACHE_TAG_DIRTY); | 481 | PAGECACHE_TAG_DIRTY); |
| 482 | spin_unlock_irq(&mapping->tree_lock); | 482 | xa_unlock_irq(&mapping->i_pages); |
| 483 | return clear_page_dirty_for_io(page); | 483 | return clear_page_dirty_for_io(page); |
| 484 | } | 484 | } |
| 485 | spin_unlock_irq(&mapping->tree_lock); | 485 | xa_unlock_irq(&mapping->i_pages); |
| 486 | return 0; | 486 | return 0; |
| 487 | } | 487 | } |
| 488 | return TestClearPageDirty(page); | 488 | return TestClearPageDirty(page); |
diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index e6cbb915ee56..09da0f124699 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h | |||
| @@ -329,7 +329,7 @@ static inline bool inode_to_wb_is_valid(struct inode *inode) | |||
| 329 | * @inode: inode of interest | 329 | * @inode: inode of interest |
| 330 | * | 330 | * |
| 331 | * Returns the wb @inode is currently associated with. The caller must be | 331 | * Returns the wb @inode is currently associated with. The caller must be |
| 332 | * holding either @inode->i_lock, @inode->i_mapping->tree_lock, or the | 332 | * holding either @inode->i_lock, the i_pages lock, or the |
| 333 | * associated wb's list_lock. | 333 | * associated wb's list_lock. |
| 334 | */ | 334 | */ |
| 335 | static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) | 335 | static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) |
| @@ -337,7 +337,7 @@ static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) | |||
| 337 | #ifdef CONFIG_LOCKDEP | 337 | #ifdef CONFIG_LOCKDEP |
| 338 | WARN_ON_ONCE(debug_locks && | 338 | WARN_ON_ONCE(debug_locks && |
| 339 | (!lockdep_is_held(&inode->i_lock) && | 339 | (!lockdep_is_held(&inode->i_lock) && |
| 340 | !lockdep_is_held(&inode->i_mapping->tree_lock) && | 340 | !lockdep_is_held(&inode->i_mapping->i_pages.xa_lock) && |
| 341 | !lockdep_is_held(&inode->i_wb->list_lock))); | 341 | !lockdep_is_held(&inode->i_wb->list_lock))); |
| 342 | #endif | 342 | #endif |
| 343 | return inode->i_wb; | 343 | return inode->i_wb; |
| @@ -349,7 +349,7 @@ static inline struct bdi_writeback *inode_to_wb(const struct inode *inode) | |||
| 349 | * @lockedp: temp bool output param, to be passed to the end function | 349 | * @lockedp: temp bool output param, to be passed to the end function |
| 350 | * | 350 | * |
| 351 | * The caller wants to access the wb associated with @inode but isn't | 351 | * The caller wants to access the wb associated with @inode but isn't |
| 352 | * holding inode->i_lock, mapping->tree_lock or wb->list_lock. This | 352 | * holding inode->i_lock, the i_pages lock or wb->list_lock. This |
| 353 | * function determines the wb associated with @inode and ensures that the | 353 | * function determines the wb associated with @inode and ensures that the |
| 354 | * association doesn't change until the transaction is finished with | 354 | * association doesn't change until the transaction is finished with |
| 355 | * unlocked_inode_to_wb_end(). | 355 | * unlocked_inode_to_wb_end(). |
| @@ -370,11 +370,11 @@ unlocked_inode_to_wb_begin(struct inode *inode, bool *lockedp) | |||
| 370 | *lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH; | 370 | *lockedp = smp_load_acquire(&inode->i_state) & I_WB_SWITCH; |
| 371 | 371 | ||
| 372 | if (unlikely(*lockedp)) | 372 | if (unlikely(*lockedp)) |
| 373 | spin_lock_irq(&inode->i_mapping->tree_lock); | 373 | xa_lock_irq(&inode->i_mapping->i_pages); |
| 374 | 374 | ||
| 375 | /* | 375 | /* |
| 376 | * Protected by either !I_WB_SWITCH + rcu_read_lock() or tree_lock. | 376 | * Protected by either !I_WB_SWITCH + rcu_read_lock() or the i_pages |
| 377 | * inode_to_wb() will bark. Deref directly. | 377 | * lock. inode_to_wb() will bark. Deref directly. |
| 378 | */ | 378 | */ |
| 379 | return inode->i_wb; | 379 | return inode->i_wb; |
| 380 | } | 380 | } |
| @@ -387,7 +387,7 @@ unlocked_inode_to_wb_begin(struct inode *inode, bool *lockedp) | |||
| 387 | static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked) | 387 | static inline void unlocked_inode_to_wb_end(struct inode *inode, bool locked) |
| 388 | { | 388 | { |
| 389 | if (unlikely(locked)) | 389 | if (unlikely(locked)) |
| 390 | spin_unlock_irq(&inode->i_mapping->tree_lock); | 390 | xa_unlock_irq(&inode->i_mapping->i_pages); |
| 391 | 391 | ||
| 392 | rcu_read_unlock(); | 392 | rcu_read_unlock(); |
| 393 | } | 393 | } |
diff --git a/include/linux/fs.h b/include/linux/fs.h index 2aa02cad94d4..92efaf1f8977 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h | |||
| @@ -13,6 +13,7 @@ | |||
| 13 | #include <linux/list_lru.h> | 13 | #include <linux/list_lru.h> |
| 14 | #include <linux/llist.h> | 14 | #include <linux/llist.h> |
| 15 | #include <linux/radix-tree.h> | 15 | #include <linux/radix-tree.h> |
| 16 | #include <linux/xarray.h> | ||
| 16 | #include <linux/rbtree.h> | 17 | #include <linux/rbtree.h> |
| 17 | #include <linux/init.h> | 18 | #include <linux/init.h> |
| 18 | #include <linux/pid.h> | 19 | #include <linux/pid.h> |
| @@ -390,12 +391,11 @@ int pagecache_write_end(struct file *, struct address_space *mapping, | |||
| 390 | 391 | ||
| 391 | struct address_space { | 392 | struct address_space { |
| 392 | struct inode *host; /* owner: inode, block_device */ | 393 | struct inode *host; /* owner: inode, block_device */ |
| 393 | struct radix_tree_root page_tree; /* radix tree of all pages */ | 394 | struct radix_tree_root i_pages; /* cached pages */ |
| 394 | spinlock_t tree_lock; /* and lock protecting it */ | ||
| 395 | atomic_t i_mmap_writable;/* count VM_SHARED mappings */ | 395 | atomic_t i_mmap_writable;/* count VM_SHARED mappings */ |
| 396 | struct rb_root_cached i_mmap; /* tree of private and shared mappings */ | 396 | struct rb_root_cached i_mmap; /* tree of private and shared mappings */ |
| 397 | struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */ | 397 | struct rw_semaphore i_mmap_rwsem; /* protect tree, count, list */ |
| 398 | /* Protected by tree_lock together with the radix tree */ | 398 | /* Protected by the i_pages lock */ |
| 399 | unsigned long nrpages; /* number of total pages */ | 399 | unsigned long nrpages; /* number of total pages */ |
| 400 | /* number of shadow or DAX exceptional entries */ | 400 | /* number of shadow or DAX exceptional entries */ |
| 401 | unsigned long nrexceptional; | 401 | unsigned long nrexceptional; |
| @@ -1989,7 +1989,7 @@ static inline void init_sync_kiocb(struct kiocb *kiocb, struct file *filp) | |||
| 1989 | * | 1989 | * |
| 1990 | * I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to | 1990 | * I_WB_SWITCH Cgroup bdi_writeback switching in progress. Used to |
| 1991 | * synchronize competing switching instances and to tell | 1991 | * synchronize competing switching instances and to tell |
| 1992 | * wb stat updates to grab mapping->tree_lock. See | 1992 | * wb stat updates to grab the i_pages lock. See |
| 1993 | * inode_switch_wb_work_fn() for details. | 1993 | * inode_switch_wb_work_fn() for details. |
| 1994 | * | 1994 | * |
| 1995 | * I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper | 1995 | * I_OVL_INUSE Used by overlayfs to get exclusive ownership on upper |
diff --git a/include/linux/mm.h b/include/linux/mm.h index f13bc25f7a9f..1ac1f06a4be6 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
| @@ -747,7 +747,7 @@ int finish_mkwrite_fault(struct vm_fault *vmf); | |||
| 747 | * refcount. The each user mapping also has a reference to the page. | 747 | * refcount. The each user mapping also has a reference to the page. |
| 748 | * | 748 | * |
| 749 | * The pagecache pages are stored in a per-mapping radix tree, which is | 749 | * The pagecache pages are stored in a per-mapping radix tree, which is |
| 750 | * rooted at mapping->page_tree, and indexed by offset. | 750 | * rooted at mapping->i_pages, and indexed by offset. |
| 751 | * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space | 751 | * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space |
| 752 | * lists, we instead now tag pages as dirty/writeback in the radix tree. | 752 | * lists, we instead now tag pages as dirty/writeback in the radix tree. |
| 753 | * | 753 | * |
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 34ce3ebf97d5..b1bd2186e6d2 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h | |||
| @@ -144,7 +144,7 @@ void release_pages(struct page **pages, int nr); | |||
| 144 | * 3. check the page is still in pagecache (if no, goto 1) | 144 | * 3. check the page is still in pagecache (if no, goto 1) |
| 145 | * | 145 | * |
| 146 | * Remove-side that cares about stability of _refcount (eg. reclaim) has the | 146 | * Remove-side that cares about stability of _refcount (eg. reclaim) has the |
| 147 | * following (with tree_lock held for write): | 147 | * following (with the i_pages lock held): |
| 148 | * A. atomically check refcount is correct and set it to 0 (atomic_cmpxchg) | 148 | * A. atomically check refcount is correct and set it to 0 (atomic_cmpxchg) |
| 149 | * B. remove page from pagecache | 149 | * B. remove page from pagecache |
| 150 | * C. free the page | 150 | * C. free the page |
| @@ -157,7 +157,7 @@ void release_pages(struct page **pages, int nr); | |||
| 157 | * | 157 | * |
| 158 | * It is possible that between 1 and 2, the page is removed then the exact same | 158 | * It is possible that between 1 and 2, the page is removed then the exact same |
| 159 | * page is inserted into the same position in pagecache. That's OK: the | 159 | * page is inserted into the same position in pagecache. That's OK: the |
| 160 | * old find_get_page using tree_lock could equally have run before or after | 160 | * old find_get_page using a lock could equally have run before or after |
| 161 | * such a re-insertion, depending on order that locks are granted. | 161 | * such a re-insertion, depending on order that locks are granted. |
| 162 | * | 162 | * |
| 163 | * Lookups racing against pagecache insertion isn't a big problem: either 1 | 163 | * Lookups racing against pagecache insertion isn't a big problem: either 1 |
diff --git a/mm/filemap.c b/mm/filemap.c index 693f62212a59..ab77e19ab09c 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
| @@ -66,7 +66,7 @@ | |||
| 66 | * ->i_mmap_rwsem (truncate_pagecache) | 66 | * ->i_mmap_rwsem (truncate_pagecache) |
| 67 | * ->private_lock (__free_pte->__set_page_dirty_buffers) | 67 | * ->private_lock (__free_pte->__set_page_dirty_buffers) |
| 68 | * ->swap_lock (exclusive_swap_page, others) | 68 | * ->swap_lock (exclusive_swap_page, others) |
| 69 | * ->mapping->tree_lock | 69 | * ->i_pages lock |
| 70 | * | 70 | * |
| 71 | * ->i_mutex | 71 | * ->i_mutex |
| 72 | * ->i_mmap_rwsem (truncate->unmap_mapping_range) | 72 | * ->i_mmap_rwsem (truncate->unmap_mapping_range) |
| @@ -74,7 +74,7 @@ | |||
| 74 | * ->mmap_sem | 74 | * ->mmap_sem |
| 75 | * ->i_mmap_rwsem | 75 | * ->i_mmap_rwsem |
| 76 | * ->page_table_lock or pte_lock (various, mainly in memory.c) | 76 | * ->page_table_lock or pte_lock (various, mainly in memory.c) |
| 77 | * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) | 77 | * ->i_pages lock (arch-dependent flush_dcache_mmap_lock) |
| 78 | * | 78 | * |
| 79 | * ->mmap_sem | 79 | * ->mmap_sem |
| 80 | * ->lock_page (access_process_vm) | 80 | * ->lock_page (access_process_vm) |
| @@ -84,7 +84,7 @@ | |||
| 84 | * | 84 | * |
| 85 | * bdi->wb.list_lock | 85 | * bdi->wb.list_lock |
| 86 | * sb_lock (fs/fs-writeback.c) | 86 | * sb_lock (fs/fs-writeback.c) |
| 87 | * ->mapping->tree_lock (__sync_single_inode) | 87 | * ->i_pages lock (__sync_single_inode) |
| 88 | * | 88 | * |
| 89 | * ->i_mmap_rwsem | 89 | * ->i_mmap_rwsem |
| 90 | * ->anon_vma.lock (vma_adjust) | 90 | * ->anon_vma.lock (vma_adjust) |
| @@ -95,11 +95,11 @@ | |||
| 95 | * ->page_table_lock or pte_lock | 95 | * ->page_table_lock or pte_lock |
| 96 | * ->swap_lock (try_to_unmap_one) | 96 | * ->swap_lock (try_to_unmap_one) |
| 97 | * ->private_lock (try_to_unmap_one) | 97 | * ->private_lock (try_to_unmap_one) |
| 98 | * ->tree_lock (try_to_unmap_one) | 98 | * ->i_pages lock (try_to_unmap_one) |
| 99 | * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) | 99 | * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) |
| 100 | * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) | 100 | * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) |
| 101 | * ->private_lock (page_remove_rmap->set_page_dirty) | 101 | * ->private_lock (page_remove_rmap->set_page_dirty) |
| 102 | * ->tree_lock (page_remove_rmap->set_page_dirty) | 102 | * ->i_pages lock (page_remove_rmap->set_page_dirty) |
| 103 | * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) | 103 | * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) |
| 104 | * ->inode->i_lock (page_remove_rmap->set_page_dirty) | 104 | * ->inode->i_lock (page_remove_rmap->set_page_dirty) |
| 105 | * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) | 105 | * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) |
| @@ -118,14 +118,15 @@ static int page_cache_tree_insert(struct address_space *mapping, | |||
| 118 | void **slot; | 118 | void **slot; |
| 119 | int error; | 119 | int error; |
| 120 | 120 | ||
| 121 | error = __radix_tree_create(&mapping->page_tree, page->index, 0, | 121 | error = __radix_tree_create(&mapping->i_pages, page->index, 0, |
| 122 | &node, &slot); | 122 | &node, &slot); |
| 123 | if (error) | 123 | if (error) |
| 124 | return error; | 124 | return error; |
| 125 | if (*slot) { | 125 | if (*slot) { |
| 126 | void *p; | 126 | void *p; |
| 127 | 127 | ||
| 128 | p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock); | 128 | p = radix_tree_deref_slot_protected(slot, |
| 129 | &mapping->i_pages.xa_lock); | ||
| 129 | if (!radix_tree_exceptional_entry(p)) | 130 | if (!radix_tree_exceptional_entry(p)) |
| 130 | return -EEXIST; | 131 | return -EEXIST; |
| 131 | 132 | ||
| @@ -133,7 +134,7 @@ static int page_cache_tree_insert(struct address_space *mapping, | |||
| 133 | if (shadowp) | 134 | if (shadowp) |
| 134 | *shadowp = p; | 135 | *shadowp = p; |
| 135 | } | 136 | } |
| 136 | __radix_tree_replace(&mapping->page_tree, node, slot, page, | 137 | __radix_tree_replace(&mapping->i_pages, node, slot, page, |
| 137 | workingset_lookup_update(mapping)); | 138 | workingset_lookup_update(mapping)); |
| 138 | mapping->nrpages++; | 139 | mapping->nrpages++; |
| 139 | return 0; | 140 | return 0; |
| @@ -155,13 +156,13 @@ static void page_cache_tree_delete(struct address_space *mapping, | |||
| 155 | struct radix_tree_node *node; | 156 | struct radix_tree_node *node; |
| 156 | void **slot; | 157 | void **slot; |
| 157 | 158 | ||
| 158 | __radix_tree_lookup(&mapping->page_tree, page->index + i, | 159 | __radix_tree_lookup(&mapping->i_pages, page->index + i, |
| 159 | &node, &slot); | 160 | &node, &slot); |
| 160 | 161 | ||
| 161 | VM_BUG_ON_PAGE(!node && nr != 1, page); | 162 | VM_BUG_ON_PAGE(!node && nr != 1, page); |
| 162 | 163 | ||
| 163 | radix_tree_clear_tags(&mapping->page_tree, node, slot); | 164 | radix_tree_clear_tags(&mapping->i_pages, node, slot); |
| 164 | __radix_tree_replace(&mapping->page_tree, node, slot, shadow, | 165 | __radix_tree_replace(&mapping->i_pages, node, slot, shadow, |
| 165 | workingset_lookup_update(mapping)); | 166 | workingset_lookup_update(mapping)); |
| 166 | } | 167 | } |
| 167 | 168 | ||
| @@ -253,7 +254,7 @@ static void unaccount_page_cache_page(struct address_space *mapping, | |||
| 253 | /* | 254 | /* |
| 254 | * Delete a page from the page cache and free it. Caller has to make | 255 | * Delete a page from the page cache and free it. Caller has to make |
| 255 | * sure the page is locked and that nobody else uses it - or that usage | 256 | * sure the page is locked and that nobody else uses it - or that usage |
| 256 | * is safe. The caller must hold the mapping's tree_lock. | 257 | * is safe. The caller must hold the i_pages lock. |
| 257 | */ | 258 | */ |
| 258 | void __delete_from_page_cache(struct page *page, void *shadow) | 259 | void __delete_from_page_cache(struct page *page, void *shadow) |
| 259 | { | 260 | { |
| @@ -296,9 +297,9 @@ void delete_from_page_cache(struct page *page) | |||
| 296 | unsigned long flags; | 297 | unsigned long flags; |
| 297 | 298 | ||
| 298 | BUG_ON(!PageLocked(page)); | 299 | BUG_ON(!PageLocked(page)); |
| 299 | spin_lock_irqsave(&mapping->tree_lock, flags); | 300 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 300 | __delete_from_page_cache(page, NULL); | 301 | __delete_from_page_cache(page, NULL); |
| 301 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 302 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 302 | 303 | ||
| 303 | page_cache_free_page(mapping, page); | 304 | page_cache_free_page(mapping, page); |
| 304 | } | 305 | } |
| @@ -309,14 +310,14 @@ EXPORT_SYMBOL(delete_from_page_cache); | |||
| 309 | * @mapping: the mapping to which pages belong | 310 | * @mapping: the mapping to which pages belong |
| 310 | * @pvec: pagevec with pages to delete | 311 | * @pvec: pagevec with pages to delete |
| 311 | * | 312 | * |
| 312 | * The function walks over mapping->page_tree and removes pages passed in @pvec | 313 | * The function walks over mapping->i_pages and removes pages passed in @pvec |
| 313 | * from the radix tree. The function expects @pvec to be sorted by page index. | 314 | * from the mapping. The function expects @pvec to be sorted by page index. |
| 314 | * It tolerates holes in @pvec (radix tree entries at those indices are not | 315 | * It tolerates holes in @pvec (mapping entries at those indices are not |
| 315 | * modified). The function expects only THP head pages to be present in the | 316 | * modified). The function expects only THP head pages to be present in the |
| 316 | * @pvec and takes care to delete all corresponding tail pages from the radix | 317 | * @pvec and takes care to delete all corresponding tail pages from the |
| 317 | * tree as well. | 318 | * mapping as well. |
| 318 | * | 319 | * |
| 319 | * The function expects mapping->tree_lock to be held. | 320 | * The function expects the i_pages lock to be held. |
| 320 | */ | 321 | */ |
| 321 | static void | 322 | static void |
| 322 | page_cache_tree_delete_batch(struct address_space *mapping, | 323 | page_cache_tree_delete_batch(struct address_space *mapping, |
| @@ -330,11 +331,11 @@ page_cache_tree_delete_batch(struct address_space *mapping, | |||
| 330 | pgoff_t start; | 331 | pgoff_t start; |
| 331 | 332 | ||
| 332 | start = pvec->pages[0]->index; | 333 | start = pvec->pages[0]->index; |
| 333 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 334 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 334 | if (i >= pagevec_count(pvec) && !tail_pages) | 335 | if (i >= pagevec_count(pvec) && !tail_pages) |
| 335 | break; | 336 | break; |
| 336 | page = radix_tree_deref_slot_protected(slot, | 337 | page = radix_tree_deref_slot_protected(slot, |
| 337 | &mapping->tree_lock); | 338 | &mapping->i_pages.xa_lock); |
| 338 | if (radix_tree_exceptional_entry(page)) | 339 | if (radix_tree_exceptional_entry(page)) |
| 339 | continue; | 340 | continue; |
| 340 | if (!tail_pages) { | 341 | if (!tail_pages) { |
| @@ -357,8 +358,8 @@ page_cache_tree_delete_batch(struct address_space *mapping, | |||
| 357 | } else { | 358 | } else { |
| 358 | tail_pages--; | 359 | tail_pages--; |
| 359 | } | 360 | } |
| 360 | radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); | 361 | radix_tree_clear_tags(&mapping->i_pages, iter.node, slot); |
| 361 | __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, | 362 | __radix_tree_replace(&mapping->i_pages, iter.node, slot, NULL, |
| 362 | workingset_lookup_update(mapping)); | 363 | workingset_lookup_update(mapping)); |
| 363 | total_pages++; | 364 | total_pages++; |
| 364 | } | 365 | } |
| @@ -374,14 +375,14 @@ void delete_from_page_cache_batch(struct address_space *mapping, | |||
| 374 | if (!pagevec_count(pvec)) | 375 | if (!pagevec_count(pvec)) |
| 375 | return; | 376 | return; |
| 376 | 377 | ||
| 377 | spin_lock_irqsave(&mapping->tree_lock, flags); | 378 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 378 | for (i = 0; i < pagevec_count(pvec); i++) { | 379 | for (i = 0; i < pagevec_count(pvec); i++) { |
| 379 | trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); | 380 | trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); |
| 380 | 381 | ||
| 381 | unaccount_page_cache_page(mapping, pvec->pages[i]); | 382 | unaccount_page_cache_page(mapping, pvec->pages[i]); |
| 382 | } | 383 | } |
| 383 | page_cache_tree_delete_batch(mapping, pvec); | 384 | page_cache_tree_delete_batch(mapping, pvec); |
| 384 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 385 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 385 | 386 | ||
| 386 | for (i = 0; i < pagevec_count(pvec); i++) | 387 | for (i = 0; i < pagevec_count(pvec); i++) |
| 387 | page_cache_free_page(mapping, pvec->pages[i]); | 388 | page_cache_free_page(mapping, pvec->pages[i]); |
| @@ -798,7 +799,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) | |||
| 798 | new->mapping = mapping; | 799 | new->mapping = mapping; |
| 799 | new->index = offset; | 800 | new->index = offset; |
| 800 | 801 | ||
| 801 | spin_lock_irqsave(&mapping->tree_lock, flags); | 802 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 802 | __delete_from_page_cache(old, NULL); | 803 | __delete_from_page_cache(old, NULL); |
| 803 | error = page_cache_tree_insert(mapping, new, NULL); | 804 | error = page_cache_tree_insert(mapping, new, NULL); |
| 804 | BUG_ON(error); | 805 | BUG_ON(error); |
| @@ -810,7 +811,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask) | |||
| 810 | __inc_node_page_state(new, NR_FILE_PAGES); | 811 | __inc_node_page_state(new, NR_FILE_PAGES); |
| 811 | if (PageSwapBacked(new)) | 812 | if (PageSwapBacked(new)) |
| 812 | __inc_node_page_state(new, NR_SHMEM); | 813 | __inc_node_page_state(new, NR_SHMEM); |
| 813 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 814 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 814 | mem_cgroup_migrate(old, new); | 815 | mem_cgroup_migrate(old, new); |
| 815 | radix_tree_preload_end(); | 816 | radix_tree_preload_end(); |
| 816 | if (freepage) | 817 | if (freepage) |
| @@ -852,7 +853,7 @@ static int __add_to_page_cache_locked(struct page *page, | |||
| 852 | page->mapping = mapping; | 853 | page->mapping = mapping; |
| 853 | page->index = offset; | 854 | page->index = offset; |
| 854 | 855 | ||
| 855 | spin_lock_irq(&mapping->tree_lock); | 856 | xa_lock_irq(&mapping->i_pages); |
| 856 | error = page_cache_tree_insert(mapping, page, shadowp); | 857 | error = page_cache_tree_insert(mapping, page, shadowp); |
| 857 | radix_tree_preload_end(); | 858 | radix_tree_preload_end(); |
| 858 | if (unlikely(error)) | 859 | if (unlikely(error)) |
| @@ -861,7 +862,7 @@ static int __add_to_page_cache_locked(struct page *page, | |||
| 861 | /* hugetlb pages do not participate in page cache accounting. */ | 862 | /* hugetlb pages do not participate in page cache accounting. */ |
| 862 | if (!huge) | 863 | if (!huge) |
| 863 | __inc_node_page_state(page, NR_FILE_PAGES); | 864 | __inc_node_page_state(page, NR_FILE_PAGES); |
| 864 | spin_unlock_irq(&mapping->tree_lock); | 865 | xa_unlock_irq(&mapping->i_pages); |
| 865 | if (!huge) | 866 | if (!huge) |
| 866 | mem_cgroup_commit_charge(page, memcg, false, false); | 867 | mem_cgroup_commit_charge(page, memcg, false, false); |
| 867 | trace_mm_filemap_add_to_page_cache(page); | 868 | trace_mm_filemap_add_to_page_cache(page); |
| @@ -869,7 +870,7 @@ static int __add_to_page_cache_locked(struct page *page, | |||
| 869 | err_insert: | 870 | err_insert: |
| 870 | page->mapping = NULL; | 871 | page->mapping = NULL; |
| 871 | /* Leave page->index set: truncation relies upon it */ | 872 | /* Leave page->index set: truncation relies upon it */ |
| 872 | spin_unlock_irq(&mapping->tree_lock); | 873 | xa_unlock_irq(&mapping->i_pages); |
| 873 | if (!huge) | 874 | if (!huge) |
| 874 | mem_cgroup_cancel_charge(page, memcg, false); | 875 | mem_cgroup_cancel_charge(page, memcg, false); |
| 875 | put_page(page); | 876 | put_page(page); |
| @@ -1353,7 +1354,7 @@ pgoff_t page_cache_next_hole(struct address_space *mapping, | |||
| 1353 | for (i = 0; i < max_scan; i++) { | 1354 | for (i = 0; i < max_scan; i++) { |
| 1354 | struct page *page; | 1355 | struct page *page; |
| 1355 | 1356 | ||
| 1356 | page = radix_tree_lookup(&mapping->page_tree, index); | 1357 | page = radix_tree_lookup(&mapping->i_pages, index); |
| 1357 | if (!page || radix_tree_exceptional_entry(page)) | 1358 | if (!page || radix_tree_exceptional_entry(page)) |
| 1358 | break; | 1359 | break; |
| 1359 | index++; | 1360 | index++; |
| @@ -1394,7 +1395,7 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping, | |||
| 1394 | for (i = 0; i < max_scan; i++) { | 1395 | for (i = 0; i < max_scan; i++) { |
| 1395 | struct page *page; | 1396 | struct page *page; |
| 1396 | 1397 | ||
| 1397 | page = radix_tree_lookup(&mapping->page_tree, index); | 1398 | page = radix_tree_lookup(&mapping->i_pages, index); |
| 1398 | if (!page || radix_tree_exceptional_entry(page)) | 1399 | if (!page || radix_tree_exceptional_entry(page)) |
| 1399 | break; | 1400 | break; |
| 1400 | index--; | 1401 | index--; |
| @@ -1427,7 +1428,7 @@ struct page *find_get_entry(struct address_space *mapping, pgoff_t offset) | |||
| 1427 | rcu_read_lock(); | 1428 | rcu_read_lock(); |
| 1428 | repeat: | 1429 | repeat: |
| 1429 | page = NULL; | 1430 | page = NULL; |
| 1430 | pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); | 1431 | pagep = radix_tree_lookup_slot(&mapping->i_pages, offset); |
| 1431 | if (pagep) { | 1432 | if (pagep) { |
| 1432 | page = radix_tree_deref_slot(pagep); | 1433 | page = radix_tree_deref_slot(pagep); |
| 1433 | if (unlikely(!page)) | 1434 | if (unlikely(!page)) |
| @@ -1633,7 +1634,7 @@ unsigned find_get_entries(struct address_space *mapping, | |||
| 1633 | return 0; | 1634 | return 0; |
| 1634 | 1635 | ||
| 1635 | rcu_read_lock(); | 1636 | rcu_read_lock(); |
| 1636 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 1637 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 1637 | struct page *head, *page; | 1638 | struct page *head, *page; |
| 1638 | repeat: | 1639 | repeat: |
| 1639 | page = radix_tree_deref_slot(slot); | 1640 | page = radix_tree_deref_slot(slot); |
| @@ -1710,7 +1711,7 @@ unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start, | |||
| 1710 | return 0; | 1711 | return 0; |
| 1711 | 1712 | ||
| 1712 | rcu_read_lock(); | 1713 | rcu_read_lock(); |
| 1713 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) { | 1714 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, *start) { |
| 1714 | struct page *head, *page; | 1715 | struct page *head, *page; |
| 1715 | 1716 | ||
| 1716 | if (iter.index > end) | 1717 | if (iter.index > end) |
| @@ -1795,7 +1796,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index, | |||
| 1795 | return 0; | 1796 | return 0; |
| 1796 | 1797 | ||
| 1797 | rcu_read_lock(); | 1798 | rcu_read_lock(); |
| 1798 | radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) { | 1799 | radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) { |
| 1799 | struct page *head, *page; | 1800 | struct page *head, *page; |
| 1800 | repeat: | 1801 | repeat: |
| 1801 | page = radix_tree_deref_slot(slot); | 1802 | page = radix_tree_deref_slot(slot); |
| @@ -1875,8 +1876,7 @@ unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index, | |||
| 1875 | return 0; | 1876 | return 0; |
| 1876 | 1877 | ||
| 1877 | rcu_read_lock(); | 1878 | rcu_read_lock(); |
| 1878 | radix_tree_for_each_tagged(slot, &mapping->page_tree, | 1879 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, *index, tag) { |
| 1879 | &iter, *index, tag) { | ||
| 1880 | struct page *head, *page; | 1880 | struct page *head, *page; |
| 1881 | 1881 | ||
| 1882 | if (iter.index > end) | 1882 | if (iter.index > end) |
| @@ -1969,8 +1969,7 @@ unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start, | |||
| 1969 | return 0; | 1969 | return 0; |
| 1970 | 1970 | ||
| 1971 | rcu_read_lock(); | 1971 | rcu_read_lock(); |
| 1972 | radix_tree_for_each_tagged(slot, &mapping->page_tree, | 1972 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, tag) { |
| 1973 | &iter, start, tag) { | ||
| 1974 | struct page *head, *page; | 1973 | struct page *head, *page; |
| 1975 | repeat: | 1974 | repeat: |
| 1976 | page = radix_tree_deref_slot(slot); | 1975 | page = radix_tree_deref_slot(slot); |
| @@ -2624,8 +2623,7 @@ void filemap_map_pages(struct vm_fault *vmf, | |||
| 2624 | struct page *head, *page; | 2623 | struct page *head, *page; |
| 2625 | 2624 | ||
| 2626 | rcu_read_lock(); | 2625 | rcu_read_lock(); |
| 2627 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, | 2626 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start_pgoff) { |
| 2628 | start_pgoff) { | ||
| 2629 | if (iter.index > end_pgoff) | 2627 | if (iter.index > end_pgoff) |
| 2630 | break; | 2628 | break; |
| 2631 | repeat: | 2629 | repeat: |
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 3f3267af4e3b..14ed6ee5e02f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c | |||
| @@ -2450,7 +2450,7 @@ static void __split_huge_page(struct page *page, struct list_head *list, | |||
| 2450 | } else { | 2450 | } else { |
| 2451 | /* Additional pin to radix tree */ | 2451 | /* Additional pin to radix tree */ |
| 2452 | page_ref_add(head, 2); | 2452 | page_ref_add(head, 2); |
| 2453 | spin_unlock(&head->mapping->tree_lock); | 2453 | xa_unlock(&head->mapping->i_pages); |
| 2454 | } | 2454 | } |
| 2455 | 2455 | ||
| 2456 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); | 2456 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); |
| @@ -2658,15 +2658,15 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) | |||
| 2658 | if (mapping) { | 2658 | if (mapping) { |
| 2659 | void **pslot; | 2659 | void **pslot; |
| 2660 | 2660 | ||
| 2661 | spin_lock(&mapping->tree_lock); | 2661 | xa_lock(&mapping->i_pages); |
| 2662 | pslot = radix_tree_lookup_slot(&mapping->page_tree, | 2662 | pslot = radix_tree_lookup_slot(&mapping->i_pages, |
| 2663 | page_index(head)); | 2663 | page_index(head)); |
| 2664 | /* | 2664 | /* |
| 2665 | * Check if the head page is present in radix tree. | 2665 | * Check if the head page is present in radix tree. |
| 2666 | * We assume all tail are present too, if head is there. | 2666 | * We assume all tail are present too, if head is there. |
| 2667 | */ | 2667 | */ |
| 2668 | if (radix_tree_deref_slot_protected(pslot, | 2668 | if (radix_tree_deref_slot_protected(pslot, |
| 2669 | &mapping->tree_lock) != head) | 2669 | &mapping->i_pages.xa_lock) != head) |
| 2670 | goto fail; | 2670 | goto fail; |
| 2671 | } | 2671 | } |
| 2672 | 2672 | ||
| @@ -2700,7 +2700,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list) | |||
| 2700 | } | 2700 | } |
| 2701 | spin_unlock(&pgdata->split_queue_lock); | 2701 | spin_unlock(&pgdata->split_queue_lock); |
| 2702 | fail: if (mapping) | 2702 | fail: if (mapping) |
| 2703 | spin_unlock(&mapping->tree_lock); | 2703 | xa_unlock(&mapping->i_pages); |
| 2704 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); | 2704 | spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); |
| 2705 | unfreeze_page(head); | 2705 | unfreeze_page(head); |
| 2706 | ret = -EBUSY; | 2706 | ret = -EBUSY; |
diff --git a/mm/khugepaged.c b/mm/khugepaged.c index eb32d0707c80..d7b2a4bf8671 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c | |||
| @@ -1344,8 +1344,8 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1344 | */ | 1344 | */ |
| 1345 | 1345 | ||
| 1346 | index = start; | 1346 | index = start; |
| 1347 | spin_lock_irq(&mapping->tree_lock); | 1347 | xa_lock_irq(&mapping->i_pages); |
| 1348 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 1348 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 1349 | int n = min(iter.index, end) - index; | 1349 | int n = min(iter.index, end) - index; |
| 1350 | 1350 | ||
| 1351 | /* | 1351 | /* |
| @@ -1358,7 +1358,7 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1358 | } | 1358 | } |
| 1359 | nr_none += n; | 1359 | nr_none += n; |
| 1360 | for (; index < min(iter.index, end); index++) { | 1360 | for (; index < min(iter.index, end); index++) { |
| 1361 | radix_tree_insert(&mapping->page_tree, index, | 1361 | radix_tree_insert(&mapping->i_pages, index, |
| 1362 | new_page + (index % HPAGE_PMD_NR)); | 1362 | new_page + (index % HPAGE_PMD_NR)); |
| 1363 | } | 1363 | } |
| 1364 | 1364 | ||
| @@ -1367,16 +1367,16 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1367 | break; | 1367 | break; |
| 1368 | 1368 | ||
| 1369 | page = radix_tree_deref_slot_protected(slot, | 1369 | page = radix_tree_deref_slot_protected(slot, |
| 1370 | &mapping->tree_lock); | 1370 | &mapping->i_pages.xa_lock); |
| 1371 | if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { | 1371 | if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { |
| 1372 | spin_unlock_irq(&mapping->tree_lock); | 1372 | xa_unlock_irq(&mapping->i_pages); |
| 1373 | /* swap in or instantiate fallocated page */ | 1373 | /* swap in or instantiate fallocated page */ |
| 1374 | if (shmem_getpage(mapping->host, index, &page, | 1374 | if (shmem_getpage(mapping->host, index, &page, |
| 1375 | SGP_NOHUGE)) { | 1375 | SGP_NOHUGE)) { |
| 1376 | result = SCAN_FAIL; | 1376 | result = SCAN_FAIL; |
| 1377 | goto tree_unlocked; | 1377 | goto tree_unlocked; |
| 1378 | } | 1378 | } |
| 1379 | spin_lock_irq(&mapping->tree_lock); | 1379 | xa_lock_irq(&mapping->i_pages); |
| 1380 | } else if (trylock_page(page)) { | 1380 | } else if (trylock_page(page)) { |
| 1381 | get_page(page); | 1381 | get_page(page); |
| 1382 | } else { | 1382 | } else { |
| @@ -1385,7 +1385,7 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1385 | } | 1385 | } |
| 1386 | 1386 | ||
| 1387 | /* | 1387 | /* |
| 1388 | * The page must be locked, so we can drop the tree_lock | 1388 | * The page must be locked, so we can drop the i_pages lock |
| 1389 | * without racing with truncate. | 1389 | * without racing with truncate. |
| 1390 | */ | 1390 | */ |
| 1391 | VM_BUG_ON_PAGE(!PageLocked(page), page); | 1391 | VM_BUG_ON_PAGE(!PageLocked(page), page); |
| @@ -1396,7 +1396,7 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1396 | result = SCAN_TRUNCATED; | 1396 | result = SCAN_TRUNCATED; |
| 1397 | goto out_unlock; | 1397 | goto out_unlock; |
| 1398 | } | 1398 | } |
| 1399 | spin_unlock_irq(&mapping->tree_lock); | 1399 | xa_unlock_irq(&mapping->i_pages); |
| 1400 | 1400 | ||
| 1401 | if (isolate_lru_page(page)) { | 1401 | if (isolate_lru_page(page)) { |
| 1402 | result = SCAN_DEL_PAGE_LRU; | 1402 | result = SCAN_DEL_PAGE_LRU; |
| @@ -1406,11 +1406,11 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1406 | if (page_mapped(page)) | 1406 | if (page_mapped(page)) |
| 1407 | unmap_mapping_pages(mapping, index, 1, false); | 1407 | unmap_mapping_pages(mapping, index, 1, false); |
| 1408 | 1408 | ||
| 1409 | spin_lock_irq(&mapping->tree_lock); | 1409 | xa_lock_irq(&mapping->i_pages); |
| 1410 | 1410 | ||
| 1411 | slot = radix_tree_lookup_slot(&mapping->page_tree, index); | 1411 | slot = radix_tree_lookup_slot(&mapping->i_pages, index); |
| 1412 | VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, | 1412 | VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, |
| 1413 | &mapping->tree_lock), page); | 1413 | &mapping->i_pages.xa_lock), page); |
| 1414 | VM_BUG_ON_PAGE(page_mapped(page), page); | 1414 | VM_BUG_ON_PAGE(page_mapped(page), page); |
| 1415 | 1415 | ||
| 1416 | /* | 1416 | /* |
| @@ -1431,14 +1431,14 @@ static void collapse_shmem(struct mm_struct *mm, | |||
| 1431 | list_add_tail(&page->lru, &pagelist); | 1431 | list_add_tail(&page->lru, &pagelist); |
| 1432 | 1432 | ||
| 1433 | /* Finally, replace with the new page. */ | 1433 | /* Finally, replace with the new page. */ |
| 1434 | radix_tree_replace_slot(&mapping->page_tree, slot, | 1434 | radix_tree_replace_slot(&mapping->i_pages, slot, |
| 1435 | new_page + (index % HPAGE_PMD_NR)); | 1435 | new_page + (index % HPAGE_PMD_NR)); |
| 1436 | 1436 | ||
| 1437 | slot = radix_tree_iter_resume(slot, &iter); | 1437 | slot = radix_tree_iter_resume(slot, &iter); |
| 1438 | index++; | 1438 | index++; |
| 1439 | continue; | 1439 | continue; |
| 1440 | out_lru: | 1440 | out_lru: |
| 1441 | spin_unlock_irq(&mapping->tree_lock); | 1441 | xa_unlock_irq(&mapping->i_pages); |
| 1442 | putback_lru_page(page); | 1442 | putback_lru_page(page); |
| 1443 | out_isolate_failed: | 1443 | out_isolate_failed: |
| 1444 | unlock_page(page); | 1444 | unlock_page(page); |
| @@ -1464,14 +1464,14 @@ out_unlock: | |||
| 1464 | } | 1464 | } |
| 1465 | 1465 | ||
| 1466 | for (; index < end; index++) { | 1466 | for (; index < end; index++) { |
| 1467 | radix_tree_insert(&mapping->page_tree, index, | 1467 | radix_tree_insert(&mapping->i_pages, index, |
| 1468 | new_page + (index % HPAGE_PMD_NR)); | 1468 | new_page + (index % HPAGE_PMD_NR)); |
| 1469 | } | 1469 | } |
| 1470 | nr_none += n; | 1470 | nr_none += n; |
| 1471 | } | 1471 | } |
| 1472 | 1472 | ||
| 1473 | tree_locked: | 1473 | tree_locked: |
| 1474 | spin_unlock_irq(&mapping->tree_lock); | 1474 | xa_unlock_irq(&mapping->i_pages); |
| 1475 | tree_unlocked: | 1475 | tree_unlocked: |
| 1476 | 1476 | ||
| 1477 | if (result == SCAN_SUCCEED) { | 1477 | if (result == SCAN_SUCCEED) { |
| @@ -1520,9 +1520,8 @@ tree_unlocked: | |||
| 1520 | } else { | 1520 | } else { |
| 1521 | /* Something went wrong: rollback changes to the radix-tree */ | 1521 | /* Something went wrong: rollback changes to the radix-tree */ |
| 1522 | shmem_uncharge(mapping->host, nr_none); | 1522 | shmem_uncharge(mapping->host, nr_none); |
| 1523 | spin_lock_irq(&mapping->tree_lock); | 1523 | xa_lock_irq(&mapping->i_pages); |
| 1524 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, | 1524 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 1525 | start) { | ||
| 1526 | if (iter.index >= end) | 1525 | if (iter.index >= end) |
| 1527 | break; | 1526 | break; |
| 1528 | page = list_first_entry_or_null(&pagelist, | 1527 | page = list_first_entry_or_null(&pagelist, |
| @@ -1532,8 +1531,7 @@ tree_unlocked: | |||
| 1532 | break; | 1531 | break; |
| 1533 | nr_none--; | 1532 | nr_none--; |
| 1534 | /* Put holes back where they were */ | 1533 | /* Put holes back where they were */ |
| 1535 | radix_tree_delete(&mapping->page_tree, | 1534 | radix_tree_delete(&mapping->i_pages, iter.index); |
| 1536 | iter.index); | ||
| 1537 | continue; | 1535 | continue; |
| 1538 | } | 1536 | } |
| 1539 | 1537 | ||
| @@ -1542,16 +1540,15 @@ tree_unlocked: | |||
| 1542 | /* Unfreeze the page. */ | 1540 | /* Unfreeze the page. */ |
| 1543 | list_del(&page->lru); | 1541 | list_del(&page->lru); |
| 1544 | page_ref_unfreeze(page, 2); | 1542 | page_ref_unfreeze(page, 2); |
| 1545 | radix_tree_replace_slot(&mapping->page_tree, | 1543 | radix_tree_replace_slot(&mapping->i_pages, slot, page); |
| 1546 | slot, page); | ||
| 1547 | slot = radix_tree_iter_resume(slot, &iter); | 1544 | slot = radix_tree_iter_resume(slot, &iter); |
| 1548 | spin_unlock_irq(&mapping->tree_lock); | 1545 | xa_unlock_irq(&mapping->i_pages); |
| 1549 | putback_lru_page(page); | 1546 | putback_lru_page(page); |
| 1550 | unlock_page(page); | 1547 | unlock_page(page); |
| 1551 | spin_lock_irq(&mapping->tree_lock); | 1548 | xa_lock_irq(&mapping->i_pages); |
| 1552 | } | 1549 | } |
| 1553 | VM_BUG_ON(nr_none); | 1550 | VM_BUG_ON(nr_none); |
| 1554 | spin_unlock_irq(&mapping->tree_lock); | 1551 | xa_unlock_irq(&mapping->i_pages); |
| 1555 | 1552 | ||
| 1556 | /* Unfreeze new_page, caller would take care about freeing it */ | 1553 | /* Unfreeze new_page, caller would take care about freeing it */ |
| 1557 | page_ref_unfreeze(new_page, 1); | 1554 | page_ref_unfreeze(new_page, 1); |
| @@ -1579,7 +1576,7 @@ static void khugepaged_scan_shmem(struct mm_struct *mm, | |||
| 1579 | swap = 0; | 1576 | swap = 0; |
| 1580 | memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); | 1577 | memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); |
| 1581 | rcu_read_lock(); | 1578 | rcu_read_lock(); |
| 1582 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 1579 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 1583 | if (iter.index >= start + HPAGE_PMD_NR) | 1580 | if (iter.index >= start + HPAGE_PMD_NR) |
| 1584 | break; | 1581 | break; |
| 1585 | 1582 | ||
diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 7978c6faae06..e074f7c637aa 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c | |||
| @@ -5974,9 +5974,9 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry) | |||
| 5974 | 5974 | ||
| 5975 | /* | 5975 | /* |
| 5976 | * Interrupts should be disabled here because the caller holds the | 5976 | * Interrupts should be disabled here because the caller holds the |
| 5977 | * mapping->tree_lock lock which is taken with interrupts-off. It is | 5977 | * i_pages lock which is taken with interrupts-off. It is |
| 5978 | * important here to have the interrupts disabled because it is the | 5978 | * important here to have the interrupts disabled because it is the |
| 5979 | * only synchronisation we have for udpating the per-CPU variables. | 5979 | * only synchronisation we have for updating the per-CPU variables. |
| 5980 | */ | 5980 | */ |
| 5981 | VM_BUG_ON(!irqs_disabled()); | 5981 | VM_BUG_ON(!irqs_disabled()); |
| 5982 | mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), | 5982 | mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), |
diff --git a/mm/migrate.c b/mm/migrate.c index 51b55f2d2db5..f65dd69e1fd1 100644 --- a/mm/migrate.c +++ b/mm/migrate.c | |||
| @@ -467,20 +467,21 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
| 467 | oldzone = page_zone(page); | 467 | oldzone = page_zone(page); |
| 468 | newzone = page_zone(newpage); | 468 | newzone = page_zone(newpage); |
| 469 | 469 | ||
| 470 | spin_lock_irq(&mapping->tree_lock); | 470 | xa_lock_irq(&mapping->i_pages); |
| 471 | 471 | ||
| 472 | pslot = radix_tree_lookup_slot(&mapping->page_tree, | 472 | pslot = radix_tree_lookup_slot(&mapping->i_pages, |
| 473 | page_index(page)); | 473 | page_index(page)); |
| 474 | 474 | ||
| 475 | expected_count += 1 + page_has_private(page); | 475 | expected_count += 1 + page_has_private(page); |
| 476 | if (page_count(page) != expected_count || | 476 | if (page_count(page) != expected_count || |
| 477 | radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { | 477 | radix_tree_deref_slot_protected(pslot, |
| 478 | spin_unlock_irq(&mapping->tree_lock); | 478 | &mapping->i_pages.xa_lock) != page) { |
| 479 | xa_unlock_irq(&mapping->i_pages); | ||
| 479 | return -EAGAIN; | 480 | return -EAGAIN; |
| 480 | } | 481 | } |
| 481 | 482 | ||
| 482 | if (!page_ref_freeze(page, expected_count)) { | 483 | if (!page_ref_freeze(page, expected_count)) { |
| 483 | spin_unlock_irq(&mapping->tree_lock); | 484 | xa_unlock_irq(&mapping->i_pages); |
| 484 | return -EAGAIN; | 485 | return -EAGAIN; |
| 485 | } | 486 | } |
| 486 | 487 | ||
| @@ -494,7 +495,7 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
| 494 | if (mode == MIGRATE_ASYNC && head && | 495 | if (mode == MIGRATE_ASYNC && head && |
| 495 | !buffer_migrate_lock_buffers(head, mode)) { | 496 | !buffer_migrate_lock_buffers(head, mode)) { |
| 496 | page_ref_unfreeze(page, expected_count); | 497 | page_ref_unfreeze(page, expected_count); |
| 497 | spin_unlock_irq(&mapping->tree_lock); | 498 | xa_unlock_irq(&mapping->i_pages); |
| 498 | return -EAGAIN; | 499 | return -EAGAIN; |
| 499 | } | 500 | } |
| 500 | 501 | ||
| @@ -522,7 +523,7 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
| 522 | SetPageDirty(newpage); | 523 | SetPageDirty(newpage); |
| 523 | } | 524 | } |
| 524 | 525 | ||
| 525 | radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); | 526 | radix_tree_replace_slot(&mapping->i_pages, pslot, newpage); |
| 526 | 527 | ||
| 527 | /* | 528 | /* |
| 528 | * Drop cache reference from old page by unfreezing | 529 | * Drop cache reference from old page by unfreezing |
| @@ -531,7 +532,7 @@ int migrate_page_move_mapping(struct address_space *mapping, | |||
| 531 | */ | 532 | */ |
| 532 | page_ref_unfreeze(page, expected_count - 1); | 533 | page_ref_unfreeze(page, expected_count - 1); |
| 533 | 534 | ||
| 534 | spin_unlock(&mapping->tree_lock); | 535 | xa_unlock(&mapping->i_pages); |
| 535 | /* Leave irq disabled to prevent preemption while updating stats */ | 536 | /* Leave irq disabled to prevent preemption while updating stats */ |
| 536 | 537 | ||
| 537 | /* | 538 | /* |
| @@ -574,20 +575,19 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, | |||
| 574 | int expected_count; | 575 | int expected_count; |
| 575 | void **pslot; | 576 | void **pslot; |
| 576 | 577 | ||
| 577 | spin_lock_irq(&mapping->tree_lock); | 578 | xa_lock_irq(&mapping->i_pages); |
| 578 | 579 | ||
| 579 | pslot = radix_tree_lookup_slot(&mapping->page_tree, | 580 | pslot = radix_tree_lookup_slot(&mapping->i_pages, page_index(page)); |
| 580 | page_index(page)); | ||
| 581 | 581 | ||
| 582 | expected_count = 2 + page_has_private(page); | 582 | expected_count = 2 + page_has_private(page); |
| 583 | if (page_count(page) != expected_count || | 583 | if (page_count(page) != expected_count || |
| 584 | radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { | 584 | radix_tree_deref_slot_protected(pslot, &mapping->i_pages.xa_lock) != page) { |
| 585 | spin_unlock_irq(&mapping->tree_lock); | 585 | xa_unlock_irq(&mapping->i_pages); |
| 586 | return -EAGAIN; | 586 | return -EAGAIN; |
| 587 | } | 587 | } |
| 588 | 588 | ||
| 589 | if (!page_ref_freeze(page, expected_count)) { | 589 | if (!page_ref_freeze(page, expected_count)) { |
| 590 | spin_unlock_irq(&mapping->tree_lock); | 590 | xa_unlock_irq(&mapping->i_pages); |
| 591 | return -EAGAIN; | 591 | return -EAGAIN; |
| 592 | } | 592 | } |
| 593 | 593 | ||
| @@ -596,11 +596,11 @@ int migrate_huge_page_move_mapping(struct address_space *mapping, | |||
| 596 | 596 | ||
| 597 | get_page(newpage); | 597 | get_page(newpage); |
| 598 | 598 | ||
| 599 | radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); | 599 | radix_tree_replace_slot(&mapping->i_pages, pslot, newpage); |
| 600 | 600 | ||
| 601 | page_ref_unfreeze(page, expected_count - 1); | 601 | page_ref_unfreeze(page, expected_count - 1); |
| 602 | 602 | ||
| 603 | spin_unlock_irq(&mapping->tree_lock); | 603 | xa_unlock_irq(&mapping->i_pages); |
| 604 | 604 | ||
| 605 | return MIGRATEPAGE_SUCCESS; | 605 | return MIGRATEPAGE_SUCCESS; |
| 606 | } | 606 | } |
diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 586f31261c83..5c1a3279e63f 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c | |||
| @@ -2099,7 +2099,8 @@ void __init page_writeback_init(void) | |||
| 2099 | * so that it can tag pages faster than a dirtying process can create them). | 2099 | * so that it can tag pages faster than a dirtying process can create them). |
| 2100 | */ | 2100 | */ |
| 2101 | /* | 2101 | /* |
| 2102 | * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency. | 2102 | * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce the i_pages lock |
| 2103 | * latency. | ||
| 2103 | */ | 2104 | */ |
| 2104 | void tag_pages_for_writeback(struct address_space *mapping, | 2105 | void tag_pages_for_writeback(struct address_space *mapping, |
| 2105 | pgoff_t start, pgoff_t end) | 2106 | pgoff_t start, pgoff_t end) |
| @@ -2109,22 +2110,22 @@ void tag_pages_for_writeback(struct address_space *mapping, | |||
| 2109 | struct radix_tree_iter iter; | 2110 | struct radix_tree_iter iter; |
| 2110 | void **slot; | 2111 | void **slot; |
| 2111 | 2112 | ||
| 2112 | spin_lock_irq(&mapping->tree_lock); | 2113 | xa_lock_irq(&mapping->i_pages); |
| 2113 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start, | 2114 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, |
| 2114 | PAGECACHE_TAG_DIRTY) { | 2115 | PAGECACHE_TAG_DIRTY) { |
| 2115 | if (iter.index > end) | 2116 | if (iter.index > end) |
| 2116 | break; | 2117 | break; |
| 2117 | radix_tree_iter_tag_set(&mapping->page_tree, &iter, | 2118 | radix_tree_iter_tag_set(&mapping->i_pages, &iter, |
| 2118 | PAGECACHE_TAG_TOWRITE); | 2119 | PAGECACHE_TAG_TOWRITE); |
| 2119 | tagged++; | 2120 | tagged++; |
| 2120 | if ((tagged % WRITEBACK_TAG_BATCH) != 0) | 2121 | if ((tagged % WRITEBACK_TAG_BATCH) != 0) |
| 2121 | continue; | 2122 | continue; |
| 2122 | slot = radix_tree_iter_resume(slot, &iter); | 2123 | slot = radix_tree_iter_resume(slot, &iter); |
| 2123 | spin_unlock_irq(&mapping->tree_lock); | 2124 | xa_unlock_irq(&mapping->i_pages); |
| 2124 | cond_resched(); | 2125 | cond_resched(); |
| 2125 | spin_lock_irq(&mapping->tree_lock); | 2126 | xa_lock_irq(&mapping->i_pages); |
| 2126 | } | 2127 | } |
| 2127 | spin_unlock_irq(&mapping->tree_lock); | 2128 | xa_unlock_irq(&mapping->i_pages); |
| 2128 | } | 2129 | } |
| 2129 | EXPORT_SYMBOL(tag_pages_for_writeback); | 2130 | EXPORT_SYMBOL(tag_pages_for_writeback); |
| 2130 | 2131 | ||
| @@ -2467,13 +2468,13 @@ int __set_page_dirty_nobuffers(struct page *page) | |||
| 2467 | return 1; | 2468 | return 1; |
| 2468 | } | 2469 | } |
| 2469 | 2470 | ||
| 2470 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2471 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 2471 | BUG_ON(page_mapping(page) != mapping); | 2472 | BUG_ON(page_mapping(page) != mapping); |
| 2472 | WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); | 2473 | WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); |
| 2473 | account_page_dirtied(page, mapping); | 2474 | account_page_dirtied(page, mapping); |
| 2474 | radix_tree_tag_set(&mapping->page_tree, page_index(page), | 2475 | radix_tree_tag_set(&mapping->i_pages, page_index(page), |
| 2475 | PAGECACHE_TAG_DIRTY); | 2476 | PAGECACHE_TAG_DIRTY); |
| 2476 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2477 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 2477 | unlock_page_memcg(page); | 2478 | unlock_page_memcg(page); |
| 2478 | 2479 | ||
| 2479 | if (mapping->host) { | 2480 | if (mapping->host) { |
| @@ -2718,11 +2719,10 @@ int test_clear_page_writeback(struct page *page) | |||
| 2718 | struct backing_dev_info *bdi = inode_to_bdi(inode); | 2719 | struct backing_dev_info *bdi = inode_to_bdi(inode); |
| 2719 | unsigned long flags; | 2720 | unsigned long flags; |
| 2720 | 2721 | ||
| 2721 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2722 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 2722 | ret = TestClearPageWriteback(page); | 2723 | ret = TestClearPageWriteback(page); |
| 2723 | if (ret) { | 2724 | if (ret) { |
| 2724 | radix_tree_tag_clear(&mapping->page_tree, | 2725 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
| 2725 | page_index(page), | ||
| 2726 | PAGECACHE_TAG_WRITEBACK); | 2726 | PAGECACHE_TAG_WRITEBACK); |
| 2727 | if (bdi_cap_account_writeback(bdi)) { | 2727 | if (bdi_cap_account_writeback(bdi)) { |
| 2728 | struct bdi_writeback *wb = inode_to_wb(inode); | 2728 | struct bdi_writeback *wb = inode_to_wb(inode); |
| @@ -2736,7 +2736,7 @@ int test_clear_page_writeback(struct page *page) | |||
| 2736 | PAGECACHE_TAG_WRITEBACK)) | 2736 | PAGECACHE_TAG_WRITEBACK)) |
| 2737 | sb_clear_inode_writeback(mapping->host); | 2737 | sb_clear_inode_writeback(mapping->host); |
| 2738 | 2738 | ||
| 2739 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2739 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 2740 | } else { | 2740 | } else { |
| 2741 | ret = TestClearPageWriteback(page); | 2741 | ret = TestClearPageWriteback(page); |
| 2742 | } | 2742 | } |
| @@ -2766,7 +2766,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) | |||
| 2766 | struct backing_dev_info *bdi = inode_to_bdi(inode); | 2766 | struct backing_dev_info *bdi = inode_to_bdi(inode); |
| 2767 | unsigned long flags; | 2767 | unsigned long flags; |
| 2768 | 2768 | ||
| 2769 | spin_lock_irqsave(&mapping->tree_lock, flags); | 2769 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 2770 | ret = TestSetPageWriteback(page); | 2770 | ret = TestSetPageWriteback(page); |
| 2771 | if (!ret) { | 2771 | if (!ret) { |
| 2772 | bool on_wblist; | 2772 | bool on_wblist; |
| @@ -2774,8 +2774,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write) | |||
| 2774 | on_wblist = mapping_tagged(mapping, | 2774 | on_wblist = mapping_tagged(mapping, |
| 2775 | PAGECACHE_TAG_WRITEBACK); | 2775 | PAGECACHE_TAG_WRITEBACK); |
| 2776 | 2776 | ||
| 2777 | radix_tree_tag_set(&mapping->page_tree, | 2777 | radix_tree_tag_set(&mapping->i_pages, page_index(page), |
| 2778 | page_index(page), | ||
| 2779 | PAGECACHE_TAG_WRITEBACK); | 2778 | PAGECACHE_TAG_WRITEBACK); |
| 2780 | if (bdi_cap_account_writeback(bdi)) | 2779 | if (bdi_cap_account_writeback(bdi)) |
| 2781 | inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); | 2780 | inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); |
| @@ -2789,14 +2788,12 @@ int __test_set_page_writeback(struct page *page, bool keep_write) | |||
| 2789 | sb_mark_inode_writeback(mapping->host); | 2788 | sb_mark_inode_writeback(mapping->host); |
| 2790 | } | 2789 | } |
| 2791 | if (!PageDirty(page)) | 2790 | if (!PageDirty(page)) |
| 2792 | radix_tree_tag_clear(&mapping->page_tree, | 2791 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
| 2793 | page_index(page), | ||
| 2794 | PAGECACHE_TAG_DIRTY); | 2792 | PAGECACHE_TAG_DIRTY); |
| 2795 | if (!keep_write) | 2793 | if (!keep_write) |
| 2796 | radix_tree_tag_clear(&mapping->page_tree, | 2794 | radix_tree_tag_clear(&mapping->i_pages, page_index(page), |
| 2797 | page_index(page), | ||
| 2798 | PAGECACHE_TAG_TOWRITE); | 2795 | PAGECACHE_TAG_TOWRITE); |
| 2799 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 2796 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 2800 | } else { | 2797 | } else { |
| 2801 | ret = TestSetPageWriteback(page); | 2798 | ret = TestSetPageWriteback(page); |
| 2802 | } | 2799 | } |
| @@ -2816,7 +2813,7 @@ EXPORT_SYMBOL(__test_set_page_writeback); | |||
| 2816 | */ | 2813 | */ |
| 2817 | int mapping_tagged(struct address_space *mapping, int tag) | 2814 | int mapping_tagged(struct address_space *mapping, int tag) |
| 2818 | { | 2815 | { |
| 2819 | return radix_tree_tagged(&mapping->page_tree, tag); | 2816 | return radix_tree_tagged(&mapping->i_pages, tag); |
| 2820 | } | 2817 | } |
| 2821 | EXPORT_SYMBOL(mapping_tagged); | 2818 | EXPORT_SYMBOL(mapping_tagged); |
| 2822 | 2819 | ||
diff --git a/mm/readahead.c b/mm/readahead.c index 4d57b4644f98..539bbb6c1fad 100644 --- a/mm/readahead.c +++ b/mm/readahead.c | |||
| @@ -175,7 +175,7 @@ int __do_page_cache_readahead(struct address_space *mapping, struct file *filp, | |||
| 175 | break; | 175 | break; |
| 176 | 176 | ||
| 177 | rcu_read_lock(); | 177 | rcu_read_lock(); |
| 178 | page = radix_tree_lookup(&mapping->page_tree, page_offset); | 178 | page = radix_tree_lookup(&mapping->i_pages, page_offset); |
| 179 | rcu_read_unlock(); | 179 | rcu_read_unlock(); |
| 180 | if (page && !radix_tree_exceptional_entry(page)) | 180 | if (page && !radix_tree_exceptional_entry(page)) |
| 181 | continue; | 181 | continue; |
| @@ -32,11 +32,11 @@ | |||
| 32 | * mmlist_lock (in mmput, drain_mmlist and others) | 32 | * mmlist_lock (in mmput, drain_mmlist and others) |
| 33 | * mapping->private_lock (in __set_page_dirty_buffers) | 33 | * mapping->private_lock (in __set_page_dirty_buffers) |
| 34 | * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) | 34 | * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) |
| 35 | * mapping->tree_lock (widely used) | 35 | * i_pages lock (widely used) |
| 36 | * inode->i_lock (in set_page_dirty's __mark_inode_dirty) | 36 | * inode->i_lock (in set_page_dirty's __mark_inode_dirty) |
| 37 | * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) | 37 | * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) |
| 38 | * sb_lock (within inode_lock in fs/fs-writeback.c) | 38 | * sb_lock (within inode_lock in fs/fs-writeback.c) |
| 39 | * mapping->tree_lock (widely used, in set_page_dirty, | 39 | * i_pages lock (widely used, in set_page_dirty, |
| 40 | * in arch-dependent flush_dcache_mmap_lock, | 40 | * in arch-dependent flush_dcache_mmap_lock, |
| 41 | * within bdi.wb->list_lock in __sync_single_inode) | 41 | * within bdi.wb->list_lock in __sync_single_inode) |
| 42 | * | 42 | * |
diff --git a/mm/shmem.c b/mm/shmem.c index 4424fc0c33aa..9d6c7e595415 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
| @@ -332,12 +332,12 @@ static int shmem_radix_tree_replace(struct address_space *mapping, | |||
| 332 | 332 | ||
| 333 | VM_BUG_ON(!expected); | 333 | VM_BUG_ON(!expected); |
| 334 | VM_BUG_ON(!replacement); | 334 | VM_BUG_ON(!replacement); |
| 335 | item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); | 335 | item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot); |
| 336 | if (!item) | 336 | if (!item) |
| 337 | return -ENOENT; | 337 | return -ENOENT; |
| 338 | if (item != expected) | 338 | if (item != expected) |
| 339 | return -ENOENT; | 339 | return -ENOENT; |
| 340 | __radix_tree_replace(&mapping->page_tree, node, pslot, | 340 | __radix_tree_replace(&mapping->i_pages, node, pslot, |
| 341 | replacement, NULL); | 341 | replacement, NULL); |
| 342 | return 0; | 342 | return 0; |
| 343 | } | 343 | } |
| @@ -355,7 +355,7 @@ static bool shmem_confirm_swap(struct address_space *mapping, | |||
| 355 | void *item; | 355 | void *item; |
| 356 | 356 | ||
| 357 | rcu_read_lock(); | 357 | rcu_read_lock(); |
| 358 | item = radix_tree_lookup(&mapping->page_tree, index); | 358 | item = radix_tree_lookup(&mapping->i_pages, index); |
| 359 | rcu_read_unlock(); | 359 | rcu_read_unlock(); |
| 360 | return item == swp_to_radix_entry(swap); | 360 | return item == swp_to_radix_entry(swap); |
| 361 | } | 361 | } |
| @@ -590,14 +590,14 @@ static int shmem_add_to_page_cache(struct page *page, | |||
| 590 | page->mapping = mapping; | 590 | page->mapping = mapping; |
| 591 | page->index = index; | 591 | page->index = index; |
| 592 | 592 | ||
| 593 | spin_lock_irq(&mapping->tree_lock); | 593 | xa_lock_irq(&mapping->i_pages); |
| 594 | if (PageTransHuge(page)) { | 594 | if (PageTransHuge(page)) { |
| 595 | void __rcu **results; | 595 | void __rcu **results; |
| 596 | pgoff_t idx; | 596 | pgoff_t idx; |
| 597 | int i; | 597 | int i; |
| 598 | 598 | ||
| 599 | error = 0; | 599 | error = 0; |
| 600 | if (radix_tree_gang_lookup_slot(&mapping->page_tree, | 600 | if (radix_tree_gang_lookup_slot(&mapping->i_pages, |
| 601 | &results, &idx, index, 1) && | 601 | &results, &idx, index, 1) && |
| 602 | idx < index + HPAGE_PMD_NR) { | 602 | idx < index + HPAGE_PMD_NR) { |
| 603 | error = -EEXIST; | 603 | error = -EEXIST; |
| @@ -605,14 +605,14 @@ static int shmem_add_to_page_cache(struct page *page, | |||
| 605 | 605 | ||
| 606 | if (!error) { | 606 | if (!error) { |
| 607 | for (i = 0; i < HPAGE_PMD_NR; i++) { | 607 | for (i = 0; i < HPAGE_PMD_NR; i++) { |
| 608 | error = radix_tree_insert(&mapping->page_tree, | 608 | error = radix_tree_insert(&mapping->i_pages, |
| 609 | index + i, page + i); | 609 | index + i, page + i); |
| 610 | VM_BUG_ON(error); | 610 | VM_BUG_ON(error); |
| 611 | } | 611 | } |
| 612 | count_vm_event(THP_FILE_ALLOC); | 612 | count_vm_event(THP_FILE_ALLOC); |
| 613 | } | 613 | } |
| 614 | } else if (!expected) { | 614 | } else if (!expected) { |
| 615 | error = radix_tree_insert(&mapping->page_tree, index, page); | 615 | error = radix_tree_insert(&mapping->i_pages, index, page); |
| 616 | } else { | 616 | } else { |
| 617 | error = shmem_radix_tree_replace(mapping, index, expected, | 617 | error = shmem_radix_tree_replace(mapping, index, expected, |
| 618 | page); | 618 | page); |
| @@ -624,10 +624,10 @@ static int shmem_add_to_page_cache(struct page *page, | |||
| 624 | __inc_node_page_state(page, NR_SHMEM_THPS); | 624 | __inc_node_page_state(page, NR_SHMEM_THPS); |
| 625 | __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); | 625 | __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); |
| 626 | __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); | 626 | __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); |
| 627 | spin_unlock_irq(&mapping->tree_lock); | 627 | xa_unlock_irq(&mapping->i_pages); |
| 628 | } else { | 628 | } else { |
| 629 | page->mapping = NULL; | 629 | page->mapping = NULL; |
| 630 | spin_unlock_irq(&mapping->tree_lock); | 630 | xa_unlock_irq(&mapping->i_pages); |
| 631 | page_ref_sub(page, nr); | 631 | page_ref_sub(page, nr); |
| 632 | } | 632 | } |
| 633 | return error; | 633 | return error; |
| @@ -643,13 +643,13 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap) | |||
| 643 | 643 | ||
| 644 | VM_BUG_ON_PAGE(PageCompound(page), page); | 644 | VM_BUG_ON_PAGE(PageCompound(page), page); |
| 645 | 645 | ||
| 646 | spin_lock_irq(&mapping->tree_lock); | 646 | xa_lock_irq(&mapping->i_pages); |
| 647 | error = shmem_radix_tree_replace(mapping, page->index, page, radswap); | 647 | error = shmem_radix_tree_replace(mapping, page->index, page, radswap); |
| 648 | page->mapping = NULL; | 648 | page->mapping = NULL; |
| 649 | mapping->nrpages--; | 649 | mapping->nrpages--; |
| 650 | __dec_node_page_state(page, NR_FILE_PAGES); | 650 | __dec_node_page_state(page, NR_FILE_PAGES); |
| 651 | __dec_node_page_state(page, NR_SHMEM); | 651 | __dec_node_page_state(page, NR_SHMEM); |
| 652 | spin_unlock_irq(&mapping->tree_lock); | 652 | xa_unlock_irq(&mapping->i_pages); |
| 653 | put_page(page); | 653 | put_page(page); |
| 654 | BUG_ON(error); | 654 | BUG_ON(error); |
| 655 | } | 655 | } |
| @@ -662,9 +662,9 @@ static int shmem_free_swap(struct address_space *mapping, | |||
| 662 | { | 662 | { |
| 663 | void *old; | 663 | void *old; |
| 664 | 664 | ||
| 665 | spin_lock_irq(&mapping->tree_lock); | 665 | xa_lock_irq(&mapping->i_pages); |
| 666 | old = radix_tree_delete_item(&mapping->page_tree, index, radswap); | 666 | old = radix_tree_delete_item(&mapping->i_pages, index, radswap); |
| 667 | spin_unlock_irq(&mapping->tree_lock); | 667 | xa_unlock_irq(&mapping->i_pages); |
| 668 | if (old != radswap) | 668 | if (old != radswap) |
| 669 | return -ENOENT; | 669 | return -ENOENT; |
| 670 | free_swap_and_cache(radix_to_swp_entry(radswap)); | 670 | free_swap_and_cache(radix_to_swp_entry(radswap)); |
| @@ -675,7 +675,7 @@ static int shmem_free_swap(struct address_space *mapping, | |||
| 675 | * Determine (in bytes) how many of the shmem object's pages mapped by the | 675 | * Determine (in bytes) how many of the shmem object's pages mapped by the |
| 676 | * given offsets are swapped out. | 676 | * given offsets are swapped out. |
| 677 | * | 677 | * |
| 678 | * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, | 678 | * This is safe to call without i_mutex or the i_pages lock thanks to RCU, |
| 679 | * as long as the inode doesn't go away and racy results are not a problem. | 679 | * as long as the inode doesn't go away and racy results are not a problem. |
| 680 | */ | 680 | */ |
| 681 | unsigned long shmem_partial_swap_usage(struct address_space *mapping, | 681 | unsigned long shmem_partial_swap_usage(struct address_space *mapping, |
| @@ -688,7 +688,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, | |||
| 688 | 688 | ||
| 689 | rcu_read_lock(); | 689 | rcu_read_lock(); |
| 690 | 690 | ||
| 691 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 691 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 692 | if (iter.index >= end) | 692 | if (iter.index >= end) |
| 693 | break; | 693 | break; |
| 694 | 694 | ||
| @@ -717,7 +717,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping, | |||
| 717 | * Determine (in bytes) how many of the shmem object's pages mapped by the | 717 | * Determine (in bytes) how many of the shmem object's pages mapped by the |
| 718 | * given vma is swapped out. | 718 | * given vma is swapped out. |
| 719 | * | 719 | * |
| 720 | * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, | 720 | * This is safe to call without i_mutex or the i_pages lock thanks to RCU, |
| 721 | * as long as the inode doesn't go away and racy results are not a problem. | 721 | * as long as the inode doesn't go away and racy results are not a problem. |
| 722 | */ | 722 | */ |
| 723 | unsigned long shmem_swap_usage(struct vm_area_struct *vma) | 723 | unsigned long shmem_swap_usage(struct vm_area_struct *vma) |
| @@ -1132,7 +1132,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, | |||
| 1132 | int error = 0; | 1132 | int error = 0; |
| 1133 | 1133 | ||
| 1134 | radswap = swp_to_radix_entry(swap); | 1134 | radswap = swp_to_radix_entry(swap); |
| 1135 | index = find_swap_entry(&mapping->page_tree, radswap); | 1135 | index = find_swap_entry(&mapping->i_pages, radswap); |
| 1136 | if (index == -1) | 1136 | if (index == -1) |
| 1137 | return -EAGAIN; /* tell shmem_unuse we found nothing */ | 1137 | return -EAGAIN; /* tell shmem_unuse we found nothing */ |
| 1138 | 1138 | ||
| @@ -1448,7 +1448,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp, | |||
| 1448 | 1448 | ||
| 1449 | hindex = round_down(index, HPAGE_PMD_NR); | 1449 | hindex = round_down(index, HPAGE_PMD_NR); |
| 1450 | rcu_read_lock(); | 1450 | rcu_read_lock(); |
| 1451 | if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, | 1451 | if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx, |
| 1452 | hindex, 1) && idx < hindex + HPAGE_PMD_NR) { | 1452 | hindex, 1) && idx < hindex + HPAGE_PMD_NR) { |
| 1453 | rcu_read_unlock(); | 1453 | rcu_read_unlock(); |
| 1454 | return NULL; | 1454 | return NULL; |
| @@ -1561,14 +1561,14 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp, | |||
| 1561 | * Our caller will very soon move newpage out of swapcache, but it's | 1561 | * Our caller will very soon move newpage out of swapcache, but it's |
| 1562 | * a nice clean interface for us to replace oldpage by newpage there. | 1562 | * a nice clean interface for us to replace oldpage by newpage there. |
| 1563 | */ | 1563 | */ |
| 1564 | spin_lock_irq(&swap_mapping->tree_lock); | 1564 | xa_lock_irq(&swap_mapping->i_pages); |
| 1565 | error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, | 1565 | error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, |
| 1566 | newpage); | 1566 | newpage); |
| 1567 | if (!error) { | 1567 | if (!error) { |
| 1568 | __inc_node_page_state(newpage, NR_FILE_PAGES); | 1568 | __inc_node_page_state(newpage, NR_FILE_PAGES); |
| 1569 | __dec_node_page_state(oldpage, NR_FILE_PAGES); | 1569 | __dec_node_page_state(oldpage, NR_FILE_PAGES); |
| 1570 | } | 1570 | } |
| 1571 | spin_unlock_irq(&swap_mapping->tree_lock); | 1571 | xa_unlock_irq(&swap_mapping->i_pages); |
| 1572 | 1572 | ||
| 1573 | if (unlikely(error)) { | 1573 | if (unlikely(error)) { |
| 1574 | /* | 1574 | /* |
| @@ -2634,7 +2634,7 @@ static void shmem_tag_pins(struct address_space *mapping) | |||
| 2634 | start = 0; | 2634 | start = 0; |
| 2635 | rcu_read_lock(); | 2635 | rcu_read_lock(); |
| 2636 | 2636 | ||
| 2637 | radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { | 2637 | radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) { |
| 2638 | page = radix_tree_deref_slot(slot); | 2638 | page = radix_tree_deref_slot(slot); |
| 2639 | if (!page || radix_tree_exception(page)) { | 2639 | if (!page || radix_tree_exception(page)) { |
| 2640 | if (radix_tree_deref_retry(page)) { | 2640 | if (radix_tree_deref_retry(page)) { |
| @@ -2642,10 +2642,10 @@ static void shmem_tag_pins(struct address_space *mapping) | |||
| 2642 | continue; | 2642 | continue; |
| 2643 | } | 2643 | } |
| 2644 | } else if (page_count(page) - page_mapcount(page) > 1) { | 2644 | } else if (page_count(page) - page_mapcount(page) > 1) { |
| 2645 | spin_lock_irq(&mapping->tree_lock); | 2645 | xa_lock_irq(&mapping->i_pages); |
| 2646 | radix_tree_tag_set(&mapping->page_tree, iter.index, | 2646 | radix_tree_tag_set(&mapping->i_pages, iter.index, |
| 2647 | SHMEM_TAG_PINNED); | 2647 | SHMEM_TAG_PINNED); |
| 2648 | spin_unlock_irq(&mapping->tree_lock); | 2648 | xa_unlock_irq(&mapping->i_pages); |
| 2649 | } | 2649 | } |
| 2650 | 2650 | ||
| 2651 | if (need_resched()) { | 2651 | if (need_resched()) { |
| @@ -2677,7 +2677,7 @@ static int shmem_wait_for_pins(struct address_space *mapping) | |||
| 2677 | 2677 | ||
| 2678 | error = 0; | 2678 | error = 0; |
| 2679 | for (scan = 0; scan <= LAST_SCAN; scan++) { | 2679 | for (scan = 0; scan <= LAST_SCAN; scan++) { |
| 2680 | if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) | 2680 | if (!radix_tree_tagged(&mapping->i_pages, SHMEM_TAG_PINNED)) |
| 2681 | break; | 2681 | break; |
| 2682 | 2682 | ||
| 2683 | if (!scan) | 2683 | if (!scan) |
| @@ -2687,7 +2687,7 @@ static int shmem_wait_for_pins(struct address_space *mapping) | |||
| 2687 | 2687 | ||
| 2688 | start = 0; | 2688 | start = 0; |
| 2689 | rcu_read_lock(); | 2689 | rcu_read_lock(); |
| 2690 | radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, | 2690 | radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, |
| 2691 | start, SHMEM_TAG_PINNED) { | 2691 | start, SHMEM_TAG_PINNED) { |
| 2692 | 2692 | ||
| 2693 | page = radix_tree_deref_slot(slot); | 2693 | page = radix_tree_deref_slot(slot); |
| @@ -2713,10 +2713,10 @@ static int shmem_wait_for_pins(struct address_space *mapping) | |||
| 2713 | error = -EBUSY; | 2713 | error = -EBUSY; |
| 2714 | } | 2714 | } |
| 2715 | 2715 | ||
| 2716 | spin_lock_irq(&mapping->tree_lock); | 2716 | xa_lock_irq(&mapping->i_pages); |
| 2717 | radix_tree_tag_clear(&mapping->page_tree, | 2717 | radix_tree_tag_clear(&mapping->i_pages, |
| 2718 | iter.index, SHMEM_TAG_PINNED); | 2718 | iter.index, SHMEM_TAG_PINNED); |
| 2719 | spin_unlock_irq(&mapping->tree_lock); | 2719 | xa_unlock_irq(&mapping->i_pages); |
| 2720 | continue_resched: | 2720 | continue_resched: |
| 2721 | if (need_resched()) { | 2721 | if (need_resched()) { |
| 2722 | slot = radix_tree_iter_resume(slot, &iter); | 2722 | slot = radix_tree_iter_resume(slot, &iter); |
diff --git a/mm/swap_state.c b/mm/swap_state.c index f233dccd3b1b..07f9aa2340c3 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c | |||
| @@ -124,10 +124,10 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) | |||
| 124 | SetPageSwapCache(page); | 124 | SetPageSwapCache(page); |
| 125 | 125 | ||
| 126 | address_space = swap_address_space(entry); | 126 | address_space = swap_address_space(entry); |
| 127 | spin_lock_irq(&address_space->tree_lock); | 127 | xa_lock_irq(&address_space->i_pages); |
| 128 | for (i = 0; i < nr; i++) { | 128 | for (i = 0; i < nr; i++) { |
| 129 | set_page_private(page + i, entry.val + i); | 129 | set_page_private(page + i, entry.val + i); |
| 130 | error = radix_tree_insert(&address_space->page_tree, | 130 | error = radix_tree_insert(&address_space->i_pages, |
| 131 | idx + i, page + i); | 131 | idx + i, page + i); |
| 132 | if (unlikely(error)) | 132 | if (unlikely(error)) |
| 133 | break; | 133 | break; |
| @@ -145,13 +145,13 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry) | |||
| 145 | VM_BUG_ON(error == -EEXIST); | 145 | VM_BUG_ON(error == -EEXIST); |
| 146 | set_page_private(page + i, 0UL); | 146 | set_page_private(page + i, 0UL); |
| 147 | while (i--) { | 147 | while (i--) { |
| 148 | radix_tree_delete(&address_space->page_tree, idx + i); | 148 | radix_tree_delete(&address_space->i_pages, idx + i); |
| 149 | set_page_private(page + i, 0UL); | 149 | set_page_private(page + i, 0UL); |
| 150 | } | 150 | } |
| 151 | ClearPageSwapCache(page); | 151 | ClearPageSwapCache(page); |
| 152 | page_ref_sub(page, nr); | 152 | page_ref_sub(page, nr); |
| 153 | } | 153 | } |
| 154 | spin_unlock_irq(&address_space->tree_lock); | 154 | xa_unlock_irq(&address_space->i_pages); |
| 155 | 155 | ||
| 156 | return error; | 156 | return error; |
| 157 | } | 157 | } |
| @@ -188,7 +188,7 @@ void __delete_from_swap_cache(struct page *page) | |||
| 188 | address_space = swap_address_space(entry); | 188 | address_space = swap_address_space(entry); |
| 189 | idx = swp_offset(entry); | 189 | idx = swp_offset(entry); |
| 190 | for (i = 0; i < nr; i++) { | 190 | for (i = 0; i < nr; i++) { |
| 191 | radix_tree_delete(&address_space->page_tree, idx + i); | 191 | radix_tree_delete(&address_space->i_pages, idx + i); |
| 192 | set_page_private(page + i, 0); | 192 | set_page_private(page + i, 0); |
| 193 | } | 193 | } |
| 194 | ClearPageSwapCache(page); | 194 | ClearPageSwapCache(page); |
| @@ -272,9 +272,9 @@ void delete_from_swap_cache(struct page *page) | |||
| 272 | entry.val = page_private(page); | 272 | entry.val = page_private(page); |
| 273 | 273 | ||
| 274 | address_space = swap_address_space(entry); | 274 | address_space = swap_address_space(entry); |
| 275 | spin_lock_irq(&address_space->tree_lock); | 275 | xa_lock_irq(&address_space->i_pages); |
| 276 | __delete_from_swap_cache(page); | 276 | __delete_from_swap_cache(page); |
| 277 | spin_unlock_irq(&address_space->tree_lock); | 277 | xa_unlock_irq(&address_space->i_pages); |
| 278 | 278 | ||
| 279 | put_swap_page(page, entry); | 279 | put_swap_page(page, entry); |
| 280 | page_ref_sub(page, hpage_nr_pages(page)); | 280 | page_ref_sub(page, hpage_nr_pages(page)); |
| @@ -628,12 +628,11 @@ int init_swap_address_space(unsigned int type, unsigned long nr_pages) | |||
| 628 | return -ENOMEM; | 628 | return -ENOMEM; |
| 629 | for (i = 0; i < nr; i++) { | 629 | for (i = 0; i < nr; i++) { |
| 630 | space = spaces + i; | 630 | space = spaces + i; |
| 631 | INIT_RADIX_TREE(&space->page_tree, GFP_ATOMIC|__GFP_NOWARN); | 631 | INIT_RADIX_TREE(&space->i_pages, GFP_ATOMIC|__GFP_NOWARN); |
| 632 | atomic_set(&space->i_mmap_writable, 0); | 632 | atomic_set(&space->i_mmap_writable, 0); |
| 633 | space->a_ops = &swap_aops; | 633 | space->a_ops = &swap_aops; |
| 634 | /* swap cache doesn't use writeback related tags */ | 634 | /* swap cache doesn't use writeback related tags */ |
| 635 | mapping_set_no_writeback_tags(space); | 635 | mapping_set_no_writeback_tags(space); |
| 636 | spin_lock_init(&space->tree_lock); | ||
| 637 | } | 636 | } |
| 638 | nr_swapper_spaces[type] = nr; | 637 | nr_swapper_spaces[type] = nr; |
| 639 | rcu_assign_pointer(swapper_spaces[type], spaces); | 638 | rcu_assign_pointer(swapper_spaces[type], spaces); |
diff --git a/mm/truncate.c b/mm/truncate.c index c34e2fd4f583..1d2fb2dca96f 100644 --- a/mm/truncate.c +++ b/mm/truncate.c | |||
| @@ -36,11 +36,11 @@ static inline void __clear_shadow_entry(struct address_space *mapping, | |||
| 36 | struct radix_tree_node *node; | 36 | struct radix_tree_node *node; |
| 37 | void **slot; | 37 | void **slot; |
| 38 | 38 | ||
| 39 | if (!__radix_tree_lookup(&mapping->page_tree, index, &node, &slot)) | 39 | if (!__radix_tree_lookup(&mapping->i_pages, index, &node, &slot)) |
| 40 | return; | 40 | return; |
| 41 | if (*slot != entry) | 41 | if (*slot != entry) |
| 42 | return; | 42 | return; |
| 43 | __radix_tree_replace(&mapping->page_tree, node, slot, NULL, | 43 | __radix_tree_replace(&mapping->i_pages, node, slot, NULL, |
| 44 | workingset_update_node); | 44 | workingset_update_node); |
| 45 | mapping->nrexceptional--; | 45 | mapping->nrexceptional--; |
| 46 | } | 46 | } |
| @@ -48,9 +48,9 @@ static inline void __clear_shadow_entry(struct address_space *mapping, | |||
| 48 | static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, | 48 | static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, |
| 49 | void *entry) | 49 | void *entry) |
| 50 | { | 50 | { |
| 51 | spin_lock_irq(&mapping->tree_lock); | 51 | xa_lock_irq(&mapping->i_pages); |
| 52 | __clear_shadow_entry(mapping, index, entry); | 52 | __clear_shadow_entry(mapping, index, entry); |
| 53 | spin_unlock_irq(&mapping->tree_lock); | 53 | xa_unlock_irq(&mapping->i_pages); |
| 54 | } | 54 | } |
| 55 | 55 | ||
| 56 | /* | 56 | /* |
| @@ -79,7 +79,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, | |||
| 79 | dax = dax_mapping(mapping); | 79 | dax = dax_mapping(mapping); |
| 80 | lock = !dax && indices[j] < end; | 80 | lock = !dax && indices[j] < end; |
| 81 | if (lock) | 81 | if (lock) |
| 82 | spin_lock_irq(&mapping->tree_lock); | 82 | xa_lock_irq(&mapping->i_pages); |
| 83 | 83 | ||
| 84 | for (i = j; i < pagevec_count(pvec); i++) { | 84 | for (i = j; i < pagevec_count(pvec); i++) { |
| 85 | struct page *page = pvec->pages[i]; | 85 | struct page *page = pvec->pages[i]; |
| @@ -102,7 +102,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping, | |||
| 102 | } | 102 | } |
| 103 | 103 | ||
| 104 | if (lock) | 104 | if (lock) |
| 105 | spin_unlock_irq(&mapping->tree_lock); | 105 | xa_unlock_irq(&mapping->i_pages); |
| 106 | pvec->nr = j; | 106 | pvec->nr = j; |
| 107 | } | 107 | } |
| 108 | 108 | ||
| @@ -518,8 +518,8 @@ void truncate_inode_pages_final(struct address_space *mapping) | |||
| 518 | * modification that does not see AS_EXITING is | 518 | * modification that does not see AS_EXITING is |
| 519 | * completed before starting the final truncate. | 519 | * completed before starting the final truncate. |
| 520 | */ | 520 | */ |
| 521 | spin_lock_irq(&mapping->tree_lock); | 521 | xa_lock_irq(&mapping->i_pages); |
| 522 | spin_unlock_irq(&mapping->tree_lock); | 522 | xa_unlock_irq(&mapping->i_pages); |
| 523 | 523 | ||
| 524 | truncate_inode_pages(mapping, 0); | 524 | truncate_inode_pages(mapping, 0); |
| 525 | } | 525 | } |
| @@ -627,13 +627,13 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page) | |||
| 627 | if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) | 627 | if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) |
| 628 | return 0; | 628 | return 0; |
| 629 | 629 | ||
| 630 | spin_lock_irqsave(&mapping->tree_lock, flags); | 630 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 631 | if (PageDirty(page)) | 631 | if (PageDirty(page)) |
| 632 | goto failed; | 632 | goto failed; |
| 633 | 633 | ||
| 634 | BUG_ON(page_has_private(page)); | 634 | BUG_ON(page_has_private(page)); |
| 635 | __delete_from_page_cache(page, NULL); | 635 | __delete_from_page_cache(page, NULL); |
| 636 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 636 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 637 | 637 | ||
| 638 | if (mapping->a_ops->freepage) | 638 | if (mapping->a_ops->freepage) |
| 639 | mapping->a_ops->freepage(page); | 639 | mapping->a_ops->freepage(page); |
| @@ -641,7 +641,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page) | |||
| 641 | put_page(page); /* pagecache ref */ | 641 | put_page(page); /* pagecache ref */ |
| 642 | return 1; | 642 | return 1; |
| 643 | failed: | 643 | failed: |
| 644 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 644 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 645 | return 0; | 645 | return 0; |
| 646 | } | 646 | } |
| 647 | 647 | ||
diff --git a/mm/vmscan.c b/mm/vmscan.c index 671597ce1ea0..8b920ce3ae02 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c | |||
| @@ -693,7 +693,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
| 693 | BUG_ON(!PageLocked(page)); | 693 | BUG_ON(!PageLocked(page)); |
| 694 | BUG_ON(mapping != page_mapping(page)); | 694 | BUG_ON(mapping != page_mapping(page)); |
| 695 | 695 | ||
| 696 | spin_lock_irqsave(&mapping->tree_lock, flags); | 696 | xa_lock_irqsave(&mapping->i_pages, flags); |
| 697 | /* | 697 | /* |
| 698 | * The non racy check for a busy page. | 698 | * The non racy check for a busy page. |
| 699 | * | 699 | * |
| @@ -717,7 +717,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
| 717 | * load is not satisfied before that of page->_refcount. | 717 | * load is not satisfied before that of page->_refcount. |
| 718 | * | 718 | * |
| 719 | * Note that if SetPageDirty is always performed via set_page_dirty, | 719 | * Note that if SetPageDirty is always performed via set_page_dirty, |
| 720 | * and thus under tree_lock, then this ordering is not required. | 720 | * and thus under the i_pages lock, then this ordering is not required. |
| 721 | */ | 721 | */ |
| 722 | if (unlikely(PageTransHuge(page)) && PageSwapCache(page)) | 722 | if (unlikely(PageTransHuge(page)) && PageSwapCache(page)) |
| 723 | refcount = 1 + HPAGE_PMD_NR; | 723 | refcount = 1 + HPAGE_PMD_NR; |
| @@ -735,7 +735,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
| 735 | swp_entry_t swap = { .val = page_private(page) }; | 735 | swp_entry_t swap = { .val = page_private(page) }; |
| 736 | mem_cgroup_swapout(page, swap); | 736 | mem_cgroup_swapout(page, swap); |
| 737 | __delete_from_swap_cache(page); | 737 | __delete_from_swap_cache(page); |
| 738 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 738 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 739 | put_swap_page(page, swap); | 739 | put_swap_page(page, swap); |
| 740 | } else { | 740 | } else { |
| 741 | void (*freepage)(struct page *); | 741 | void (*freepage)(struct page *); |
| @@ -756,13 +756,13 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
| 756 | * only page cache pages found in these are zero pages | 756 | * only page cache pages found in these are zero pages |
| 757 | * covering holes, and because we don't want to mix DAX | 757 | * covering holes, and because we don't want to mix DAX |
| 758 | * exceptional entries and shadow exceptional entries in the | 758 | * exceptional entries and shadow exceptional entries in the |
| 759 | * same page_tree. | 759 | * same address_space. |
| 760 | */ | 760 | */ |
| 761 | if (reclaimed && page_is_file_cache(page) && | 761 | if (reclaimed && page_is_file_cache(page) && |
| 762 | !mapping_exiting(mapping) && !dax_mapping(mapping)) | 762 | !mapping_exiting(mapping) && !dax_mapping(mapping)) |
| 763 | shadow = workingset_eviction(mapping, page); | 763 | shadow = workingset_eviction(mapping, page); |
| 764 | __delete_from_page_cache(page, shadow); | 764 | __delete_from_page_cache(page, shadow); |
| 765 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 765 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 766 | 766 | ||
| 767 | if (freepage != NULL) | 767 | if (freepage != NULL) |
| 768 | freepage(page); | 768 | freepage(page); |
| @@ -771,7 +771,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page, | |||
| 771 | return 1; | 771 | return 1; |
| 772 | 772 | ||
| 773 | cannot_free: | 773 | cannot_free: |
| 774 | spin_unlock_irqrestore(&mapping->tree_lock, flags); | 774 | xa_unlock_irqrestore(&mapping->i_pages, flags); |
| 775 | return 0; | 775 | return 0; |
| 776 | } | 776 | } |
| 777 | 777 | ||
diff --git a/mm/workingset.c b/mm/workingset.c index b7d616a3bbbe..40ee02c83978 100644 --- a/mm/workingset.c +++ b/mm/workingset.c | |||
| @@ -202,7 +202,7 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat, | |||
| 202 | * @mapping: address space the page was backing | 202 | * @mapping: address space the page was backing |
| 203 | * @page: the page being evicted | 203 | * @page: the page being evicted |
| 204 | * | 204 | * |
| 205 | * Returns a shadow entry to be stored in @mapping->page_tree in place | 205 | * Returns a shadow entry to be stored in @mapping->i_pages in place |
| 206 | * of the evicted @page so that a later refault can be detected. | 206 | * of the evicted @page so that a later refault can be detected. |
| 207 | */ | 207 | */ |
| 208 | void *workingset_eviction(struct address_space *mapping, struct page *page) | 208 | void *workingset_eviction(struct address_space *mapping, struct page *page) |
| @@ -348,7 +348,7 @@ void workingset_update_node(struct radix_tree_node *node) | |||
| 348 | * | 348 | * |
| 349 | * Avoid acquiring the list_lru lock when the nodes are | 349 | * Avoid acquiring the list_lru lock when the nodes are |
| 350 | * already where they should be. The list_empty() test is safe | 350 | * already where they should be. The list_empty() test is safe |
| 351 | * as node->private_list is protected by &mapping->tree_lock. | 351 | * as node->private_list is protected by the i_pages lock. |
| 352 | */ | 352 | */ |
| 353 | if (node->count && node->count == node->exceptional) { | 353 | if (node->count && node->count == node->exceptional) { |
| 354 | if (list_empty(&node->private_list)) | 354 | if (list_empty(&node->private_list)) |
| @@ -366,7 +366,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker, | |||
| 366 | unsigned long nodes; | 366 | unsigned long nodes; |
| 367 | unsigned long cache; | 367 | unsigned long cache; |
| 368 | 368 | ||
| 369 | /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ | 369 | /* list_lru lock nests inside the IRQ-safe i_pages lock */ |
| 370 | local_irq_disable(); | 370 | local_irq_disable(); |
| 371 | nodes = list_lru_shrink_count(&shadow_nodes, sc); | 371 | nodes = list_lru_shrink_count(&shadow_nodes, sc); |
| 372 | local_irq_enable(); | 372 | local_irq_enable(); |
| @@ -419,21 +419,21 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, | |||
| 419 | 419 | ||
| 420 | /* | 420 | /* |
| 421 | * Page cache insertions and deletions synchroneously maintain | 421 | * Page cache insertions and deletions synchroneously maintain |
| 422 | * the shadow node LRU under the mapping->tree_lock and the | 422 | * the shadow node LRU under the i_pages lock and the |
| 423 | * lru_lock. Because the page cache tree is emptied before | 423 | * lru_lock. Because the page cache tree is emptied before |
| 424 | * the inode can be destroyed, holding the lru_lock pins any | 424 | * the inode can be destroyed, holding the lru_lock pins any |
| 425 | * address_space that has radix tree nodes on the LRU. | 425 | * address_space that has radix tree nodes on the LRU. |
| 426 | * | 426 | * |
| 427 | * We can then safely transition to the mapping->tree_lock to | 427 | * We can then safely transition to the i_pages lock to |
| 428 | * pin only the address_space of the particular node we want | 428 | * pin only the address_space of the particular node we want |
| 429 | * to reclaim, take the node off-LRU, and drop the lru_lock. | 429 | * to reclaim, take the node off-LRU, and drop the lru_lock. |
| 430 | */ | 430 | */ |
| 431 | 431 | ||
| 432 | node = container_of(item, struct radix_tree_node, private_list); | 432 | node = container_of(item, struct radix_tree_node, private_list); |
| 433 | mapping = container_of(node->root, struct address_space, page_tree); | 433 | mapping = container_of(node->root, struct address_space, i_pages); |
| 434 | 434 | ||
| 435 | /* Coming from the list, invert the lock order */ | 435 | /* Coming from the list, invert the lock order */ |
| 436 | if (!spin_trylock(&mapping->tree_lock)) { | 436 | if (!xa_trylock(&mapping->i_pages)) { |
| 437 | spin_unlock(lru_lock); | 437 | spin_unlock(lru_lock); |
| 438 | ret = LRU_RETRY; | 438 | ret = LRU_RETRY; |
| 439 | goto out; | 439 | goto out; |
| @@ -468,11 +468,11 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, | |||
| 468 | if (WARN_ON_ONCE(node->exceptional)) | 468 | if (WARN_ON_ONCE(node->exceptional)) |
| 469 | goto out_invalid; | 469 | goto out_invalid; |
| 470 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); | 470 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); |
| 471 | __radix_tree_delete_node(&mapping->page_tree, node, | 471 | __radix_tree_delete_node(&mapping->i_pages, node, |
| 472 | workingset_lookup_update(mapping)); | 472 | workingset_lookup_update(mapping)); |
| 473 | 473 | ||
| 474 | out_invalid: | 474 | out_invalid: |
| 475 | spin_unlock(&mapping->tree_lock); | 475 | xa_unlock(&mapping->i_pages); |
| 476 | ret = LRU_REMOVED_RETRY; | 476 | ret = LRU_REMOVED_RETRY; |
| 477 | out: | 477 | out: |
| 478 | local_irq_enable(); | 478 | local_irq_enable(); |
| @@ -487,7 +487,7 @@ static unsigned long scan_shadow_nodes(struct shrinker *shrinker, | |||
| 487 | { | 487 | { |
| 488 | unsigned long ret; | 488 | unsigned long ret; |
| 489 | 489 | ||
| 490 | /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ | 490 | /* list_lru lock nests inside the IRQ-safe i_pages lock */ |
| 491 | local_irq_disable(); | 491 | local_irq_disable(); |
| 492 | ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); | 492 | ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); |
| 493 | local_irq_enable(); | 493 | local_irq_enable(); |
| @@ -503,7 +503,7 @@ static struct shrinker workingset_shadow_shrinker = { | |||
| 503 | 503 | ||
| 504 | /* | 504 | /* |
| 505 | * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe | 505 | * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe |
| 506 | * mapping->tree_lock. | 506 | * i_pages lock. |
| 507 | */ | 507 | */ |
| 508 | static struct lock_class_key shadow_nodes_key; | 508 | static struct lock_class_key shadow_nodes_key; |
| 509 | 509 | ||
