aboutsummaryrefslogtreecommitdiffstats
path: root/mm
diff options
context:
space:
mode:
authorMatthew Wilcox <mawilcox@microsoft.com>2018-04-10 19:36:56 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2018-04-11 13:28:39 -0400
commitb93b016313b3ba8003c3b8bb71f569af91f19fc7 (patch)
treead4be96414189dcdf8c972f351ba430996e9fdff /mm
parentf6bb2a2c0b81c47282ddb7883f92e65a063c27dd (diff)
page cache: use xa_lock
Remove the address_space ->tree_lock and use the xa_lock newly added to the radix_tree_root. Rename the address_space ->page_tree to ->i_pages, since we don't really care that it's a tree. [willy@infradead.org: fix nds32, fs/dax.c] Link: http://lkml.kernel.org/r/20180406145415.GB20605@bombadil.infradead.orgLink: http://lkml.kernel.org/r/20180313132639.17387-9-willy@infradead.org Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com> Acked-by: Jeff Layton <jlayton@redhat.com> Cc: Darrick J. Wong <darrick.wong@oracle.com> Cc: Dave Chinner <david@fromorbit.com> Cc: Ryusuke Konishi <konishi.ryusuke@lab.ntt.co.jp> Cc: Will Deacon <will.deacon@arm.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'mm')
-rw-r--r--mm/filemap.c84
-rw-r--r--mm/huge_memory.c10
-rw-r--r--mm/khugepaged.c49
-rw-r--r--mm/memcontrol.c4
-rw-r--r--mm/migrate.c32
-rw-r--r--mm/page-writeback.c43
-rw-r--r--mm/readahead.c2
-rw-r--r--mm/rmap.c4
-rw-r--r--mm/shmem.c60
-rw-r--r--mm/swap_state.c17
-rw-r--r--mm/truncate.c22
-rw-r--r--mm/vmscan.c12
-rw-r--r--mm/workingset.c22
13 files changed, 176 insertions, 185 deletions
diff --git a/mm/filemap.c b/mm/filemap.c
index 693f62212a59..ab77e19ab09c 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -66,7 +66,7 @@
66 * ->i_mmap_rwsem (truncate_pagecache) 66 * ->i_mmap_rwsem (truncate_pagecache)
67 * ->private_lock (__free_pte->__set_page_dirty_buffers) 67 * ->private_lock (__free_pte->__set_page_dirty_buffers)
68 * ->swap_lock (exclusive_swap_page, others) 68 * ->swap_lock (exclusive_swap_page, others)
69 * ->mapping->tree_lock 69 * ->i_pages lock
70 * 70 *
71 * ->i_mutex 71 * ->i_mutex
72 * ->i_mmap_rwsem (truncate->unmap_mapping_range) 72 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
@@ -74,7 +74,7 @@
74 * ->mmap_sem 74 * ->mmap_sem
75 * ->i_mmap_rwsem 75 * ->i_mmap_rwsem
76 * ->page_table_lock or pte_lock (various, mainly in memory.c) 76 * ->page_table_lock or pte_lock (various, mainly in memory.c)
77 * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock) 77 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
78 * 78 *
79 * ->mmap_sem 79 * ->mmap_sem
80 * ->lock_page (access_process_vm) 80 * ->lock_page (access_process_vm)
@@ -84,7 +84,7 @@
84 * 84 *
85 * bdi->wb.list_lock 85 * bdi->wb.list_lock
86 * sb_lock (fs/fs-writeback.c) 86 * sb_lock (fs/fs-writeback.c)
87 * ->mapping->tree_lock (__sync_single_inode) 87 * ->i_pages lock (__sync_single_inode)
88 * 88 *
89 * ->i_mmap_rwsem 89 * ->i_mmap_rwsem
90 * ->anon_vma.lock (vma_adjust) 90 * ->anon_vma.lock (vma_adjust)
@@ -95,11 +95,11 @@
95 * ->page_table_lock or pte_lock 95 * ->page_table_lock or pte_lock
96 * ->swap_lock (try_to_unmap_one) 96 * ->swap_lock (try_to_unmap_one)
97 * ->private_lock (try_to_unmap_one) 97 * ->private_lock (try_to_unmap_one)
98 * ->tree_lock (try_to_unmap_one) 98 * ->i_pages lock (try_to_unmap_one)
99 * ->zone_lru_lock(zone) (follow_page->mark_page_accessed) 99 * ->zone_lru_lock(zone) (follow_page->mark_page_accessed)
100 * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page) 100 * ->zone_lru_lock(zone) (check_pte_range->isolate_lru_page)
101 * ->private_lock (page_remove_rmap->set_page_dirty) 101 * ->private_lock (page_remove_rmap->set_page_dirty)
102 * ->tree_lock (page_remove_rmap->set_page_dirty) 102 * ->i_pages lock (page_remove_rmap->set_page_dirty)
103 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty) 103 * bdi.wb->list_lock (page_remove_rmap->set_page_dirty)
104 * ->inode->i_lock (page_remove_rmap->set_page_dirty) 104 * ->inode->i_lock (page_remove_rmap->set_page_dirty)
105 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg) 105 * ->memcg->move_lock (page_remove_rmap->lock_page_memcg)
@@ -118,14 +118,15 @@ static int page_cache_tree_insert(struct address_space *mapping,
118 void **slot; 118 void **slot;
119 int error; 119 int error;
120 120
121 error = __radix_tree_create(&mapping->page_tree, page->index, 0, 121 error = __radix_tree_create(&mapping->i_pages, page->index, 0,
122 &node, &slot); 122 &node, &slot);
123 if (error) 123 if (error)
124 return error; 124 return error;
125 if (*slot) { 125 if (*slot) {
126 void *p; 126 void *p;
127 127
128 p = radix_tree_deref_slot_protected(slot, &mapping->tree_lock); 128 p = radix_tree_deref_slot_protected(slot,
129 &mapping->i_pages.xa_lock);
129 if (!radix_tree_exceptional_entry(p)) 130 if (!radix_tree_exceptional_entry(p))
130 return -EEXIST; 131 return -EEXIST;
131 132
@@ -133,7 +134,7 @@ static int page_cache_tree_insert(struct address_space *mapping,
133 if (shadowp) 134 if (shadowp)
134 *shadowp = p; 135 *shadowp = p;
135 } 136 }
136 __radix_tree_replace(&mapping->page_tree, node, slot, page, 137 __radix_tree_replace(&mapping->i_pages, node, slot, page,
137 workingset_lookup_update(mapping)); 138 workingset_lookup_update(mapping));
138 mapping->nrpages++; 139 mapping->nrpages++;
139 return 0; 140 return 0;
@@ -155,13 +156,13 @@ static void page_cache_tree_delete(struct address_space *mapping,
155 struct radix_tree_node *node; 156 struct radix_tree_node *node;
156 void **slot; 157 void **slot;
157 158
158 __radix_tree_lookup(&mapping->page_tree, page->index + i, 159 __radix_tree_lookup(&mapping->i_pages, page->index + i,
159 &node, &slot); 160 &node, &slot);
160 161
161 VM_BUG_ON_PAGE(!node && nr != 1, page); 162 VM_BUG_ON_PAGE(!node && nr != 1, page);
162 163
163 radix_tree_clear_tags(&mapping->page_tree, node, slot); 164 radix_tree_clear_tags(&mapping->i_pages, node, slot);
164 __radix_tree_replace(&mapping->page_tree, node, slot, shadow, 165 __radix_tree_replace(&mapping->i_pages, node, slot, shadow,
165 workingset_lookup_update(mapping)); 166 workingset_lookup_update(mapping));
166 } 167 }
167 168
@@ -253,7 +254,7 @@ static void unaccount_page_cache_page(struct address_space *mapping,
253/* 254/*
254 * Delete a page from the page cache and free it. Caller has to make 255 * Delete a page from the page cache and free it. Caller has to make
255 * sure the page is locked and that nobody else uses it - or that usage 256 * sure the page is locked and that nobody else uses it - or that usage
256 * is safe. The caller must hold the mapping's tree_lock. 257 * is safe. The caller must hold the i_pages lock.
257 */ 258 */
258void __delete_from_page_cache(struct page *page, void *shadow) 259void __delete_from_page_cache(struct page *page, void *shadow)
259{ 260{
@@ -296,9 +297,9 @@ void delete_from_page_cache(struct page *page)
296 unsigned long flags; 297 unsigned long flags;
297 298
298 BUG_ON(!PageLocked(page)); 299 BUG_ON(!PageLocked(page));
299 spin_lock_irqsave(&mapping->tree_lock, flags); 300 xa_lock_irqsave(&mapping->i_pages, flags);
300 __delete_from_page_cache(page, NULL); 301 __delete_from_page_cache(page, NULL);
301 spin_unlock_irqrestore(&mapping->tree_lock, flags); 302 xa_unlock_irqrestore(&mapping->i_pages, flags);
302 303
303 page_cache_free_page(mapping, page); 304 page_cache_free_page(mapping, page);
304} 305}
@@ -309,14 +310,14 @@ EXPORT_SYMBOL(delete_from_page_cache);
309 * @mapping: the mapping to which pages belong 310 * @mapping: the mapping to which pages belong
310 * @pvec: pagevec with pages to delete 311 * @pvec: pagevec with pages to delete
311 * 312 *
312 * The function walks over mapping->page_tree and removes pages passed in @pvec 313 * The function walks over mapping->i_pages and removes pages passed in @pvec
313 * from the radix tree. The function expects @pvec to be sorted by page index. 314 * from the mapping. The function expects @pvec to be sorted by page index.
314 * It tolerates holes in @pvec (radix tree entries at those indices are not 315 * It tolerates holes in @pvec (mapping entries at those indices are not
315 * modified). The function expects only THP head pages to be present in the 316 * modified). The function expects only THP head pages to be present in the
316 * @pvec and takes care to delete all corresponding tail pages from the radix 317 * @pvec and takes care to delete all corresponding tail pages from the
317 * tree as well. 318 * mapping as well.
318 * 319 *
319 * The function expects mapping->tree_lock to be held. 320 * The function expects the i_pages lock to be held.
320 */ 321 */
321static void 322static void
322page_cache_tree_delete_batch(struct address_space *mapping, 323page_cache_tree_delete_batch(struct address_space *mapping,
@@ -330,11 +331,11 @@ page_cache_tree_delete_batch(struct address_space *mapping,
330 pgoff_t start; 331 pgoff_t start;
331 332
332 start = pvec->pages[0]->index; 333 start = pvec->pages[0]->index;
333 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 334 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
334 if (i >= pagevec_count(pvec) && !tail_pages) 335 if (i >= pagevec_count(pvec) && !tail_pages)
335 break; 336 break;
336 page = radix_tree_deref_slot_protected(slot, 337 page = radix_tree_deref_slot_protected(slot,
337 &mapping->tree_lock); 338 &mapping->i_pages.xa_lock);
338 if (radix_tree_exceptional_entry(page)) 339 if (radix_tree_exceptional_entry(page))
339 continue; 340 continue;
340 if (!tail_pages) { 341 if (!tail_pages) {
@@ -357,8 +358,8 @@ page_cache_tree_delete_batch(struct address_space *mapping,
357 } else { 358 } else {
358 tail_pages--; 359 tail_pages--;
359 } 360 }
360 radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); 361 radix_tree_clear_tags(&mapping->i_pages, iter.node, slot);
361 __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, 362 __radix_tree_replace(&mapping->i_pages, iter.node, slot, NULL,
362 workingset_lookup_update(mapping)); 363 workingset_lookup_update(mapping));
363 total_pages++; 364 total_pages++;
364 } 365 }
@@ -374,14 +375,14 @@ void delete_from_page_cache_batch(struct address_space *mapping,
374 if (!pagevec_count(pvec)) 375 if (!pagevec_count(pvec))
375 return; 376 return;
376 377
377 spin_lock_irqsave(&mapping->tree_lock, flags); 378 xa_lock_irqsave(&mapping->i_pages, flags);
378 for (i = 0; i < pagevec_count(pvec); i++) { 379 for (i = 0; i < pagevec_count(pvec); i++) {
379 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]); 380 trace_mm_filemap_delete_from_page_cache(pvec->pages[i]);
380 381
381 unaccount_page_cache_page(mapping, pvec->pages[i]); 382 unaccount_page_cache_page(mapping, pvec->pages[i]);
382 } 383 }
383 page_cache_tree_delete_batch(mapping, pvec); 384 page_cache_tree_delete_batch(mapping, pvec);
384 spin_unlock_irqrestore(&mapping->tree_lock, flags); 385 xa_unlock_irqrestore(&mapping->i_pages, flags);
385 386
386 for (i = 0; i < pagevec_count(pvec); i++) 387 for (i = 0; i < pagevec_count(pvec); i++)
387 page_cache_free_page(mapping, pvec->pages[i]); 388 page_cache_free_page(mapping, pvec->pages[i]);
@@ -798,7 +799,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
798 new->mapping = mapping; 799 new->mapping = mapping;
799 new->index = offset; 800 new->index = offset;
800 801
801 spin_lock_irqsave(&mapping->tree_lock, flags); 802 xa_lock_irqsave(&mapping->i_pages, flags);
802 __delete_from_page_cache(old, NULL); 803 __delete_from_page_cache(old, NULL);
803 error = page_cache_tree_insert(mapping, new, NULL); 804 error = page_cache_tree_insert(mapping, new, NULL);
804 BUG_ON(error); 805 BUG_ON(error);
@@ -810,7 +811,7 @@ int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask)
810 __inc_node_page_state(new, NR_FILE_PAGES); 811 __inc_node_page_state(new, NR_FILE_PAGES);
811 if (PageSwapBacked(new)) 812 if (PageSwapBacked(new))
812 __inc_node_page_state(new, NR_SHMEM); 813 __inc_node_page_state(new, NR_SHMEM);
813 spin_unlock_irqrestore(&mapping->tree_lock, flags); 814 xa_unlock_irqrestore(&mapping->i_pages, flags);
814 mem_cgroup_migrate(old, new); 815 mem_cgroup_migrate(old, new);
815 radix_tree_preload_end(); 816 radix_tree_preload_end();
816 if (freepage) 817 if (freepage)
@@ -852,7 +853,7 @@ static int __add_to_page_cache_locked(struct page *page,
852 page->mapping = mapping; 853 page->mapping = mapping;
853 page->index = offset; 854 page->index = offset;
854 855
855 spin_lock_irq(&mapping->tree_lock); 856 xa_lock_irq(&mapping->i_pages);
856 error = page_cache_tree_insert(mapping, page, shadowp); 857 error = page_cache_tree_insert(mapping, page, shadowp);
857 radix_tree_preload_end(); 858 radix_tree_preload_end();
858 if (unlikely(error)) 859 if (unlikely(error))
@@ -861,7 +862,7 @@ static int __add_to_page_cache_locked(struct page *page,
861 /* hugetlb pages do not participate in page cache accounting. */ 862 /* hugetlb pages do not participate in page cache accounting. */
862 if (!huge) 863 if (!huge)
863 __inc_node_page_state(page, NR_FILE_PAGES); 864 __inc_node_page_state(page, NR_FILE_PAGES);
864 spin_unlock_irq(&mapping->tree_lock); 865 xa_unlock_irq(&mapping->i_pages);
865 if (!huge) 866 if (!huge)
866 mem_cgroup_commit_charge(page, memcg, false, false); 867 mem_cgroup_commit_charge(page, memcg, false, false);
867 trace_mm_filemap_add_to_page_cache(page); 868 trace_mm_filemap_add_to_page_cache(page);
@@ -869,7 +870,7 @@ static int __add_to_page_cache_locked(struct page *page,
869err_insert: 870err_insert:
870 page->mapping = NULL; 871 page->mapping = NULL;
871 /* Leave page->index set: truncation relies upon it */ 872 /* Leave page->index set: truncation relies upon it */
872 spin_unlock_irq(&mapping->tree_lock); 873 xa_unlock_irq(&mapping->i_pages);
873 if (!huge) 874 if (!huge)
874 mem_cgroup_cancel_charge(page, memcg, false); 875 mem_cgroup_cancel_charge(page, memcg, false);
875 put_page(page); 876 put_page(page);
@@ -1353,7 +1354,7 @@ pgoff_t page_cache_next_hole(struct address_space *mapping,
1353 for (i = 0; i < max_scan; i++) { 1354 for (i = 0; i < max_scan; i++) {
1354 struct page *page; 1355 struct page *page;
1355 1356
1356 page = radix_tree_lookup(&mapping->page_tree, index); 1357 page = radix_tree_lookup(&mapping->i_pages, index);
1357 if (!page || radix_tree_exceptional_entry(page)) 1358 if (!page || radix_tree_exceptional_entry(page))
1358 break; 1359 break;
1359 index++; 1360 index++;
@@ -1394,7 +1395,7 @@ pgoff_t page_cache_prev_hole(struct address_space *mapping,
1394 for (i = 0; i < max_scan; i++) { 1395 for (i = 0; i < max_scan; i++) {
1395 struct page *page; 1396 struct page *page;
1396 1397
1397 page = radix_tree_lookup(&mapping->page_tree, index); 1398 page = radix_tree_lookup(&mapping->i_pages, index);
1398 if (!page || radix_tree_exceptional_entry(page)) 1399 if (!page || radix_tree_exceptional_entry(page))
1399 break; 1400 break;
1400 index--; 1401 index--;
@@ -1427,7 +1428,7 @@ struct page *find_get_entry(struct address_space *mapping, pgoff_t offset)
1427 rcu_read_lock(); 1428 rcu_read_lock();
1428repeat: 1429repeat:
1429 page = NULL; 1430 page = NULL;
1430 pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); 1431 pagep = radix_tree_lookup_slot(&mapping->i_pages, offset);
1431 if (pagep) { 1432 if (pagep) {
1432 page = radix_tree_deref_slot(pagep); 1433 page = radix_tree_deref_slot(pagep);
1433 if (unlikely(!page)) 1434 if (unlikely(!page))
@@ -1633,7 +1634,7 @@ unsigned find_get_entries(struct address_space *mapping,
1633 return 0; 1634 return 0;
1634 1635
1635 rcu_read_lock(); 1636 rcu_read_lock();
1636 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1637 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1637 struct page *head, *page; 1638 struct page *head, *page;
1638repeat: 1639repeat:
1639 page = radix_tree_deref_slot(slot); 1640 page = radix_tree_deref_slot(slot);
@@ -1710,7 +1711,7 @@ unsigned find_get_pages_range(struct address_space *mapping, pgoff_t *start,
1710 return 0; 1711 return 0;
1711 1712
1712 rcu_read_lock(); 1713 rcu_read_lock();
1713 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, *start) { 1714 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, *start) {
1714 struct page *head, *page; 1715 struct page *head, *page;
1715 1716
1716 if (iter.index > end) 1717 if (iter.index > end)
@@ -1795,7 +1796,7 @@ unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
1795 return 0; 1796 return 0;
1796 1797
1797 rcu_read_lock(); 1798 rcu_read_lock();
1798 radix_tree_for_each_contig(slot, &mapping->page_tree, &iter, index) { 1799 radix_tree_for_each_contig(slot, &mapping->i_pages, &iter, index) {
1799 struct page *head, *page; 1800 struct page *head, *page;
1800repeat: 1801repeat:
1801 page = radix_tree_deref_slot(slot); 1802 page = radix_tree_deref_slot(slot);
@@ -1875,8 +1876,7 @@ unsigned find_get_pages_range_tag(struct address_space *mapping, pgoff_t *index,
1875 return 0; 1876 return 0;
1876 1877
1877 rcu_read_lock(); 1878 rcu_read_lock();
1878 radix_tree_for_each_tagged(slot, &mapping->page_tree, 1879 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, *index, tag) {
1879 &iter, *index, tag) {
1880 struct page *head, *page; 1880 struct page *head, *page;
1881 1881
1882 if (iter.index > end) 1882 if (iter.index > end)
@@ -1969,8 +1969,7 @@ unsigned find_get_entries_tag(struct address_space *mapping, pgoff_t start,
1969 return 0; 1969 return 0;
1970 1970
1971 rcu_read_lock(); 1971 rcu_read_lock();
1972 radix_tree_for_each_tagged(slot, &mapping->page_tree, 1972 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start, tag) {
1973 &iter, start, tag) {
1974 struct page *head, *page; 1973 struct page *head, *page;
1975repeat: 1974repeat:
1976 page = radix_tree_deref_slot(slot); 1975 page = radix_tree_deref_slot(slot);
@@ -2624,8 +2623,7 @@ void filemap_map_pages(struct vm_fault *vmf,
2624 struct page *head, *page; 2623 struct page *head, *page;
2625 2624
2626 rcu_read_lock(); 2625 rcu_read_lock();
2627 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, 2626 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start_pgoff) {
2628 start_pgoff) {
2629 if (iter.index > end_pgoff) 2627 if (iter.index > end_pgoff)
2630 break; 2628 break;
2631repeat: 2629repeat:
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 3f3267af4e3b..14ed6ee5e02f 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2450,7 +2450,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
2450 } else { 2450 } else {
2451 /* Additional pin to radix tree */ 2451 /* Additional pin to radix tree */
2452 page_ref_add(head, 2); 2452 page_ref_add(head, 2);
2453 spin_unlock(&head->mapping->tree_lock); 2453 xa_unlock(&head->mapping->i_pages);
2454 } 2454 }
2455 2455
2456 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); 2456 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
@@ -2658,15 +2658,15 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
2658 if (mapping) { 2658 if (mapping) {
2659 void **pslot; 2659 void **pslot;
2660 2660
2661 spin_lock(&mapping->tree_lock); 2661 xa_lock(&mapping->i_pages);
2662 pslot = radix_tree_lookup_slot(&mapping->page_tree, 2662 pslot = radix_tree_lookup_slot(&mapping->i_pages,
2663 page_index(head)); 2663 page_index(head));
2664 /* 2664 /*
2665 * Check if the head page is present in radix tree. 2665 * Check if the head page is present in radix tree.
2666 * We assume all tail are present too, if head is there. 2666 * We assume all tail are present too, if head is there.
2667 */ 2667 */
2668 if (radix_tree_deref_slot_protected(pslot, 2668 if (radix_tree_deref_slot_protected(pslot,
2669 &mapping->tree_lock) != head) 2669 &mapping->i_pages.xa_lock) != head)
2670 goto fail; 2670 goto fail;
2671 } 2671 }
2672 2672
@@ -2700,7 +2700,7 @@ int split_huge_page_to_list(struct page *page, struct list_head *list)
2700 } 2700 }
2701 spin_unlock(&pgdata->split_queue_lock); 2701 spin_unlock(&pgdata->split_queue_lock);
2702fail: if (mapping) 2702fail: if (mapping)
2703 spin_unlock(&mapping->tree_lock); 2703 xa_unlock(&mapping->i_pages);
2704 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags); 2704 spin_unlock_irqrestore(zone_lru_lock(page_zone(head)), flags);
2705 unfreeze_page(head); 2705 unfreeze_page(head);
2706 ret = -EBUSY; 2706 ret = -EBUSY;
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index eb32d0707c80..d7b2a4bf8671 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -1344,8 +1344,8 @@ static void collapse_shmem(struct mm_struct *mm,
1344 */ 1344 */
1345 1345
1346 index = start; 1346 index = start;
1347 spin_lock_irq(&mapping->tree_lock); 1347 xa_lock_irq(&mapping->i_pages);
1348 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1348 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1349 int n = min(iter.index, end) - index; 1349 int n = min(iter.index, end) - index;
1350 1350
1351 /* 1351 /*
@@ -1358,7 +1358,7 @@ static void collapse_shmem(struct mm_struct *mm,
1358 } 1358 }
1359 nr_none += n; 1359 nr_none += n;
1360 for (; index < min(iter.index, end); index++) { 1360 for (; index < min(iter.index, end); index++) {
1361 radix_tree_insert(&mapping->page_tree, index, 1361 radix_tree_insert(&mapping->i_pages, index,
1362 new_page + (index % HPAGE_PMD_NR)); 1362 new_page + (index % HPAGE_PMD_NR));
1363 } 1363 }
1364 1364
@@ -1367,16 +1367,16 @@ static void collapse_shmem(struct mm_struct *mm,
1367 break; 1367 break;
1368 1368
1369 page = radix_tree_deref_slot_protected(slot, 1369 page = radix_tree_deref_slot_protected(slot,
1370 &mapping->tree_lock); 1370 &mapping->i_pages.xa_lock);
1371 if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) { 1371 if (radix_tree_exceptional_entry(page) || !PageUptodate(page)) {
1372 spin_unlock_irq(&mapping->tree_lock); 1372 xa_unlock_irq(&mapping->i_pages);
1373 /* swap in or instantiate fallocated page */ 1373 /* swap in or instantiate fallocated page */
1374 if (shmem_getpage(mapping->host, index, &page, 1374 if (shmem_getpage(mapping->host, index, &page,
1375 SGP_NOHUGE)) { 1375 SGP_NOHUGE)) {
1376 result = SCAN_FAIL; 1376 result = SCAN_FAIL;
1377 goto tree_unlocked; 1377 goto tree_unlocked;
1378 } 1378 }
1379 spin_lock_irq(&mapping->tree_lock); 1379 xa_lock_irq(&mapping->i_pages);
1380 } else if (trylock_page(page)) { 1380 } else if (trylock_page(page)) {
1381 get_page(page); 1381 get_page(page);
1382 } else { 1382 } else {
@@ -1385,7 +1385,7 @@ static void collapse_shmem(struct mm_struct *mm,
1385 } 1385 }
1386 1386
1387 /* 1387 /*
1388 * The page must be locked, so we can drop the tree_lock 1388 * The page must be locked, so we can drop the i_pages lock
1389 * without racing with truncate. 1389 * without racing with truncate.
1390 */ 1390 */
1391 VM_BUG_ON_PAGE(!PageLocked(page), page); 1391 VM_BUG_ON_PAGE(!PageLocked(page), page);
@@ -1396,7 +1396,7 @@ static void collapse_shmem(struct mm_struct *mm,
1396 result = SCAN_TRUNCATED; 1396 result = SCAN_TRUNCATED;
1397 goto out_unlock; 1397 goto out_unlock;
1398 } 1398 }
1399 spin_unlock_irq(&mapping->tree_lock); 1399 xa_unlock_irq(&mapping->i_pages);
1400 1400
1401 if (isolate_lru_page(page)) { 1401 if (isolate_lru_page(page)) {
1402 result = SCAN_DEL_PAGE_LRU; 1402 result = SCAN_DEL_PAGE_LRU;
@@ -1406,11 +1406,11 @@ static void collapse_shmem(struct mm_struct *mm,
1406 if (page_mapped(page)) 1406 if (page_mapped(page))
1407 unmap_mapping_pages(mapping, index, 1, false); 1407 unmap_mapping_pages(mapping, index, 1, false);
1408 1408
1409 spin_lock_irq(&mapping->tree_lock); 1409 xa_lock_irq(&mapping->i_pages);
1410 1410
1411 slot = radix_tree_lookup_slot(&mapping->page_tree, index); 1411 slot = radix_tree_lookup_slot(&mapping->i_pages, index);
1412 VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot, 1412 VM_BUG_ON_PAGE(page != radix_tree_deref_slot_protected(slot,
1413 &mapping->tree_lock), page); 1413 &mapping->i_pages.xa_lock), page);
1414 VM_BUG_ON_PAGE(page_mapped(page), page); 1414 VM_BUG_ON_PAGE(page_mapped(page), page);
1415 1415
1416 /* 1416 /*
@@ -1431,14 +1431,14 @@ static void collapse_shmem(struct mm_struct *mm,
1431 list_add_tail(&page->lru, &pagelist); 1431 list_add_tail(&page->lru, &pagelist);
1432 1432
1433 /* Finally, replace with the new page. */ 1433 /* Finally, replace with the new page. */
1434 radix_tree_replace_slot(&mapping->page_tree, slot, 1434 radix_tree_replace_slot(&mapping->i_pages, slot,
1435 new_page + (index % HPAGE_PMD_NR)); 1435 new_page + (index % HPAGE_PMD_NR));
1436 1436
1437 slot = radix_tree_iter_resume(slot, &iter); 1437 slot = radix_tree_iter_resume(slot, &iter);
1438 index++; 1438 index++;
1439 continue; 1439 continue;
1440out_lru: 1440out_lru:
1441 spin_unlock_irq(&mapping->tree_lock); 1441 xa_unlock_irq(&mapping->i_pages);
1442 putback_lru_page(page); 1442 putback_lru_page(page);
1443out_isolate_failed: 1443out_isolate_failed:
1444 unlock_page(page); 1444 unlock_page(page);
@@ -1464,14 +1464,14 @@ out_unlock:
1464 } 1464 }
1465 1465
1466 for (; index < end; index++) { 1466 for (; index < end; index++) {
1467 radix_tree_insert(&mapping->page_tree, index, 1467 radix_tree_insert(&mapping->i_pages, index,
1468 new_page + (index % HPAGE_PMD_NR)); 1468 new_page + (index % HPAGE_PMD_NR));
1469 } 1469 }
1470 nr_none += n; 1470 nr_none += n;
1471 } 1471 }
1472 1472
1473tree_locked: 1473tree_locked:
1474 spin_unlock_irq(&mapping->tree_lock); 1474 xa_unlock_irq(&mapping->i_pages);
1475tree_unlocked: 1475tree_unlocked:
1476 1476
1477 if (result == SCAN_SUCCEED) { 1477 if (result == SCAN_SUCCEED) {
@@ -1520,9 +1520,8 @@ tree_unlocked:
1520 } else { 1520 } else {
1521 /* Something went wrong: rollback changes to the radix-tree */ 1521 /* Something went wrong: rollback changes to the radix-tree */
1522 shmem_uncharge(mapping->host, nr_none); 1522 shmem_uncharge(mapping->host, nr_none);
1523 spin_lock_irq(&mapping->tree_lock); 1523 xa_lock_irq(&mapping->i_pages);
1524 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, 1524 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1525 start) {
1526 if (iter.index >= end) 1525 if (iter.index >= end)
1527 break; 1526 break;
1528 page = list_first_entry_or_null(&pagelist, 1527 page = list_first_entry_or_null(&pagelist,
@@ -1532,8 +1531,7 @@ tree_unlocked:
1532 break; 1531 break;
1533 nr_none--; 1532 nr_none--;
1534 /* Put holes back where they were */ 1533 /* Put holes back where they were */
1535 radix_tree_delete(&mapping->page_tree, 1534 radix_tree_delete(&mapping->i_pages, iter.index);
1536 iter.index);
1537 continue; 1535 continue;
1538 } 1536 }
1539 1537
@@ -1542,16 +1540,15 @@ tree_unlocked:
1542 /* Unfreeze the page. */ 1540 /* Unfreeze the page. */
1543 list_del(&page->lru); 1541 list_del(&page->lru);
1544 page_ref_unfreeze(page, 2); 1542 page_ref_unfreeze(page, 2);
1545 radix_tree_replace_slot(&mapping->page_tree, 1543 radix_tree_replace_slot(&mapping->i_pages, slot, page);
1546 slot, page);
1547 slot = radix_tree_iter_resume(slot, &iter); 1544 slot = radix_tree_iter_resume(slot, &iter);
1548 spin_unlock_irq(&mapping->tree_lock); 1545 xa_unlock_irq(&mapping->i_pages);
1549 putback_lru_page(page); 1546 putback_lru_page(page);
1550 unlock_page(page); 1547 unlock_page(page);
1551 spin_lock_irq(&mapping->tree_lock); 1548 xa_lock_irq(&mapping->i_pages);
1552 } 1549 }
1553 VM_BUG_ON(nr_none); 1550 VM_BUG_ON(nr_none);
1554 spin_unlock_irq(&mapping->tree_lock); 1551 xa_unlock_irq(&mapping->i_pages);
1555 1552
1556 /* Unfreeze new_page, caller would take care about freeing it */ 1553 /* Unfreeze new_page, caller would take care about freeing it */
1557 page_ref_unfreeze(new_page, 1); 1554 page_ref_unfreeze(new_page, 1);
@@ -1579,7 +1576,7 @@ static void khugepaged_scan_shmem(struct mm_struct *mm,
1579 swap = 0; 1576 swap = 0;
1580 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load)); 1577 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
1581 rcu_read_lock(); 1578 rcu_read_lock();
1582 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 1579 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
1583 if (iter.index >= start + HPAGE_PMD_NR) 1580 if (iter.index >= start + HPAGE_PMD_NR)
1584 break; 1581 break;
1585 1582
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 7978c6faae06..e074f7c637aa 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -5974,9 +5974,9 @@ void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
5974 5974
5975 /* 5975 /*
5976 * Interrupts should be disabled here because the caller holds the 5976 * Interrupts should be disabled here because the caller holds the
5977 * mapping->tree_lock lock which is taken with interrupts-off. It is 5977 * i_pages lock which is taken with interrupts-off. It is
5978 * important here to have the interrupts disabled because it is the 5978 * important here to have the interrupts disabled because it is the
5979 * only synchronisation we have for udpating the per-CPU variables. 5979 * only synchronisation we have for updating the per-CPU variables.
5980 */ 5980 */
5981 VM_BUG_ON(!irqs_disabled()); 5981 VM_BUG_ON(!irqs_disabled());
5982 mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page), 5982 mem_cgroup_charge_statistics(memcg, page, PageTransHuge(page),
diff --git a/mm/migrate.c b/mm/migrate.c
index 51b55f2d2db5..f65dd69e1fd1 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -467,20 +467,21 @@ int migrate_page_move_mapping(struct address_space *mapping,
467 oldzone = page_zone(page); 467 oldzone = page_zone(page);
468 newzone = page_zone(newpage); 468 newzone = page_zone(newpage);
469 469
470 spin_lock_irq(&mapping->tree_lock); 470 xa_lock_irq(&mapping->i_pages);
471 471
472 pslot = radix_tree_lookup_slot(&mapping->page_tree, 472 pslot = radix_tree_lookup_slot(&mapping->i_pages,
473 page_index(page)); 473 page_index(page));
474 474
475 expected_count += 1 + page_has_private(page); 475 expected_count += 1 + page_has_private(page);
476 if (page_count(page) != expected_count || 476 if (page_count(page) != expected_count ||
477 radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 477 radix_tree_deref_slot_protected(pslot,
478 spin_unlock_irq(&mapping->tree_lock); 478 &mapping->i_pages.xa_lock) != page) {
479 xa_unlock_irq(&mapping->i_pages);
479 return -EAGAIN; 480 return -EAGAIN;
480 } 481 }
481 482
482 if (!page_ref_freeze(page, expected_count)) { 483 if (!page_ref_freeze(page, expected_count)) {
483 spin_unlock_irq(&mapping->tree_lock); 484 xa_unlock_irq(&mapping->i_pages);
484 return -EAGAIN; 485 return -EAGAIN;
485 } 486 }
486 487
@@ -494,7 +495,7 @@ int migrate_page_move_mapping(struct address_space *mapping,
494 if (mode == MIGRATE_ASYNC && head && 495 if (mode == MIGRATE_ASYNC && head &&
495 !buffer_migrate_lock_buffers(head, mode)) { 496 !buffer_migrate_lock_buffers(head, mode)) {
496 page_ref_unfreeze(page, expected_count); 497 page_ref_unfreeze(page, expected_count);
497 spin_unlock_irq(&mapping->tree_lock); 498 xa_unlock_irq(&mapping->i_pages);
498 return -EAGAIN; 499 return -EAGAIN;
499 } 500 }
500 501
@@ -522,7 +523,7 @@ int migrate_page_move_mapping(struct address_space *mapping,
522 SetPageDirty(newpage); 523 SetPageDirty(newpage);
523 } 524 }
524 525
525 radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); 526 radix_tree_replace_slot(&mapping->i_pages, pslot, newpage);
526 527
527 /* 528 /*
528 * Drop cache reference from old page by unfreezing 529 * Drop cache reference from old page by unfreezing
@@ -531,7 +532,7 @@ int migrate_page_move_mapping(struct address_space *mapping,
531 */ 532 */
532 page_ref_unfreeze(page, expected_count - 1); 533 page_ref_unfreeze(page, expected_count - 1);
533 534
534 spin_unlock(&mapping->tree_lock); 535 xa_unlock(&mapping->i_pages);
535 /* Leave irq disabled to prevent preemption while updating stats */ 536 /* Leave irq disabled to prevent preemption while updating stats */
536 537
537 /* 538 /*
@@ -574,20 +575,19 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
574 int expected_count; 575 int expected_count;
575 void **pslot; 576 void **pslot;
576 577
577 spin_lock_irq(&mapping->tree_lock); 578 xa_lock_irq(&mapping->i_pages);
578 579
579 pslot = radix_tree_lookup_slot(&mapping->page_tree, 580 pslot = radix_tree_lookup_slot(&mapping->i_pages, page_index(page));
580 page_index(page));
581 581
582 expected_count = 2 + page_has_private(page); 582 expected_count = 2 + page_has_private(page);
583 if (page_count(page) != expected_count || 583 if (page_count(page) != expected_count ||
584 radix_tree_deref_slot_protected(pslot, &mapping->tree_lock) != page) { 584 radix_tree_deref_slot_protected(pslot, &mapping->i_pages.xa_lock) != page) {
585 spin_unlock_irq(&mapping->tree_lock); 585 xa_unlock_irq(&mapping->i_pages);
586 return -EAGAIN; 586 return -EAGAIN;
587 } 587 }
588 588
589 if (!page_ref_freeze(page, expected_count)) { 589 if (!page_ref_freeze(page, expected_count)) {
590 spin_unlock_irq(&mapping->tree_lock); 590 xa_unlock_irq(&mapping->i_pages);
591 return -EAGAIN; 591 return -EAGAIN;
592 } 592 }
593 593
@@ -596,11 +596,11 @@ int migrate_huge_page_move_mapping(struct address_space *mapping,
596 596
597 get_page(newpage); 597 get_page(newpage);
598 598
599 radix_tree_replace_slot(&mapping->page_tree, pslot, newpage); 599 radix_tree_replace_slot(&mapping->i_pages, pslot, newpage);
600 600
601 page_ref_unfreeze(page, expected_count - 1); 601 page_ref_unfreeze(page, expected_count - 1);
602 602
603 spin_unlock_irq(&mapping->tree_lock); 603 xa_unlock_irq(&mapping->i_pages);
604 604
605 return MIGRATEPAGE_SUCCESS; 605 return MIGRATEPAGE_SUCCESS;
606} 606}
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 586f31261c83..5c1a3279e63f 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -2099,7 +2099,8 @@ void __init page_writeback_init(void)
2099 * so that it can tag pages faster than a dirtying process can create them). 2099 * so that it can tag pages faster than a dirtying process can create them).
2100 */ 2100 */
2101/* 2101/*
2102 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency. 2102 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce the i_pages lock
2103 * latency.
2103 */ 2104 */
2104void tag_pages_for_writeback(struct address_space *mapping, 2105void tag_pages_for_writeback(struct address_space *mapping,
2105 pgoff_t start, pgoff_t end) 2106 pgoff_t start, pgoff_t end)
@@ -2109,22 +2110,22 @@ void tag_pages_for_writeback(struct address_space *mapping,
2109 struct radix_tree_iter iter; 2110 struct radix_tree_iter iter;
2110 void **slot; 2111 void **slot;
2111 2112
2112 spin_lock_irq(&mapping->tree_lock); 2113 xa_lock_irq(&mapping->i_pages);
2113 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start, 2114 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start,
2114 PAGECACHE_TAG_DIRTY) { 2115 PAGECACHE_TAG_DIRTY) {
2115 if (iter.index > end) 2116 if (iter.index > end)
2116 break; 2117 break;
2117 radix_tree_iter_tag_set(&mapping->page_tree, &iter, 2118 radix_tree_iter_tag_set(&mapping->i_pages, &iter,
2118 PAGECACHE_TAG_TOWRITE); 2119 PAGECACHE_TAG_TOWRITE);
2119 tagged++; 2120 tagged++;
2120 if ((tagged % WRITEBACK_TAG_BATCH) != 0) 2121 if ((tagged % WRITEBACK_TAG_BATCH) != 0)
2121 continue; 2122 continue;
2122 slot = radix_tree_iter_resume(slot, &iter); 2123 slot = radix_tree_iter_resume(slot, &iter);
2123 spin_unlock_irq(&mapping->tree_lock); 2124 xa_unlock_irq(&mapping->i_pages);
2124 cond_resched(); 2125 cond_resched();
2125 spin_lock_irq(&mapping->tree_lock); 2126 xa_lock_irq(&mapping->i_pages);
2126 } 2127 }
2127 spin_unlock_irq(&mapping->tree_lock); 2128 xa_unlock_irq(&mapping->i_pages);
2128} 2129}
2129EXPORT_SYMBOL(tag_pages_for_writeback); 2130EXPORT_SYMBOL(tag_pages_for_writeback);
2130 2131
@@ -2467,13 +2468,13 @@ int __set_page_dirty_nobuffers(struct page *page)
2467 return 1; 2468 return 1;
2468 } 2469 }
2469 2470
2470 spin_lock_irqsave(&mapping->tree_lock, flags); 2471 xa_lock_irqsave(&mapping->i_pages, flags);
2471 BUG_ON(page_mapping(page) != mapping); 2472 BUG_ON(page_mapping(page) != mapping);
2472 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page)); 2473 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
2473 account_page_dirtied(page, mapping); 2474 account_page_dirtied(page, mapping);
2474 radix_tree_tag_set(&mapping->page_tree, page_index(page), 2475 radix_tree_tag_set(&mapping->i_pages, page_index(page),
2475 PAGECACHE_TAG_DIRTY); 2476 PAGECACHE_TAG_DIRTY);
2476 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2477 xa_unlock_irqrestore(&mapping->i_pages, flags);
2477 unlock_page_memcg(page); 2478 unlock_page_memcg(page);
2478 2479
2479 if (mapping->host) { 2480 if (mapping->host) {
@@ -2718,11 +2719,10 @@ int test_clear_page_writeback(struct page *page)
2718 struct backing_dev_info *bdi = inode_to_bdi(inode); 2719 struct backing_dev_info *bdi = inode_to_bdi(inode);
2719 unsigned long flags; 2720 unsigned long flags;
2720 2721
2721 spin_lock_irqsave(&mapping->tree_lock, flags); 2722 xa_lock_irqsave(&mapping->i_pages, flags);
2722 ret = TestClearPageWriteback(page); 2723 ret = TestClearPageWriteback(page);
2723 if (ret) { 2724 if (ret) {
2724 radix_tree_tag_clear(&mapping->page_tree, 2725 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2725 page_index(page),
2726 PAGECACHE_TAG_WRITEBACK); 2726 PAGECACHE_TAG_WRITEBACK);
2727 if (bdi_cap_account_writeback(bdi)) { 2727 if (bdi_cap_account_writeback(bdi)) {
2728 struct bdi_writeback *wb = inode_to_wb(inode); 2728 struct bdi_writeback *wb = inode_to_wb(inode);
@@ -2736,7 +2736,7 @@ int test_clear_page_writeback(struct page *page)
2736 PAGECACHE_TAG_WRITEBACK)) 2736 PAGECACHE_TAG_WRITEBACK))
2737 sb_clear_inode_writeback(mapping->host); 2737 sb_clear_inode_writeback(mapping->host);
2738 2738
2739 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2739 xa_unlock_irqrestore(&mapping->i_pages, flags);
2740 } else { 2740 } else {
2741 ret = TestClearPageWriteback(page); 2741 ret = TestClearPageWriteback(page);
2742 } 2742 }
@@ -2766,7 +2766,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write)
2766 struct backing_dev_info *bdi = inode_to_bdi(inode); 2766 struct backing_dev_info *bdi = inode_to_bdi(inode);
2767 unsigned long flags; 2767 unsigned long flags;
2768 2768
2769 spin_lock_irqsave(&mapping->tree_lock, flags); 2769 xa_lock_irqsave(&mapping->i_pages, flags);
2770 ret = TestSetPageWriteback(page); 2770 ret = TestSetPageWriteback(page);
2771 if (!ret) { 2771 if (!ret) {
2772 bool on_wblist; 2772 bool on_wblist;
@@ -2774,8 +2774,7 @@ int __test_set_page_writeback(struct page *page, bool keep_write)
2774 on_wblist = mapping_tagged(mapping, 2774 on_wblist = mapping_tagged(mapping,
2775 PAGECACHE_TAG_WRITEBACK); 2775 PAGECACHE_TAG_WRITEBACK);
2776 2776
2777 radix_tree_tag_set(&mapping->page_tree, 2777 radix_tree_tag_set(&mapping->i_pages, page_index(page),
2778 page_index(page),
2779 PAGECACHE_TAG_WRITEBACK); 2778 PAGECACHE_TAG_WRITEBACK);
2780 if (bdi_cap_account_writeback(bdi)) 2779 if (bdi_cap_account_writeback(bdi))
2781 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK); 2780 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
@@ -2789,14 +2788,12 @@ int __test_set_page_writeback(struct page *page, bool keep_write)
2789 sb_mark_inode_writeback(mapping->host); 2788 sb_mark_inode_writeback(mapping->host);
2790 } 2789 }
2791 if (!PageDirty(page)) 2790 if (!PageDirty(page))
2792 radix_tree_tag_clear(&mapping->page_tree, 2791 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2793 page_index(page),
2794 PAGECACHE_TAG_DIRTY); 2792 PAGECACHE_TAG_DIRTY);
2795 if (!keep_write) 2793 if (!keep_write)
2796 radix_tree_tag_clear(&mapping->page_tree, 2794 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2797 page_index(page),
2798 PAGECACHE_TAG_TOWRITE); 2795 PAGECACHE_TAG_TOWRITE);
2799 spin_unlock_irqrestore(&mapping->tree_lock, flags); 2796 xa_unlock_irqrestore(&mapping->i_pages, flags);
2800 } else { 2797 } else {
2801 ret = TestSetPageWriteback(page); 2798 ret = TestSetPageWriteback(page);
2802 } 2799 }
@@ -2816,7 +2813,7 @@ EXPORT_SYMBOL(__test_set_page_writeback);
2816 */ 2813 */
2817int mapping_tagged(struct address_space *mapping, int tag) 2814int mapping_tagged(struct address_space *mapping, int tag)
2818{ 2815{
2819 return radix_tree_tagged(&mapping->page_tree, tag); 2816 return radix_tree_tagged(&mapping->i_pages, tag);
2820} 2817}
2821EXPORT_SYMBOL(mapping_tagged); 2818EXPORT_SYMBOL(mapping_tagged);
2822 2819
diff --git a/mm/readahead.c b/mm/readahead.c
index 4d57b4644f98..539bbb6c1fad 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -175,7 +175,7 @@ int __do_page_cache_readahead(struct address_space *mapping, struct file *filp,
175 break; 175 break;
176 176
177 rcu_read_lock(); 177 rcu_read_lock();
178 page = radix_tree_lookup(&mapping->page_tree, page_offset); 178 page = radix_tree_lookup(&mapping->i_pages, page_offset);
179 rcu_read_unlock(); 179 rcu_read_unlock();
180 if (page && !radix_tree_exceptional_entry(page)) 180 if (page && !radix_tree_exceptional_entry(page))
181 continue; 181 continue;
diff --git a/mm/rmap.c b/mm/rmap.c
index 9122787c4947..f0dd4e4565bc 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -32,11 +32,11 @@
32 * mmlist_lock (in mmput, drain_mmlist and others) 32 * mmlist_lock (in mmput, drain_mmlist and others)
33 * mapping->private_lock (in __set_page_dirty_buffers) 33 * mapping->private_lock (in __set_page_dirty_buffers)
34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock) 34 * mem_cgroup_{begin,end}_page_stat (memcg->move_lock)
35 * mapping->tree_lock (widely used) 35 * i_pages lock (widely used)
36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty) 36 * inode->i_lock (in set_page_dirty's __mark_inode_dirty)
37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty) 37 * bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
38 * sb_lock (within inode_lock in fs/fs-writeback.c) 38 * sb_lock (within inode_lock in fs/fs-writeback.c)
39 * mapping->tree_lock (widely used, in set_page_dirty, 39 * i_pages lock (widely used, in set_page_dirty,
40 * in arch-dependent flush_dcache_mmap_lock, 40 * in arch-dependent flush_dcache_mmap_lock,
41 * within bdi.wb->list_lock in __sync_single_inode) 41 * within bdi.wb->list_lock in __sync_single_inode)
42 * 42 *
diff --git a/mm/shmem.c b/mm/shmem.c
index 4424fc0c33aa..9d6c7e595415 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -332,12 +332,12 @@ static int shmem_radix_tree_replace(struct address_space *mapping,
332 332
333 VM_BUG_ON(!expected); 333 VM_BUG_ON(!expected);
334 VM_BUG_ON(!replacement); 334 VM_BUG_ON(!replacement);
335 item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); 335 item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot);
336 if (!item) 336 if (!item)
337 return -ENOENT; 337 return -ENOENT;
338 if (item != expected) 338 if (item != expected)
339 return -ENOENT; 339 return -ENOENT;
340 __radix_tree_replace(&mapping->page_tree, node, pslot, 340 __radix_tree_replace(&mapping->i_pages, node, pslot,
341 replacement, NULL); 341 replacement, NULL);
342 return 0; 342 return 0;
343} 343}
@@ -355,7 +355,7 @@ static bool shmem_confirm_swap(struct address_space *mapping,
355 void *item; 355 void *item;
356 356
357 rcu_read_lock(); 357 rcu_read_lock();
358 item = radix_tree_lookup(&mapping->page_tree, index); 358 item = radix_tree_lookup(&mapping->i_pages, index);
359 rcu_read_unlock(); 359 rcu_read_unlock();
360 return item == swp_to_radix_entry(swap); 360 return item == swp_to_radix_entry(swap);
361} 361}
@@ -590,14 +590,14 @@ static int shmem_add_to_page_cache(struct page *page,
590 page->mapping = mapping; 590 page->mapping = mapping;
591 page->index = index; 591 page->index = index;
592 592
593 spin_lock_irq(&mapping->tree_lock); 593 xa_lock_irq(&mapping->i_pages);
594 if (PageTransHuge(page)) { 594 if (PageTransHuge(page)) {
595 void __rcu **results; 595 void __rcu **results;
596 pgoff_t idx; 596 pgoff_t idx;
597 int i; 597 int i;
598 598
599 error = 0; 599 error = 0;
600 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 600 if (radix_tree_gang_lookup_slot(&mapping->i_pages,
601 &results, &idx, index, 1) && 601 &results, &idx, index, 1) &&
602 idx < index + HPAGE_PMD_NR) { 602 idx < index + HPAGE_PMD_NR) {
603 error = -EEXIST; 603 error = -EEXIST;
@@ -605,14 +605,14 @@ static int shmem_add_to_page_cache(struct page *page,
605 605
606 if (!error) { 606 if (!error) {
607 for (i = 0; i < HPAGE_PMD_NR; i++) { 607 for (i = 0; i < HPAGE_PMD_NR; i++) {
608 error = radix_tree_insert(&mapping->page_tree, 608 error = radix_tree_insert(&mapping->i_pages,
609 index + i, page + i); 609 index + i, page + i);
610 VM_BUG_ON(error); 610 VM_BUG_ON(error);
611 } 611 }
612 count_vm_event(THP_FILE_ALLOC); 612 count_vm_event(THP_FILE_ALLOC);
613 } 613 }
614 } else if (!expected) { 614 } else if (!expected) {
615 error = radix_tree_insert(&mapping->page_tree, index, page); 615 error = radix_tree_insert(&mapping->i_pages, index, page);
616 } else { 616 } else {
617 error = shmem_radix_tree_replace(mapping, index, expected, 617 error = shmem_radix_tree_replace(mapping, index, expected,
618 page); 618 page);
@@ -624,10 +624,10 @@ static int shmem_add_to_page_cache(struct page *page,
624 __inc_node_page_state(page, NR_SHMEM_THPS); 624 __inc_node_page_state(page, NR_SHMEM_THPS);
625 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 625 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
626 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 626 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
627 spin_unlock_irq(&mapping->tree_lock); 627 xa_unlock_irq(&mapping->i_pages);
628 } else { 628 } else {
629 page->mapping = NULL; 629 page->mapping = NULL;
630 spin_unlock_irq(&mapping->tree_lock); 630 xa_unlock_irq(&mapping->i_pages);
631 page_ref_sub(page, nr); 631 page_ref_sub(page, nr);
632 } 632 }
633 return error; 633 return error;
@@ -643,13 +643,13 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap)
643 643
644 VM_BUG_ON_PAGE(PageCompound(page), page); 644 VM_BUG_ON_PAGE(PageCompound(page), page);
645 645
646 spin_lock_irq(&mapping->tree_lock); 646 xa_lock_irq(&mapping->i_pages);
647 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 647 error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
648 page->mapping = NULL; 648 page->mapping = NULL;
649 mapping->nrpages--; 649 mapping->nrpages--;
650 __dec_node_page_state(page, NR_FILE_PAGES); 650 __dec_node_page_state(page, NR_FILE_PAGES);
651 __dec_node_page_state(page, NR_SHMEM); 651 __dec_node_page_state(page, NR_SHMEM);
652 spin_unlock_irq(&mapping->tree_lock); 652 xa_unlock_irq(&mapping->i_pages);
653 put_page(page); 653 put_page(page);
654 BUG_ON(error); 654 BUG_ON(error);
655} 655}
@@ -662,9 +662,9 @@ static int shmem_free_swap(struct address_space *mapping,
662{ 662{
663 void *old; 663 void *old;
664 664
665 spin_lock_irq(&mapping->tree_lock); 665 xa_lock_irq(&mapping->i_pages);
666 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 666 old = radix_tree_delete_item(&mapping->i_pages, index, radswap);
667 spin_unlock_irq(&mapping->tree_lock); 667 xa_unlock_irq(&mapping->i_pages);
668 if (old != radswap) 668 if (old != radswap)
669 return -ENOENT; 669 return -ENOENT;
670 free_swap_and_cache(radix_to_swp_entry(radswap)); 670 free_swap_and_cache(radix_to_swp_entry(radswap));
@@ -675,7 +675,7 @@ static int shmem_free_swap(struct address_space *mapping,
675 * Determine (in bytes) how many of the shmem object's pages mapped by the 675 * Determine (in bytes) how many of the shmem object's pages mapped by the
676 * given offsets are swapped out. 676 * given offsets are swapped out.
677 * 677 *
678 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 678 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
679 * as long as the inode doesn't go away and racy results are not a problem. 679 * as long as the inode doesn't go away and racy results are not a problem.
680 */ 680 */
681unsigned long shmem_partial_swap_usage(struct address_space *mapping, 681unsigned long shmem_partial_swap_usage(struct address_space *mapping,
@@ -688,7 +688,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
688 688
689 rcu_read_lock(); 689 rcu_read_lock();
690 690
691 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 691 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
692 if (iter.index >= end) 692 if (iter.index >= end)
693 break; 693 break;
694 694
@@ -717,7 +717,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
717 * Determine (in bytes) how many of the shmem object's pages mapped by the 717 * Determine (in bytes) how many of the shmem object's pages mapped by the
718 * given vma is swapped out. 718 * given vma is swapped out.
719 * 719 *
720 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 720 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
721 * as long as the inode doesn't go away and racy results are not a problem. 721 * as long as the inode doesn't go away and racy results are not a problem.
722 */ 722 */
723unsigned long shmem_swap_usage(struct vm_area_struct *vma) 723unsigned long shmem_swap_usage(struct vm_area_struct *vma)
@@ -1132,7 +1132,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info,
1132 int error = 0; 1132 int error = 0;
1133 1133
1134 radswap = swp_to_radix_entry(swap); 1134 radswap = swp_to_radix_entry(swap);
1135 index = find_swap_entry(&mapping->page_tree, radswap); 1135 index = find_swap_entry(&mapping->i_pages, radswap);
1136 if (index == -1) 1136 if (index == -1)
1137 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1137 return -EAGAIN; /* tell shmem_unuse we found nothing */
1138 1138
@@ -1448,7 +1448,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp,
1448 1448
1449 hindex = round_down(index, HPAGE_PMD_NR); 1449 hindex = round_down(index, HPAGE_PMD_NR);
1450 rcu_read_lock(); 1450 rcu_read_lock();
1451 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1451 if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx,
1452 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1452 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1453 rcu_read_unlock(); 1453 rcu_read_unlock();
1454 return NULL; 1454 return NULL;
@@ -1561,14 +1561,14 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1561 * Our caller will very soon move newpage out of swapcache, but it's 1561 * Our caller will very soon move newpage out of swapcache, but it's
1562 * a nice clean interface for us to replace oldpage by newpage there. 1562 * a nice clean interface for us to replace oldpage by newpage there.
1563 */ 1563 */
1564 spin_lock_irq(&swap_mapping->tree_lock); 1564 xa_lock_irq(&swap_mapping->i_pages);
1565 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1565 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1566 newpage); 1566 newpage);
1567 if (!error) { 1567 if (!error) {
1568 __inc_node_page_state(newpage, NR_FILE_PAGES); 1568 __inc_node_page_state(newpage, NR_FILE_PAGES);
1569 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1569 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1570 } 1570 }
1571 spin_unlock_irq(&swap_mapping->tree_lock); 1571 xa_unlock_irq(&swap_mapping->i_pages);
1572 1572
1573 if (unlikely(error)) { 1573 if (unlikely(error)) {
1574 /* 1574 /*
@@ -2634,7 +2634,7 @@ static void shmem_tag_pins(struct address_space *mapping)
2634 start = 0; 2634 start = 0;
2635 rcu_read_lock(); 2635 rcu_read_lock();
2636 2636
2637 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2637 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
2638 page = radix_tree_deref_slot(slot); 2638 page = radix_tree_deref_slot(slot);
2639 if (!page || radix_tree_exception(page)) { 2639 if (!page || radix_tree_exception(page)) {
2640 if (radix_tree_deref_retry(page)) { 2640 if (radix_tree_deref_retry(page)) {
@@ -2642,10 +2642,10 @@ static void shmem_tag_pins(struct address_space *mapping)
2642 continue; 2642 continue;
2643 } 2643 }
2644 } else if (page_count(page) - page_mapcount(page) > 1) { 2644 } else if (page_count(page) - page_mapcount(page) > 1) {
2645 spin_lock_irq(&mapping->tree_lock); 2645 xa_lock_irq(&mapping->i_pages);
2646 radix_tree_tag_set(&mapping->page_tree, iter.index, 2646 radix_tree_tag_set(&mapping->i_pages, iter.index,
2647 SHMEM_TAG_PINNED); 2647 SHMEM_TAG_PINNED);
2648 spin_unlock_irq(&mapping->tree_lock); 2648 xa_unlock_irq(&mapping->i_pages);
2649 } 2649 }
2650 2650
2651 if (need_resched()) { 2651 if (need_resched()) {
@@ -2677,7 +2677,7 @@ static int shmem_wait_for_pins(struct address_space *mapping)
2677 2677
2678 error = 0; 2678 error = 0;
2679 for (scan = 0; scan <= LAST_SCAN; scan++) { 2679 for (scan = 0; scan <= LAST_SCAN; scan++) {
2680 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2680 if (!radix_tree_tagged(&mapping->i_pages, SHMEM_TAG_PINNED))
2681 break; 2681 break;
2682 2682
2683 if (!scan) 2683 if (!scan)
@@ -2687,7 +2687,7 @@ static int shmem_wait_for_pins(struct address_space *mapping)
2687 2687
2688 start = 0; 2688 start = 0;
2689 rcu_read_lock(); 2689 rcu_read_lock();
2690 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2690 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter,
2691 start, SHMEM_TAG_PINNED) { 2691 start, SHMEM_TAG_PINNED) {
2692 2692
2693 page = radix_tree_deref_slot(slot); 2693 page = radix_tree_deref_slot(slot);
@@ -2713,10 +2713,10 @@ static int shmem_wait_for_pins(struct address_space *mapping)
2713 error = -EBUSY; 2713 error = -EBUSY;
2714 } 2714 }
2715 2715
2716 spin_lock_irq(&mapping->tree_lock); 2716 xa_lock_irq(&mapping->i_pages);
2717 radix_tree_tag_clear(&mapping->page_tree, 2717 radix_tree_tag_clear(&mapping->i_pages,
2718 iter.index, SHMEM_TAG_PINNED); 2718 iter.index, SHMEM_TAG_PINNED);
2719 spin_unlock_irq(&mapping->tree_lock); 2719 xa_unlock_irq(&mapping->i_pages);
2720continue_resched: 2720continue_resched:
2721 if (need_resched()) { 2721 if (need_resched()) {
2722 slot = radix_tree_iter_resume(slot, &iter); 2722 slot = radix_tree_iter_resume(slot, &iter);
diff --git a/mm/swap_state.c b/mm/swap_state.c
index f233dccd3b1b..07f9aa2340c3 100644
--- a/mm/swap_state.c
+++ b/mm/swap_state.c
@@ -124,10 +124,10 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry)
124 SetPageSwapCache(page); 124 SetPageSwapCache(page);
125 125
126 address_space = swap_address_space(entry); 126 address_space = swap_address_space(entry);
127 spin_lock_irq(&address_space->tree_lock); 127 xa_lock_irq(&address_space->i_pages);
128 for (i = 0; i < nr; i++) { 128 for (i = 0; i < nr; i++) {
129 set_page_private(page + i, entry.val + i); 129 set_page_private(page + i, entry.val + i);
130 error = radix_tree_insert(&address_space->page_tree, 130 error = radix_tree_insert(&address_space->i_pages,
131 idx + i, page + i); 131 idx + i, page + i);
132 if (unlikely(error)) 132 if (unlikely(error))
133 break; 133 break;
@@ -145,13 +145,13 @@ int __add_to_swap_cache(struct page *page, swp_entry_t entry)
145 VM_BUG_ON(error == -EEXIST); 145 VM_BUG_ON(error == -EEXIST);
146 set_page_private(page + i, 0UL); 146 set_page_private(page + i, 0UL);
147 while (i--) { 147 while (i--) {
148 radix_tree_delete(&address_space->page_tree, idx + i); 148 radix_tree_delete(&address_space->i_pages, idx + i);
149 set_page_private(page + i, 0UL); 149 set_page_private(page + i, 0UL);
150 } 150 }
151 ClearPageSwapCache(page); 151 ClearPageSwapCache(page);
152 page_ref_sub(page, nr); 152 page_ref_sub(page, nr);
153 } 153 }
154 spin_unlock_irq(&address_space->tree_lock); 154 xa_unlock_irq(&address_space->i_pages);
155 155
156 return error; 156 return error;
157} 157}
@@ -188,7 +188,7 @@ void __delete_from_swap_cache(struct page *page)
188 address_space = swap_address_space(entry); 188 address_space = swap_address_space(entry);
189 idx = swp_offset(entry); 189 idx = swp_offset(entry);
190 for (i = 0; i < nr; i++) { 190 for (i = 0; i < nr; i++) {
191 radix_tree_delete(&address_space->page_tree, idx + i); 191 radix_tree_delete(&address_space->i_pages, idx + i);
192 set_page_private(page + i, 0); 192 set_page_private(page + i, 0);
193 } 193 }
194 ClearPageSwapCache(page); 194 ClearPageSwapCache(page);
@@ -272,9 +272,9 @@ void delete_from_swap_cache(struct page *page)
272 entry.val = page_private(page); 272 entry.val = page_private(page);
273 273
274 address_space = swap_address_space(entry); 274 address_space = swap_address_space(entry);
275 spin_lock_irq(&address_space->tree_lock); 275 xa_lock_irq(&address_space->i_pages);
276 __delete_from_swap_cache(page); 276 __delete_from_swap_cache(page);
277 spin_unlock_irq(&address_space->tree_lock); 277 xa_unlock_irq(&address_space->i_pages);
278 278
279 put_swap_page(page, entry); 279 put_swap_page(page, entry);
280 page_ref_sub(page, hpage_nr_pages(page)); 280 page_ref_sub(page, hpage_nr_pages(page));
@@ -628,12 +628,11 @@ int init_swap_address_space(unsigned int type, unsigned long nr_pages)
628 return -ENOMEM; 628 return -ENOMEM;
629 for (i = 0; i < nr; i++) { 629 for (i = 0; i < nr; i++) {
630 space = spaces + i; 630 space = spaces + i;
631 INIT_RADIX_TREE(&space->page_tree, GFP_ATOMIC|__GFP_NOWARN); 631 INIT_RADIX_TREE(&space->i_pages, GFP_ATOMIC|__GFP_NOWARN);
632 atomic_set(&space->i_mmap_writable, 0); 632 atomic_set(&space->i_mmap_writable, 0);
633 space->a_ops = &swap_aops; 633 space->a_ops = &swap_aops;
634 /* swap cache doesn't use writeback related tags */ 634 /* swap cache doesn't use writeback related tags */
635 mapping_set_no_writeback_tags(space); 635 mapping_set_no_writeback_tags(space);
636 spin_lock_init(&space->tree_lock);
637 } 636 }
638 nr_swapper_spaces[type] = nr; 637 nr_swapper_spaces[type] = nr;
639 rcu_assign_pointer(swapper_spaces[type], spaces); 638 rcu_assign_pointer(swapper_spaces[type], spaces);
diff --git a/mm/truncate.c b/mm/truncate.c
index c34e2fd4f583..1d2fb2dca96f 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -36,11 +36,11 @@ static inline void __clear_shadow_entry(struct address_space *mapping,
36 struct radix_tree_node *node; 36 struct radix_tree_node *node;
37 void **slot; 37 void **slot;
38 38
39 if (!__radix_tree_lookup(&mapping->page_tree, index, &node, &slot)) 39 if (!__radix_tree_lookup(&mapping->i_pages, index, &node, &slot))
40 return; 40 return;
41 if (*slot != entry) 41 if (*slot != entry)
42 return; 42 return;
43 __radix_tree_replace(&mapping->page_tree, node, slot, NULL, 43 __radix_tree_replace(&mapping->i_pages, node, slot, NULL,
44 workingset_update_node); 44 workingset_update_node);
45 mapping->nrexceptional--; 45 mapping->nrexceptional--;
46} 46}
@@ -48,9 +48,9 @@ static inline void __clear_shadow_entry(struct address_space *mapping,
48static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, 48static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
49 void *entry) 49 void *entry)
50{ 50{
51 spin_lock_irq(&mapping->tree_lock); 51 xa_lock_irq(&mapping->i_pages);
52 __clear_shadow_entry(mapping, index, entry); 52 __clear_shadow_entry(mapping, index, entry);
53 spin_unlock_irq(&mapping->tree_lock); 53 xa_unlock_irq(&mapping->i_pages);
54} 54}
55 55
56/* 56/*
@@ -79,7 +79,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping,
79 dax = dax_mapping(mapping); 79 dax = dax_mapping(mapping);
80 lock = !dax && indices[j] < end; 80 lock = !dax && indices[j] < end;
81 if (lock) 81 if (lock)
82 spin_lock_irq(&mapping->tree_lock); 82 xa_lock_irq(&mapping->i_pages);
83 83
84 for (i = j; i < pagevec_count(pvec); i++) { 84 for (i = j; i < pagevec_count(pvec); i++) {
85 struct page *page = pvec->pages[i]; 85 struct page *page = pvec->pages[i];
@@ -102,7 +102,7 @@ static void truncate_exceptional_pvec_entries(struct address_space *mapping,
102 } 102 }
103 103
104 if (lock) 104 if (lock)
105 spin_unlock_irq(&mapping->tree_lock); 105 xa_unlock_irq(&mapping->i_pages);
106 pvec->nr = j; 106 pvec->nr = j;
107} 107}
108 108
@@ -518,8 +518,8 @@ void truncate_inode_pages_final(struct address_space *mapping)
518 * modification that does not see AS_EXITING is 518 * modification that does not see AS_EXITING is
519 * completed before starting the final truncate. 519 * completed before starting the final truncate.
520 */ 520 */
521 spin_lock_irq(&mapping->tree_lock); 521 xa_lock_irq(&mapping->i_pages);
522 spin_unlock_irq(&mapping->tree_lock); 522 xa_unlock_irq(&mapping->i_pages);
523 523
524 truncate_inode_pages(mapping, 0); 524 truncate_inode_pages(mapping, 0);
525 } 525 }
@@ -627,13 +627,13 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
627 if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL)) 627 if (page_has_private(page) && !try_to_release_page(page, GFP_KERNEL))
628 return 0; 628 return 0;
629 629
630 spin_lock_irqsave(&mapping->tree_lock, flags); 630 xa_lock_irqsave(&mapping->i_pages, flags);
631 if (PageDirty(page)) 631 if (PageDirty(page))
632 goto failed; 632 goto failed;
633 633
634 BUG_ON(page_has_private(page)); 634 BUG_ON(page_has_private(page));
635 __delete_from_page_cache(page, NULL); 635 __delete_from_page_cache(page, NULL);
636 spin_unlock_irqrestore(&mapping->tree_lock, flags); 636 xa_unlock_irqrestore(&mapping->i_pages, flags);
637 637
638 if (mapping->a_ops->freepage) 638 if (mapping->a_ops->freepage)
639 mapping->a_ops->freepage(page); 639 mapping->a_ops->freepage(page);
@@ -641,7 +641,7 @@ invalidate_complete_page2(struct address_space *mapping, struct page *page)
641 put_page(page); /* pagecache ref */ 641 put_page(page); /* pagecache ref */
642 return 1; 642 return 1;
643failed: 643failed:
644 spin_unlock_irqrestore(&mapping->tree_lock, flags); 644 xa_unlock_irqrestore(&mapping->i_pages, flags);
645 return 0; 645 return 0;
646} 646}
647 647
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 671597ce1ea0..8b920ce3ae02 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -693,7 +693,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
693 BUG_ON(!PageLocked(page)); 693 BUG_ON(!PageLocked(page));
694 BUG_ON(mapping != page_mapping(page)); 694 BUG_ON(mapping != page_mapping(page));
695 695
696 spin_lock_irqsave(&mapping->tree_lock, flags); 696 xa_lock_irqsave(&mapping->i_pages, flags);
697 /* 697 /*
698 * The non racy check for a busy page. 698 * The non racy check for a busy page.
699 * 699 *
@@ -717,7 +717,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
717 * load is not satisfied before that of page->_refcount. 717 * load is not satisfied before that of page->_refcount.
718 * 718 *
719 * Note that if SetPageDirty is always performed via set_page_dirty, 719 * Note that if SetPageDirty is always performed via set_page_dirty,
720 * and thus under tree_lock, then this ordering is not required. 720 * and thus under the i_pages lock, then this ordering is not required.
721 */ 721 */
722 if (unlikely(PageTransHuge(page)) && PageSwapCache(page)) 722 if (unlikely(PageTransHuge(page)) && PageSwapCache(page))
723 refcount = 1 + HPAGE_PMD_NR; 723 refcount = 1 + HPAGE_PMD_NR;
@@ -735,7 +735,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
735 swp_entry_t swap = { .val = page_private(page) }; 735 swp_entry_t swap = { .val = page_private(page) };
736 mem_cgroup_swapout(page, swap); 736 mem_cgroup_swapout(page, swap);
737 __delete_from_swap_cache(page); 737 __delete_from_swap_cache(page);
738 spin_unlock_irqrestore(&mapping->tree_lock, flags); 738 xa_unlock_irqrestore(&mapping->i_pages, flags);
739 put_swap_page(page, swap); 739 put_swap_page(page, swap);
740 } else { 740 } else {
741 void (*freepage)(struct page *); 741 void (*freepage)(struct page *);
@@ -756,13 +756,13 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
756 * only page cache pages found in these are zero pages 756 * only page cache pages found in these are zero pages
757 * covering holes, and because we don't want to mix DAX 757 * covering holes, and because we don't want to mix DAX
758 * exceptional entries and shadow exceptional entries in the 758 * exceptional entries and shadow exceptional entries in the
759 * same page_tree. 759 * same address_space.
760 */ 760 */
761 if (reclaimed && page_is_file_cache(page) && 761 if (reclaimed && page_is_file_cache(page) &&
762 !mapping_exiting(mapping) && !dax_mapping(mapping)) 762 !mapping_exiting(mapping) && !dax_mapping(mapping))
763 shadow = workingset_eviction(mapping, page); 763 shadow = workingset_eviction(mapping, page);
764 __delete_from_page_cache(page, shadow); 764 __delete_from_page_cache(page, shadow);
765 spin_unlock_irqrestore(&mapping->tree_lock, flags); 765 xa_unlock_irqrestore(&mapping->i_pages, flags);
766 766
767 if (freepage != NULL) 767 if (freepage != NULL)
768 freepage(page); 768 freepage(page);
@@ -771,7 +771,7 @@ static int __remove_mapping(struct address_space *mapping, struct page *page,
771 return 1; 771 return 1;
772 772
773cannot_free: 773cannot_free:
774 spin_unlock_irqrestore(&mapping->tree_lock, flags); 774 xa_unlock_irqrestore(&mapping->i_pages, flags);
775 return 0; 775 return 0;
776} 776}
777 777
diff --git a/mm/workingset.c b/mm/workingset.c
index b7d616a3bbbe..40ee02c83978 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -202,7 +202,7 @@ static void unpack_shadow(void *shadow, int *memcgidp, pg_data_t **pgdat,
202 * @mapping: address space the page was backing 202 * @mapping: address space the page was backing
203 * @page: the page being evicted 203 * @page: the page being evicted
204 * 204 *
205 * Returns a shadow entry to be stored in @mapping->page_tree in place 205 * Returns a shadow entry to be stored in @mapping->i_pages in place
206 * of the evicted @page so that a later refault can be detected. 206 * of the evicted @page so that a later refault can be detected.
207 */ 207 */
208void *workingset_eviction(struct address_space *mapping, struct page *page) 208void *workingset_eviction(struct address_space *mapping, struct page *page)
@@ -348,7 +348,7 @@ void workingset_update_node(struct radix_tree_node *node)
348 * 348 *
349 * Avoid acquiring the list_lru lock when the nodes are 349 * Avoid acquiring the list_lru lock when the nodes are
350 * already where they should be. The list_empty() test is safe 350 * already where they should be. The list_empty() test is safe
351 * as node->private_list is protected by &mapping->tree_lock. 351 * as node->private_list is protected by the i_pages lock.
352 */ 352 */
353 if (node->count && node->count == node->exceptional) { 353 if (node->count && node->count == node->exceptional) {
354 if (list_empty(&node->private_list)) 354 if (list_empty(&node->private_list))
@@ -366,7 +366,7 @@ static unsigned long count_shadow_nodes(struct shrinker *shrinker,
366 unsigned long nodes; 366 unsigned long nodes;
367 unsigned long cache; 367 unsigned long cache;
368 368
369 /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ 369 /* list_lru lock nests inside the IRQ-safe i_pages lock */
370 local_irq_disable(); 370 local_irq_disable();
371 nodes = list_lru_shrink_count(&shadow_nodes, sc); 371 nodes = list_lru_shrink_count(&shadow_nodes, sc);
372 local_irq_enable(); 372 local_irq_enable();
@@ -419,21 +419,21 @@ static enum lru_status shadow_lru_isolate(struct list_head *item,
419 419
420 /* 420 /*
421 * Page cache insertions and deletions synchroneously maintain 421 * Page cache insertions and deletions synchroneously maintain
422 * the shadow node LRU under the mapping->tree_lock and the 422 * the shadow node LRU under the i_pages lock and the
423 * lru_lock. Because the page cache tree is emptied before 423 * lru_lock. Because the page cache tree is emptied before
424 * the inode can be destroyed, holding the lru_lock pins any 424 * the inode can be destroyed, holding the lru_lock pins any
425 * address_space that has radix tree nodes on the LRU. 425 * address_space that has radix tree nodes on the LRU.
426 * 426 *
427 * We can then safely transition to the mapping->tree_lock to 427 * We can then safely transition to the i_pages lock to
428 * pin only the address_space of the particular node we want 428 * pin only the address_space of the particular node we want
429 * to reclaim, take the node off-LRU, and drop the lru_lock. 429 * to reclaim, take the node off-LRU, and drop the lru_lock.
430 */ 430 */
431 431
432 node = container_of(item, struct radix_tree_node, private_list); 432 node = container_of(item, struct radix_tree_node, private_list);
433 mapping = container_of(node->root, struct address_space, page_tree); 433 mapping = container_of(node->root, struct address_space, i_pages);
434 434
435 /* Coming from the list, invert the lock order */ 435 /* Coming from the list, invert the lock order */
436 if (!spin_trylock(&mapping->tree_lock)) { 436 if (!xa_trylock(&mapping->i_pages)) {
437 spin_unlock(lru_lock); 437 spin_unlock(lru_lock);
438 ret = LRU_RETRY; 438 ret = LRU_RETRY;
439 goto out; 439 goto out;
@@ -468,11 +468,11 @@ static enum lru_status shadow_lru_isolate(struct list_head *item,
468 if (WARN_ON_ONCE(node->exceptional)) 468 if (WARN_ON_ONCE(node->exceptional))
469 goto out_invalid; 469 goto out_invalid;
470 inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); 470 inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM);
471 __radix_tree_delete_node(&mapping->page_tree, node, 471 __radix_tree_delete_node(&mapping->i_pages, node,
472 workingset_lookup_update(mapping)); 472 workingset_lookup_update(mapping));
473 473
474out_invalid: 474out_invalid:
475 spin_unlock(&mapping->tree_lock); 475 xa_unlock(&mapping->i_pages);
476 ret = LRU_REMOVED_RETRY; 476 ret = LRU_REMOVED_RETRY;
477out: 477out:
478 local_irq_enable(); 478 local_irq_enable();
@@ -487,7 +487,7 @@ static unsigned long scan_shadow_nodes(struct shrinker *shrinker,
487{ 487{
488 unsigned long ret; 488 unsigned long ret;
489 489
490 /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ 490 /* list_lru lock nests inside the IRQ-safe i_pages lock */
491 local_irq_disable(); 491 local_irq_disable();
492 ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL); 492 ret = list_lru_shrink_walk(&shadow_nodes, sc, shadow_lru_isolate, NULL);
493 local_irq_enable(); 493 local_irq_enable();
@@ -503,7 +503,7 @@ static struct shrinker workingset_shadow_shrinker = {
503 503
504/* 504/*
505 * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe 505 * Our list_lru->lock is IRQ-safe as it nests inside the IRQ-safe
506 * mapping->tree_lock. 506 * i_pages lock.
507 */ 507 */
508static struct lock_class_key shadow_nodes_key; 508static struct lock_class_key shadow_nodes_key;
509 509
generated by cgit v1.2.2 (git 2.25.0) at 2026-05-27 07:55:41 -0400