aboutsummaryrefslogtreecommitdiffstats
path: root/mm/shmem.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/shmem.c')
-rw-r--r--mm/shmem.c60
1 files changed, 30 insertions, 30 deletions
diff --git a/mm/shmem.c b/mm/shmem.c
index 4424fc0c33aa..9d6c7e595415 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -332,12 +332,12 @@ static int shmem_radix_tree_replace(struct address_space *mapping,
332 332
333 VM_BUG_ON(!expected); 333 VM_BUG_ON(!expected);
334 VM_BUG_ON(!replacement); 334 VM_BUG_ON(!replacement);
335 item = __radix_tree_lookup(&mapping->page_tree, index, &node, &pslot); 335 item = __radix_tree_lookup(&mapping->i_pages, index, &node, &pslot);
336 if (!item) 336 if (!item)
337 return -ENOENT; 337 return -ENOENT;
338 if (item != expected) 338 if (item != expected)
339 return -ENOENT; 339 return -ENOENT;
340 __radix_tree_replace(&mapping->page_tree, node, pslot, 340 __radix_tree_replace(&mapping->i_pages, node, pslot,
341 replacement, NULL); 341 replacement, NULL);
342 return 0; 342 return 0;
343} 343}
@@ -355,7 +355,7 @@ static bool shmem_confirm_swap(struct address_space *mapping,
355 void *item; 355 void *item;
356 356
357 rcu_read_lock(); 357 rcu_read_lock();
358 item = radix_tree_lookup(&mapping->page_tree, index); 358 item = radix_tree_lookup(&mapping->i_pages, index);
359 rcu_read_unlock(); 359 rcu_read_unlock();
360 return item == swp_to_radix_entry(swap); 360 return item == swp_to_radix_entry(swap);
361} 361}
@@ -590,14 +590,14 @@ static int shmem_add_to_page_cache(struct page *page,
590 page->mapping = mapping; 590 page->mapping = mapping;
591 page->index = index; 591 page->index = index;
592 592
593 spin_lock_irq(&mapping->tree_lock); 593 xa_lock_irq(&mapping->i_pages);
594 if (PageTransHuge(page)) { 594 if (PageTransHuge(page)) {
595 void __rcu **results; 595 void __rcu **results;
596 pgoff_t idx; 596 pgoff_t idx;
597 int i; 597 int i;
598 598
599 error = 0; 599 error = 0;
600 if (radix_tree_gang_lookup_slot(&mapping->page_tree, 600 if (radix_tree_gang_lookup_slot(&mapping->i_pages,
601 &results, &idx, index, 1) && 601 &results, &idx, index, 1) &&
602 idx < index + HPAGE_PMD_NR) { 602 idx < index + HPAGE_PMD_NR) {
603 error = -EEXIST; 603 error = -EEXIST;
@@ -605,14 +605,14 @@ static int shmem_add_to_page_cache(struct page *page,
605 605
606 if (!error) { 606 if (!error) {
607 for (i = 0; i < HPAGE_PMD_NR; i++) { 607 for (i = 0; i < HPAGE_PMD_NR; i++) {
608 error = radix_tree_insert(&mapping->page_tree, 608 error = radix_tree_insert(&mapping->i_pages,
609 index + i, page + i); 609 index + i, page + i);
610 VM_BUG_ON(error); 610 VM_BUG_ON(error);
611 } 611 }
612 count_vm_event(THP_FILE_ALLOC); 612 count_vm_event(THP_FILE_ALLOC);
613 } 613 }
614 } else if (!expected) { 614 } else if (!expected) {
615 error = radix_tree_insert(&mapping->page_tree, index, page); 615 error = radix_tree_insert(&mapping->i_pages, index, page);
616 } else { 616 } else {
617 error = shmem_radix_tree_replace(mapping, index, expected, 617 error = shmem_radix_tree_replace(mapping, index, expected,
618 page); 618 page);
@@ -624,10 +624,10 @@ static int shmem_add_to_page_cache(struct page *page,
624 __inc_node_page_state(page, NR_SHMEM_THPS); 624 __inc_node_page_state(page, NR_SHMEM_THPS);
625 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 625 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
626 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 626 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
627 spin_unlock_irq(&mapping->tree_lock); 627 xa_unlock_irq(&mapping->i_pages);
628 } else { 628 } else {
629 page->mapping = NULL; 629 page->mapping = NULL;
630 spin_unlock_irq(&mapping->tree_lock); 630 xa_unlock_irq(&mapping->i_pages);
631 page_ref_sub(page, nr); 631 page_ref_sub(page, nr);
632 } 632 }
633 return error; 633 return error;
@@ -643,13 +643,13 @@ static void shmem_delete_from_page_cache(struct page *page, void *radswap)
643 643
644 VM_BUG_ON_PAGE(PageCompound(page), page); 644 VM_BUG_ON_PAGE(PageCompound(page), page);
645 645
646 spin_lock_irq(&mapping->tree_lock); 646 xa_lock_irq(&mapping->i_pages);
647 error = shmem_radix_tree_replace(mapping, page->index, page, radswap); 647 error = shmem_radix_tree_replace(mapping, page->index, page, radswap);
648 page->mapping = NULL; 648 page->mapping = NULL;
649 mapping->nrpages--; 649 mapping->nrpages--;
650 __dec_node_page_state(page, NR_FILE_PAGES); 650 __dec_node_page_state(page, NR_FILE_PAGES);
651 __dec_node_page_state(page, NR_SHMEM); 651 __dec_node_page_state(page, NR_SHMEM);
652 spin_unlock_irq(&mapping->tree_lock); 652 xa_unlock_irq(&mapping->i_pages);
653 put_page(page); 653 put_page(page);
654 BUG_ON(error); 654 BUG_ON(error);
655} 655}
@@ -662,9 +662,9 @@ static int shmem_free_swap(struct address_space *mapping,
662{ 662{
663 void *old; 663 void *old;
664 664
665 spin_lock_irq(&mapping->tree_lock); 665 xa_lock_irq(&mapping->i_pages);
666 old = radix_tree_delete_item(&mapping->page_tree, index, radswap); 666 old = radix_tree_delete_item(&mapping->i_pages, index, radswap);
667 spin_unlock_irq(&mapping->tree_lock); 667 xa_unlock_irq(&mapping->i_pages);
668 if (old != radswap) 668 if (old != radswap)
669 return -ENOENT; 669 return -ENOENT;
670 free_swap_and_cache(radix_to_swp_entry(radswap)); 670 free_swap_and_cache(radix_to_swp_entry(radswap));
@@ -675,7 +675,7 @@ static int shmem_free_swap(struct address_space *mapping,
675 * Determine (in bytes) how many of the shmem object's pages mapped by the 675 * Determine (in bytes) how many of the shmem object's pages mapped by the
676 * given offsets are swapped out. 676 * given offsets are swapped out.
677 * 677 *
678 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 678 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
679 * as long as the inode doesn't go away and racy results are not a problem. 679 * as long as the inode doesn't go away and racy results are not a problem.
680 */ 680 */
681unsigned long shmem_partial_swap_usage(struct address_space *mapping, 681unsigned long shmem_partial_swap_usage(struct address_space *mapping,
@@ -688,7 +688,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
688 688
689 rcu_read_lock(); 689 rcu_read_lock();
690 690
691 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 691 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
692 if (iter.index >= end) 692 if (iter.index >= end)
693 break; 693 break;
694 694
@@ -717,7 +717,7 @@ unsigned long shmem_partial_swap_usage(struct address_space *mapping,
717 * Determine (in bytes) how many of the shmem object's pages mapped by the 717 * Determine (in bytes) how many of the shmem object's pages mapped by the
718 * given vma is swapped out. 718 * given vma is swapped out.
719 * 719 *
720 * This is safe to call without i_mutex or mapping->tree_lock thanks to RCU, 720 * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
721 * as long as the inode doesn't go away and racy results are not a problem. 721 * as long as the inode doesn't go away and racy results are not a problem.
722 */ 722 */
723unsigned long shmem_swap_usage(struct vm_area_struct *vma) 723unsigned long shmem_swap_usage(struct vm_area_struct *vma)
@@ -1132,7 +1132,7 @@ static int shmem_unuse_inode(struct shmem_inode_info *info,
1132 int error = 0; 1132 int error = 0;
1133 1133
1134 radswap = swp_to_radix_entry(swap); 1134 radswap = swp_to_radix_entry(swap);
1135 index = find_swap_entry(&mapping->page_tree, radswap); 1135 index = find_swap_entry(&mapping->i_pages, radswap);
1136 if (index == -1) 1136 if (index == -1)
1137 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1137 return -EAGAIN; /* tell shmem_unuse we found nothing */
1138 1138
@@ -1448,7 +1448,7 @@ static struct page *shmem_alloc_hugepage(gfp_t gfp,
1448 1448
1449 hindex = round_down(index, HPAGE_PMD_NR); 1449 hindex = round_down(index, HPAGE_PMD_NR);
1450 rcu_read_lock(); 1450 rcu_read_lock();
1451 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx, 1451 if (radix_tree_gang_lookup_slot(&mapping->i_pages, &results, &idx,
1452 hindex, 1) && idx < hindex + HPAGE_PMD_NR) { 1452 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1453 rcu_read_unlock(); 1453 rcu_read_unlock();
1454 return NULL; 1454 return NULL;
@@ -1561,14 +1561,14 @@ static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1561 * Our caller will very soon move newpage out of swapcache, but it's 1561 * Our caller will very soon move newpage out of swapcache, but it's
1562 * a nice clean interface for us to replace oldpage by newpage there. 1562 * a nice clean interface for us to replace oldpage by newpage there.
1563 */ 1563 */
1564 spin_lock_irq(&swap_mapping->tree_lock); 1564 xa_lock_irq(&swap_mapping->i_pages);
1565 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage, 1565 error = shmem_radix_tree_replace(swap_mapping, swap_index, oldpage,
1566 newpage); 1566 newpage);
1567 if (!error) { 1567 if (!error) {
1568 __inc_node_page_state(newpage, NR_FILE_PAGES); 1568 __inc_node_page_state(newpage, NR_FILE_PAGES);
1569 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1569 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1570 } 1570 }
1571 spin_unlock_irq(&swap_mapping->tree_lock); 1571 xa_unlock_irq(&swap_mapping->i_pages);
1572 1572
1573 if (unlikely(error)) { 1573 if (unlikely(error)) {
1574 /* 1574 /*
@@ -2634,7 +2634,7 @@ static void shmem_tag_pins(struct address_space *mapping)
2634 start = 0; 2634 start = 0;
2635 rcu_read_lock(); 2635 rcu_read_lock();
2636 2636
2637 radix_tree_for_each_slot(slot, &mapping->page_tree, &iter, start) { 2637 radix_tree_for_each_slot(slot, &mapping->i_pages, &iter, start) {
2638 page = radix_tree_deref_slot(slot); 2638 page = radix_tree_deref_slot(slot);
2639 if (!page || radix_tree_exception(page)) { 2639 if (!page || radix_tree_exception(page)) {
2640 if (radix_tree_deref_retry(page)) { 2640 if (radix_tree_deref_retry(page)) {
@@ -2642,10 +2642,10 @@ static void shmem_tag_pins(struct address_space *mapping)
2642 continue; 2642 continue;
2643 } 2643 }
2644 } else if (page_count(page) - page_mapcount(page) > 1) { 2644 } else if (page_count(page) - page_mapcount(page) > 1) {
2645 spin_lock_irq(&mapping->tree_lock); 2645 xa_lock_irq(&mapping->i_pages);
2646 radix_tree_tag_set(&mapping->page_tree, iter.index, 2646 radix_tree_tag_set(&mapping->i_pages, iter.index,
2647 SHMEM_TAG_PINNED); 2647 SHMEM_TAG_PINNED);
2648 spin_unlock_irq(&mapping->tree_lock); 2648 xa_unlock_irq(&mapping->i_pages);
2649 } 2649 }
2650 2650
2651 if (need_resched()) { 2651 if (need_resched()) {
@@ -2677,7 +2677,7 @@ static int shmem_wait_for_pins(struct address_space *mapping)
2677 2677
2678 error = 0; 2678 error = 0;
2679 for (scan = 0; scan <= LAST_SCAN; scan++) { 2679 for (scan = 0; scan <= LAST_SCAN; scan++) {
2680 if (!radix_tree_tagged(&mapping->page_tree, SHMEM_TAG_PINNED)) 2680 if (!radix_tree_tagged(&mapping->i_pages, SHMEM_TAG_PINNED))
2681 break; 2681 break;
2682 2682
2683 if (!scan) 2683 if (!scan)
@@ -2687,7 +2687,7 @@ static int shmem_wait_for_pins(struct address_space *mapping)
2687 2687
2688 start = 0; 2688 start = 0;
2689 rcu_read_lock(); 2689 rcu_read_lock();
2690 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, 2690 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter,
2691 start, SHMEM_TAG_PINNED) { 2691 start, SHMEM_TAG_PINNED) {
2692 2692
2693 page = radix_tree_deref_slot(slot); 2693 page = radix_tree_deref_slot(slot);
@@ -2713,10 +2713,10 @@ static int shmem_wait_for_pins(struct address_space *mapping)
2713 error = -EBUSY; 2713 error = -EBUSY;
2714 } 2714 }
2715 2715
2716 spin_lock_irq(&mapping->tree_lock); 2716 xa_lock_irq(&mapping->i_pages);
2717 radix_tree_tag_clear(&mapping->page_tree, 2717 radix_tree_tag_clear(&mapping->i_pages,
2718 iter.index, SHMEM_TAG_PINNED); 2718 iter.index, SHMEM_TAG_PINNED);
2719 spin_unlock_irq(&mapping->tree_lock); 2719 xa_unlock_irq(&mapping->i_pages);
2720continue_resched: 2720continue_resched:
2721 if (need_resched()) { 2721 if (need_resched()) {
2722 slot = radix_tree_iter_resume(slot, &iter); 2722 slot = radix_tree_iter_resume(slot, &iter);