summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--fs/dax.c2
-rw-r--r--include/linux/radix-tree.h7
-rw-r--r--include/linux/swap.h13
-rw-r--r--lib/idr.c2
-rw-r--r--lib/radix-tree.c30
-rw-r--r--mm/filemap.c7
-rw-r--r--mm/shmem.c2
-rw-r--r--mm/truncate.c2
-rw-r--r--mm/workingset.c10
-rw-r--r--tools/testing/radix-tree/multiorder.c2
10 files changed, 39 insertions, 38 deletions
diff --git a/fs/dax.c b/fs/dax.c
index 9ec797424e4f..165fdfb6e508 100644
--- a/fs/dax.c
+++ b/fs/dax.c
@@ -565,7 +565,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping,
565 ret = __radix_tree_lookup(page_tree, index, &node, &slot); 565 ret = __radix_tree_lookup(page_tree, index, &node, &slot);
566 WARN_ON_ONCE(ret != entry); 566 WARN_ON_ONCE(ret != entry);
567 __radix_tree_replace(page_tree, node, slot, 567 __radix_tree_replace(page_tree, node, slot,
568 new_entry, NULL, NULL); 568 new_entry, NULL);
569 entry = new_entry; 569 entry = new_entry;
570 } 570 }
571 571
diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h
index 567ebb5eaab0..0ca448c1cb42 100644
--- a/include/linux/radix-tree.h
+++ b/include/linux/radix-tree.h
@@ -301,18 +301,17 @@ void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index,
301void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); 301void *radix_tree_lookup(const struct radix_tree_root *, unsigned long);
302void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, 302void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *,
303 unsigned long index); 303 unsigned long index);
304typedef void (*radix_tree_update_node_t)(struct radix_tree_node *, void *); 304typedef void (*radix_tree_update_node_t)(struct radix_tree_node *);
305void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, 305void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *,
306 void __rcu **slot, void *entry, 306 void __rcu **slot, void *entry,
307 radix_tree_update_node_t update_node, void *private); 307 radix_tree_update_node_t update_node);
308void radix_tree_iter_replace(struct radix_tree_root *, 308void radix_tree_iter_replace(struct radix_tree_root *,
309 const struct radix_tree_iter *, void __rcu **slot, void *entry); 309 const struct radix_tree_iter *, void __rcu **slot, void *entry);
310void radix_tree_replace_slot(struct radix_tree_root *, 310void radix_tree_replace_slot(struct radix_tree_root *,
311 void __rcu **slot, void *entry); 311 void __rcu **slot, void *entry);
312void __radix_tree_delete_node(struct radix_tree_root *, 312void __radix_tree_delete_node(struct radix_tree_root *,
313 struct radix_tree_node *, 313 struct radix_tree_node *,
314 radix_tree_update_node_t update_node, 314 radix_tree_update_node_t update_node);
315 void *private);
316void radix_tree_iter_delete(struct radix_tree_root *, 315void radix_tree_iter_delete(struct radix_tree_root *,
317 struct radix_tree_iter *iter, void __rcu **slot); 316 struct radix_tree_iter *iter, void __rcu **slot);
318void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); 317void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *);
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 8b8a6f965785..454f042bcdd5 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -298,7 +298,18 @@ struct vma_swap_readahead {
298void *workingset_eviction(struct address_space *mapping, struct page *page); 298void *workingset_eviction(struct address_space *mapping, struct page *page);
299bool workingset_refault(void *shadow); 299bool workingset_refault(void *shadow);
300void workingset_activation(struct page *page); 300void workingset_activation(struct page *page);
301void workingset_update_node(struct radix_tree_node *node, void *private); 301
302/* Do not use directly, use workingset_lookup_update */
303void workingset_update_node(struct radix_tree_node *node);
304
305/* Returns workingset_update_node() if the mapping has shadow entries. */
306#define workingset_lookup_update(mapping) \
307({ \
308 radix_tree_update_node_t __helper = workingset_update_node; \
309 if (dax_mapping(mapping) || shmem_mapping(mapping)) \
310 __helper = NULL; \
311 __helper; \
312})
302 313
303/* linux/mm/page_alloc.c */ 314/* linux/mm/page_alloc.c */
304extern unsigned long totalram_pages; 315extern unsigned long totalram_pages;
diff --git a/lib/idr.c b/lib/idr.c
index edd9b2be1651..2593ce513a18 100644
--- a/lib/idr.c
+++ b/lib/idr.c
@@ -171,7 +171,7 @@ void *idr_replace_ext(struct idr *idr, void *ptr, unsigned long id)
171 if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) 171 if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE))
172 return ERR_PTR(-ENOENT); 172 return ERR_PTR(-ENOENT);
173 173
174 __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL, NULL); 174 __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL);
175 175
176 return entry; 176 return entry;
177} 177}
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index 8b1feca1230a..c8d55565fafa 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -677,8 +677,7 @@ out:
677 * @root radix tree root 677 * @root radix tree root
678 */ 678 */
679static inline bool radix_tree_shrink(struct radix_tree_root *root, 679static inline bool radix_tree_shrink(struct radix_tree_root *root,
680 radix_tree_update_node_t update_node, 680 radix_tree_update_node_t update_node)
681 void *private)
682{ 681{
683 bool shrunk = false; 682 bool shrunk = false;
684 683
@@ -739,7 +738,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root,
739 if (!radix_tree_is_internal_node(child)) { 738 if (!radix_tree_is_internal_node(child)) {
740 node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; 739 node->slots[0] = (void __rcu *)RADIX_TREE_RETRY;
741 if (update_node) 740 if (update_node)
742 update_node(node, private); 741 update_node(node);
743 } 742 }
744 743
745 WARN_ON_ONCE(!list_empty(&node->private_list)); 744 WARN_ON_ONCE(!list_empty(&node->private_list));
@@ -752,7 +751,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root,
752 751
753static bool delete_node(struct radix_tree_root *root, 752static bool delete_node(struct radix_tree_root *root,
754 struct radix_tree_node *node, 753 struct radix_tree_node *node,
755 radix_tree_update_node_t update_node, void *private) 754 radix_tree_update_node_t update_node)
756{ 755{
757 bool deleted = false; 756 bool deleted = false;
758 757
@@ -762,8 +761,8 @@ static bool delete_node(struct radix_tree_root *root,
762 if (node->count) { 761 if (node->count) {
763 if (node_to_entry(node) == 762 if (node_to_entry(node) ==
764 rcu_dereference_raw(root->rnode)) 763 rcu_dereference_raw(root->rnode))
765 deleted |= radix_tree_shrink(root, update_node, 764 deleted |= radix_tree_shrink(root,
766 private); 765 update_node);
767 return deleted; 766 return deleted;
768 } 767 }
769 768
@@ -1173,7 +1172,6 @@ static int calculate_count(struct radix_tree_root *root,
1173 * @slot: pointer to slot in @node 1172 * @slot: pointer to slot in @node
1174 * @item: new item to store in the slot. 1173 * @item: new item to store in the slot.
1175 * @update_node: callback for changing leaf nodes 1174 * @update_node: callback for changing leaf nodes
1176 * @private: private data to pass to @update_node
1177 * 1175 *
1178 * For use with __radix_tree_lookup(). Caller must hold tree write locked 1176 * For use with __radix_tree_lookup(). Caller must hold tree write locked
1179 * across slot lookup and replacement. 1177 * across slot lookup and replacement.
@@ -1181,7 +1179,7 @@ static int calculate_count(struct radix_tree_root *root,
1181void __radix_tree_replace(struct radix_tree_root *root, 1179void __radix_tree_replace(struct radix_tree_root *root,
1182 struct radix_tree_node *node, 1180 struct radix_tree_node *node,
1183 void __rcu **slot, void *item, 1181 void __rcu **slot, void *item,
1184 radix_tree_update_node_t update_node, void *private) 1182 radix_tree_update_node_t update_node)
1185{ 1183{
1186 void *old = rcu_dereference_raw(*slot); 1184 void *old = rcu_dereference_raw(*slot);
1187 int exceptional = !!radix_tree_exceptional_entry(item) - 1185 int exceptional = !!radix_tree_exceptional_entry(item) -
@@ -1201,9 +1199,9 @@ void __radix_tree_replace(struct radix_tree_root *root,
1201 return; 1199 return;
1202 1200
1203 if (update_node) 1201 if (update_node)
1204 update_node(node, private); 1202 update_node(node);
1205 1203
1206 delete_node(root, node, update_node, private); 1204 delete_node(root, node, update_node);
1207} 1205}
1208 1206
1209/** 1207/**
@@ -1225,7 +1223,7 @@ void __radix_tree_replace(struct radix_tree_root *root,
1225void radix_tree_replace_slot(struct radix_tree_root *root, 1223void radix_tree_replace_slot(struct radix_tree_root *root,
1226 void __rcu **slot, void *item) 1224 void __rcu **slot, void *item)
1227{ 1225{
1228 __radix_tree_replace(root, NULL, slot, item, NULL, NULL); 1226 __radix_tree_replace(root, NULL, slot, item, NULL);
1229} 1227}
1230EXPORT_SYMBOL(radix_tree_replace_slot); 1228EXPORT_SYMBOL(radix_tree_replace_slot);
1231 1229
@@ -1242,7 +1240,7 @@ void radix_tree_iter_replace(struct radix_tree_root *root,
1242 const struct radix_tree_iter *iter, 1240 const struct radix_tree_iter *iter,
1243 void __rcu **slot, void *item) 1241 void __rcu **slot, void *item)
1244{ 1242{
1245 __radix_tree_replace(root, iter->node, slot, item, NULL, NULL); 1243 __radix_tree_replace(root, iter->node, slot, item, NULL);
1246} 1244}
1247 1245
1248#ifdef CONFIG_RADIX_TREE_MULTIORDER 1246#ifdef CONFIG_RADIX_TREE_MULTIORDER
@@ -1972,7 +1970,6 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot);
1972 * @root: radix tree root 1970 * @root: radix tree root
1973 * @node: node containing @index 1971 * @node: node containing @index
1974 * @update_node: callback for changing leaf nodes 1972 * @update_node: callback for changing leaf nodes
1975 * @private: private data to pass to @update_node
1976 * 1973 *
1977 * After clearing the slot at @index in @node from radix tree 1974 * After clearing the slot at @index in @node from radix tree
1978 * rooted at @root, call this function to attempt freeing the 1975 * rooted at @root, call this function to attempt freeing the
@@ -1980,10 +1977,9 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot);
1980 */ 1977 */
1981void __radix_tree_delete_node(struct radix_tree_root *root, 1978void __radix_tree_delete_node(struct radix_tree_root *root,
1982 struct radix_tree_node *node, 1979 struct radix_tree_node *node,
1983 radix_tree_update_node_t update_node, 1980 radix_tree_update_node_t update_node)
1984 void *private)
1985{ 1981{
1986 delete_node(root, node, update_node, private); 1982 delete_node(root, node, update_node);
1987} 1983}
1988 1984
1989static bool __radix_tree_delete(struct radix_tree_root *root, 1985static bool __radix_tree_delete(struct radix_tree_root *root,
@@ -2001,7 +1997,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root,
2001 node_tag_clear(root, node, tag, offset); 1997 node_tag_clear(root, node, tag, offset);
2002 1998
2003 replace_slot(slot, NULL, node, -1, exceptional); 1999 replace_slot(slot, NULL, node, -1, exceptional);
2004 return node && delete_node(root, node, NULL, NULL); 2000 return node && delete_node(root, node, NULL);
2005} 2001}
2006 2002
2007/** 2003/**
diff --git a/mm/filemap.c b/mm/filemap.c
index a470dd8cd05b..155370fc87f2 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -35,6 +35,7 @@
35#include <linux/hugetlb.h> 35#include <linux/hugetlb.h>
36#include <linux/memcontrol.h> 36#include <linux/memcontrol.h>
37#include <linux/cleancache.h> 37#include <linux/cleancache.h>
38#include <linux/shmem_fs.h>
38#include <linux/rmap.h> 39#include <linux/rmap.h>
39#include "internal.h" 40#include "internal.h"
40 41
@@ -134,7 +135,7 @@ static int page_cache_tree_insert(struct address_space *mapping,
134 *shadowp = p; 135 *shadowp = p;
135 } 136 }
136 __radix_tree_replace(&mapping->page_tree, node, slot, page, 137 __radix_tree_replace(&mapping->page_tree, node, slot, page,
137 workingset_update_node, mapping); 138 workingset_lookup_update(mapping));
138 mapping->nrpages++; 139 mapping->nrpages++;
139 return 0; 140 return 0;
140} 141}
@@ -162,7 +163,7 @@ static void page_cache_tree_delete(struct address_space *mapping,
162 163
163 radix_tree_clear_tags(&mapping->page_tree, node, slot); 164 radix_tree_clear_tags(&mapping->page_tree, node, slot);
164 __radix_tree_replace(&mapping->page_tree, node, slot, shadow, 165 __radix_tree_replace(&mapping->page_tree, node, slot, shadow,
165 workingset_update_node, mapping); 166 workingset_lookup_update(mapping));
166 } 167 }
167 168
168 page->mapping = NULL; 169 page->mapping = NULL;
@@ -359,7 +360,7 @@ page_cache_tree_delete_batch(struct address_space *mapping,
359 } 360 }
360 radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); 361 radix_tree_clear_tags(&mapping->page_tree, iter.node, slot);
361 __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, 362 __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL,
362 workingset_update_node, mapping); 363 workingset_lookup_update(mapping));
363 total_pages++; 364 total_pages++;
364 } 365 }
365 mapping->nrpages -= total_pages; 366 mapping->nrpages -= total_pages;
diff --git a/mm/shmem.c b/mm/shmem.c
index 07a1d22807be..a72f68aee6a4 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -338,7 +338,7 @@ static int shmem_radix_tree_replace(struct address_space *mapping,
338 if (item != expected) 338 if (item != expected)
339 return -ENOENT; 339 return -ENOENT;
340 __radix_tree_replace(&mapping->page_tree, node, pslot, 340 __radix_tree_replace(&mapping->page_tree, node, pslot,
341 replacement, NULL, NULL); 341 replacement, NULL);
342 return 0; 342 return 0;
343} 343}
344 344
diff --git a/mm/truncate.c b/mm/truncate.c
index 4a39a3150ee2..02a0c0466c78 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -42,7 +42,7 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index,
42 if (*slot != entry) 42 if (*slot != entry)
43 goto unlock; 43 goto unlock;
44 __radix_tree_replace(&mapping->page_tree, node, slot, NULL, 44 __radix_tree_replace(&mapping->page_tree, node, slot, NULL,
45 workingset_update_node, mapping); 45 workingset_update_node);
46 mapping->nrexceptional--; 46 mapping->nrexceptional--;
47unlock: 47unlock:
48 spin_unlock_irq(&mapping->tree_lock); 48 spin_unlock_irq(&mapping->tree_lock);
diff --git a/mm/workingset.c b/mm/workingset.c
index b997c9de28f6..b7d616a3bbbe 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -340,14 +340,8 @@ out:
340 340
341static struct list_lru shadow_nodes; 341static struct list_lru shadow_nodes;
342 342
343void workingset_update_node(struct radix_tree_node *node, void *private) 343void workingset_update_node(struct radix_tree_node *node)
344{ 344{
345 struct address_space *mapping = private;
346
347 /* Only regular page cache has shadow entries */
348 if (dax_mapping(mapping) || shmem_mapping(mapping))
349 return;
350
351 /* 345 /*
352 * Track non-empty nodes that contain only shadow entries; 346 * Track non-empty nodes that contain only shadow entries;
353 * unlink those that contain pages or are being freed. 347 * unlink those that contain pages or are being freed.
@@ -475,7 +469,7 @@ static enum lru_status shadow_lru_isolate(struct list_head *item,
475 goto out_invalid; 469 goto out_invalid;
476 inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); 470 inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM);
477 __radix_tree_delete_node(&mapping->page_tree, node, 471 __radix_tree_delete_node(&mapping->page_tree, node,
478 workingset_update_node, mapping); 472 workingset_lookup_update(mapping));
479 473
480out_invalid: 474out_invalid:
481 spin_unlock(&mapping->tree_lock); 475 spin_unlock(&mapping->tree_lock);
diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c
index 06c71178d07d..59245b3d587c 100644
--- a/tools/testing/radix-tree/multiorder.c
+++ b/tools/testing/radix-tree/multiorder.c
@@ -618,7 +618,7 @@ static void multiorder_account(void)
618 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); 618 __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12);
619 __radix_tree_lookup(&tree, 1 << 5, &node, &slot); 619 __radix_tree_lookup(&tree, 1 << 5, &node, &slot);
620 assert(node->count == node->exceptional * 2); 620 assert(node->count == node->exceptional * 2);
621 __radix_tree_replace(&tree, node, slot, NULL, NULL, NULL); 621 __radix_tree_replace(&tree, node, slot, NULL, NULL);
622 assert(node->exceptional == 0); 622 assert(node->exceptional == 0);
623 623
624 item_kill_tree(&tree); 624 item_kill_tree(&tree);