diff options
-rw-r--r-- | fs/dax.c | 2 | ||||
-rw-r--r-- | include/linux/radix-tree.h | 7 | ||||
-rw-r--r-- | include/linux/swap.h | 13 | ||||
-rw-r--r-- | lib/idr.c | 2 | ||||
-rw-r--r-- | lib/radix-tree.c | 30 | ||||
-rw-r--r-- | mm/filemap.c | 7 | ||||
-rw-r--r-- | mm/shmem.c | 2 | ||||
-rw-r--r-- | mm/truncate.c | 2 | ||||
-rw-r--r-- | mm/workingset.c | 10 | ||||
-rw-r--r-- | tools/testing/radix-tree/multiorder.c | 2 |
10 files changed, 39 insertions, 38 deletions
@@ -565,7 +565,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
565 | ret = __radix_tree_lookup(page_tree, index, &node, &slot); | 565 | ret = __radix_tree_lookup(page_tree, index, &node, &slot); |
566 | WARN_ON_ONCE(ret != entry); | 566 | WARN_ON_ONCE(ret != entry); |
567 | __radix_tree_replace(page_tree, node, slot, | 567 | __radix_tree_replace(page_tree, node, slot, |
568 | new_entry, NULL, NULL); | 568 | new_entry, NULL); |
569 | entry = new_entry; | 569 | entry = new_entry; |
570 | } | 570 | } |
571 | 571 | ||
diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 567ebb5eaab0..0ca448c1cb42 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h | |||
@@ -301,18 +301,17 @@ void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index, | |||
301 | void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); | 301 | void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); |
302 | void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, | 302 | void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, |
303 | unsigned long index); | 303 | unsigned long index); |
304 | typedef void (*radix_tree_update_node_t)(struct radix_tree_node *, void *); | 304 | typedef void (*radix_tree_update_node_t)(struct radix_tree_node *); |
305 | void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, | 305 | void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, |
306 | void __rcu **slot, void *entry, | 306 | void __rcu **slot, void *entry, |
307 | radix_tree_update_node_t update_node, void *private); | 307 | radix_tree_update_node_t update_node); |
308 | void radix_tree_iter_replace(struct radix_tree_root *, | 308 | void radix_tree_iter_replace(struct radix_tree_root *, |
309 | const struct radix_tree_iter *, void __rcu **slot, void *entry); | 309 | const struct radix_tree_iter *, void __rcu **slot, void *entry); |
310 | void radix_tree_replace_slot(struct radix_tree_root *, | 310 | void radix_tree_replace_slot(struct radix_tree_root *, |
311 | void __rcu **slot, void *entry); | 311 | void __rcu **slot, void *entry); |
312 | void __radix_tree_delete_node(struct radix_tree_root *, | 312 | void __radix_tree_delete_node(struct radix_tree_root *, |
313 | struct radix_tree_node *, | 313 | struct radix_tree_node *, |
314 | radix_tree_update_node_t update_node, | 314 | radix_tree_update_node_t update_node); |
315 | void *private); | ||
316 | void radix_tree_iter_delete(struct radix_tree_root *, | 315 | void radix_tree_iter_delete(struct radix_tree_root *, |
317 | struct radix_tree_iter *iter, void __rcu **slot); | 316 | struct radix_tree_iter *iter, void __rcu **slot); |
318 | void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); | 317 | void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); |
diff --git a/include/linux/swap.h b/include/linux/swap.h index 8b8a6f965785..454f042bcdd5 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h | |||
@@ -298,7 +298,18 @@ struct vma_swap_readahead { | |||
298 | void *workingset_eviction(struct address_space *mapping, struct page *page); | 298 | void *workingset_eviction(struct address_space *mapping, struct page *page); |
299 | bool workingset_refault(void *shadow); | 299 | bool workingset_refault(void *shadow); |
300 | void workingset_activation(struct page *page); | 300 | void workingset_activation(struct page *page); |
301 | void workingset_update_node(struct radix_tree_node *node, void *private); | 301 | |
302 | /* Do not use directly, use workingset_lookup_update */ | ||
303 | void workingset_update_node(struct radix_tree_node *node); | ||
304 | |||
305 | /* Returns workingset_update_node() if the mapping has shadow entries. */ | ||
306 | #define workingset_lookup_update(mapping) \ | ||
307 | ({ \ | ||
308 | radix_tree_update_node_t __helper = workingset_update_node; \ | ||
309 | if (dax_mapping(mapping) || shmem_mapping(mapping)) \ | ||
310 | __helper = NULL; \ | ||
311 | __helper; \ | ||
312 | }) | ||
302 | 313 | ||
303 | /* linux/mm/page_alloc.c */ | 314 | /* linux/mm/page_alloc.c */ |
304 | extern unsigned long totalram_pages; | 315 | extern unsigned long totalram_pages; |
@@ -171,7 +171,7 @@ void *idr_replace_ext(struct idr *idr, void *ptr, unsigned long id) | |||
171 | if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) | 171 | if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) |
172 | return ERR_PTR(-ENOENT); | 172 | return ERR_PTR(-ENOENT); |
173 | 173 | ||
174 | __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL, NULL); | 174 | __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL); |
175 | 175 | ||
176 | return entry; | 176 | return entry; |
177 | } | 177 | } |
diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 8b1feca1230a..c8d55565fafa 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c | |||
@@ -677,8 +677,7 @@ out: | |||
677 | * @root radix tree root | 677 | * @root radix tree root |
678 | */ | 678 | */ |
679 | static inline bool radix_tree_shrink(struct radix_tree_root *root, | 679 | static inline bool radix_tree_shrink(struct radix_tree_root *root, |
680 | radix_tree_update_node_t update_node, | 680 | radix_tree_update_node_t update_node) |
681 | void *private) | ||
682 | { | 681 | { |
683 | bool shrunk = false; | 682 | bool shrunk = false; |
684 | 683 | ||
@@ -739,7 +738,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, | |||
739 | if (!radix_tree_is_internal_node(child)) { | 738 | if (!radix_tree_is_internal_node(child)) { |
740 | node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; | 739 | node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; |
741 | if (update_node) | 740 | if (update_node) |
742 | update_node(node, private); | 741 | update_node(node); |
743 | } | 742 | } |
744 | 743 | ||
745 | WARN_ON_ONCE(!list_empty(&node->private_list)); | 744 | WARN_ON_ONCE(!list_empty(&node->private_list)); |
@@ -752,7 +751,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, | |||
752 | 751 | ||
753 | static bool delete_node(struct radix_tree_root *root, | 752 | static bool delete_node(struct radix_tree_root *root, |
754 | struct radix_tree_node *node, | 753 | struct radix_tree_node *node, |
755 | radix_tree_update_node_t update_node, void *private) | 754 | radix_tree_update_node_t update_node) |
756 | { | 755 | { |
757 | bool deleted = false; | 756 | bool deleted = false; |
758 | 757 | ||
@@ -762,8 +761,8 @@ static bool delete_node(struct radix_tree_root *root, | |||
762 | if (node->count) { | 761 | if (node->count) { |
763 | if (node_to_entry(node) == | 762 | if (node_to_entry(node) == |
764 | rcu_dereference_raw(root->rnode)) | 763 | rcu_dereference_raw(root->rnode)) |
765 | deleted |= radix_tree_shrink(root, update_node, | 764 | deleted |= radix_tree_shrink(root, |
766 | private); | 765 | update_node); |
767 | return deleted; | 766 | return deleted; |
768 | } | 767 | } |
769 | 768 | ||
@@ -1173,7 +1172,6 @@ static int calculate_count(struct radix_tree_root *root, | |||
1173 | * @slot: pointer to slot in @node | 1172 | * @slot: pointer to slot in @node |
1174 | * @item: new item to store in the slot. | 1173 | * @item: new item to store in the slot. |
1175 | * @update_node: callback for changing leaf nodes | 1174 | * @update_node: callback for changing leaf nodes |
1176 | * @private: private data to pass to @update_node | ||
1177 | * | 1175 | * |
1178 | * For use with __radix_tree_lookup(). Caller must hold tree write locked | 1176 | * For use with __radix_tree_lookup(). Caller must hold tree write locked |
1179 | * across slot lookup and replacement. | 1177 | * across slot lookup and replacement. |
@@ -1181,7 +1179,7 @@ static int calculate_count(struct radix_tree_root *root, | |||
1181 | void __radix_tree_replace(struct radix_tree_root *root, | 1179 | void __radix_tree_replace(struct radix_tree_root *root, |
1182 | struct radix_tree_node *node, | 1180 | struct radix_tree_node *node, |
1183 | void __rcu **slot, void *item, | 1181 | void __rcu **slot, void *item, |
1184 | radix_tree_update_node_t update_node, void *private) | 1182 | radix_tree_update_node_t update_node) |
1185 | { | 1183 | { |
1186 | void *old = rcu_dereference_raw(*slot); | 1184 | void *old = rcu_dereference_raw(*slot); |
1187 | int exceptional = !!radix_tree_exceptional_entry(item) - | 1185 | int exceptional = !!radix_tree_exceptional_entry(item) - |
@@ -1201,9 +1199,9 @@ void __radix_tree_replace(struct radix_tree_root *root, | |||
1201 | return; | 1199 | return; |
1202 | 1200 | ||
1203 | if (update_node) | 1201 | if (update_node) |
1204 | update_node(node, private); | 1202 | update_node(node); |
1205 | 1203 | ||
1206 | delete_node(root, node, update_node, private); | 1204 | delete_node(root, node, update_node); |
1207 | } | 1205 | } |
1208 | 1206 | ||
1209 | /** | 1207 | /** |
@@ -1225,7 +1223,7 @@ void __radix_tree_replace(struct radix_tree_root *root, | |||
1225 | void radix_tree_replace_slot(struct radix_tree_root *root, | 1223 | void radix_tree_replace_slot(struct radix_tree_root *root, |
1226 | void __rcu **slot, void *item) | 1224 | void __rcu **slot, void *item) |
1227 | { | 1225 | { |
1228 | __radix_tree_replace(root, NULL, slot, item, NULL, NULL); | 1226 | __radix_tree_replace(root, NULL, slot, item, NULL); |
1229 | } | 1227 | } |
1230 | EXPORT_SYMBOL(radix_tree_replace_slot); | 1228 | EXPORT_SYMBOL(radix_tree_replace_slot); |
1231 | 1229 | ||
@@ -1242,7 +1240,7 @@ void radix_tree_iter_replace(struct radix_tree_root *root, | |||
1242 | const struct radix_tree_iter *iter, | 1240 | const struct radix_tree_iter *iter, |
1243 | void __rcu **slot, void *item) | 1241 | void __rcu **slot, void *item) |
1244 | { | 1242 | { |
1245 | __radix_tree_replace(root, iter->node, slot, item, NULL, NULL); | 1243 | __radix_tree_replace(root, iter->node, slot, item, NULL); |
1246 | } | 1244 | } |
1247 | 1245 | ||
1248 | #ifdef CONFIG_RADIX_TREE_MULTIORDER | 1246 | #ifdef CONFIG_RADIX_TREE_MULTIORDER |
@@ -1972,7 +1970,6 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot); | |||
1972 | * @root: radix tree root | 1970 | * @root: radix tree root |
1973 | * @node: node containing @index | 1971 | * @node: node containing @index |
1974 | * @update_node: callback for changing leaf nodes | 1972 | * @update_node: callback for changing leaf nodes |
1975 | * @private: private data to pass to @update_node | ||
1976 | * | 1973 | * |
1977 | * After clearing the slot at @index in @node from radix tree | 1974 | * After clearing the slot at @index in @node from radix tree |
1978 | * rooted at @root, call this function to attempt freeing the | 1975 | * rooted at @root, call this function to attempt freeing the |
@@ -1980,10 +1977,9 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot); | |||
1980 | */ | 1977 | */ |
1981 | void __radix_tree_delete_node(struct radix_tree_root *root, | 1978 | void __radix_tree_delete_node(struct radix_tree_root *root, |
1982 | struct radix_tree_node *node, | 1979 | struct radix_tree_node *node, |
1983 | radix_tree_update_node_t update_node, | 1980 | radix_tree_update_node_t update_node) |
1984 | void *private) | ||
1985 | { | 1981 | { |
1986 | delete_node(root, node, update_node, private); | 1982 | delete_node(root, node, update_node); |
1987 | } | 1983 | } |
1988 | 1984 | ||
1989 | static bool __radix_tree_delete(struct radix_tree_root *root, | 1985 | static bool __radix_tree_delete(struct radix_tree_root *root, |
@@ -2001,7 +1997,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, | |||
2001 | node_tag_clear(root, node, tag, offset); | 1997 | node_tag_clear(root, node, tag, offset); |
2002 | 1998 | ||
2003 | replace_slot(slot, NULL, node, -1, exceptional); | 1999 | replace_slot(slot, NULL, node, -1, exceptional); |
2004 | return node && delete_node(root, node, NULL, NULL); | 2000 | return node && delete_node(root, node, NULL); |
2005 | } | 2001 | } |
2006 | 2002 | ||
2007 | /** | 2003 | /** |
diff --git a/mm/filemap.c b/mm/filemap.c index a470dd8cd05b..155370fc87f2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
@@ -35,6 +35,7 @@ | |||
35 | #include <linux/hugetlb.h> | 35 | #include <linux/hugetlb.h> |
36 | #include <linux/memcontrol.h> | 36 | #include <linux/memcontrol.h> |
37 | #include <linux/cleancache.h> | 37 | #include <linux/cleancache.h> |
38 | #include <linux/shmem_fs.h> | ||
38 | #include <linux/rmap.h> | 39 | #include <linux/rmap.h> |
39 | #include "internal.h" | 40 | #include "internal.h" |
40 | 41 | ||
@@ -134,7 +135,7 @@ static int page_cache_tree_insert(struct address_space *mapping, | |||
134 | *shadowp = p; | 135 | *shadowp = p; |
135 | } | 136 | } |
136 | __radix_tree_replace(&mapping->page_tree, node, slot, page, | 137 | __radix_tree_replace(&mapping->page_tree, node, slot, page, |
137 | workingset_update_node, mapping); | 138 | workingset_lookup_update(mapping)); |
138 | mapping->nrpages++; | 139 | mapping->nrpages++; |
139 | return 0; | 140 | return 0; |
140 | } | 141 | } |
@@ -162,7 +163,7 @@ static void page_cache_tree_delete(struct address_space *mapping, | |||
162 | 163 | ||
163 | radix_tree_clear_tags(&mapping->page_tree, node, slot); | 164 | radix_tree_clear_tags(&mapping->page_tree, node, slot); |
164 | __radix_tree_replace(&mapping->page_tree, node, slot, shadow, | 165 | __radix_tree_replace(&mapping->page_tree, node, slot, shadow, |
165 | workingset_update_node, mapping); | 166 | workingset_lookup_update(mapping)); |
166 | } | 167 | } |
167 | 168 | ||
168 | page->mapping = NULL; | 169 | page->mapping = NULL; |
@@ -359,7 +360,7 @@ page_cache_tree_delete_batch(struct address_space *mapping, | |||
359 | } | 360 | } |
360 | radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); | 361 | radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); |
361 | __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, | 362 | __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, |
362 | workingset_update_node, mapping); | 363 | workingset_lookup_update(mapping)); |
363 | total_pages++; | 364 | total_pages++; |
364 | } | 365 | } |
365 | mapping->nrpages -= total_pages; | 366 | mapping->nrpages -= total_pages; |
diff --git a/mm/shmem.c b/mm/shmem.c index 07a1d22807be..a72f68aee6a4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
@@ -338,7 +338,7 @@ static int shmem_radix_tree_replace(struct address_space *mapping, | |||
338 | if (item != expected) | 338 | if (item != expected) |
339 | return -ENOENT; | 339 | return -ENOENT; |
340 | __radix_tree_replace(&mapping->page_tree, node, pslot, | 340 | __radix_tree_replace(&mapping->page_tree, node, pslot, |
341 | replacement, NULL, NULL); | 341 | replacement, NULL); |
342 | return 0; | 342 | return 0; |
343 | } | 343 | } |
344 | 344 | ||
diff --git a/mm/truncate.c b/mm/truncate.c index 4a39a3150ee2..02a0c0466c78 100644 --- a/mm/truncate.c +++ b/mm/truncate.c | |||
@@ -42,7 +42,7 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, | |||
42 | if (*slot != entry) | 42 | if (*slot != entry) |
43 | goto unlock; | 43 | goto unlock; |
44 | __radix_tree_replace(&mapping->page_tree, node, slot, NULL, | 44 | __radix_tree_replace(&mapping->page_tree, node, slot, NULL, |
45 | workingset_update_node, mapping); | 45 | workingset_update_node); |
46 | mapping->nrexceptional--; | 46 | mapping->nrexceptional--; |
47 | unlock: | 47 | unlock: |
48 | spin_unlock_irq(&mapping->tree_lock); | 48 | spin_unlock_irq(&mapping->tree_lock); |
diff --git a/mm/workingset.c b/mm/workingset.c index b997c9de28f6..b7d616a3bbbe 100644 --- a/mm/workingset.c +++ b/mm/workingset.c | |||
@@ -340,14 +340,8 @@ out: | |||
340 | 340 | ||
341 | static struct list_lru shadow_nodes; | 341 | static struct list_lru shadow_nodes; |
342 | 342 | ||
343 | void workingset_update_node(struct radix_tree_node *node, void *private) | 343 | void workingset_update_node(struct radix_tree_node *node) |
344 | { | 344 | { |
345 | struct address_space *mapping = private; | ||
346 | |||
347 | /* Only regular page cache has shadow entries */ | ||
348 | if (dax_mapping(mapping) || shmem_mapping(mapping)) | ||
349 | return; | ||
350 | |||
351 | /* | 345 | /* |
352 | * Track non-empty nodes that contain only shadow entries; | 346 | * Track non-empty nodes that contain only shadow entries; |
353 | * unlink those that contain pages or are being freed. | 347 | * unlink those that contain pages or are being freed. |
@@ -475,7 +469,7 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, | |||
475 | goto out_invalid; | 469 | goto out_invalid; |
476 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); | 470 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); |
477 | __radix_tree_delete_node(&mapping->page_tree, node, | 471 | __radix_tree_delete_node(&mapping->page_tree, node, |
478 | workingset_update_node, mapping); | 472 | workingset_lookup_update(mapping)); |
479 | 473 | ||
480 | out_invalid: | 474 | out_invalid: |
481 | spin_unlock(&mapping->tree_lock); | 475 | spin_unlock(&mapping->tree_lock); |
diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 06c71178d07d..59245b3d587c 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c | |||
@@ -618,7 +618,7 @@ static void multiorder_account(void) | |||
618 | __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); | 618 | __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); |
619 | __radix_tree_lookup(&tree, 1 << 5, &node, &slot); | 619 | __radix_tree_lookup(&tree, 1 << 5, &node, &slot); |
620 | assert(node->count == node->exceptional * 2); | 620 | assert(node->count == node->exceptional * 2); |
621 | __radix_tree_replace(&tree, node, slot, NULL, NULL, NULL); | 621 | __radix_tree_replace(&tree, node, slot, NULL, NULL); |
622 | assert(node->exceptional == 0); | 622 | assert(node->exceptional == 0); |
623 | 623 | ||
624 | item_kill_tree(&tree); | 624 | item_kill_tree(&tree); |