diff options
| -rw-r--r-- | fs/dax.c | 2 | ||||
| -rw-r--r-- | include/linux/radix-tree.h | 7 | ||||
| -rw-r--r-- | include/linux/swap.h | 13 | ||||
| -rw-r--r-- | lib/idr.c | 2 | ||||
| -rw-r--r-- | lib/radix-tree.c | 30 | ||||
| -rw-r--r-- | mm/filemap.c | 7 | ||||
| -rw-r--r-- | mm/shmem.c | 2 | ||||
| -rw-r--r-- | mm/truncate.c | 2 | ||||
| -rw-r--r-- | mm/workingset.c | 10 | ||||
| -rw-r--r-- | tools/testing/radix-tree/multiorder.c | 2 |
10 files changed, 39 insertions, 38 deletions
| @@ -565,7 +565,7 @@ static void *dax_insert_mapping_entry(struct address_space *mapping, | |||
| 565 | ret = __radix_tree_lookup(page_tree, index, &node, &slot); | 565 | ret = __radix_tree_lookup(page_tree, index, &node, &slot); |
| 566 | WARN_ON_ONCE(ret != entry); | 566 | WARN_ON_ONCE(ret != entry); |
| 567 | __radix_tree_replace(page_tree, node, slot, | 567 | __radix_tree_replace(page_tree, node, slot, |
| 568 | new_entry, NULL, NULL); | 568 | new_entry, NULL); |
| 569 | entry = new_entry; | 569 | entry = new_entry; |
| 570 | } | 570 | } |
| 571 | 571 | ||
diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index 567ebb5eaab0..0ca448c1cb42 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h | |||
| @@ -301,18 +301,17 @@ void *__radix_tree_lookup(const struct radix_tree_root *, unsigned long index, | |||
| 301 | void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); | 301 | void *radix_tree_lookup(const struct radix_tree_root *, unsigned long); |
| 302 | void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, | 302 | void __rcu **radix_tree_lookup_slot(const struct radix_tree_root *, |
| 303 | unsigned long index); | 303 | unsigned long index); |
| 304 | typedef void (*radix_tree_update_node_t)(struct radix_tree_node *, void *); | 304 | typedef void (*radix_tree_update_node_t)(struct radix_tree_node *); |
| 305 | void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, | 305 | void __radix_tree_replace(struct radix_tree_root *, struct radix_tree_node *, |
| 306 | void __rcu **slot, void *entry, | 306 | void __rcu **slot, void *entry, |
| 307 | radix_tree_update_node_t update_node, void *private); | 307 | radix_tree_update_node_t update_node); |
| 308 | void radix_tree_iter_replace(struct radix_tree_root *, | 308 | void radix_tree_iter_replace(struct radix_tree_root *, |
| 309 | const struct radix_tree_iter *, void __rcu **slot, void *entry); | 309 | const struct radix_tree_iter *, void __rcu **slot, void *entry); |
| 310 | void radix_tree_replace_slot(struct radix_tree_root *, | 310 | void radix_tree_replace_slot(struct radix_tree_root *, |
| 311 | void __rcu **slot, void *entry); | 311 | void __rcu **slot, void *entry); |
| 312 | void __radix_tree_delete_node(struct radix_tree_root *, | 312 | void __radix_tree_delete_node(struct radix_tree_root *, |
| 313 | struct radix_tree_node *, | 313 | struct radix_tree_node *, |
| 314 | radix_tree_update_node_t update_node, | 314 | radix_tree_update_node_t update_node); |
| 315 | void *private); | ||
| 316 | void radix_tree_iter_delete(struct radix_tree_root *, | 315 | void radix_tree_iter_delete(struct radix_tree_root *, |
| 317 | struct radix_tree_iter *iter, void __rcu **slot); | 316 | struct radix_tree_iter *iter, void __rcu **slot); |
| 318 | void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); | 317 | void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *); |
diff --git a/include/linux/swap.h b/include/linux/swap.h index 8b8a6f965785..454f042bcdd5 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h | |||
| @@ -298,7 +298,18 @@ struct vma_swap_readahead { | |||
| 298 | void *workingset_eviction(struct address_space *mapping, struct page *page); | 298 | void *workingset_eviction(struct address_space *mapping, struct page *page); |
| 299 | bool workingset_refault(void *shadow); | 299 | bool workingset_refault(void *shadow); |
| 300 | void workingset_activation(struct page *page); | 300 | void workingset_activation(struct page *page); |
| 301 | void workingset_update_node(struct radix_tree_node *node, void *private); | 301 | |
| 302 | /* Do not use directly, use workingset_lookup_update */ | ||
| 303 | void workingset_update_node(struct radix_tree_node *node); | ||
| 304 | |||
| 305 | /* Returns workingset_update_node() if the mapping has shadow entries. */ | ||
| 306 | #define workingset_lookup_update(mapping) \ | ||
| 307 | ({ \ | ||
| 308 | radix_tree_update_node_t __helper = workingset_update_node; \ | ||
| 309 | if (dax_mapping(mapping) || shmem_mapping(mapping)) \ | ||
| 310 | __helper = NULL; \ | ||
| 311 | __helper; \ | ||
| 312 | }) | ||
| 302 | 313 | ||
| 303 | /* linux/mm/page_alloc.c */ | 314 | /* linux/mm/page_alloc.c */ |
| 304 | extern unsigned long totalram_pages; | 315 | extern unsigned long totalram_pages; |
| @@ -171,7 +171,7 @@ void *idr_replace_ext(struct idr *idr, void *ptr, unsigned long id) | |||
| 171 | if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) | 171 | if (!slot || radix_tree_tag_get(&idr->idr_rt, id, IDR_FREE)) |
| 172 | return ERR_PTR(-ENOENT); | 172 | return ERR_PTR(-ENOENT); |
| 173 | 173 | ||
| 174 | __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL, NULL); | 174 | __radix_tree_replace(&idr->idr_rt, node, slot, ptr, NULL); |
| 175 | 175 | ||
| 176 | return entry; | 176 | return entry; |
| 177 | } | 177 | } |
diff --git a/lib/radix-tree.c b/lib/radix-tree.c index 8b1feca1230a..c8d55565fafa 100644 --- a/lib/radix-tree.c +++ b/lib/radix-tree.c | |||
| @@ -677,8 +677,7 @@ out: | |||
| 677 | * @root radix tree root | 677 | * @root radix tree root |
| 678 | */ | 678 | */ |
| 679 | static inline bool radix_tree_shrink(struct radix_tree_root *root, | 679 | static inline bool radix_tree_shrink(struct radix_tree_root *root, |
| 680 | radix_tree_update_node_t update_node, | 680 | radix_tree_update_node_t update_node) |
| 681 | void *private) | ||
| 682 | { | 681 | { |
| 683 | bool shrunk = false; | 682 | bool shrunk = false; |
| 684 | 683 | ||
| @@ -739,7 +738,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, | |||
| 739 | if (!radix_tree_is_internal_node(child)) { | 738 | if (!radix_tree_is_internal_node(child)) { |
| 740 | node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; | 739 | node->slots[0] = (void __rcu *)RADIX_TREE_RETRY; |
| 741 | if (update_node) | 740 | if (update_node) |
| 742 | update_node(node, private); | 741 | update_node(node); |
| 743 | } | 742 | } |
| 744 | 743 | ||
| 745 | WARN_ON_ONCE(!list_empty(&node->private_list)); | 744 | WARN_ON_ONCE(!list_empty(&node->private_list)); |
| @@ -752,7 +751,7 @@ static inline bool radix_tree_shrink(struct radix_tree_root *root, | |||
| 752 | 751 | ||
| 753 | static bool delete_node(struct radix_tree_root *root, | 752 | static bool delete_node(struct radix_tree_root *root, |
| 754 | struct radix_tree_node *node, | 753 | struct radix_tree_node *node, |
| 755 | radix_tree_update_node_t update_node, void *private) | 754 | radix_tree_update_node_t update_node) |
| 756 | { | 755 | { |
| 757 | bool deleted = false; | 756 | bool deleted = false; |
| 758 | 757 | ||
| @@ -762,8 +761,8 @@ static bool delete_node(struct radix_tree_root *root, | |||
| 762 | if (node->count) { | 761 | if (node->count) { |
| 763 | if (node_to_entry(node) == | 762 | if (node_to_entry(node) == |
| 764 | rcu_dereference_raw(root->rnode)) | 763 | rcu_dereference_raw(root->rnode)) |
| 765 | deleted |= radix_tree_shrink(root, update_node, | 764 | deleted |= radix_tree_shrink(root, |
| 766 | private); | 765 | update_node); |
| 767 | return deleted; | 766 | return deleted; |
| 768 | } | 767 | } |
| 769 | 768 | ||
| @@ -1173,7 +1172,6 @@ static int calculate_count(struct radix_tree_root *root, | |||
| 1173 | * @slot: pointer to slot in @node | 1172 | * @slot: pointer to slot in @node |
| 1174 | * @item: new item to store in the slot. | 1173 | * @item: new item to store in the slot. |
| 1175 | * @update_node: callback for changing leaf nodes | 1174 | * @update_node: callback for changing leaf nodes |
| 1176 | * @private: private data to pass to @update_node | ||
| 1177 | * | 1175 | * |
| 1178 | * For use with __radix_tree_lookup(). Caller must hold tree write locked | 1176 | * For use with __radix_tree_lookup(). Caller must hold tree write locked |
| 1179 | * across slot lookup and replacement. | 1177 | * across slot lookup and replacement. |
| @@ -1181,7 +1179,7 @@ static int calculate_count(struct radix_tree_root *root, | |||
| 1181 | void __radix_tree_replace(struct radix_tree_root *root, | 1179 | void __radix_tree_replace(struct radix_tree_root *root, |
| 1182 | struct radix_tree_node *node, | 1180 | struct radix_tree_node *node, |
| 1183 | void __rcu **slot, void *item, | 1181 | void __rcu **slot, void *item, |
| 1184 | radix_tree_update_node_t update_node, void *private) | 1182 | radix_tree_update_node_t update_node) |
| 1185 | { | 1183 | { |
| 1186 | void *old = rcu_dereference_raw(*slot); | 1184 | void *old = rcu_dereference_raw(*slot); |
| 1187 | int exceptional = !!radix_tree_exceptional_entry(item) - | 1185 | int exceptional = !!radix_tree_exceptional_entry(item) - |
| @@ -1201,9 +1199,9 @@ void __radix_tree_replace(struct radix_tree_root *root, | |||
| 1201 | return; | 1199 | return; |
| 1202 | 1200 | ||
| 1203 | if (update_node) | 1201 | if (update_node) |
| 1204 | update_node(node, private); | 1202 | update_node(node); |
| 1205 | 1203 | ||
| 1206 | delete_node(root, node, update_node, private); | 1204 | delete_node(root, node, update_node); |
| 1207 | } | 1205 | } |
| 1208 | 1206 | ||
| 1209 | /** | 1207 | /** |
| @@ -1225,7 +1223,7 @@ void __radix_tree_replace(struct radix_tree_root *root, | |||
| 1225 | void radix_tree_replace_slot(struct radix_tree_root *root, | 1223 | void radix_tree_replace_slot(struct radix_tree_root *root, |
| 1226 | void __rcu **slot, void *item) | 1224 | void __rcu **slot, void *item) |
| 1227 | { | 1225 | { |
| 1228 | __radix_tree_replace(root, NULL, slot, item, NULL, NULL); | 1226 | __radix_tree_replace(root, NULL, slot, item, NULL); |
| 1229 | } | 1227 | } |
| 1230 | EXPORT_SYMBOL(radix_tree_replace_slot); | 1228 | EXPORT_SYMBOL(radix_tree_replace_slot); |
| 1231 | 1229 | ||
| @@ -1242,7 +1240,7 @@ void radix_tree_iter_replace(struct radix_tree_root *root, | |||
| 1242 | const struct radix_tree_iter *iter, | 1240 | const struct radix_tree_iter *iter, |
| 1243 | void __rcu **slot, void *item) | 1241 | void __rcu **slot, void *item) |
| 1244 | { | 1242 | { |
| 1245 | __radix_tree_replace(root, iter->node, slot, item, NULL, NULL); | 1243 | __radix_tree_replace(root, iter->node, slot, item, NULL); |
| 1246 | } | 1244 | } |
| 1247 | 1245 | ||
| 1248 | #ifdef CONFIG_RADIX_TREE_MULTIORDER | 1246 | #ifdef CONFIG_RADIX_TREE_MULTIORDER |
| @@ -1972,7 +1970,6 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot); | |||
| 1972 | * @root: radix tree root | 1970 | * @root: radix tree root |
| 1973 | * @node: node containing @index | 1971 | * @node: node containing @index |
| 1974 | * @update_node: callback for changing leaf nodes | 1972 | * @update_node: callback for changing leaf nodes |
| 1975 | * @private: private data to pass to @update_node | ||
| 1976 | * | 1973 | * |
| 1977 | * After clearing the slot at @index in @node from radix tree | 1974 | * After clearing the slot at @index in @node from radix tree |
| 1978 | * rooted at @root, call this function to attempt freeing the | 1975 | * rooted at @root, call this function to attempt freeing the |
| @@ -1980,10 +1977,9 @@ EXPORT_SYMBOL(radix_tree_gang_lookup_tag_slot); | |||
| 1980 | */ | 1977 | */ |
| 1981 | void __radix_tree_delete_node(struct radix_tree_root *root, | 1978 | void __radix_tree_delete_node(struct radix_tree_root *root, |
| 1982 | struct radix_tree_node *node, | 1979 | struct radix_tree_node *node, |
| 1983 | radix_tree_update_node_t update_node, | 1980 | radix_tree_update_node_t update_node) |
| 1984 | void *private) | ||
| 1985 | { | 1981 | { |
| 1986 | delete_node(root, node, update_node, private); | 1982 | delete_node(root, node, update_node); |
| 1987 | } | 1983 | } |
| 1988 | 1984 | ||
| 1989 | static bool __radix_tree_delete(struct radix_tree_root *root, | 1985 | static bool __radix_tree_delete(struct radix_tree_root *root, |
| @@ -2001,7 +1997,7 @@ static bool __radix_tree_delete(struct radix_tree_root *root, | |||
| 2001 | node_tag_clear(root, node, tag, offset); | 1997 | node_tag_clear(root, node, tag, offset); |
| 2002 | 1998 | ||
| 2003 | replace_slot(slot, NULL, node, -1, exceptional); | 1999 | replace_slot(slot, NULL, node, -1, exceptional); |
| 2004 | return node && delete_node(root, node, NULL, NULL); | 2000 | return node && delete_node(root, node, NULL); |
| 2005 | } | 2001 | } |
| 2006 | 2002 | ||
| 2007 | /** | 2003 | /** |
diff --git a/mm/filemap.c b/mm/filemap.c index a470dd8cd05b..155370fc87f2 100644 --- a/mm/filemap.c +++ b/mm/filemap.c | |||
| @@ -35,6 +35,7 @@ | |||
| 35 | #include <linux/hugetlb.h> | 35 | #include <linux/hugetlb.h> |
| 36 | #include <linux/memcontrol.h> | 36 | #include <linux/memcontrol.h> |
| 37 | #include <linux/cleancache.h> | 37 | #include <linux/cleancache.h> |
| 38 | #include <linux/shmem_fs.h> | ||
| 38 | #include <linux/rmap.h> | 39 | #include <linux/rmap.h> |
| 39 | #include "internal.h" | 40 | #include "internal.h" |
| 40 | 41 | ||
| @@ -134,7 +135,7 @@ static int page_cache_tree_insert(struct address_space *mapping, | |||
| 134 | *shadowp = p; | 135 | *shadowp = p; |
| 135 | } | 136 | } |
| 136 | __radix_tree_replace(&mapping->page_tree, node, slot, page, | 137 | __radix_tree_replace(&mapping->page_tree, node, slot, page, |
| 137 | workingset_update_node, mapping); | 138 | workingset_lookup_update(mapping)); |
| 138 | mapping->nrpages++; | 139 | mapping->nrpages++; |
| 139 | return 0; | 140 | return 0; |
| 140 | } | 141 | } |
| @@ -162,7 +163,7 @@ static void page_cache_tree_delete(struct address_space *mapping, | |||
| 162 | 163 | ||
| 163 | radix_tree_clear_tags(&mapping->page_tree, node, slot); | 164 | radix_tree_clear_tags(&mapping->page_tree, node, slot); |
| 164 | __radix_tree_replace(&mapping->page_tree, node, slot, shadow, | 165 | __radix_tree_replace(&mapping->page_tree, node, slot, shadow, |
| 165 | workingset_update_node, mapping); | 166 | workingset_lookup_update(mapping)); |
| 166 | } | 167 | } |
| 167 | 168 | ||
| 168 | page->mapping = NULL; | 169 | page->mapping = NULL; |
| @@ -359,7 +360,7 @@ page_cache_tree_delete_batch(struct address_space *mapping, | |||
| 359 | } | 360 | } |
| 360 | radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); | 361 | radix_tree_clear_tags(&mapping->page_tree, iter.node, slot); |
| 361 | __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, | 362 | __radix_tree_replace(&mapping->page_tree, iter.node, slot, NULL, |
| 362 | workingset_update_node, mapping); | 363 | workingset_lookup_update(mapping)); |
| 363 | total_pages++; | 364 | total_pages++; |
| 364 | } | 365 | } |
| 365 | mapping->nrpages -= total_pages; | 366 | mapping->nrpages -= total_pages; |
diff --git a/mm/shmem.c b/mm/shmem.c index 07a1d22807be..a72f68aee6a4 100644 --- a/mm/shmem.c +++ b/mm/shmem.c | |||
| @@ -338,7 +338,7 @@ static int shmem_radix_tree_replace(struct address_space *mapping, | |||
| 338 | if (item != expected) | 338 | if (item != expected) |
| 339 | return -ENOENT; | 339 | return -ENOENT; |
| 340 | __radix_tree_replace(&mapping->page_tree, node, pslot, | 340 | __radix_tree_replace(&mapping->page_tree, node, pslot, |
| 341 | replacement, NULL, NULL); | 341 | replacement, NULL); |
| 342 | return 0; | 342 | return 0; |
| 343 | } | 343 | } |
| 344 | 344 | ||
diff --git a/mm/truncate.c b/mm/truncate.c index 4a39a3150ee2..02a0c0466c78 100644 --- a/mm/truncate.c +++ b/mm/truncate.c | |||
| @@ -42,7 +42,7 @@ static void clear_shadow_entry(struct address_space *mapping, pgoff_t index, | |||
| 42 | if (*slot != entry) | 42 | if (*slot != entry) |
| 43 | goto unlock; | 43 | goto unlock; |
| 44 | __radix_tree_replace(&mapping->page_tree, node, slot, NULL, | 44 | __radix_tree_replace(&mapping->page_tree, node, slot, NULL, |
| 45 | workingset_update_node, mapping); | 45 | workingset_update_node); |
| 46 | mapping->nrexceptional--; | 46 | mapping->nrexceptional--; |
| 47 | unlock: | 47 | unlock: |
| 48 | spin_unlock_irq(&mapping->tree_lock); | 48 | spin_unlock_irq(&mapping->tree_lock); |
diff --git a/mm/workingset.c b/mm/workingset.c index b997c9de28f6..b7d616a3bbbe 100644 --- a/mm/workingset.c +++ b/mm/workingset.c | |||
| @@ -340,14 +340,8 @@ out: | |||
| 340 | 340 | ||
| 341 | static struct list_lru shadow_nodes; | 341 | static struct list_lru shadow_nodes; |
| 342 | 342 | ||
| 343 | void workingset_update_node(struct radix_tree_node *node, void *private) | 343 | void workingset_update_node(struct radix_tree_node *node) |
| 344 | { | 344 | { |
| 345 | struct address_space *mapping = private; | ||
| 346 | |||
| 347 | /* Only regular page cache has shadow entries */ | ||
| 348 | if (dax_mapping(mapping) || shmem_mapping(mapping)) | ||
| 349 | return; | ||
| 350 | |||
| 351 | /* | 345 | /* |
| 352 | * Track non-empty nodes that contain only shadow entries; | 346 | * Track non-empty nodes that contain only shadow entries; |
| 353 | * unlink those that contain pages or are being freed. | 347 | * unlink those that contain pages or are being freed. |
| @@ -475,7 +469,7 @@ static enum lru_status shadow_lru_isolate(struct list_head *item, | |||
| 475 | goto out_invalid; | 469 | goto out_invalid; |
| 476 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); | 470 | inc_lruvec_page_state(virt_to_page(node), WORKINGSET_NODERECLAIM); |
| 477 | __radix_tree_delete_node(&mapping->page_tree, node, | 471 | __radix_tree_delete_node(&mapping->page_tree, node, |
| 478 | workingset_update_node, mapping); | 472 | workingset_lookup_update(mapping)); |
| 479 | 473 | ||
| 480 | out_invalid: | 474 | out_invalid: |
| 481 | spin_unlock(&mapping->tree_lock); | 475 | spin_unlock(&mapping->tree_lock); |
diff --git a/tools/testing/radix-tree/multiorder.c b/tools/testing/radix-tree/multiorder.c index 06c71178d07d..59245b3d587c 100644 --- a/tools/testing/radix-tree/multiorder.c +++ b/tools/testing/radix-tree/multiorder.c | |||
| @@ -618,7 +618,7 @@ static void multiorder_account(void) | |||
| 618 | __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); | 618 | __radix_tree_insert(&tree, 1 << 5, 5, (void *)0x12); |
| 619 | __radix_tree_lookup(&tree, 1 << 5, &node, &slot); | 619 | __radix_tree_lookup(&tree, 1 << 5, &node, &slot); |
| 620 | assert(node->count == node->exceptional * 2); | 620 | assert(node->count == node->exceptional * 2); |
| 621 | __radix_tree_replace(&tree, node, slot, NULL, NULL, NULL); | 621 | __radix_tree_replace(&tree, node, slot, NULL, NULL); |
| 622 | assert(node->exceptional == 0); | 622 | assert(node->exceptional == 0); |
| 623 | 623 | ||
| 624 | item_kill_tree(&tree); | 624 | item_kill_tree(&tree); |
