aboutsummaryrefslogtreecommitdiffstats
path: root/mm/filemap.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/filemap.c')
-rw-r--r--mm/filemap.c68
1 files changed, 10 insertions, 58 deletions
diff --git a/mm/filemap.c b/mm/filemap.c
index 50b52fe51937..5b4dd03130da 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -132,44 +132,29 @@ static int page_cache_tree_insert(struct address_space *mapping,
132 if (!dax_mapping(mapping)) { 132 if (!dax_mapping(mapping)) {
133 if (shadowp) 133 if (shadowp)
134 *shadowp = p; 134 *shadowp = p;
135 if (node)
136 workingset_node_shadows_dec(node);
137 } else { 135 } else {
138 /* DAX can replace empty locked entry with a hole */ 136 /* DAX can replace empty locked entry with a hole */
139 WARN_ON_ONCE(p != 137 WARN_ON_ONCE(p !=
140 (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY | 138 (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
141 RADIX_DAX_ENTRY_LOCK)); 139 RADIX_DAX_ENTRY_LOCK));
142 /* DAX accounts exceptional entries as normal pages */
143 if (node)
144 workingset_node_pages_dec(node);
145 /* Wakeup waiters for exceptional entry lock */ 140 /* Wakeup waiters for exceptional entry lock */
146 dax_wake_mapping_entry_waiter(mapping, page->index, 141 dax_wake_mapping_entry_waiter(mapping, page->index,
147 false); 142 false);
148 } 143 }
149 } 144 }
150 radix_tree_replace_slot(slot, page); 145 __radix_tree_replace(&mapping->page_tree, node, slot, page,
146 workingset_update_node, mapping);
151 mapping->nrpages++; 147 mapping->nrpages++;
152 if (node) {
153 workingset_node_pages_inc(node);
154 /*
155 * Don't track node that contains actual pages.
156 *
157 * Avoid acquiring the list_lru lock if already
158 * untracked. The list_empty() test is safe as
159 * node->private_list is protected by
160 * mapping->tree_lock.
161 */
162 if (!list_empty(&node->private_list))
163 list_lru_del(&workingset_shadow_nodes,
164 &node->private_list);
165 }
166 return 0; 148 return 0;
167} 149}
168 150
169static void page_cache_tree_delete(struct address_space *mapping, 151static void page_cache_tree_delete(struct address_space *mapping,
170 struct page *page, void *shadow) 152 struct page *page, void *shadow)
171{ 153{
172 int i, nr = PageHuge(page) ? 1 : hpage_nr_pages(page); 154 int i, nr;
155
156 /* hugetlb pages are represented by one entry in the radix tree */
157 nr = PageHuge(page) ? 1 : hpage_nr_pages(page);
173 158
174 VM_BUG_ON_PAGE(!PageLocked(page), page); 159 VM_BUG_ON_PAGE(!PageLocked(page), page);
175 VM_BUG_ON_PAGE(PageTail(page), page); 160 VM_BUG_ON_PAGE(PageTail(page), page);
@@ -182,44 +167,11 @@ static void page_cache_tree_delete(struct address_space *mapping,
182 __radix_tree_lookup(&mapping->page_tree, page->index + i, 167 __radix_tree_lookup(&mapping->page_tree, page->index + i,
183 &node, &slot); 168 &node, &slot);
184 169
185 radix_tree_clear_tags(&mapping->page_tree, node, slot); 170 VM_BUG_ON_PAGE(!node && nr != 1, page);
186
187 if (!node) {
188 VM_BUG_ON_PAGE(nr != 1, page);
189 /*
190 * We need a node to properly account shadow
191 * entries. Don't plant any without. XXX
192 */
193 shadow = NULL;
194 }
195
196 radix_tree_replace_slot(slot, shadow);
197 171
198 if (!node) 172 radix_tree_clear_tags(&mapping->page_tree, node, slot);
199 break; 173 __radix_tree_replace(&mapping->page_tree, node, slot, shadow,
200 174 workingset_update_node, mapping);
201 workingset_node_pages_dec(node);
202 if (shadow)
203 workingset_node_shadows_inc(node);
204 else
205 if (__radix_tree_delete_node(&mapping->page_tree, node))
206 continue;
207
208 /*
209 * Track node that only contains shadow entries. DAX mappings
210 * contain no shadow entries and may contain other exceptional
211 * entries so skip those.
212 *
213 * Avoid acquiring the list_lru lock if already tracked.
214 * The list_empty() test is safe as node->private_list is
215 * protected by mapping->tree_lock.
216 */
217 if (!dax_mapping(mapping) && !workingset_node_pages(node) &&
218 list_empty(&node->private_list)) {
219 node->private_data = mapping;
220 list_lru_add(&workingset_shadow_nodes,
221 &node->private_list);
222 }
223 } 175 }
224 176
225 if (shadow) { 177 if (shadow) {