aboutsummaryrefslogtreecommitdiffstats
path: root/fs/btrfs/extent_io.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/btrfs/extent_io.c')
-rw-r--r--fs/btrfs/extent_io.c439
1 files changed, 249 insertions, 190 deletions
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 96577e8bf9fd..2e993cf1766e 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -2,7 +2,6 @@
2#include <linux/slab.h> 2#include <linux/slab.h>
3#include <linux/bio.h> 3#include <linux/bio.h>
4#include <linux/mm.h> 4#include <linux/mm.h>
5#include <linux/gfp.h>
6#include <linux/pagemap.h> 5#include <linux/pagemap.h>
7#include <linux/page-flags.h> 6#include <linux/page-flags.h>
8#include <linux/module.h> 7#include <linux/module.h>
@@ -104,8 +103,8 @@ void extent_io_exit(void)
104void extent_io_tree_init(struct extent_io_tree *tree, 103void extent_io_tree_init(struct extent_io_tree *tree,
105 struct address_space *mapping, gfp_t mask) 104 struct address_space *mapping, gfp_t mask)
106{ 105{
107 tree->state.rb_node = NULL; 106 tree->state = RB_ROOT;
108 tree->buffer.rb_node = NULL; 107 INIT_RADIX_TREE(&tree->buffer, GFP_ATOMIC);
109 tree->ops = NULL; 108 tree->ops = NULL;
110 tree->dirty_bytes = 0; 109 tree->dirty_bytes = 0;
111 spin_lock_init(&tree->lock); 110 spin_lock_init(&tree->lock);
@@ -136,7 +135,7 @@ static struct extent_state *alloc_extent_state(gfp_t mask)
136 return state; 135 return state;
137} 136}
138 137
139static void free_extent_state(struct extent_state *state) 138void free_extent_state(struct extent_state *state)
140{ 139{
141 if (!state) 140 if (!state)
142 return; 141 return;
@@ -236,50 +235,6 @@ static inline struct rb_node *tree_search(struct extent_io_tree *tree,
236 return ret; 235 return ret;
237} 236}
238 237
239static struct extent_buffer *buffer_tree_insert(struct extent_io_tree *tree,
240 u64 offset, struct rb_node *node)
241{
242 struct rb_root *root = &tree->buffer;
243 struct rb_node **p = &root->rb_node;
244 struct rb_node *parent = NULL;
245 struct extent_buffer *eb;
246
247 while (*p) {
248 parent = *p;
249 eb = rb_entry(parent, struct extent_buffer, rb_node);
250
251 if (offset < eb->start)
252 p = &(*p)->rb_left;
253 else if (offset > eb->start)
254 p = &(*p)->rb_right;
255 else
256 return eb;
257 }
258
259 rb_link_node(node, parent, p);
260 rb_insert_color(node, root);
261 return NULL;
262}
263
264static struct extent_buffer *buffer_search(struct extent_io_tree *tree,
265 u64 offset)
266{
267 struct rb_root *root = &tree->buffer;
268 struct rb_node *n = root->rb_node;
269 struct extent_buffer *eb;
270
271 while (n) {
272 eb = rb_entry(n, struct extent_buffer, rb_node);
273 if (offset < eb->start)
274 n = n->rb_left;
275 else if (offset > eb->start)
276 n = n->rb_right;
277 else
278 return eb;
279 }
280 return NULL;
281}
282
283static void merge_cb(struct extent_io_tree *tree, struct extent_state *new, 238static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
284 struct extent_state *other) 239 struct extent_state *other)
285{ 240{
@@ -336,21 +291,18 @@ static int merge_state(struct extent_io_tree *tree,
336} 291}
337 292
338static int set_state_cb(struct extent_io_tree *tree, 293static int set_state_cb(struct extent_io_tree *tree,
339 struct extent_state *state, 294 struct extent_state *state, int *bits)
340 unsigned long bits)
341{ 295{
342 if (tree->ops && tree->ops->set_bit_hook) { 296 if (tree->ops && tree->ops->set_bit_hook) {
343 return tree->ops->set_bit_hook(tree->mapping->host, 297 return tree->ops->set_bit_hook(tree->mapping->host,
344 state->start, state->end, 298 state, bits);
345 state->state, bits);
346 } 299 }
347 300
348 return 0; 301 return 0;
349} 302}
350 303
351static void clear_state_cb(struct extent_io_tree *tree, 304static void clear_state_cb(struct extent_io_tree *tree,
352 struct extent_state *state, 305 struct extent_state *state, int *bits)
353 unsigned long bits)
354{ 306{
355 if (tree->ops && tree->ops->clear_bit_hook) 307 if (tree->ops && tree->ops->clear_bit_hook)
356 tree->ops->clear_bit_hook(tree->mapping->host, state, bits); 308 tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
@@ -368,9 +320,10 @@ static void clear_state_cb(struct extent_io_tree *tree,
368 */ 320 */
369static int insert_state(struct extent_io_tree *tree, 321static int insert_state(struct extent_io_tree *tree,
370 struct extent_state *state, u64 start, u64 end, 322 struct extent_state *state, u64 start, u64 end,
371 int bits) 323 int *bits)
372{ 324{
373 struct rb_node *node; 325 struct rb_node *node;
326 int bits_to_set = *bits & ~EXTENT_CTLBITS;
374 int ret; 327 int ret;
375 328
376 if (end < start) { 329 if (end < start) {
@@ -385,9 +338,9 @@ static int insert_state(struct extent_io_tree *tree,
385 if (ret) 338 if (ret)
386 return ret; 339 return ret;
387 340
388 if (bits & EXTENT_DIRTY) 341 if (bits_to_set & EXTENT_DIRTY)
389 tree->dirty_bytes += end - start + 1; 342 tree->dirty_bytes += end - start + 1;
390 state->state |= bits; 343 state->state |= bits_to_set;
391 node = tree_insert(&tree->state, end, &state->rb_node); 344 node = tree_insert(&tree->state, end, &state->rb_node);
392 if (node) { 345 if (node) {
393 struct extent_state *found; 346 struct extent_state *found;
@@ -457,13 +410,13 @@ static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
457 * struct is freed and removed from the tree 410 * struct is freed and removed from the tree
458 */ 411 */
459static int clear_state_bit(struct extent_io_tree *tree, 412static int clear_state_bit(struct extent_io_tree *tree,
460 struct extent_state *state, int bits, int wake, 413 struct extent_state *state,
461 int delete) 414 int *bits, int wake)
462{ 415{
463 int bits_to_clear = bits & ~EXTENT_DO_ACCOUNTING; 416 int bits_to_clear = *bits & ~EXTENT_CTLBITS;
464 int ret = state->state & bits_to_clear; 417 int ret = state->state & bits_to_clear;
465 418
466 if ((bits & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) { 419 if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
467 u64 range = state->end - state->start + 1; 420 u64 range = state->end - state->start + 1;
468 WARN_ON(range > tree->dirty_bytes); 421 WARN_ON(range > tree->dirty_bytes);
469 tree->dirty_bytes -= range; 422 tree->dirty_bytes -= range;
@@ -472,9 +425,8 @@ static int clear_state_bit(struct extent_io_tree *tree,
472 state->state &= ~bits_to_clear; 425 state->state &= ~bits_to_clear;
473 if (wake) 426 if (wake)
474 wake_up(&state->wq); 427 wake_up(&state->wq);
475 if (delete || state->state == 0) { 428 if (state->state == 0) {
476 if (state->tree) { 429 if (state->tree) {
477 clear_state_cb(tree, state, state->state);
478 rb_erase(&state->rb_node, &tree->state); 430 rb_erase(&state->rb_node, &tree->state);
479 state->tree = NULL; 431 state->tree = NULL;
480 free_extent_state(state); 432 free_extent_state(state);
@@ -513,7 +465,14 @@ int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
513 u64 last_end; 465 u64 last_end;
514 int err; 466 int err;
515 int set = 0; 467 int set = 0;
468 int clear = 0;
469
470 if (delete)
471 bits |= ~EXTENT_CTLBITS;
472 bits |= EXTENT_FIRST_DELALLOC;
516 473
474 if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
475 clear = 1;
517again: 476again:
518 if (!prealloc && (mask & __GFP_WAIT)) { 477 if (!prealloc && (mask & __GFP_WAIT)) {
519 prealloc = alloc_extent_state(mask); 478 prealloc = alloc_extent_state(mask);
@@ -524,14 +483,20 @@ again:
524 spin_lock(&tree->lock); 483 spin_lock(&tree->lock);
525 if (cached_state) { 484 if (cached_state) {
526 cached = *cached_state; 485 cached = *cached_state;
527 *cached_state = NULL; 486
528 cached_state = NULL; 487 if (clear) {
488 *cached_state = NULL;
489 cached_state = NULL;
490 }
491
529 if (cached && cached->tree && cached->start == start) { 492 if (cached && cached->tree && cached->start == start) {
530 atomic_dec(&cached->refs); 493 if (clear)
494 atomic_dec(&cached->refs);
531 state = cached; 495 state = cached;
532 goto hit_next; 496 goto hit_next;
533 } 497 }
534 free_extent_state(cached); 498 if (clear)
499 free_extent_state(cached);
535 } 500 }
536 /* 501 /*
537 * this search will find the extents that end after 502 * this search will find the extents that end after
@@ -572,8 +537,7 @@ hit_next:
572 if (err) 537 if (err)
573 goto out; 538 goto out;
574 if (state->end <= end) { 539 if (state->end <= end) {
575 set |= clear_state_bit(tree, state, bits, wake, 540 set |= clear_state_bit(tree, state, &bits, wake);
576 delete);
577 if (last_end == (u64)-1) 541 if (last_end == (u64)-1)
578 goto out; 542 goto out;
579 start = last_end + 1; 543 start = last_end + 1;
@@ -594,7 +558,7 @@ hit_next:
594 if (wake) 558 if (wake)
595 wake_up(&state->wq); 559 wake_up(&state->wq);
596 560
597 set |= clear_state_bit(tree, prealloc, bits, wake, delete); 561 set |= clear_state_bit(tree, prealloc, &bits, wake);
598 562
599 prealloc = NULL; 563 prealloc = NULL;
600 goto out; 564 goto out;
@@ -605,7 +569,7 @@ hit_next:
605 else 569 else
606 next_node = NULL; 570 next_node = NULL;
607 571
608 set |= clear_state_bit(tree, state, bits, wake, delete); 572 set |= clear_state_bit(tree, state, &bits, wake);
609 if (last_end == (u64)-1) 573 if (last_end == (u64)-1)
610 goto out; 574 goto out;
611 start = last_end + 1; 575 start = last_end + 1;
@@ -698,19 +662,19 @@ out:
698 662
699static int set_state_bits(struct extent_io_tree *tree, 663static int set_state_bits(struct extent_io_tree *tree,
700 struct extent_state *state, 664 struct extent_state *state,
701 int bits) 665 int *bits)
702{ 666{
703 int ret; 667 int ret;
668 int bits_to_set = *bits & ~EXTENT_CTLBITS;
704 669
705 ret = set_state_cb(tree, state, bits); 670 ret = set_state_cb(tree, state, bits);
706 if (ret) 671 if (ret)
707 return ret; 672 return ret;
708 673 if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
709 if ((bits & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
710 u64 range = state->end - state->start + 1; 674 u64 range = state->end - state->start + 1;
711 tree->dirty_bytes += range; 675 tree->dirty_bytes += range;
712 } 676 }
713 state->state |= bits; 677 state->state |= bits_to_set;
714 678
715 return 0; 679 return 0;
716} 680}
@@ -737,10 +701,9 @@ static void cache_state(struct extent_state *state,
737 * [start, end] is inclusive This takes the tree lock. 701 * [start, end] is inclusive This takes the tree lock.
738 */ 702 */
739 703
740static int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 704int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
741 int bits, int exclusive_bits, u64 *failed_start, 705 int bits, int exclusive_bits, u64 *failed_start,
742 struct extent_state **cached_state, 706 struct extent_state **cached_state, gfp_t mask)
743 gfp_t mask)
744{ 707{
745 struct extent_state *state; 708 struct extent_state *state;
746 struct extent_state *prealloc = NULL; 709 struct extent_state *prealloc = NULL;
@@ -749,6 +712,7 @@ static int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
749 u64 last_start; 712 u64 last_start;
750 u64 last_end; 713 u64 last_end;
751 714
715 bits |= EXTENT_FIRST_DELALLOC;
752again: 716again:
753 if (!prealloc && (mask & __GFP_WAIT)) { 717 if (!prealloc && (mask & __GFP_WAIT)) {
754 prealloc = alloc_extent_state(mask); 718 prealloc = alloc_extent_state(mask);
@@ -770,7 +734,7 @@ again:
770 */ 734 */
771 node = tree_search(tree, start); 735 node = tree_search(tree, start);
772 if (!node) { 736 if (!node) {
773 err = insert_state(tree, prealloc, start, end, bits); 737 err = insert_state(tree, prealloc, start, end, &bits);
774 prealloc = NULL; 738 prealloc = NULL;
775 BUG_ON(err == -EEXIST); 739 BUG_ON(err == -EEXIST);
776 goto out; 740 goto out;
@@ -794,7 +758,7 @@ hit_next:
794 goto out; 758 goto out;
795 } 759 }
796 760
797 err = set_state_bits(tree, state, bits); 761 err = set_state_bits(tree, state, &bits);
798 if (err) 762 if (err)
799 goto out; 763 goto out;
800 764
@@ -844,7 +808,7 @@ hit_next:
844 if (err) 808 if (err)
845 goto out; 809 goto out;
846 if (state->end <= end) { 810 if (state->end <= end) {
847 err = set_state_bits(tree, state, bits); 811 err = set_state_bits(tree, state, &bits);
848 if (err) 812 if (err)
849 goto out; 813 goto out;
850 cache_state(state, cached_state); 814 cache_state(state, cached_state);
@@ -869,7 +833,7 @@ hit_next:
869 else 833 else
870 this_end = last_start - 1; 834 this_end = last_start - 1;
871 err = insert_state(tree, prealloc, start, this_end, 835 err = insert_state(tree, prealloc, start, this_end,
872 bits); 836 &bits);
873 BUG_ON(err == -EEXIST); 837 BUG_ON(err == -EEXIST);
874 if (err) { 838 if (err) {
875 prealloc = NULL; 839 prealloc = NULL;
@@ -895,7 +859,7 @@ hit_next:
895 err = split_state(tree, state, prealloc, end + 1); 859 err = split_state(tree, state, prealloc, end + 1);
896 BUG_ON(err == -EEXIST); 860 BUG_ON(err == -EEXIST);
897 861
898 err = set_state_bits(tree, prealloc, bits); 862 err = set_state_bits(tree, prealloc, &bits);
899 if (err) { 863 if (err) {
900 prealloc = NULL; 864 prealloc = NULL;
901 goto out; 865 goto out;
@@ -946,11 +910,11 @@ int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
946} 910}
947 911
948int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end, 912int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
949 gfp_t mask) 913 struct extent_state **cached_state, gfp_t mask)
950{ 914{
951 return set_extent_bit(tree, start, end, 915 return set_extent_bit(tree, start, end,
952 EXTENT_DELALLOC | EXTENT_DIRTY | EXTENT_UPTODATE, 916 EXTENT_DELALLOC | EXTENT_DIRTY | EXTENT_UPTODATE,
953 0, NULL, NULL, mask); 917 0, NULL, cached_state, mask);
954} 918}
955 919
956int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, 920int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
@@ -958,8 +922,7 @@ int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
958{ 922{
959 return clear_extent_bit(tree, start, end, 923 return clear_extent_bit(tree, start, end,
960 EXTENT_DIRTY | EXTENT_DELALLOC | 924 EXTENT_DIRTY | EXTENT_DELALLOC |
961 EXTENT_DO_ACCOUNTING, 0, 0, 925 EXTENT_DO_ACCOUNTING, 0, 0, NULL, mask);
962 NULL, mask);
963} 926}
964 927
965int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end, 928int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
@@ -984,10 +947,11 @@ int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
984} 947}
985 948
986static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, 949static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start,
987 u64 end, gfp_t mask) 950 u64 end, struct extent_state **cached_state,
951 gfp_t mask)
988{ 952{
989 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, 953 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
990 NULL, mask); 954 cached_state, mask);
991} 955}
992 956
993int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end) 957int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
@@ -1171,7 +1135,8 @@ out:
1171 * 1 is returned if we find something, 0 if nothing was in the tree 1135 * 1 is returned if we find something, 0 if nothing was in the tree
1172 */ 1136 */
1173static noinline u64 find_delalloc_range(struct extent_io_tree *tree, 1137static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1174 u64 *start, u64 *end, u64 max_bytes) 1138 u64 *start, u64 *end, u64 max_bytes,
1139 struct extent_state **cached_state)
1175{ 1140{
1176 struct rb_node *node; 1141 struct rb_node *node;
1177 struct extent_state *state; 1142 struct extent_state *state;
@@ -1203,8 +1168,11 @@ static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1203 *end = state->end; 1168 *end = state->end;
1204 goto out; 1169 goto out;
1205 } 1170 }
1206 if (!found) 1171 if (!found) {
1207 *start = state->start; 1172 *start = state->start;
1173 *cached_state = state;
1174 atomic_inc(&state->refs);
1175 }
1208 found++; 1176 found++;
1209 *end = state->end; 1177 *end = state->end;
1210 cur_start = state->end + 1; 1178 cur_start = state->end + 1;
@@ -1336,10 +1304,11 @@ again:
1336 delalloc_start = *start; 1304 delalloc_start = *start;
1337 delalloc_end = 0; 1305 delalloc_end = 0;
1338 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end, 1306 found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1339 max_bytes); 1307 max_bytes, &cached_state);
1340 if (!found || delalloc_end <= *start) { 1308 if (!found || delalloc_end <= *start) {
1341 *start = delalloc_start; 1309 *start = delalloc_start;
1342 *end = delalloc_end; 1310 *end = delalloc_end;
1311 free_extent_state(cached_state);
1343 return found; 1312 return found;
1344 } 1313 }
1345 1314
@@ -1421,9 +1390,6 @@ int extent_clear_unlock_delalloc(struct inode *inode,
1421 if (op & EXTENT_CLEAR_DELALLOC) 1390 if (op & EXTENT_CLEAR_DELALLOC)
1422 clear_bits |= EXTENT_DELALLOC; 1391 clear_bits |= EXTENT_DELALLOC;
1423 1392
1424 if (op & EXTENT_CLEAR_ACCOUNTING)
1425 clear_bits |= EXTENT_DO_ACCOUNTING;
1426
1427 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS); 1393 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1428 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY | 1394 if (!(op & (EXTENT_CLEAR_UNLOCK_PAGE | EXTENT_CLEAR_DIRTY |
1429 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK | 1395 EXTENT_SET_WRITEBACK | EXTENT_END_WRITEBACK |
@@ -1722,7 +1688,7 @@ static void end_bio_extent_writepage(struct bio *bio, int err)
1722 } 1688 }
1723 1689
1724 if (!uptodate) { 1690 if (!uptodate) {
1725 clear_extent_uptodate(tree, start, end, GFP_NOFS); 1691 clear_extent_uptodate(tree, start, end, NULL, GFP_NOFS);
1726 ClearPageUptodate(page); 1692 ClearPageUptodate(page);
1727 SetPageError(page); 1693 SetPageError(page);
1728 } 1694 }
@@ -1750,7 +1716,8 @@ static void end_bio_extent_writepage(struct bio *bio, int err)
1750static void end_bio_extent_readpage(struct bio *bio, int err) 1716static void end_bio_extent_readpage(struct bio *bio, int err)
1751{ 1717{
1752 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags); 1718 int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1753 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1; 1719 struct bio_vec *bvec_end = bio->bi_io_vec + bio->bi_vcnt - 1;
1720 struct bio_vec *bvec = bio->bi_io_vec;
1754 struct extent_io_tree *tree; 1721 struct extent_io_tree *tree;
1755 u64 start; 1722 u64 start;
1756 u64 end; 1723 u64 end;
@@ -1773,7 +1740,7 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
1773 else 1740 else
1774 whole_page = 0; 1741 whole_page = 0;
1775 1742
1776 if (--bvec >= bio->bi_io_vec) 1743 if (++bvec <= bvec_end)
1777 prefetchw(&bvec->bv_page->flags); 1744 prefetchw(&bvec->bv_page->flags);
1778 1745
1779 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) { 1746 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
@@ -1818,7 +1785,7 @@ static void end_bio_extent_readpage(struct bio *bio, int err)
1818 } 1785 }
1819 check_page_locked(tree, page); 1786 check_page_locked(tree, page);
1820 } 1787 }
1821 } while (bvec >= bio->bi_io_vec); 1788 } while (bvec <= bvec_end);
1822 1789
1823 bio_put(bio); 1790 bio_put(bio);
1824} 1791}
@@ -1861,9 +1828,9 @@ static void end_bio_extent_preparewrite(struct bio *bio, int err)
1861 bio_put(bio); 1828 bio_put(bio);
1862} 1829}
1863 1830
1864static struct bio * 1831struct bio *
1865extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs, 1832btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
1866 gfp_t gfp_flags) 1833 gfp_t gfp_flags)
1867{ 1834{
1868 struct bio *bio; 1835 struct bio *bio;
1869 1836
@@ -1890,10 +1857,8 @@ static int submit_one_bio(int rw, struct bio *bio, int mirror_num,
1890 struct page *page = bvec->bv_page; 1857 struct page *page = bvec->bv_page;
1891 struct extent_io_tree *tree = bio->bi_private; 1858 struct extent_io_tree *tree = bio->bi_private;
1892 u64 start; 1859 u64 start;
1893 u64 end;
1894 1860
1895 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset; 1861 start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
1896 end = start + bvec->bv_len - 1;
1897 1862
1898 bio->bi_private = NULL; 1863 bio->bi_private = NULL;
1899 1864
@@ -1901,7 +1866,7 @@ static int submit_one_bio(int rw, struct bio *bio, int mirror_num,
1901 1866
1902 if (tree->ops && tree->ops->submit_bio_hook) 1867 if (tree->ops && tree->ops->submit_bio_hook)
1903 tree->ops->submit_bio_hook(page->mapping->host, rw, bio, 1868 tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
1904 mirror_num, bio_flags); 1869 mirror_num, bio_flags, start);
1905 else 1870 else
1906 submit_bio(rw, bio); 1871 submit_bio(rw, bio);
1907 if (bio_flagged(bio, BIO_EOPNOTSUPP)) 1872 if (bio_flagged(bio, BIO_EOPNOTSUPP))
@@ -1954,7 +1919,7 @@ static int submit_extent_page(int rw, struct extent_io_tree *tree,
1954 else 1919 else
1955 nr = bio_get_nr_vecs(bdev); 1920 nr = bio_get_nr_vecs(bdev);
1956 1921
1957 bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH); 1922 bio = btrfs_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
1958 1923
1959 bio_add_page(bio, page, page_size, offset); 1924 bio_add_page(bio, page, page_size, offset);
1960 bio->bi_end_io = end_io_func; 1925 bio->bi_end_io = end_io_func;
@@ -2005,6 +1970,7 @@ static int __extent_read_full_page(struct extent_io_tree *tree,
2005 sector_t sector; 1970 sector_t sector;
2006 struct extent_map *em; 1971 struct extent_map *em;
2007 struct block_device *bdev; 1972 struct block_device *bdev;
1973 struct btrfs_ordered_extent *ordered;
2008 int ret; 1974 int ret;
2009 int nr = 0; 1975 int nr = 0;
2010 size_t page_offset = 0; 1976 size_t page_offset = 0;
@@ -2016,7 +1982,15 @@ static int __extent_read_full_page(struct extent_io_tree *tree,
2016 set_page_extent_mapped(page); 1982 set_page_extent_mapped(page);
2017 1983
2018 end = page_end; 1984 end = page_end;
2019 lock_extent(tree, start, end, GFP_NOFS); 1985 while (1) {
1986 lock_extent(tree, start, end, GFP_NOFS);
1987 ordered = btrfs_lookup_ordered_extent(inode, start);
1988 if (!ordered)
1989 break;
1990 unlock_extent(tree, start, end, GFP_NOFS);
1991 btrfs_start_ordered_extent(inode, ordered, 1);
1992 btrfs_put_ordered_extent(ordered);
1993 }
2020 1994
2021 if (page->index == last_byte >> PAGE_CACHE_SHIFT) { 1995 if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
2022 char *userpage; 1996 char *userpage;
@@ -2054,8 +2028,11 @@ static int __extent_read_full_page(struct extent_io_tree *tree,
2054 BUG_ON(extent_map_end(em) <= cur); 2028 BUG_ON(extent_map_end(em) <= cur);
2055 BUG_ON(end < cur); 2029 BUG_ON(end < cur);
2056 2030
2057 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 2031 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2058 this_bio_flag = EXTENT_BIO_COMPRESSED; 2032 this_bio_flag = EXTENT_BIO_COMPRESSED;
2033 extent_set_compress_type(&this_bio_flag,
2034 em->compress_type);
2035 }
2059 2036
2060 iosize = min(extent_map_end(em) - cur, end - cur + 1); 2037 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2061 cur_end = min(extent_map_end(em) - 1, end); 2038 cur_end = min(extent_map_end(em) - 1, end);
@@ -2184,7 +2161,6 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2184 u64 last_byte = i_size_read(inode); 2161 u64 last_byte = i_size_read(inode);
2185 u64 block_start; 2162 u64 block_start;
2186 u64 iosize; 2163 u64 iosize;
2187 u64 unlock_start;
2188 sector_t sector; 2164 sector_t sector;
2189 struct extent_state *cached_state = NULL; 2165 struct extent_state *cached_state = NULL;
2190 struct extent_map *em; 2166 struct extent_map *em;
@@ -2309,7 +2285,6 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2309 if (tree->ops && tree->ops->writepage_end_io_hook) 2285 if (tree->ops && tree->ops->writepage_end_io_hook)
2310 tree->ops->writepage_end_io_hook(page, start, 2286 tree->ops->writepage_end_io_hook(page, start,
2311 page_end, NULL, 1); 2287 page_end, NULL, 1);
2312 unlock_start = page_end + 1;
2313 goto done; 2288 goto done;
2314 } 2289 }
2315 2290
@@ -2320,7 +2295,6 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2320 if (tree->ops && tree->ops->writepage_end_io_hook) 2295 if (tree->ops && tree->ops->writepage_end_io_hook)
2321 tree->ops->writepage_end_io_hook(page, cur, 2296 tree->ops->writepage_end_io_hook(page, cur,
2322 page_end, NULL, 1); 2297 page_end, NULL, 1);
2323 unlock_start = page_end + 1;
2324 break; 2298 break;
2325 } 2299 }
2326 em = epd->get_extent(inode, page, pg_offset, cur, 2300 em = epd->get_extent(inode, page, pg_offset, cur,
@@ -2367,7 +2341,6 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2367 2341
2368 cur += iosize; 2342 cur += iosize;
2369 pg_offset += iosize; 2343 pg_offset += iosize;
2370 unlock_start = cur;
2371 continue; 2344 continue;
2372 } 2345 }
2373 /* leave this out until we have a page_mkwrite call */ 2346 /* leave this out until we have a page_mkwrite call */
@@ -2453,7 +2426,6 @@ static int extent_write_cache_pages(struct extent_io_tree *tree,
2453 pgoff_t index; 2426 pgoff_t index;
2454 pgoff_t end; /* Inclusive */ 2427 pgoff_t end; /* Inclusive */
2455 int scanned = 0; 2428 int scanned = 0;
2456 int range_whole = 0;
2457 2429
2458 pagevec_init(&pvec, 0); 2430 pagevec_init(&pvec, 0);
2459 if (wbc->range_cyclic) { 2431 if (wbc->range_cyclic) {
@@ -2462,8 +2434,6 @@ static int extent_write_cache_pages(struct extent_io_tree *tree,
2462 } else { 2434 } else {
2463 index = wbc->range_start >> PAGE_CACHE_SHIFT; 2435 index = wbc->range_start >> PAGE_CACHE_SHIFT;
2464 end = wbc->range_end >> PAGE_CACHE_SHIFT; 2436 end = wbc->range_end >> PAGE_CACHE_SHIFT;
2465 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2466 range_whole = 1;
2467 scanned = 1; 2437 scanned = 1;
2468 } 2438 }
2469retry: 2439retry:
@@ -2574,7 +2544,6 @@ int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
2574 .sync_io = wbc->sync_mode == WB_SYNC_ALL, 2544 .sync_io = wbc->sync_mode == WB_SYNC_ALL,
2575 }; 2545 };
2576 struct writeback_control wbc_writepages = { 2546 struct writeback_control wbc_writepages = {
2577 .bdi = wbc->bdi,
2578 .sync_mode = wbc->sync_mode, 2547 .sync_mode = wbc->sync_mode,
2579 .older_than_this = NULL, 2548 .older_than_this = NULL,
2580 .nr_to_write = 64, 2549 .nr_to_write = 64,
@@ -2608,7 +2577,6 @@ int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
2608 .sync_io = mode == WB_SYNC_ALL, 2577 .sync_io = mode == WB_SYNC_ALL,
2609 }; 2578 };
2610 struct writeback_control wbc_writepages = { 2579 struct writeback_control wbc_writepages = {
2611 .bdi = inode->i_mapping->backing_dev_info,
2612 .sync_mode = mode, 2580 .sync_mode = mode,
2613 .older_than_this = NULL, 2581 .older_than_this = NULL,
2614 .nr_to_write = nr_pages * 2, 2582 .nr_to_write = nr_pages * 2,
@@ -2663,33 +2631,20 @@ int extent_readpages(struct extent_io_tree *tree,
2663{ 2631{
2664 struct bio *bio = NULL; 2632 struct bio *bio = NULL;
2665 unsigned page_idx; 2633 unsigned page_idx;
2666 struct pagevec pvec;
2667 unsigned long bio_flags = 0; 2634 unsigned long bio_flags = 0;
2668 2635
2669 pagevec_init(&pvec, 0);
2670 for (page_idx = 0; page_idx < nr_pages; page_idx++) { 2636 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
2671 struct page *page = list_entry(pages->prev, struct page, lru); 2637 struct page *page = list_entry(pages->prev, struct page, lru);
2672 2638
2673 prefetchw(&page->flags); 2639 prefetchw(&page->flags);
2674 list_del(&page->lru); 2640 list_del(&page->lru);
2675 /* 2641 if (!add_to_page_cache_lru(page, mapping,
2676 * what we want to do here is call add_to_page_cache_lru,
2677 * but that isn't exported, so we reproduce it here
2678 */
2679 if (!add_to_page_cache(page, mapping,
2680 page->index, GFP_KERNEL)) { 2642 page->index, GFP_KERNEL)) {
2681
2682 /* open coding of lru_cache_add, also not exported */
2683 page_cache_get(page);
2684 if (!pagevec_add(&pvec, page))
2685 __pagevec_lru_add_file(&pvec);
2686 __extent_read_full_page(tree, page, get_extent, 2643 __extent_read_full_page(tree, page, get_extent,
2687 &bio, 0, &bio_flags); 2644 &bio, 0, &bio_flags);
2688 } 2645 }
2689 page_cache_release(page); 2646 page_cache_release(page);
2690 } 2647 }
2691 if (pagevec_count(&pvec))
2692 __pagevec_lru_add_file(&pvec);
2693 BUG_ON(!list_empty(pages)); 2648 BUG_ON(!list_empty(pages));
2694 if (bio) 2649 if (bio)
2695 submit_one_bio(READ, bio, 0, bio_flags); 2650 submit_one_bio(READ, bio, 0, bio_flags);
@@ -2704,6 +2659,7 @@ int extent_readpages(struct extent_io_tree *tree,
2704int extent_invalidatepage(struct extent_io_tree *tree, 2659int extent_invalidatepage(struct extent_io_tree *tree,
2705 struct page *page, unsigned long offset) 2660 struct page *page, unsigned long offset)
2706{ 2661{
2662 struct extent_state *cached_state = NULL;
2707 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT); 2663 u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
2708 u64 end = start + PAGE_CACHE_SIZE - 1; 2664 u64 end = start + PAGE_CACHE_SIZE - 1;
2709 size_t blocksize = page->mapping->host->i_sb->s_blocksize; 2665 size_t blocksize = page->mapping->host->i_sb->s_blocksize;
@@ -2712,12 +2668,12 @@ int extent_invalidatepage(struct extent_io_tree *tree,
2712 if (start > end) 2668 if (start > end)
2713 return 0; 2669 return 0;
2714 2670
2715 lock_extent(tree, start, end, GFP_NOFS); 2671 lock_extent_bits(tree, start, end, 0, &cached_state, GFP_NOFS);
2716 wait_on_page_writeback(page); 2672 wait_on_page_writeback(page);
2717 clear_extent_bit(tree, start, end, 2673 clear_extent_bit(tree, start, end,
2718 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC | 2674 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
2719 EXTENT_DO_ACCOUNTING, 2675 EXTENT_DO_ACCOUNTING,
2720 1, 1, NULL, GFP_NOFS); 2676 1, 1, &cached_state, GFP_NOFS);
2721 return 0; 2677 return 0;
2722} 2678}
2723 2679
@@ -2817,6 +2773,8 @@ int extent_prepare_write(struct extent_io_tree *tree,
2817 NULL, 1, 2773 NULL, 1,
2818 end_bio_extent_preparewrite, 0, 2774 end_bio_extent_preparewrite, 0,
2819 0, 0); 2775 0, 0);
2776 if (ret && !err)
2777 err = ret;
2820 iocount++; 2778 iocount++;
2821 block_start = block_start + iosize; 2779 block_start = block_start + iosize;
2822 } else { 2780 } else {
@@ -2920,16 +2878,17 @@ sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
2920 get_extent_t *get_extent) 2878 get_extent_t *get_extent)
2921{ 2879{
2922 struct inode *inode = mapping->host; 2880 struct inode *inode = mapping->host;
2881 struct extent_state *cached_state = NULL;
2923 u64 start = iblock << inode->i_blkbits; 2882 u64 start = iblock << inode->i_blkbits;
2924 sector_t sector = 0; 2883 sector_t sector = 0;
2925 size_t blksize = (1 << inode->i_blkbits); 2884 size_t blksize = (1 << inode->i_blkbits);
2926 struct extent_map *em; 2885 struct extent_map *em;
2927 2886
2928 lock_extent(&BTRFS_I(inode)->io_tree, start, start + blksize - 1, 2887 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + blksize - 1,
2929 GFP_NOFS); 2888 0, &cached_state, GFP_NOFS);
2930 em = get_extent(inode, NULL, 0, start, blksize, 0); 2889 em = get_extent(inode, NULL, 0, start, blksize, 0);
2931 unlock_extent(&BTRFS_I(inode)->io_tree, start, start + blksize - 1, 2890 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start,
2932 GFP_NOFS); 2891 start + blksize - 1, &cached_state, GFP_NOFS);
2933 if (!em || IS_ERR(em)) 2892 if (!em || IS_ERR(em))
2934 return 0; 2893 return 0;
2935 2894
@@ -2945,22 +2904,55 @@ out:
2945int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 2904int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
2946 __u64 start, __u64 len, get_extent_t *get_extent) 2905 __u64 start, __u64 len, get_extent_t *get_extent)
2947{ 2906{
2948 int ret; 2907 int ret = 0;
2949 u64 off = start; 2908 u64 off = start;
2950 u64 max = start + len; 2909 u64 max = start + len;
2951 u32 flags = 0; 2910 u32 flags = 0;
2911 u32 found_type;
2912 u64 last;
2952 u64 disko = 0; 2913 u64 disko = 0;
2914 struct btrfs_key found_key;
2953 struct extent_map *em = NULL; 2915 struct extent_map *em = NULL;
2916 struct extent_state *cached_state = NULL;
2917 struct btrfs_path *path;
2918 struct btrfs_file_extent_item *item;
2954 int end = 0; 2919 int end = 0;
2955 u64 em_start = 0, em_len = 0; 2920 u64 em_start = 0, em_len = 0;
2956 unsigned long emflags; 2921 unsigned long emflags;
2957 ret = 0; 2922 int hole = 0;
2958 2923
2959 if (len == 0) 2924 if (len == 0)
2960 return -EINVAL; 2925 return -EINVAL;
2961 2926
2962 lock_extent(&BTRFS_I(inode)->io_tree, start, start + len, 2927 path = btrfs_alloc_path();
2963 GFP_NOFS); 2928 if (!path)
2929 return -ENOMEM;
2930 path->leave_spinning = 1;
2931
2932 ret = btrfs_lookup_file_extent(NULL, BTRFS_I(inode)->root,
2933 path, inode->i_ino, -1, 0);
2934 if (ret < 0) {
2935 btrfs_free_path(path);
2936 return ret;
2937 }
2938 WARN_ON(!ret);
2939 path->slots[0]--;
2940 item = btrfs_item_ptr(path->nodes[0], path->slots[0],
2941 struct btrfs_file_extent_item);
2942 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
2943 found_type = btrfs_key_type(&found_key);
2944
2945 /* No extents, just return */
2946 if (found_key.objectid != inode->i_ino ||
2947 found_type != BTRFS_EXTENT_DATA_KEY) {
2948 btrfs_free_path(path);
2949 return 0;
2950 }
2951 last = found_key.offset;
2952 btrfs_free_path(path);
2953
2954 lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len, 0,
2955 &cached_state, GFP_NOFS);
2964 em = get_extent(inode, NULL, 0, off, max - off, 0); 2956 em = get_extent(inode, NULL, 0, off, max - off, 0);
2965 if (!em) 2957 if (!em)
2966 goto out; 2958 goto out;
@@ -2968,11 +2960,18 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
2968 ret = PTR_ERR(em); 2960 ret = PTR_ERR(em);
2969 goto out; 2961 goto out;
2970 } 2962 }
2963
2971 while (!end) { 2964 while (!end) {
2965 hole = 0;
2972 off = em->start + em->len; 2966 off = em->start + em->len;
2973 if (off >= max) 2967 if (off >= max)
2974 end = 1; 2968 end = 1;
2975 2969
2970 if (em->block_start == EXTENT_MAP_HOLE) {
2971 hole = 1;
2972 goto next;
2973 }
2974
2976 em_start = em->start; 2975 em_start = em->start;
2977 em_len = em->len; 2976 em_len = em->len;
2978 2977
@@ -2982,8 +2981,6 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
2982 if (em->block_start == EXTENT_MAP_LAST_BYTE) { 2981 if (em->block_start == EXTENT_MAP_LAST_BYTE) {
2983 end = 1; 2982 end = 1;
2984 flags |= FIEMAP_EXTENT_LAST; 2983 flags |= FIEMAP_EXTENT_LAST;
2985 } else if (em->block_start == EXTENT_MAP_HOLE) {
2986 flags |= FIEMAP_EXTENT_UNWRITTEN;
2987 } else if (em->block_start == EXTENT_MAP_INLINE) { 2984 } else if (em->block_start == EXTENT_MAP_INLINE) {
2988 flags |= (FIEMAP_EXTENT_DATA_INLINE | 2985 flags |= (FIEMAP_EXTENT_DATA_INLINE |
2989 FIEMAP_EXTENT_NOT_ALIGNED); 2986 FIEMAP_EXTENT_NOT_ALIGNED);
@@ -2996,10 +2993,10 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
2996 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) 2993 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
2997 flags |= FIEMAP_EXTENT_ENCODED; 2994 flags |= FIEMAP_EXTENT_ENCODED;
2998 2995
2996next:
2999 emflags = em->flags; 2997 emflags = em->flags;
3000 free_extent_map(em); 2998 free_extent_map(em);
3001 em = NULL; 2999 em = NULL;
3002
3003 if (!end) { 3000 if (!end) {
3004 em = get_extent(inode, NULL, 0, off, max - off, 0); 3001 em = get_extent(inode, NULL, 0, off, max - off, 0);
3005 if (!em) 3002 if (!em)
@@ -3010,21 +3007,29 @@ int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
3010 } 3007 }
3011 emflags = em->flags; 3008 emflags = em->flags;
3012 } 3009 }
3010
3013 if (test_bit(EXTENT_FLAG_VACANCY, &emflags)) { 3011 if (test_bit(EXTENT_FLAG_VACANCY, &emflags)) {
3014 flags |= FIEMAP_EXTENT_LAST; 3012 flags |= FIEMAP_EXTENT_LAST;
3015 end = 1; 3013 end = 1;
3016 } 3014 }
3017 3015
3018 ret = fiemap_fill_next_extent(fieinfo, em_start, disko, 3016 if (em_start == last) {
3019 em_len, flags); 3017 flags |= FIEMAP_EXTENT_LAST;
3020 if (ret) 3018 end = 1;
3021 goto out_free; 3019 }
3020
3021 if (!hole) {
3022 ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
3023 em_len, flags);
3024 if (ret)
3025 goto out_free;
3026 }
3022 } 3027 }
3023out_free: 3028out_free:
3024 free_extent_map(em); 3029 free_extent_map(em);
3025out: 3030out:
3026 unlock_extent(&BTRFS_I(inode)->io_tree, start, start + len, 3031 unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len,
3027 GFP_NOFS); 3032 &cached_state, GFP_NOFS);
3028 return ret; 3033 return ret;
3029} 3034}
3030 3035
@@ -3070,6 +3075,8 @@ static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
3070#endif 3075#endif
3071 3076
3072 eb = kmem_cache_zalloc(extent_buffer_cache, mask); 3077 eb = kmem_cache_zalloc(extent_buffer_cache, mask);
3078 if (eb == NULL)
3079 return NULL;
3073 eb->start = start; 3080 eb->start = start;
3074 eb->len = len; 3081 eb->len = len;
3075 spin_lock_init(&eb->lock); 3082 spin_lock_init(&eb->lock);
@@ -3096,6 +3103,39 @@ static void __free_extent_buffer(struct extent_buffer *eb)
3096 kmem_cache_free(extent_buffer_cache, eb); 3103 kmem_cache_free(extent_buffer_cache, eb);
3097} 3104}
3098 3105
3106/*
3107 * Helper for releasing extent buffer page.
3108 */
3109static void btrfs_release_extent_buffer_page(struct extent_buffer *eb,
3110 unsigned long start_idx)
3111{
3112 unsigned long index;
3113 struct page *page;
3114
3115 if (!eb->first_page)
3116 return;
3117
3118 index = num_extent_pages(eb->start, eb->len);
3119 if (start_idx >= index)
3120 return;
3121
3122 do {
3123 index--;
3124 page = extent_buffer_page(eb, index);
3125 if (page)
3126 page_cache_release(page);
3127 } while (index != start_idx);
3128}
3129
3130/*
3131 * Helper for releasing the extent buffer.
3132 */
3133static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
3134{
3135 btrfs_release_extent_buffer_page(eb, 0);
3136 __free_extent_buffer(eb);
3137}
3138
3099struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree, 3139struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
3100 u64 start, unsigned long len, 3140 u64 start, unsigned long len,
3101 struct page *page0, 3141 struct page *page0,
@@ -3109,16 +3149,16 @@ struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
3109 struct page *p; 3149 struct page *p;
3110 struct address_space *mapping = tree->mapping; 3150 struct address_space *mapping = tree->mapping;
3111 int uptodate = 1; 3151 int uptodate = 1;
3152 int ret;
3112 3153
3113 spin_lock(&tree->buffer_lock); 3154 rcu_read_lock();
3114 eb = buffer_search(tree, start); 3155 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
3115 if (eb) { 3156 if (eb && atomic_inc_not_zero(&eb->refs)) {
3116 atomic_inc(&eb->refs); 3157 rcu_read_unlock();
3117 spin_unlock(&tree->buffer_lock);
3118 mark_page_accessed(eb->first_page); 3158 mark_page_accessed(eb->first_page);
3119 return eb; 3159 return eb;
3120 } 3160 }
3121 spin_unlock(&tree->buffer_lock); 3161 rcu_read_unlock();
3122 3162
3123 eb = __alloc_extent_buffer(tree, start, len, mask); 3163 eb = __alloc_extent_buffer(tree, start, len, mask);
3124 if (!eb) 3164 if (!eb)
@@ -3157,27 +3197,31 @@ struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
3157 if (uptodate) 3197 if (uptodate)
3158 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 3198 set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3159 3199
3200 ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
3201 if (ret)
3202 goto free_eb;
3203
3160 spin_lock(&tree->buffer_lock); 3204 spin_lock(&tree->buffer_lock);
3161 exists = buffer_tree_insert(tree, start, &eb->rb_node); 3205 ret = radix_tree_insert(&tree->buffer, start >> PAGE_CACHE_SHIFT, eb);
3162 if (exists) { 3206 if (ret == -EEXIST) {
3207 exists = radix_tree_lookup(&tree->buffer,
3208 start >> PAGE_CACHE_SHIFT);
3163 /* add one reference for the caller */ 3209 /* add one reference for the caller */
3164 atomic_inc(&exists->refs); 3210 atomic_inc(&exists->refs);
3165 spin_unlock(&tree->buffer_lock); 3211 spin_unlock(&tree->buffer_lock);
3212 radix_tree_preload_end();
3166 goto free_eb; 3213 goto free_eb;
3167 } 3214 }
3168 spin_unlock(&tree->buffer_lock);
3169
3170 /* add one reference for the tree */ 3215 /* add one reference for the tree */
3171 atomic_inc(&eb->refs); 3216 atomic_inc(&eb->refs);
3217 spin_unlock(&tree->buffer_lock);
3218 radix_tree_preload_end();
3172 return eb; 3219 return eb;
3173 3220
3174free_eb: 3221free_eb:
3175 if (!atomic_dec_and_test(&eb->refs)) 3222 if (!atomic_dec_and_test(&eb->refs))
3176 return exists; 3223 return exists;
3177 for (index = 1; index < i; index++) 3224 btrfs_release_extent_buffer(eb);
3178 page_cache_release(extent_buffer_page(eb, index));
3179 page_cache_release(extent_buffer_page(eb, 0));
3180 __free_extent_buffer(eb);
3181 return exists; 3225 return exists;
3182} 3226}
3183 3227
@@ -3187,16 +3231,16 @@ struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
3187{ 3231{
3188 struct extent_buffer *eb; 3232 struct extent_buffer *eb;
3189 3233
3190 spin_lock(&tree->buffer_lock); 3234 rcu_read_lock();
3191 eb = buffer_search(tree, start); 3235 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
3192 if (eb) 3236 if (eb && atomic_inc_not_zero(&eb->refs)) {
3193 atomic_inc(&eb->refs); 3237 rcu_read_unlock();
3194 spin_unlock(&tree->buffer_lock);
3195
3196 if (eb)
3197 mark_page_accessed(eb->first_page); 3238 mark_page_accessed(eb->first_page);
3239 return eb;
3240 }
3241 rcu_read_unlock();
3198 3242
3199 return eb; 3243 return NULL;
3200} 3244}
3201 3245
3202void free_extent_buffer(struct extent_buffer *eb) 3246void free_extent_buffer(struct extent_buffer *eb)
@@ -3265,7 +3309,8 @@ int set_extent_buffer_dirty(struct extent_io_tree *tree,
3265} 3309}
3266 3310
3267int clear_extent_buffer_uptodate(struct extent_io_tree *tree, 3311int clear_extent_buffer_uptodate(struct extent_io_tree *tree,
3268 struct extent_buffer *eb) 3312 struct extent_buffer *eb,
3313 struct extent_state **cached_state)
3269{ 3314{
3270 unsigned long i; 3315 unsigned long i;
3271 struct page *page; 3316 struct page *page;
@@ -3275,7 +3320,7 @@ int clear_extent_buffer_uptodate(struct extent_io_tree *tree,
3275 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags); 3320 clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3276 3321
3277 clear_extent_uptodate(tree, eb->start, eb->start + eb->len - 1, 3322 clear_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
3278 GFP_NOFS); 3323 cached_state, GFP_NOFS);
3279 for (i = 0; i < num_pages; i++) { 3324 for (i = 0; i < num_pages; i++) {
3280 page = extent_buffer_page(eb, i); 3325 page = extent_buffer_page(eb, i);
3281 if (page) 3326 if (page)
@@ -3335,7 +3380,8 @@ int extent_range_uptodate(struct extent_io_tree *tree,
3335} 3380}
3336 3381
3337int extent_buffer_uptodate(struct extent_io_tree *tree, 3382int extent_buffer_uptodate(struct extent_io_tree *tree,
3338 struct extent_buffer *eb) 3383 struct extent_buffer *eb,
3384 struct extent_state *cached_state)
3339{ 3385{
3340 int ret = 0; 3386 int ret = 0;
3341 unsigned long num_pages; 3387 unsigned long num_pages;
@@ -3347,7 +3393,7 @@ int extent_buffer_uptodate(struct extent_io_tree *tree,
3347 return 1; 3393 return 1;
3348 3394
3349 ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1, 3395 ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1,
3350 EXTENT_UPTODATE, 1, NULL); 3396 EXTENT_UPTODATE, 1, cached_state);
3351 if (ret) 3397 if (ret)
3352 return ret; 3398 return ret;
3353 3399
@@ -3824,34 +3870,47 @@ void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3824 } 3870 }
3825} 3871}
3826 3872
3873static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3874{
3875 struct extent_buffer *eb =
3876 container_of(head, struct extent_buffer, rcu_head);
3877
3878 btrfs_release_extent_buffer(eb);
3879}
3880
3827int try_release_extent_buffer(struct extent_io_tree *tree, struct page *page) 3881int try_release_extent_buffer(struct extent_io_tree *tree, struct page *page)
3828{ 3882{
3829 u64 start = page_offset(page); 3883 u64 start = page_offset(page);
3830 struct extent_buffer *eb; 3884 struct extent_buffer *eb;
3831 int ret = 1; 3885 int ret = 1;
3832 unsigned long i;
3833 unsigned long num_pages;
3834 3886
3835 spin_lock(&tree->buffer_lock); 3887 spin_lock(&tree->buffer_lock);
3836 eb = buffer_search(tree, start); 3888 eb = radix_tree_lookup(&tree->buffer, start >> PAGE_CACHE_SHIFT);
3837 if (!eb) 3889 if (!eb) {
3838 goto out; 3890 spin_unlock(&tree->buffer_lock);
3891 return ret;
3892 }
3839 3893
3840 if (atomic_read(&eb->refs) > 1) { 3894 if (test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3841 ret = 0; 3895 ret = 0;
3842 goto out; 3896 goto out;
3843 } 3897 }
3844 if (test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) { 3898
3899 /*
3900 * set @eb->refs to 0 if it is already 1, and then release the @eb.
3901 * Or go back.
3902 */
3903 if (atomic_cmpxchg(&eb->refs, 1, 0) != 1) {
3845 ret = 0; 3904 ret = 0;
3846 goto out; 3905 goto out;
3847 } 3906 }
3848 /* at this point we can safely release the extent buffer */ 3907
3849 num_pages = num_extent_pages(eb->start, eb->len); 3908 radix_tree_delete(&tree->buffer, start >> PAGE_CACHE_SHIFT);
3850 for (i = 0; i < num_pages; i++)
3851 page_cache_release(extent_buffer_page(eb, i));
3852 rb_erase(&eb->rb_node, &tree->buffer);
3853 __free_extent_buffer(eb);
3854out: 3909out:
3855 spin_unlock(&tree->buffer_lock); 3910 spin_unlock(&tree->buffer_lock);
3911
3912 /* at this point we can safely release the extent buffer */
3913 if (atomic_read(&eb->refs) == 0)
3914 call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3856 return ret; 3915 return ret;
3857} 3916}
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-03 12:01:28 -0400