aboutsummaryrefslogtreecommitdiffstats
path: root/fs/btrfs/extent_io.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/btrfs/extent_io.c')
-rw-r--r--fs/btrfs/extent_io.c293
1 files changed, 167 insertions, 126 deletions
diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c
index 68260180f587..a102422cd92e 100644
--- a/fs/btrfs/extent_io.c
+++ b/fs/btrfs/extent_io.c
@@ -367,10 +367,10 @@ static int insert_state(struct extent_io_tree *tree,
367 } 367 }
368 if (bits & EXTENT_DIRTY) 368 if (bits & EXTENT_DIRTY)
369 tree->dirty_bytes += end - start + 1; 369 tree->dirty_bytes += end - start + 1;
370 set_state_cb(tree, state, bits);
371 state->state |= bits;
372 state->start = start; 370 state->start = start;
373 state->end = end; 371 state->end = end;
372 set_state_cb(tree, state, bits);
373 state->state |= bits;
374 node = tree_insert(&tree->state, end, &state->rb_node); 374 node = tree_insert(&tree->state, end, &state->rb_node);
375 if (node) { 375 if (node) {
376 struct extent_state *found; 376 struct extent_state *found;
@@ -471,10 +471,14 @@ static int clear_state_bit(struct extent_io_tree *tree,
471 * bits were already set, or zero if none of the bits were already set. 471 * bits were already set, or zero if none of the bits were already set.
472 */ 472 */
473int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 473int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
474 int bits, int wake, int delete, gfp_t mask) 474 int bits, int wake, int delete,
475 struct extent_state **cached_state,
476 gfp_t mask)
475{ 477{
476 struct extent_state *state; 478 struct extent_state *state;
479 struct extent_state *cached;
477 struct extent_state *prealloc = NULL; 480 struct extent_state *prealloc = NULL;
481 struct rb_node *next_node;
478 struct rb_node *node; 482 struct rb_node *node;
479 u64 last_end; 483 u64 last_end;
480 int err; 484 int err;
@@ -488,6 +492,17 @@ again:
488 } 492 }
489 493
490 spin_lock(&tree->lock); 494 spin_lock(&tree->lock);
495 if (cached_state) {
496 cached = *cached_state;
497 *cached_state = NULL;
498 if (cached->tree && cached->start == start) {
499 atomic_dec(&cached->refs);
500 state = cached;
501 last_end = state->end;
502 goto found;
503 }
504 free_extent_state(cached);
505 }
491 /* 506 /*
492 * this search will find the extents that end after 507 * this search will find the extents that end after
493 * our range starts 508 * our range starts
@@ -496,6 +511,7 @@ again:
496 if (!node) 511 if (!node)
497 goto out; 512 goto out;
498 state = rb_entry(node, struct extent_state, rb_node); 513 state = rb_entry(node, struct extent_state, rb_node);
514hit_next:
499 if (state->start > end) 515 if (state->start > end)
500 goto out; 516 goto out;
501 WARN_ON(state->end < start); 517 WARN_ON(state->end < start);
@@ -555,11 +571,21 @@ again:
555 prealloc = NULL; 571 prealloc = NULL;
556 goto out; 572 goto out;
557 } 573 }
558 574found:
575 if (state->end < end && prealloc && !need_resched())
576 next_node = rb_next(&state->rb_node);
577 else
578 next_node = NULL;
559 set |= clear_state_bit(tree, state, bits, wake, delete); 579 set |= clear_state_bit(tree, state, bits, wake, delete);
560 if (last_end == (u64)-1) 580 if (last_end == (u64)-1)
561 goto out; 581 goto out;
562 start = last_end + 1; 582 start = last_end + 1;
583 if (start <= end && next_node) {
584 state = rb_entry(next_node, struct extent_state,
585 rb_node);
586 if (state->start == start)
587 goto hit_next;
588 }
563 goto search_again; 589 goto search_again;
564 590
565out: 591out:
@@ -653,26 +679,37 @@ static void set_state_bits(struct extent_io_tree *tree,
653 state->state |= bits; 679 state->state |= bits;
654} 680}
655 681
682static void cache_state(struct extent_state *state,
683 struct extent_state **cached_ptr)
684{
685 if (cached_ptr && !(*cached_ptr)) {
686 if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY)) {
687 *cached_ptr = state;
688 atomic_inc(&state->refs);
689 }
690 }
691}
692
656/* 693/*
657 * set some bits on a range in the tree. This may require allocations 694 * set some bits on a range in the tree. This may require allocations or
658 * or sleeping, so the gfp mask is used to indicate what is allowed. 695 * sleeping, so the gfp mask is used to indicate what is allowed.
659 * 696 *
660 * If 'exclusive' == 1, this will fail with -EEXIST if some part of the 697 * If any of the exclusive bits are set, this will fail with -EEXIST if some
661 * range already has the desired bits set. The start of the existing 698 * part of the range already has the desired bits set. The start of the
662 * range is returned in failed_start in this case. 699 * existing range is returned in failed_start in this case.
663 * 700 *
664 * [start, end] is inclusive 701 * [start, end] is inclusive This takes the tree lock.
665 * This takes the tree lock.
666 */ 702 */
703
667static int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, 704static int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
668 int bits, int exclusive, u64 *failed_start, 705 int bits, int exclusive_bits, u64 *failed_start,
706 struct extent_state **cached_state,
669 gfp_t mask) 707 gfp_t mask)
670{ 708{
671 struct extent_state *state; 709 struct extent_state *state;
672 struct extent_state *prealloc = NULL; 710 struct extent_state *prealloc = NULL;
673 struct rb_node *node; 711 struct rb_node *node;
674 int err = 0; 712 int err = 0;
675 int set;
676 u64 last_start; 713 u64 last_start;
677 u64 last_end; 714 u64 last_end;
678again: 715again:
@@ -683,6 +720,13 @@ again:
683 } 720 }
684 721
685 spin_lock(&tree->lock); 722 spin_lock(&tree->lock);
723 if (cached_state && *cached_state) {
724 state = *cached_state;
725 if (state->start == start && state->tree) {
726 node = &state->rb_node;
727 goto hit_next;
728 }
729 }
686 /* 730 /*
687 * this search will find all the extents that end after 731 * this search will find all the extents that end after
688 * our range starts. 732 * our range starts.
@@ -694,8 +738,8 @@ again:
694 BUG_ON(err == -EEXIST); 738 BUG_ON(err == -EEXIST);
695 goto out; 739 goto out;
696 } 740 }
697
698 state = rb_entry(node, struct extent_state, rb_node); 741 state = rb_entry(node, struct extent_state, rb_node);
742hit_next:
699 last_start = state->start; 743 last_start = state->start;
700 last_end = state->end; 744 last_end = state->end;
701 745
@@ -706,17 +750,28 @@ again:
706 * Just lock what we found and keep going 750 * Just lock what we found and keep going
707 */ 751 */
708 if (state->start == start && state->end <= end) { 752 if (state->start == start && state->end <= end) {
709 set = state->state & bits; 753 struct rb_node *next_node;
710 if (set && exclusive) { 754 if (state->state & exclusive_bits) {
711 *failed_start = state->start; 755 *failed_start = state->start;
712 err = -EEXIST; 756 err = -EEXIST;
713 goto out; 757 goto out;
714 } 758 }
715 set_state_bits(tree, state, bits); 759 set_state_bits(tree, state, bits);
760 cache_state(state, cached_state);
716 merge_state(tree, state); 761 merge_state(tree, state);
717 if (last_end == (u64)-1) 762 if (last_end == (u64)-1)
718 goto out; 763 goto out;
764
719 start = last_end + 1; 765 start = last_end + 1;
766 if (start < end && prealloc && !need_resched()) {
767 next_node = rb_next(node);
768 if (next_node) {
769 state = rb_entry(next_node, struct extent_state,
770 rb_node);
771 if (state->start == start)
772 goto hit_next;
773 }
774 }
720 goto search_again; 775 goto search_again;
721 } 776 }
722 777
@@ -737,8 +792,7 @@ again:
737 * desired bit on it. 792 * desired bit on it.
738 */ 793 */
739 if (state->start < start) { 794 if (state->start < start) {
740 set = state->state & bits; 795 if (state->state & exclusive_bits) {
741 if (exclusive && set) {
742 *failed_start = start; 796 *failed_start = start;
743 err = -EEXIST; 797 err = -EEXIST;
744 goto out; 798 goto out;
@@ -750,6 +804,7 @@ again:
750 goto out; 804 goto out;
751 if (state->end <= end) { 805 if (state->end <= end) {
752 set_state_bits(tree, state, bits); 806 set_state_bits(tree, state, bits);
807 cache_state(state, cached_state);
753 merge_state(tree, state); 808 merge_state(tree, state);
754 if (last_end == (u64)-1) 809 if (last_end == (u64)-1)
755 goto out; 810 goto out;
@@ -774,6 +829,7 @@ again:
774 this_end = last_start - 1; 829 this_end = last_start - 1;
775 err = insert_state(tree, prealloc, start, this_end, 830 err = insert_state(tree, prealloc, start, this_end,
776 bits); 831 bits);
832 cache_state(prealloc, cached_state);
777 prealloc = NULL; 833 prealloc = NULL;
778 BUG_ON(err == -EEXIST); 834 BUG_ON(err == -EEXIST);
779 if (err) 835 if (err)
@@ -788,8 +844,7 @@ again:
788 * on the first half 844 * on the first half
789 */ 845 */
790 if (state->start <= end && state->end > end) { 846 if (state->start <= end && state->end > end) {
791 set = state->state & bits; 847 if (state->state & exclusive_bits) {
792 if (exclusive && set) {
793 *failed_start = start; 848 *failed_start = start;
794 err = -EEXIST; 849 err = -EEXIST;
795 goto out; 850 goto out;
@@ -798,6 +853,7 @@ again:
798 BUG_ON(err == -EEXIST); 853 BUG_ON(err == -EEXIST);
799 854
800 set_state_bits(tree, prealloc, bits); 855 set_state_bits(tree, prealloc, bits);
856 cache_state(prealloc, cached_state);
801 merge_state(tree, prealloc); 857 merge_state(tree, prealloc);
802 prealloc = NULL; 858 prealloc = NULL;
803 goto out; 859 goto out;
@@ -826,86 +882,64 @@ int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
826 gfp_t mask) 882 gfp_t mask)
827{ 883{
828 return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL, 884 return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
829 mask); 885 NULL, mask);
830}
831
832int set_extent_ordered(struct extent_io_tree *tree, u64 start, u64 end,
833 gfp_t mask)
834{
835 return set_extent_bit(tree, start, end, EXTENT_ORDERED, 0, NULL, mask);
836} 886}
837 887
838int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 888int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
839 int bits, gfp_t mask) 889 int bits, gfp_t mask)
840{ 890{
841 return set_extent_bit(tree, start, end, bits, 0, NULL, 891 return set_extent_bit(tree, start, end, bits, 0, NULL,
842 mask); 892 NULL, mask);
843} 893}
844 894
845int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end, 895int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
846 int bits, gfp_t mask) 896 int bits, gfp_t mask)
847{ 897{
848 return clear_extent_bit(tree, start, end, bits, 0, 0, mask); 898 return clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask);
849} 899}
850 900
851int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end, 901int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
852 gfp_t mask) 902 gfp_t mask)
853{ 903{
854 return set_extent_bit(tree, start, end, 904 return set_extent_bit(tree, start, end,
855 EXTENT_DELALLOC | EXTENT_DIRTY, 905 EXTENT_DELALLOC | EXTENT_DIRTY | EXTENT_UPTODATE,
856 0, NULL, mask); 906 0, NULL, NULL, mask);
857} 907}
858 908
859int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end, 909int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
860 gfp_t mask) 910 gfp_t mask)
861{ 911{
862 return clear_extent_bit(tree, start, end, 912 return clear_extent_bit(tree, start, end,
863 EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask); 913 EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0,
864} 914 NULL, mask);
865
866int clear_extent_ordered(struct extent_io_tree *tree, u64 start, u64 end,
867 gfp_t mask)
868{
869 return clear_extent_bit(tree, start, end, EXTENT_ORDERED, 1, 0, mask);
870} 915}
871 916
872int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end, 917int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
873 gfp_t mask) 918 gfp_t mask)
874{ 919{
875 return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL, 920 return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
876 mask); 921 NULL, mask);
877} 922}
878 923
879static int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end, 924static int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
880 gfp_t mask) 925 gfp_t mask)
881{ 926{
882 return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask); 927 return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0,
928 NULL, mask);
883} 929}
884 930
885int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end, 931int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
886 gfp_t mask) 932 gfp_t mask)
887{ 933{
888 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL, 934 return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
889 mask); 935 NULL, mask);
890} 936}
891 937
892static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, 938static int clear_extent_uptodate(struct extent_io_tree *tree, u64 start,
893 u64 end, gfp_t mask) 939 u64 end, gfp_t mask)
894{ 940{
895 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask); 941 return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0,
896} 942 NULL, mask);
897
898static int set_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
899 gfp_t mask)
900{
901 return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
902 0, NULL, mask);
903}
904
905static int clear_extent_writeback(struct extent_io_tree *tree, u64 start,
906 u64 end, gfp_t mask)
907{
908 return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
909} 943}
910 944
911int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end) 945int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
@@ -917,13 +951,15 @@ int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
917 * either insert or lock state struct between start and end use mask to tell 951 * either insert or lock state struct between start and end use mask to tell
918 * us if waiting is desired. 952 * us if waiting is desired.
919 */ 953 */
920int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask) 954int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
955 int bits, struct extent_state **cached_state, gfp_t mask)
921{ 956{
922 int err; 957 int err;
923 u64 failed_start; 958 u64 failed_start;
924 while (1) { 959 while (1) {
925 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 960 err = set_extent_bit(tree, start, end, EXTENT_LOCKED | bits,
926 &failed_start, mask); 961 EXTENT_LOCKED, &failed_start,
962 cached_state, mask);
927 if (err == -EEXIST && (mask & __GFP_WAIT)) { 963 if (err == -EEXIST && (mask & __GFP_WAIT)) {
928 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED); 964 wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
929 start = failed_start; 965 start = failed_start;
@@ -935,27 +971,40 @@ int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
935 return err; 971 return err;
936} 972}
937 973
974int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
975{
976 return lock_extent_bits(tree, start, end, 0, NULL, mask);
977}
978
938int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end, 979int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end,
939 gfp_t mask) 980 gfp_t mask)
940{ 981{
941 int err; 982 int err;
942 u64 failed_start; 983 u64 failed_start;
943 984
944 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 985 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
945 &failed_start, mask); 986 &failed_start, NULL, mask);
946 if (err == -EEXIST) { 987 if (err == -EEXIST) {
947 if (failed_start > start) 988 if (failed_start > start)
948 clear_extent_bit(tree, start, failed_start - 1, 989 clear_extent_bit(tree, start, failed_start - 1,
949 EXTENT_LOCKED, 1, 0, mask); 990 EXTENT_LOCKED, 1, 0, NULL, mask);
950 return 0; 991 return 0;
951 } 992 }
952 return 1; 993 return 1;
953} 994}
954 995
996int unlock_extent_cached(struct extent_io_tree *tree, u64 start, u64 end,
997 struct extent_state **cached, gfp_t mask)
998{
999 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, cached,
1000 mask);
1001}
1002
955int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end, 1003int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end,
956 gfp_t mask) 1004 gfp_t mask)
957{ 1005{
958 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask); 1006 return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, NULL,
1007 mask);
959} 1008}
960 1009
961/* 1010/*
@@ -974,7 +1023,6 @@ int set_range_dirty(struct extent_io_tree *tree, u64 start, u64 end)
974 page_cache_release(page); 1023 page_cache_release(page);
975 index++; 1024 index++;
976 } 1025 }
977 set_extent_dirty(tree, start, end, GFP_NOFS);
978 return 0; 1026 return 0;
979} 1027}
980 1028
@@ -994,7 +1042,6 @@ static int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
994 page_cache_release(page); 1042 page_cache_release(page);
995 index++; 1043 index++;
996 } 1044 }
997 set_extent_writeback(tree, start, end, GFP_NOFS);
998 return 0; 1045 return 0;
999} 1046}
1000 1047
@@ -1232,6 +1279,7 @@ static noinline u64 find_lock_delalloc_range(struct inode *inode,
1232 u64 delalloc_start; 1279 u64 delalloc_start;
1233 u64 delalloc_end; 1280 u64 delalloc_end;
1234 u64 found; 1281 u64 found;
1282 struct extent_state *cached_state = NULL;
1235 int ret; 1283 int ret;
1236 int loops = 0; 1284 int loops = 0;
1237 1285
@@ -1269,6 +1317,7 @@ again:
1269 /* some of the pages are gone, lets avoid looping by 1317 /* some of the pages are gone, lets avoid looping by
1270 * shortening the size of the delalloc range we're searching 1318 * shortening the size of the delalloc range we're searching
1271 */ 1319 */
1320 free_extent_state(cached_state);
1272 if (!loops) { 1321 if (!loops) {
1273 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1); 1322 unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1274 max_bytes = PAGE_CACHE_SIZE - offset; 1323 max_bytes = PAGE_CACHE_SIZE - offset;
@@ -1282,18 +1331,21 @@ again:
1282 BUG_ON(ret); 1331 BUG_ON(ret);
1283 1332
1284 /* step three, lock the state bits for the whole range */ 1333 /* step three, lock the state bits for the whole range */
1285 lock_extent(tree, delalloc_start, delalloc_end, GFP_NOFS); 1334 lock_extent_bits(tree, delalloc_start, delalloc_end,
1335 0, &cached_state, GFP_NOFS);
1286 1336
1287 /* then test to make sure it is all still delalloc */ 1337 /* then test to make sure it is all still delalloc */
1288 ret = test_range_bit(tree, delalloc_start, delalloc_end, 1338 ret = test_range_bit(tree, delalloc_start, delalloc_end,
1289 EXTENT_DELALLOC, 1); 1339 EXTENT_DELALLOC, 1, cached_state);
1290 if (!ret) { 1340 if (!ret) {
1291 unlock_extent(tree, delalloc_start, delalloc_end, GFP_NOFS); 1341 unlock_extent_cached(tree, delalloc_start, delalloc_end,
1342 &cached_state, GFP_NOFS);
1292 __unlock_for_delalloc(inode, locked_page, 1343 __unlock_for_delalloc(inode, locked_page,
1293 delalloc_start, delalloc_end); 1344 delalloc_start, delalloc_end);
1294 cond_resched(); 1345 cond_resched();
1295 goto again; 1346 goto again;
1296 } 1347 }
1348 free_extent_state(cached_state);
1297 *start = delalloc_start; 1349 *start = delalloc_start;
1298 *end = delalloc_end; 1350 *end = delalloc_end;
1299out_failed: 1351out_failed:
@@ -1307,7 +1359,8 @@ int extent_clear_unlock_delalloc(struct inode *inode,
1307 int clear_unlock, 1359 int clear_unlock,
1308 int clear_delalloc, int clear_dirty, 1360 int clear_delalloc, int clear_dirty,
1309 int set_writeback, 1361 int set_writeback,
1310 int end_writeback) 1362 int end_writeback,
1363 int set_private2)
1311{ 1364{
1312 int ret; 1365 int ret;
1313 struct page *pages[16]; 1366 struct page *pages[16];
@@ -1325,8 +1378,9 @@ int extent_clear_unlock_delalloc(struct inode *inode,
1325 if (clear_delalloc) 1378 if (clear_delalloc)
1326 clear_bits |= EXTENT_DELALLOC; 1379 clear_bits |= EXTENT_DELALLOC;
1327 1380
1328 clear_extent_bit(tree, start, end, clear_bits, 1, 0, GFP_NOFS); 1381 clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1329 if (!(unlock_pages || clear_dirty || set_writeback || end_writeback)) 1382 if (!(unlock_pages || clear_dirty || set_writeback || end_writeback ||
1383 set_private2))
1330 return 0; 1384 return 0;
1331 1385
1332 while (nr_pages > 0) { 1386 while (nr_pages > 0) {
@@ -1334,6 +1388,10 @@ int extent_clear_unlock_delalloc(struct inode *inode,
1334 min_t(unsigned long, 1388 min_t(unsigned long,
1335 nr_pages, ARRAY_SIZE(pages)), pages); 1389 nr_pages, ARRAY_SIZE(pages)), pages);
1336 for (i = 0; i < ret; i++) { 1390 for (i = 0; i < ret; i++) {
1391
1392 if (set_private2)
1393 SetPagePrivate2(pages[i]);
1394
1337 if (pages[i] == locked_page) { 1395 if (pages[i] == locked_page) {
1338 page_cache_release(pages[i]); 1396 page_cache_release(pages[i]);
1339 continue; 1397 continue;
@@ -1476,14 +1534,17 @@ out:
1476 * range is found set. 1534 * range is found set.
1477 */ 1535 */
1478int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end, 1536int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1479 int bits, int filled) 1537 int bits, int filled, struct extent_state *cached)
1480{ 1538{
1481 struct extent_state *state = NULL; 1539 struct extent_state *state = NULL;
1482 struct rb_node *node; 1540 struct rb_node *node;
1483 int bitset = 0; 1541 int bitset = 0;
1484 1542
1485 spin_lock(&tree->lock); 1543 spin_lock(&tree->lock);
1486 node = tree_search(tree, start); 1544 if (cached && cached->tree && cached->start == start)
1545 node = &cached->rb_node;
1546 else
1547 node = tree_search(tree, start);
1487 while (node && start <= end) { 1548 while (node && start <= end) {
1488 state = rb_entry(node, struct extent_state, rb_node); 1549 state = rb_entry(node, struct extent_state, rb_node);
1489 1550
@@ -1526,7 +1587,7 @@ static int check_page_uptodate(struct extent_io_tree *tree,
1526{ 1587{
1527 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 1588 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1528 u64 end = start + PAGE_CACHE_SIZE - 1; 1589 u64 end = start + PAGE_CACHE_SIZE - 1;
1529 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1)) 1590 if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1530 SetPageUptodate(page); 1591 SetPageUptodate(page);
1531 return 0; 1592 return 0;
1532} 1593}
@@ -1540,7 +1601,7 @@ static int check_page_locked(struct extent_io_tree *tree,
1540{ 1601{
1541 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 1602 u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1542 u64 end = start + PAGE_CACHE_SIZE - 1; 1603 u64 end = start + PAGE_CACHE_SIZE - 1;
1543 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0)) 1604 if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0, NULL))
1544 unlock_page(page); 1605 unlock_page(page);
1545 return 0; 1606 return 0;
1546} 1607}
@@ -1552,10 +1613,7 @@ static int check_page_locked(struct extent_io_tree *tree,
1552static int check_page_writeback(struct extent_io_tree *tree, 1613static int check_page_writeback(struct extent_io_tree *tree,
1553 struct page *page) 1614 struct page *page)
1554{ 1615{
1555 u64 start = (u64)page->index << PAGE_CACHE_SHIFT; 1616 end_page_writeback(page);
1556 u64 end = start + PAGE_CACHE_SIZE - 1;
1557 if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
1558 end_page_writeback(page);
1559 return 0; 1617 return 0;
1560} 1618}
1561 1619
@@ -1613,13 +1671,11 @@ static void end_bio_extent_writepage(struct bio *bio, int err)
1613 } 1671 }
1614 1672
1615 if (!uptodate) { 1673 if (!uptodate) {
1616 clear_extent_uptodate(tree, start, end, GFP_ATOMIC); 1674 clear_extent_uptodate(tree, start, end, GFP_NOFS);
1617 ClearPageUptodate(page); 1675 ClearPageUptodate(page);
1618 SetPageError(page); 1676 SetPageError(page);
1619 } 1677 }
1620 1678
1621 clear_extent_writeback(tree, start, end, GFP_ATOMIC);
1622
1623 if (whole_page) 1679 if (whole_page)
1624 end_page_writeback(page); 1680 end_page_writeback(page);
1625 else 1681 else
@@ -1983,7 +2039,8 @@ static int __extent_read_full_page(struct extent_io_tree *tree,
1983 continue; 2039 continue;
1984 } 2040 }
1985 /* the get_extent function already copied into the page */ 2041 /* the get_extent function already copied into the page */
1986 if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) { 2042 if (test_range_bit(tree, cur, cur_end,
2043 EXTENT_UPTODATE, 1, NULL)) {
1987 check_page_uptodate(tree, page); 2044 check_page_uptodate(tree, page);
1988 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS); 2045 unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1989 cur = cur + iosize; 2046 cur = cur + iosize;
@@ -2078,6 +2135,7 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2078 u64 iosize; 2135 u64 iosize;
2079 u64 unlock_start; 2136 u64 unlock_start;
2080 sector_t sector; 2137 sector_t sector;
2138 struct extent_state *cached_state = NULL;
2081 struct extent_map *em; 2139 struct extent_map *em;
2082 struct block_device *bdev; 2140 struct block_device *bdev;
2083 int ret; 2141 int ret;
@@ -2124,6 +2182,7 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2124 delalloc_end = 0; 2182 delalloc_end = 0;
2125 page_started = 0; 2183 page_started = 0;
2126 if (!epd->extent_locked) { 2184 if (!epd->extent_locked) {
2185 u64 delalloc_to_write;
2127 /* 2186 /*
2128 * make sure the wbc mapping index is at least updated 2187 * make sure the wbc mapping index is at least updated
2129 * to this page. 2188 * to this page.
@@ -2143,6 +2202,14 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2143 tree->ops->fill_delalloc(inode, page, delalloc_start, 2202 tree->ops->fill_delalloc(inode, page, delalloc_start,
2144 delalloc_end, &page_started, 2203 delalloc_end, &page_started,
2145 &nr_written); 2204 &nr_written);
2205 delalloc_to_write = (delalloc_end -
2206 max_t(u64, page_offset(page),
2207 delalloc_start) + 1) >>
2208 PAGE_CACHE_SHIFT;
2209 if (wbc->nr_to_write < delalloc_to_write) {
2210 wbc->nr_to_write = min_t(long, 8192,
2211 delalloc_to_write);
2212 }
2146 delalloc_start = delalloc_end + 1; 2213 delalloc_start = delalloc_end + 1;
2147 } 2214 }
2148 2215
@@ -2160,15 +2227,10 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2160 goto done_unlocked; 2227 goto done_unlocked;
2161 } 2228 }
2162 } 2229 }
2163 lock_extent(tree, start, page_end, GFP_NOFS);
2164
2165 unlock_start = start;
2166
2167 if (tree->ops && tree->ops->writepage_start_hook) { 2230 if (tree->ops && tree->ops->writepage_start_hook) {
2168 ret = tree->ops->writepage_start_hook(page, start, 2231 ret = tree->ops->writepage_start_hook(page, start,
2169 page_end); 2232 page_end);
2170 if (ret == -EAGAIN) { 2233 if (ret == -EAGAIN) {
2171 unlock_extent(tree, start, page_end, GFP_NOFS);
2172 redirty_page_for_writepage(wbc, page); 2234 redirty_page_for_writepage(wbc, page);
2173 update_nr_written(page, wbc, nr_written); 2235 update_nr_written(page, wbc, nr_written);
2174 unlock_page(page); 2236 unlock_page(page);
@@ -2184,12 +2246,7 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2184 update_nr_written(page, wbc, nr_written + 1); 2246 update_nr_written(page, wbc, nr_written + 1);
2185 2247
2186 end = page_end; 2248 end = page_end;
2187 if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0))
2188 printk(KERN_ERR "btrfs delalloc bits after lock_extent\n");
2189
2190 if (last_byte <= start) { 2249 if (last_byte <= start) {
2191 clear_extent_dirty(tree, start, page_end, GFP_NOFS);
2192 unlock_extent(tree, start, page_end, GFP_NOFS);
2193 if (tree->ops && tree->ops->writepage_end_io_hook) 2250 if (tree->ops && tree->ops->writepage_end_io_hook)
2194 tree->ops->writepage_end_io_hook(page, start, 2251 tree->ops->writepage_end_io_hook(page, start,
2195 page_end, NULL, 1); 2252 page_end, NULL, 1);
@@ -2197,13 +2254,10 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2197 goto done; 2254 goto done;
2198 } 2255 }
2199 2256
2200 set_extent_uptodate(tree, start, page_end, GFP_NOFS);
2201 blocksize = inode->i_sb->s_blocksize; 2257 blocksize = inode->i_sb->s_blocksize;
2202 2258
2203 while (cur <= end) { 2259 while (cur <= end) {
2204 if (cur >= last_byte) { 2260 if (cur >= last_byte) {
2205 clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
2206 unlock_extent(tree, unlock_start, page_end, GFP_NOFS);
2207 if (tree->ops && tree->ops->writepage_end_io_hook) 2261 if (tree->ops && tree->ops->writepage_end_io_hook)
2208 tree->ops->writepage_end_io_hook(page, cur, 2262 tree->ops->writepage_end_io_hook(page, cur,
2209 page_end, NULL, 1); 2263 page_end, NULL, 1);
@@ -2235,12 +2289,6 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2235 */ 2289 */
2236 if (compressed || block_start == EXTENT_MAP_HOLE || 2290 if (compressed || block_start == EXTENT_MAP_HOLE ||
2237 block_start == EXTENT_MAP_INLINE) { 2291 block_start == EXTENT_MAP_INLINE) {
2238 clear_extent_dirty(tree, cur,
2239 cur + iosize - 1, GFP_NOFS);
2240
2241 unlock_extent(tree, unlock_start, cur + iosize - 1,
2242 GFP_NOFS);
2243
2244 /* 2292 /*
2245 * end_io notification does not happen here for 2293 * end_io notification does not happen here for
2246 * compressed extents 2294 * compressed extents
@@ -2265,13 +2313,12 @@ static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2265 } 2313 }
2266 /* leave this out until we have a page_mkwrite call */ 2314 /* leave this out until we have a page_mkwrite call */
2267 if (0 && !test_range_bit(tree, cur, cur + iosize - 1, 2315 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
2268 EXTENT_DIRTY, 0)) { 2316 EXTENT_DIRTY, 0, NULL)) {
2269 cur = cur + iosize; 2317 cur = cur + iosize;
2270 pg_offset += iosize; 2318 pg_offset += iosize;
2271 continue; 2319 continue;
2272 } 2320 }
2273 2321
2274 clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
2275 if (tree->ops && tree->ops->writepage_io_hook) { 2322 if (tree->ops && tree->ops->writepage_io_hook) {
2276 ret = tree->ops->writepage_io_hook(page, cur, 2323 ret = tree->ops->writepage_io_hook(page, cur,
2277 cur + iosize - 1); 2324 cur + iosize - 1);
@@ -2309,12 +2356,12 @@ done:
2309 set_page_writeback(page); 2356 set_page_writeback(page);
2310 end_page_writeback(page); 2357 end_page_writeback(page);
2311 } 2358 }
2312 if (unlock_start <= page_end)
2313 unlock_extent(tree, unlock_start, page_end, GFP_NOFS);
2314 unlock_page(page); 2359 unlock_page(page);
2315 2360
2316done_unlocked: 2361done_unlocked:
2317 2362
2363 /* drop our reference on any cached states */
2364 free_extent_state(cached_state);
2318 return 0; 2365 return 0;
2319} 2366}
2320 2367
@@ -2339,7 +2386,6 @@ static int extent_write_cache_pages(struct extent_io_tree *tree,
2339 writepage_t writepage, void *data, 2386 writepage_t writepage, void *data,
2340 void (*flush_fn)(void *)) 2387 void (*flush_fn)(void *))
2341{ 2388{
2342 struct backing_dev_info *bdi = mapping->backing_dev_info;
2343 int ret = 0; 2389 int ret = 0;
2344 int done = 0; 2390 int done = 0;
2345 struct pagevec pvec; 2391 struct pagevec pvec;
@@ -2414,10 +2460,6 @@ retry:
2414 } 2460 }
2415 if (ret || wbc->nr_to_write <= 0) 2461 if (ret || wbc->nr_to_write <= 0)
2416 done = 1; 2462 done = 1;
2417 if (wbc->nonblocking && bdi_write_congested(bdi)) {
2418 wbc->encountered_congestion = 1;
2419 done = 1;
2420 }
2421 } 2463 }
2422 pagevec_release(&pvec); 2464 pagevec_release(&pvec);
2423 cond_resched(); 2465 cond_resched();
@@ -2604,10 +2646,10 @@ int extent_invalidatepage(struct extent_io_tree *tree,
2604 return 0; 2646 return 0;
2605 2647
2606 lock_extent(tree, start, end, GFP_NOFS); 2648 lock_extent(tree, start, end, GFP_NOFS);
2607 wait_on_extent_writeback(tree, start, end); 2649 wait_on_page_writeback(page);
2608 clear_extent_bit(tree, start, end, 2650 clear_extent_bit(tree, start, end,
2609 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC, 2651 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
2610 1, 1, GFP_NOFS); 2652 1, 1, NULL, GFP_NOFS);
2611 return 0; 2653 return 0;
2612} 2654}
2613 2655
@@ -2687,7 +2729,7 @@ int extent_prepare_write(struct extent_io_tree *tree,
2687 !isnew && !PageUptodate(page) && 2729 !isnew && !PageUptodate(page) &&
2688 (block_off_end > to || block_off_start < from) && 2730 (block_off_end > to || block_off_start < from) &&
2689 !test_range_bit(tree, block_start, cur_end, 2731 !test_range_bit(tree, block_start, cur_end,
2690 EXTENT_UPTODATE, 1)) { 2732 EXTENT_UPTODATE, 1, NULL)) {
2691 u64 sector; 2733 u64 sector;
2692 u64 extent_offset = block_start - em->start; 2734 u64 extent_offset = block_start - em->start;
2693 size_t iosize; 2735 size_t iosize;
@@ -2701,7 +2743,7 @@ int extent_prepare_write(struct extent_io_tree *tree,
2701 */ 2743 */
2702 set_extent_bit(tree, block_start, 2744 set_extent_bit(tree, block_start,
2703 block_start + iosize - 1, 2745 block_start + iosize - 1,
2704 EXTENT_LOCKED, 0, NULL, GFP_NOFS); 2746 EXTENT_LOCKED, 0, NULL, NULL, GFP_NOFS);
2705 ret = submit_extent_page(READ, tree, page, 2747 ret = submit_extent_page(READ, tree, page,
2706 sector, iosize, page_offset, em->bdev, 2748 sector, iosize, page_offset, em->bdev,
2707 NULL, 1, 2749 NULL, 1,
@@ -2742,13 +2784,13 @@ int try_release_extent_state(struct extent_map_tree *map,
2742 int ret = 1; 2784 int ret = 1;
2743 2785
2744 if (test_range_bit(tree, start, end, 2786 if (test_range_bit(tree, start, end,
2745 EXTENT_IOBITS | EXTENT_ORDERED, 0)) 2787 EXTENT_IOBITS, 0, NULL))
2746 ret = 0; 2788 ret = 0;
2747 else { 2789 else {
2748 if ((mask & GFP_NOFS) == GFP_NOFS) 2790 if ((mask & GFP_NOFS) == GFP_NOFS)
2749 mask = GFP_NOFS; 2791 mask = GFP_NOFS;
2750 clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 2792 clear_extent_bit(tree, start, end, EXTENT_UPTODATE,
2751 1, 1, mask); 2793 1, 1, NULL, mask);
2752 } 2794 }
2753 return ret; 2795 return ret;
2754} 2796}
@@ -2771,29 +2813,28 @@ int try_release_extent_mapping(struct extent_map_tree *map,
2771 u64 len; 2813 u64 len;
2772 while (start <= end) { 2814 while (start <= end) {
2773 len = end - start + 1; 2815 len = end - start + 1;
2774 spin_lock(&map->lock); 2816 write_lock(&map->lock);
2775 em = lookup_extent_mapping(map, start, len); 2817 em = lookup_extent_mapping(map, start, len);
2776 if (!em || IS_ERR(em)) { 2818 if (!em || IS_ERR(em)) {
2777 spin_unlock(&map->lock); 2819 write_unlock(&map->lock);
2778 break; 2820 break;
2779 } 2821 }
2780 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) || 2822 if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2781 em->start != start) { 2823 em->start != start) {
2782 spin_unlock(&map->lock); 2824 write_unlock(&map->lock);
2783 free_extent_map(em); 2825 free_extent_map(em);
2784 break; 2826 break;
2785 } 2827 }
2786 if (!test_range_bit(tree, em->start, 2828 if (!test_range_bit(tree, em->start,
2787 extent_map_end(em) - 1, 2829 extent_map_end(em) - 1,
2788 EXTENT_LOCKED | EXTENT_WRITEBACK | 2830 EXTENT_LOCKED | EXTENT_WRITEBACK,
2789 EXTENT_ORDERED, 2831 0, NULL)) {
2790 0)) {
2791 remove_extent_mapping(map, em); 2832 remove_extent_mapping(map, em);
2792 /* once for the rb tree */ 2833 /* once for the rb tree */
2793 free_extent_map(em); 2834 free_extent_map(em);
2794 } 2835 }
2795 start = extent_map_end(em); 2836 start = extent_map_end(em);
2796 spin_unlock(&map->lock); 2837 write_unlock(&map->lock);
2797 2838
2798 /* once for us */ 2839 /* once for us */
2799 free_extent_map(em); 2840 free_extent_map(em);
@@ -3203,7 +3244,7 @@ int extent_range_uptodate(struct extent_io_tree *tree,
3203 int uptodate; 3244 int uptodate;
3204 unsigned long index; 3245 unsigned long index;
3205 3246
3206 ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1); 3247 ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL);
3207 if (ret) 3248 if (ret)
3208 return 1; 3249 return 1;
3209 while (start <= end) { 3250 while (start <= end) {
@@ -3233,7 +3274,7 @@ int extent_buffer_uptodate(struct extent_io_tree *tree,
3233 return 1; 3274 return 1;
3234 3275
3235 ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1, 3276 ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1,
3236 EXTENT_UPTODATE, 1); 3277 EXTENT_UPTODATE, 1, NULL);
3237 if (ret) 3278 if (ret)
3238 return ret; 3279 return ret;
3239 3280
@@ -3269,7 +3310,7 @@ int read_extent_buffer_pages(struct extent_io_tree *tree,
3269 return 0; 3310 return 0;
3270 3311
3271 if (test_range_bit(tree, eb->start, eb->start + eb->len - 1, 3312 if (test_range_bit(tree, eb->start, eb->start + eb->len - 1,
3272 EXTENT_UPTODATE, 1)) { 3313 EXTENT_UPTODATE, 1, NULL)) {
3273 return 0; 3314 return 0;
3274 } 3315 }
3275 3316