aboutsummaryrefslogtreecommitdiffstats
path: root/fs/btrfs/ctree.c
diff options
context:
space:
mode:
Diffstat (limited to 'fs/btrfs/ctree.c')
-rw-r--r--fs/btrfs/ctree.c588
1 files changed, 316 insertions, 272 deletions
diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index 37f31b5529a..dbb72412463 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -254,18 +254,13 @@ int btrfs_copy_root(struct btrfs_trans_handle *trans,
254 * empty_size -- a hint that you plan on doing more cow. This is the size in 254 * empty_size -- a hint that you plan on doing more cow. This is the size in
255 * bytes the allocator should try to find free next to the block it returns. 255 * bytes the allocator should try to find free next to the block it returns.
256 * This is just a hint and may be ignored by the allocator. 256 * This is just a hint and may be ignored by the allocator.
257 *
258 * prealloc_dest -- if you have already reserved a destination for the cow,
259 * this uses that block instead of allocating a new one.
260 * btrfs_alloc_reserved_extent is used to finish the allocation.
261 */ 257 */
262static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans, 258static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
263 struct btrfs_root *root, 259 struct btrfs_root *root,
264 struct extent_buffer *buf, 260 struct extent_buffer *buf,
265 struct extent_buffer *parent, int parent_slot, 261 struct extent_buffer *parent, int parent_slot,
266 struct extent_buffer **cow_ret, 262 struct extent_buffer **cow_ret,
267 u64 search_start, u64 empty_size, 263 u64 search_start, u64 empty_size)
268 u64 prealloc_dest)
269{ 264{
270 u64 parent_start; 265 u64 parent_start;
271 struct extent_buffer *cow; 266 struct extent_buffer *cow;
@@ -291,26 +286,10 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
291 level = btrfs_header_level(buf); 286 level = btrfs_header_level(buf);
292 nritems = btrfs_header_nritems(buf); 287 nritems = btrfs_header_nritems(buf);
293 288
294 if (prealloc_dest) { 289 cow = btrfs_alloc_free_block(trans, root, buf->len,
295 struct btrfs_key ins; 290 parent_start, root->root_key.objectid,
296 291 trans->transid, level,
297 ins.objectid = prealloc_dest; 292 search_start, empty_size);
298 ins.offset = buf->len;
299 ins.type = BTRFS_EXTENT_ITEM_KEY;
300
301 ret = btrfs_alloc_reserved_extent(trans, root, parent_start,
302 root->root_key.objectid,
303 trans->transid, level, &ins);
304 BUG_ON(ret);
305 cow = btrfs_init_new_buffer(trans, root, prealloc_dest,
306 buf->len, level);
307 } else {
308 cow = btrfs_alloc_free_block(trans, root, buf->len,
309 parent_start,
310 root->root_key.objectid,
311 trans->transid, level,
312 search_start, empty_size);
313 }
314 if (IS_ERR(cow)) 293 if (IS_ERR(cow))
315 return PTR_ERR(cow); 294 return PTR_ERR(cow);
316 295
@@ -413,7 +392,7 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
413noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, 392noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
414 struct btrfs_root *root, struct extent_buffer *buf, 393 struct btrfs_root *root, struct extent_buffer *buf,
415 struct extent_buffer *parent, int parent_slot, 394 struct extent_buffer *parent, int parent_slot,
416 struct extent_buffer **cow_ret, u64 prealloc_dest) 395 struct extent_buffer **cow_ret)
417{ 396{
418 u64 search_start; 397 u64 search_start;
419 int ret; 398 int ret;
@@ -436,7 +415,6 @@ noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
436 btrfs_header_owner(buf) == root->root_key.objectid && 415 btrfs_header_owner(buf) == root->root_key.objectid &&
437 !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) { 416 !btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
438 *cow_ret = buf; 417 *cow_ret = buf;
439 WARN_ON(prealloc_dest);
440 return 0; 418 return 0;
441 } 419 }
442 420
@@ -447,8 +425,7 @@ noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
447 btrfs_set_lock_blocking(buf); 425 btrfs_set_lock_blocking(buf);
448 426
449 ret = __btrfs_cow_block(trans, root, buf, parent, 427 ret = __btrfs_cow_block(trans, root, buf, parent,
450 parent_slot, cow_ret, search_start, 0, 428 parent_slot, cow_ret, search_start, 0);
451 prealloc_dest);
452 return ret; 429 return ret;
453} 430}
454 431
@@ -617,7 +594,7 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans,
617 err = __btrfs_cow_block(trans, root, cur, parent, i, 594 err = __btrfs_cow_block(trans, root, cur, parent, i,
618 &cur, search_start, 595 &cur, search_start,
619 min(16 * blocksize, 596 min(16 * blocksize,
620 (end_slot - i) * blocksize), 0); 597 (end_slot - i) * blocksize));
621 if (err) { 598 if (err) {
622 btrfs_tree_unlock(cur); 599 btrfs_tree_unlock(cur);
623 free_extent_buffer(cur); 600 free_extent_buffer(cur);
@@ -937,7 +914,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
937 BUG_ON(!child); 914 BUG_ON(!child);
938 btrfs_tree_lock(child); 915 btrfs_tree_lock(child);
939 btrfs_set_lock_blocking(child); 916 btrfs_set_lock_blocking(child);
940 ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 0); 917 ret = btrfs_cow_block(trans, root, child, mid, 0, &child);
941 BUG_ON(ret); 918 BUG_ON(ret);
942 919
943 spin_lock(&root->node_lock); 920 spin_lock(&root->node_lock);
@@ -945,6 +922,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
945 spin_unlock(&root->node_lock); 922 spin_unlock(&root->node_lock);
946 923
947 ret = btrfs_update_extent_ref(trans, root, child->start, 924 ret = btrfs_update_extent_ref(trans, root, child->start,
925 child->len,
948 mid->start, child->start, 926 mid->start, child->start,
949 root->root_key.objectid, 927 root->root_key.objectid,
950 trans->transid, level - 1); 928 trans->transid, level - 1);
@@ -971,6 +949,10 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
971 BTRFS_NODEPTRS_PER_BLOCK(root) / 4) 949 BTRFS_NODEPTRS_PER_BLOCK(root) / 4)
972 return 0; 950 return 0;
973 951
952 if (trans->transaction->delayed_refs.flushing &&
953 btrfs_header_nritems(mid) > 2)
954 return 0;
955
974 if (btrfs_header_nritems(mid) < 2) 956 if (btrfs_header_nritems(mid) < 2)
975 err_on_enospc = 1; 957 err_on_enospc = 1;
976 958
@@ -979,7 +961,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
979 btrfs_tree_lock(left); 961 btrfs_tree_lock(left);
980 btrfs_set_lock_blocking(left); 962 btrfs_set_lock_blocking(left);
981 wret = btrfs_cow_block(trans, root, left, 963 wret = btrfs_cow_block(trans, root, left,
982 parent, pslot - 1, &left, 0); 964 parent, pslot - 1, &left);
983 if (wret) { 965 if (wret) {
984 ret = wret; 966 ret = wret;
985 goto enospc; 967 goto enospc;
@@ -990,7 +972,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
990 btrfs_tree_lock(right); 972 btrfs_tree_lock(right);
991 btrfs_set_lock_blocking(right); 973 btrfs_set_lock_blocking(right);
992 wret = btrfs_cow_block(trans, root, right, 974 wret = btrfs_cow_block(trans, root, right,
993 parent, pslot + 1, &right, 0); 975 parent, pslot + 1, &right);
994 if (wret) { 976 if (wret) {
995 ret = wret; 977 ret = wret;
996 goto enospc; 978 goto enospc;
@@ -1171,7 +1153,7 @@ static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
1171 wret = 1; 1153 wret = 1;
1172 } else { 1154 } else {
1173 ret = btrfs_cow_block(trans, root, left, parent, 1155 ret = btrfs_cow_block(trans, root, left, parent,
1174 pslot - 1, &left, 0); 1156 pslot - 1, &left);
1175 if (ret) 1157 if (ret)
1176 wret = 1; 1158 wret = 1;
1177 else { 1159 else {
@@ -1222,7 +1204,7 @@ static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
1222 } else { 1204 } else {
1223 ret = btrfs_cow_block(trans, root, right, 1205 ret = btrfs_cow_block(trans, root, right,
1224 parent, pslot + 1, 1206 parent, pslot + 1,
1225 &right, 0); 1207 &right);
1226 if (ret) 1208 if (ret)
1227 wret = 1; 1209 wret = 1;
1228 else { 1210 else {
@@ -1492,7 +1474,6 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
1492 u8 lowest_level = 0; 1474 u8 lowest_level = 0;
1493 u64 blocknr; 1475 u64 blocknr;
1494 u64 gen; 1476 u64 gen;
1495 struct btrfs_key prealloc_block;
1496 1477
1497 lowest_level = p->lowest_level; 1478 lowest_level = p->lowest_level;
1498 WARN_ON(lowest_level && ins_len > 0); 1479 WARN_ON(lowest_level && ins_len > 0);
@@ -1501,8 +1482,6 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root
1501 if (ins_len < 0) 1482 if (ins_len < 0)
1502 lowest_unlock = 2; 1483 lowest_unlock = 2;
1503 1484
1504 prealloc_block.objectid = 0;
1505
1506again: 1485again:
1507 if (p->skip_locking) 1486 if (p->skip_locking)
1508 b = btrfs_root_node(root); 1487 b = btrfs_root_node(root);
@@ -1529,44 +1508,11 @@ again:
1529 !btrfs_header_flag(b, BTRFS_HEADER_FLAG_WRITTEN)) { 1508 !btrfs_header_flag(b, BTRFS_HEADER_FLAG_WRITTEN)) {
1530 goto cow_done; 1509 goto cow_done;
1531 } 1510 }
1532
1533 /* ok, we have to cow, is our old prealloc the right
1534 * size?
1535 */
1536 if (prealloc_block.objectid &&
1537 prealloc_block.offset != b->len) {
1538 btrfs_release_path(root, p);
1539 btrfs_free_reserved_extent(root,
1540 prealloc_block.objectid,
1541 prealloc_block.offset);
1542 prealloc_block.objectid = 0;
1543 goto again;
1544 }
1545
1546 /*
1547 * for higher level blocks, try not to allocate blocks
1548 * with the block and the parent locks held.
1549 */
1550 if (level > 0 && !prealloc_block.objectid) {
1551 u32 size = b->len;
1552 u64 hint = b->start;
1553
1554 btrfs_release_path(root, p);
1555 ret = btrfs_reserve_extent(trans, root,
1556 size, size, 0,
1557 hint, (u64)-1,
1558 &prealloc_block, 0);
1559 BUG_ON(ret);
1560 goto again;
1561 }
1562
1563 btrfs_set_path_blocking(p); 1511 btrfs_set_path_blocking(p);
1564 1512
1565 wret = btrfs_cow_block(trans, root, b, 1513 wret = btrfs_cow_block(trans, root, b,
1566 p->nodes[level + 1], 1514 p->nodes[level + 1],
1567 p->slots[level + 1], 1515 p->slots[level + 1], &b);
1568 &b, prealloc_block.objectid);
1569 prealloc_block.objectid = 0;
1570 if (wret) { 1516 if (wret) {
1571 free_extent_buffer(b); 1517 free_extent_buffer(b);
1572 ret = wret; 1518 ret = wret;
@@ -1742,12 +1688,8 @@ done:
1742 * we don't really know what they plan on doing with the path 1688 * we don't really know what they plan on doing with the path
1743 * from here on, so for now just mark it as blocking 1689 * from here on, so for now just mark it as blocking
1744 */ 1690 */
1745 btrfs_set_path_blocking(p); 1691 if (!p->leave_spinning)
1746 if (prealloc_block.objectid) { 1692 btrfs_set_path_blocking(p);
1747 btrfs_free_reserved_extent(root,
1748 prealloc_block.objectid,
1749 prealloc_block.offset);
1750 }
1751 return ret; 1693 return ret;
1752} 1694}
1753 1695
@@ -1768,7 +1710,7 @@ int btrfs_merge_path(struct btrfs_trans_handle *trans,
1768 int ret; 1710 int ret;
1769 1711
1770 eb = btrfs_lock_root_node(root); 1712 eb = btrfs_lock_root_node(root);
1771 ret = btrfs_cow_block(trans, root, eb, NULL, 0, &eb, 0); 1713 ret = btrfs_cow_block(trans, root, eb, NULL, 0, &eb);
1772 BUG_ON(ret); 1714 BUG_ON(ret);
1773 1715
1774 btrfs_set_lock_blocking(eb); 1716 btrfs_set_lock_blocking(eb);
@@ -1826,7 +1768,7 @@ int btrfs_merge_path(struct btrfs_trans_handle *trans,
1826 } 1768 }
1827 1769
1828 ret = btrfs_cow_block(trans, root, eb, parent, slot, 1770 ret = btrfs_cow_block(trans, root, eb, parent, slot,
1829 &eb, 0); 1771 &eb);
1830 BUG_ON(ret); 1772 BUG_ON(ret);
1831 1773
1832 if (root->root_key.objectid == 1774 if (root->root_key.objectid ==
@@ -2139,7 +2081,7 @@ static noinline int insert_new_root(struct btrfs_trans_handle *trans,
2139 spin_unlock(&root->node_lock); 2081 spin_unlock(&root->node_lock);
2140 2082
2141 ret = btrfs_update_extent_ref(trans, root, lower->start, 2083 ret = btrfs_update_extent_ref(trans, root, lower->start,
2142 lower->start, c->start, 2084 lower->len, lower->start, c->start,
2143 root->root_key.objectid, 2085 root->root_key.objectid,
2144 trans->transid, level - 1); 2086 trans->transid, level - 1);
2145 BUG_ON(ret); 2087 BUG_ON(ret);
@@ -2221,7 +2163,7 @@ static noinline int split_node(struct btrfs_trans_handle *trans,
2221 ret = insert_new_root(trans, root, path, level + 1); 2163 ret = insert_new_root(trans, root, path, level + 1);
2222 if (ret) 2164 if (ret)
2223 return ret; 2165 return ret;
2224 } else { 2166 } else if (!trans->transaction->delayed_refs.flushing) {
2225 ret = push_nodes_for_insert(trans, root, path, level); 2167 ret = push_nodes_for_insert(trans, root, path, level);
2226 c = path->nodes[level]; 2168 c = path->nodes[level];
2227 if (!ret && btrfs_header_nritems(c) < 2169 if (!ret && btrfs_header_nritems(c) <
@@ -2329,66 +2271,27 @@ noinline int btrfs_leaf_free_space(struct btrfs_root *root,
2329 return ret; 2271 return ret;
2330} 2272}
2331 2273
2332/* 2274static noinline int __push_leaf_right(struct btrfs_trans_handle *trans,
2333 * push some data in the path leaf to the right, trying to free up at 2275 struct btrfs_root *root,
2334 * least data_size bytes. returns zero if the push worked, nonzero otherwise 2276 struct btrfs_path *path,
2335 * 2277 int data_size, int empty,
2336 * returns 1 if the push failed because the other node didn't have enough 2278 struct extent_buffer *right,
2337 * room, 0 if everything worked out and < 0 if there were major errors. 2279 int free_space, u32 left_nritems)
2338 */
2339static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
2340 *root, struct btrfs_path *path, int data_size,
2341 int empty)
2342{ 2280{
2343 struct extent_buffer *left = path->nodes[0]; 2281 struct extent_buffer *left = path->nodes[0];
2344 struct extent_buffer *right; 2282 struct extent_buffer *upper = path->nodes[1];
2345 struct extent_buffer *upper;
2346 struct btrfs_disk_key disk_key; 2283 struct btrfs_disk_key disk_key;
2347 int slot; 2284 int slot;
2348 u32 i; 2285 u32 i;
2349 int free_space;
2350 int push_space = 0; 2286 int push_space = 0;
2351 int push_items = 0; 2287 int push_items = 0;
2352 struct btrfs_item *item; 2288 struct btrfs_item *item;
2353 u32 left_nritems;
2354 u32 nr; 2289 u32 nr;
2355 u32 right_nritems; 2290 u32 right_nritems;
2356 u32 data_end; 2291 u32 data_end;
2357 u32 this_item_size; 2292 u32 this_item_size;
2358 int ret; 2293 int ret;
2359 2294
2360 slot = path->slots[1];
2361 if (!path->nodes[1])
2362 return 1;
2363
2364 upper = path->nodes[1];
2365 if (slot >= btrfs_header_nritems(upper) - 1)
2366 return 1;
2367
2368 btrfs_assert_tree_locked(path->nodes[1]);
2369
2370 right = read_node_slot(root, upper, slot + 1);
2371 btrfs_tree_lock(right);
2372 btrfs_set_lock_blocking(right);
2373
2374 free_space = btrfs_leaf_free_space(root, right);
2375 if (free_space < data_size)
2376 goto out_unlock;
2377
2378 /* cow and double check */
2379 ret = btrfs_cow_block(trans, root, right, upper,
2380 slot + 1, &right, 0);
2381 if (ret)
2382 goto out_unlock;
2383
2384 free_space = btrfs_leaf_free_space(root, right);
2385 if (free_space < data_size)
2386 goto out_unlock;
2387
2388 left_nritems = btrfs_header_nritems(left);
2389 if (left_nritems == 0)
2390 goto out_unlock;
2391
2392 if (empty) 2295 if (empty)
2393 nr = 0; 2296 nr = 0;
2394 else 2297 else
@@ -2397,6 +2300,7 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
2397 if (path->slots[0] >= left_nritems) 2300 if (path->slots[0] >= left_nritems)
2398 push_space += data_size; 2301 push_space += data_size;
2399 2302
2303 slot = path->slots[1];
2400 i = left_nritems - 1; 2304 i = left_nritems - 1;
2401 while (i >= nr) { 2305 while (i >= nr) {
2402 item = btrfs_item_nr(left, i); 2306 item = btrfs_item_nr(left, i);
@@ -2528,24 +2432,82 @@ out_unlock:
2528} 2432}
2529 2433
2530/* 2434/*
2435 * push some data in the path leaf to the right, trying to free up at
2436 * least data_size bytes. returns zero if the push worked, nonzero otherwise
2437 *
2438 * returns 1 if the push failed because the other node didn't have enough
2439 * room, 0 if everything worked out and < 0 if there were major errors.
2440 */
2441static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
2442 *root, struct btrfs_path *path, int data_size,
2443 int empty)
2444{
2445 struct extent_buffer *left = path->nodes[0];
2446 struct extent_buffer *right;
2447 struct extent_buffer *upper;
2448 int slot;
2449 int free_space;
2450 u32 left_nritems;
2451 int ret;
2452
2453 if (!path->nodes[1])
2454 return 1;
2455
2456 slot = path->slots[1];
2457 upper = path->nodes[1];
2458 if (slot >= btrfs_header_nritems(upper) - 1)
2459 return 1;
2460
2461 btrfs_assert_tree_locked(path->nodes[1]);
2462
2463 right = read_node_slot(root, upper, slot + 1);
2464 btrfs_tree_lock(right);
2465 btrfs_set_lock_blocking(right);
2466
2467 free_space = btrfs_leaf_free_space(root, right);
2468 if (free_space < data_size)
2469 goto out_unlock;
2470
2471 /* cow and double check */
2472 ret = btrfs_cow_block(trans, root, right, upper,
2473 slot + 1, &right);
2474 if (ret)
2475 goto out_unlock;
2476
2477 free_space = btrfs_leaf_free_space(root, right);
2478 if (free_space < data_size)
2479 goto out_unlock;
2480
2481 left_nritems = btrfs_header_nritems(left);
2482 if (left_nritems == 0)
2483 goto out_unlock;
2484
2485 return __push_leaf_right(trans, root, path, data_size, empty,
2486 right, free_space, left_nritems);
2487out_unlock:
2488 btrfs_tree_unlock(right);
2489 free_extent_buffer(right);
2490 return 1;
2491}
2492
2493/*
2531 * push some data in the path leaf to the left, trying to free up at 2494 * push some data in the path leaf to the left, trying to free up at
2532 * least data_size bytes. returns zero if the push worked, nonzero otherwise 2495 * least data_size bytes. returns zero if the push worked, nonzero otherwise
2533 */ 2496 */
2534static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root 2497static noinline int __push_leaf_left(struct btrfs_trans_handle *trans,
2535 *root, struct btrfs_path *path, int data_size, 2498 struct btrfs_root *root,
2536 int empty) 2499 struct btrfs_path *path, int data_size,
2500 int empty, struct extent_buffer *left,
2501 int free_space, int right_nritems)
2537{ 2502{
2538 struct btrfs_disk_key disk_key; 2503 struct btrfs_disk_key disk_key;
2539 struct extent_buffer *right = path->nodes[0]; 2504 struct extent_buffer *right = path->nodes[0];
2540 struct extent_buffer *left;
2541 int slot; 2505 int slot;
2542 int i; 2506 int i;
2543 int free_space;
2544 int push_space = 0; 2507 int push_space = 0;
2545 int push_items = 0; 2508 int push_items = 0;
2546 struct btrfs_item *item; 2509 struct btrfs_item *item;
2547 u32 old_left_nritems; 2510 u32 old_left_nritems;
2548 u32 right_nritems;
2549 u32 nr; 2511 u32 nr;
2550 int ret = 0; 2512 int ret = 0;
2551 int wret; 2513 int wret;
@@ -2553,41 +2515,6 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
2553 u32 old_left_item_size; 2515 u32 old_left_item_size;
2554 2516
2555 slot = path->slots[1]; 2517 slot = path->slots[1];
2556 if (slot == 0)
2557 return 1;
2558 if (!path->nodes[1])
2559 return 1;
2560
2561 right_nritems = btrfs_header_nritems(right);
2562 if (right_nritems == 0)
2563 return 1;
2564
2565 btrfs_assert_tree_locked(path->nodes[1]);
2566
2567 left = read_node_slot(root, path->nodes[1], slot - 1);
2568 btrfs_tree_lock(left);
2569 btrfs_set_lock_blocking(left);
2570
2571 free_space = btrfs_leaf_free_space(root, left);
2572 if (free_space < data_size) {
2573 ret = 1;
2574 goto out;
2575 }
2576
2577 /* cow and double check */
2578 ret = btrfs_cow_block(trans, root, left,
2579 path->nodes[1], slot - 1, &left, 0);
2580 if (ret) {
2581 /* we hit -ENOSPC, but it isn't fatal here */
2582 ret = 1;
2583 goto out;
2584 }
2585
2586 free_space = btrfs_leaf_free_space(root, left);
2587 if (free_space < data_size) {
2588 ret = 1;
2589 goto out;
2590 }
2591 2518
2592 if (empty) 2519 if (empty)
2593 nr = right_nritems; 2520 nr = right_nritems;
@@ -2755,6 +2682,154 @@ out:
2755} 2682}
2756 2683
2757/* 2684/*
2685 * push some data in the path leaf to the left, trying to free up at
2686 * least data_size bytes. returns zero if the push worked, nonzero otherwise
2687 */
2688static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
2689 *root, struct btrfs_path *path, int data_size,
2690 int empty)
2691{
2692 struct extent_buffer *right = path->nodes[0];
2693 struct extent_buffer *left;
2694 int slot;
2695 int free_space;
2696 u32 right_nritems;
2697 int ret = 0;
2698
2699 slot = path->slots[1];
2700 if (slot == 0)
2701 return 1;
2702 if (!path->nodes[1])
2703 return 1;
2704
2705 right_nritems = btrfs_header_nritems(right);
2706 if (right_nritems == 0)
2707 return 1;
2708
2709 btrfs_assert_tree_locked(path->nodes[1]);
2710
2711 left = read_node_slot(root, path->nodes[1], slot - 1);
2712 btrfs_tree_lock(left);
2713 btrfs_set_lock_blocking(left);
2714
2715 free_space = btrfs_leaf_free_space(root, left);
2716 if (free_space < data_size) {
2717 ret = 1;
2718 goto out;
2719 }
2720
2721 /* cow and double check */
2722 ret = btrfs_cow_block(trans, root, left,
2723 path->nodes[1], slot - 1, &left);
2724 if (ret) {
2725 /* we hit -ENOSPC, but it isn't fatal here */
2726 ret = 1;
2727 goto out;
2728 }
2729
2730 free_space = btrfs_leaf_free_space(root, left);
2731 if (free_space < data_size) {
2732 ret = 1;
2733 goto out;
2734 }
2735
2736 return __push_leaf_left(trans, root, path, data_size,
2737 empty, left, free_space, right_nritems);
2738out:
2739 btrfs_tree_unlock(left);
2740 free_extent_buffer(left);
2741 return ret;
2742}
2743
2744/*
2745 * split the path's leaf in two, making sure there is at least data_size
2746 * available for the resulting leaf level of the path.
2747 *
2748 * returns 0 if all went well and < 0 on failure.
2749 */
2750static noinline int copy_for_split(struct btrfs_trans_handle *trans,
2751 struct btrfs_root *root,
2752 struct btrfs_path *path,
2753 struct extent_buffer *l,
2754 struct extent_buffer *right,
2755 int slot, int mid, int nritems)
2756{
2757 int data_copy_size;
2758 int rt_data_off;
2759 int i;
2760 int ret = 0;
2761 int wret;
2762 struct btrfs_disk_key disk_key;
2763
2764 nritems = nritems - mid;
2765 btrfs_set_header_nritems(right, nritems);
2766 data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
2767
2768 copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
2769 btrfs_item_nr_offset(mid),
2770 nritems * sizeof(struct btrfs_item));
2771
2772 copy_extent_buffer(right, l,
2773 btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
2774 data_copy_size, btrfs_leaf_data(l) +
2775 leaf_data_end(root, l), data_copy_size);
2776
2777 rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
2778 btrfs_item_end_nr(l, mid);
2779
2780 for (i = 0; i < nritems; i++) {
2781 struct btrfs_item *item = btrfs_item_nr(right, i);
2782 u32 ioff;
2783
2784 if (!right->map_token) {
2785 map_extent_buffer(right, (unsigned long)item,
2786 sizeof(struct btrfs_item),
2787 &right->map_token, &right->kaddr,
2788 &right->map_start, &right->map_len,
2789 KM_USER1);
2790 }
2791
2792 ioff = btrfs_item_offset(right, item);
2793 btrfs_set_item_offset(right, item, ioff + rt_data_off);
2794 }
2795
2796 if (right->map_token) {
2797 unmap_extent_buffer(right, right->map_token, KM_USER1);
2798 right->map_token = NULL;
2799 }
2800
2801 btrfs_set_header_nritems(l, mid);
2802 ret = 0;
2803 btrfs_item_key(right, &disk_key, 0);
2804 wret = insert_ptr(trans, root, path, &disk_key, right->start,
2805 path->slots[1] + 1, 1);
2806 if (wret)
2807 ret = wret;
2808
2809 btrfs_mark_buffer_dirty(right);
2810 btrfs_mark_buffer_dirty(l);
2811 BUG_ON(path->slots[0] != slot);
2812
2813 ret = btrfs_update_ref(trans, root, l, right, 0, nritems);
2814 BUG_ON(ret);
2815
2816 if (mid <= slot) {
2817 btrfs_tree_unlock(path->nodes[0]);
2818 free_extent_buffer(path->nodes[0]);
2819 path->nodes[0] = right;
2820 path->slots[0] -= mid;
2821 path->slots[1] += 1;
2822 } else {
2823 btrfs_tree_unlock(right);
2824 free_extent_buffer(right);
2825 }
2826
2827 BUG_ON(path->slots[0] < 0);
2828
2829 return ret;
2830}
2831
2832/*
2758 * split the path's leaf in two, making sure there is at least data_size 2833 * split the path's leaf in two, making sure there is at least data_size
2759 * available for the resulting leaf level of the path. 2834 * available for the resulting leaf level of the path.
2760 * 2835 *
@@ -2771,17 +2846,14 @@ static noinline int split_leaf(struct btrfs_trans_handle *trans,
2771 int mid; 2846 int mid;
2772 int slot; 2847 int slot;
2773 struct extent_buffer *right; 2848 struct extent_buffer *right;
2774 int data_copy_size;
2775 int rt_data_off;
2776 int i;
2777 int ret = 0; 2849 int ret = 0;
2778 int wret; 2850 int wret;
2779 int double_split; 2851 int double_split;
2780 int num_doubles = 0; 2852 int num_doubles = 0;
2781 struct btrfs_disk_key disk_key;
2782 2853
2783 /* first try to make some room by pushing left and right */ 2854 /* first try to make some room by pushing left and right */
2784 if (data_size && ins_key->type != BTRFS_DIR_ITEM_KEY) { 2855 if (data_size && ins_key->type != BTRFS_DIR_ITEM_KEY &&
2856 !trans->transaction->delayed_refs.flushing) {
2785 wret = push_leaf_right(trans, root, path, data_size, 0); 2857 wret = push_leaf_right(trans, root, path, data_size, 0);
2786 if (wret < 0) 2858 if (wret < 0)
2787 return wret; 2859 return wret;
@@ -2830,11 +2902,14 @@ again:
2830 write_extent_buffer(right, root->fs_info->chunk_tree_uuid, 2902 write_extent_buffer(right, root->fs_info->chunk_tree_uuid,
2831 (unsigned long)btrfs_header_chunk_tree_uuid(right), 2903 (unsigned long)btrfs_header_chunk_tree_uuid(right),
2832 BTRFS_UUID_SIZE); 2904 BTRFS_UUID_SIZE);
2905
2833 if (mid <= slot) { 2906 if (mid <= slot) {
2834 if (nritems == 1 || 2907 if (nritems == 1 ||
2835 leaf_space_used(l, mid, nritems - mid) + data_size > 2908 leaf_space_used(l, mid, nritems - mid) + data_size >
2836 BTRFS_LEAF_DATA_SIZE(root)) { 2909 BTRFS_LEAF_DATA_SIZE(root)) {
2837 if (slot >= nritems) { 2910 if (slot >= nritems) {
2911 struct btrfs_disk_key disk_key;
2912
2838 btrfs_cpu_key_to_disk(&disk_key, ins_key); 2913 btrfs_cpu_key_to_disk(&disk_key, ins_key);
2839 btrfs_set_header_nritems(right, 0); 2914 btrfs_set_header_nritems(right, 0);
2840 wret = insert_ptr(trans, root, path, 2915 wret = insert_ptr(trans, root, path,
@@ -2862,6 +2937,8 @@ again:
2862 if (leaf_space_used(l, 0, mid) + data_size > 2937 if (leaf_space_used(l, 0, mid) + data_size >
2863 BTRFS_LEAF_DATA_SIZE(root)) { 2938 BTRFS_LEAF_DATA_SIZE(root)) {
2864 if (!extend && data_size && slot == 0) { 2939 if (!extend && data_size && slot == 0) {
2940 struct btrfs_disk_key disk_key;
2941
2865 btrfs_cpu_key_to_disk(&disk_key, ins_key); 2942 btrfs_cpu_key_to_disk(&disk_key, ins_key);
2866 btrfs_set_header_nritems(right, 0); 2943 btrfs_set_header_nritems(right, 0);
2867 wret = insert_ptr(trans, root, path, 2944 wret = insert_ptr(trans, root, path,
@@ -2894,76 +2971,16 @@ again:
2894 } 2971 }
2895 } 2972 }
2896 } 2973 }
2897 nritems = nritems - mid;
2898 btrfs_set_header_nritems(right, nritems);
2899 data_copy_size = btrfs_item_end_nr(l, mid) - leaf_data_end(root, l);
2900
2901 copy_extent_buffer(right, l, btrfs_item_nr_offset(0),
2902 btrfs_item_nr_offset(mid),
2903 nritems * sizeof(struct btrfs_item));
2904
2905 copy_extent_buffer(right, l,
2906 btrfs_leaf_data(right) + BTRFS_LEAF_DATA_SIZE(root) -
2907 data_copy_size, btrfs_leaf_data(l) +
2908 leaf_data_end(root, l), data_copy_size);
2909
2910 rt_data_off = BTRFS_LEAF_DATA_SIZE(root) -
2911 btrfs_item_end_nr(l, mid);
2912
2913 for (i = 0; i < nritems; i++) {
2914 struct btrfs_item *item = btrfs_item_nr(right, i);
2915 u32 ioff;
2916
2917 if (!right->map_token) {
2918 map_extent_buffer(right, (unsigned long)item,
2919 sizeof(struct btrfs_item),
2920 &right->map_token, &right->kaddr,
2921 &right->map_start, &right->map_len,
2922 KM_USER1);
2923 }
2924
2925 ioff = btrfs_item_offset(right, item);
2926 btrfs_set_item_offset(right, item, ioff + rt_data_off);
2927 }
2928
2929 if (right->map_token) {
2930 unmap_extent_buffer(right, right->map_token, KM_USER1);
2931 right->map_token = NULL;
2932 }
2933
2934 btrfs_set_header_nritems(l, mid);
2935 ret = 0;
2936 btrfs_item_key(right, &disk_key, 0);
2937 wret = insert_ptr(trans, root, path, &disk_key, right->start,
2938 path->slots[1] + 1, 1);
2939 if (wret)
2940 ret = wret;
2941
2942 btrfs_mark_buffer_dirty(right);
2943 btrfs_mark_buffer_dirty(l);
2944 BUG_ON(path->slots[0] != slot);
2945 2974
2946 ret = btrfs_update_ref(trans, root, l, right, 0, nritems); 2975 ret = copy_for_split(trans, root, path, l, right, slot, mid, nritems);
2947 BUG_ON(ret); 2976 BUG_ON(ret);
2948 2977
2949 if (mid <= slot) {
2950 btrfs_tree_unlock(path->nodes[0]);
2951 free_extent_buffer(path->nodes[0]);
2952 path->nodes[0] = right;
2953 path->slots[0] -= mid;
2954 path->slots[1] += 1;
2955 } else {
2956 btrfs_tree_unlock(right);
2957 free_extent_buffer(right);
2958 }
2959
2960 BUG_ON(path->slots[0] < 0);
2961
2962 if (double_split) { 2978 if (double_split) {
2963 BUG_ON(num_doubles != 0); 2979 BUG_ON(num_doubles != 0);
2964 num_doubles++; 2980 num_doubles++;
2965 goto again; 2981 goto again;
2966 } 2982 }
2983
2967 return ret; 2984 return ret;
2968} 2985}
2969 2986
@@ -3021,26 +3038,27 @@ int btrfs_split_item(struct btrfs_trans_handle *trans,
3021 return -EAGAIN; 3038 return -EAGAIN;
3022 } 3039 }
3023 3040
3041 btrfs_set_path_blocking(path);
3024 ret = split_leaf(trans, root, &orig_key, path, 3042 ret = split_leaf(trans, root, &orig_key, path,
3025 sizeof(struct btrfs_item), 1); 3043 sizeof(struct btrfs_item), 1);
3026 path->keep_locks = 0; 3044 path->keep_locks = 0;
3027 BUG_ON(ret); 3045 BUG_ON(ret);
3028 3046
3047 btrfs_unlock_up_safe(path, 1);
3048 leaf = path->nodes[0];
3049 BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item));
3050
3051split:
3029 /* 3052 /*
3030 * make sure any changes to the path from split_leaf leave it 3053 * make sure any changes to the path from split_leaf leave it
3031 * in a blocking state 3054 * in a blocking state
3032 */ 3055 */
3033 btrfs_set_path_blocking(path); 3056 btrfs_set_path_blocking(path);
3034 3057
3035 leaf = path->nodes[0];
3036 BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item));
3037
3038split:
3039 item = btrfs_item_nr(leaf, path->slots[0]); 3058 item = btrfs_item_nr(leaf, path->slots[0]);
3040 orig_offset = btrfs_item_offset(leaf, item); 3059 orig_offset = btrfs_item_offset(leaf, item);
3041 item_size = btrfs_item_size(leaf, item); 3060 item_size = btrfs_item_size(leaf, item);
3042 3061
3043
3044 buf = kmalloc(item_size, GFP_NOFS); 3062 buf = kmalloc(item_size, GFP_NOFS);
3045 read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf, 3063 read_extent_buffer(leaf, buf, btrfs_item_ptr_offset(leaf,
3046 path->slots[0]), item_size); 3064 path->slots[0]), item_size);
@@ -3445,39 +3463,27 @@ out:
3445} 3463}
3446 3464
3447/* 3465/*
3448 * Given a key and some data, insert items into the tree. 3466 * this is a helper for btrfs_insert_empty_items, the main goal here is
3449 * This does all the path init required, making room in the tree if needed. 3467 * to save stack depth by doing the bulk of the work in a function
3468 * that doesn't call btrfs_search_slot
3450 */ 3469 */
3451int btrfs_insert_empty_items(struct btrfs_trans_handle *trans, 3470static noinline_for_stack int
3452 struct btrfs_root *root, 3471setup_items_for_insert(struct btrfs_trans_handle *trans,
3453 struct btrfs_path *path, 3472 struct btrfs_root *root, struct btrfs_path *path,
3454 struct btrfs_key *cpu_key, u32 *data_size, 3473 struct btrfs_key *cpu_key, u32 *data_size,
3455 int nr) 3474 u32 total_data, u32 total_size, int nr)
3456{ 3475{
3457 struct extent_buffer *leaf;
3458 struct btrfs_item *item; 3476 struct btrfs_item *item;
3459 int ret = 0;
3460 int slot;
3461 int slot_orig;
3462 int i; 3477 int i;
3463 u32 nritems; 3478 u32 nritems;
3464 u32 total_size = 0;
3465 u32 total_data = 0;
3466 unsigned int data_end; 3479 unsigned int data_end;
3467 struct btrfs_disk_key disk_key; 3480 struct btrfs_disk_key disk_key;
3481 int ret;
3482 struct extent_buffer *leaf;
3483 int slot;
3468 3484
3469 for (i = 0; i < nr; i++)
3470 total_data += data_size[i];
3471
3472 total_size = total_data + (nr * sizeof(struct btrfs_item));
3473 ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
3474 if (ret == 0)
3475 return -EEXIST;
3476 if (ret < 0)
3477 goto out;
3478
3479 slot_orig = path->slots[0];
3480 leaf = path->nodes[0]; 3485 leaf = path->nodes[0];
3486 slot = path->slots[0];
3481 3487
3482 nritems = btrfs_header_nritems(leaf); 3488 nritems = btrfs_header_nritems(leaf);
3483 data_end = leaf_data_end(root, leaf); 3489 data_end = leaf_data_end(root, leaf);
@@ -3489,9 +3495,6 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
3489 BUG(); 3495 BUG();
3490 } 3496 }
3491 3497
3492 slot = path->slots[0];
3493 BUG_ON(slot < 0);
3494
3495 if (slot != nritems) { 3498 if (slot != nritems) {
3496 unsigned int old_data = btrfs_item_end_nr(leaf, slot); 3499 unsigned int old_data = btrfs_item_end_nr(leaf, slot);
3497 3500
@@ -3547,21 +3550,60 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
3547 data_end -= data_size[i]; 3550 data_end -= data_size[i];
3548 btrfs_set_item_size(leaf, item, data_size[i]); 3551 btrfs_set_item_size(leaf, item, data_size[i]);
3549 } 3552 }
3553
3550 btrfs_set_header_nritems(leaf, nritems + nr); 3554 btrfs_set_header_nritems(leaf, nritems + nr);
3551 btrfs_mark_buffer_dirty(leaf);
3552 3555
3553 ret = 0; 3556 ret = 0;
3554 if (slot == 0) { 3557 if (slot == 0) {
3558 struct btrfs_disk_key disk_key;
3555 btrfs_cpu_key_to_disk(&disk_key, cpu_key); 3559 btrfs_cpu_key_to_disk(&disk_key, cpu_key);
3556 ret = fixup_low_keys(trans, root, path, &disk_key, 1); 3560 ret = fixup_low_keys(trans, root, path, &disk_key, 1);
3557 } 3561 }
3562 btrfs_unlock_up_safe(path, 1);
3563 btrfs_mark_buffer_dirty(leaf);
3558 3564
3559 if (btrfs_leaf_free_space(root, leaf) < 0) { 3565 if (btrfs_leaf_free_space(root, leaf) < 0) {
3560 btrfs_print_leaf(root, leaf); 3566 btrfs_print_leaf(root, leaf);
3561 BUG(); 3567 BUG();
3562 } 3568 }
3569 return ret;
3570}
3571
3572/*
3573 * Given a key and some data, insert items into the tree.
3574 * This does all the path init required, making room in the tree if needed.
3575 */
3576int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
3577 struct btrfs_root *root,
3578 struct btrfs_path *path,
3579 struct btrfs_key *cpu_key, u32 *data_size,
3580 int nr)
3581{
3582 struct extent_buffer *leaf;
3583 int ret = 0;
3584 int slot;
3585 int i;
3586 u32 total_size = 0;
3587 u32 total_data = 0;
3588
3589 for (i = 0; i < nr; i++)
3590 total_data += data_size[i];
3591
3592 total_size = total_data + (nr * sizeof(struct btrfs_item));
3593 ret = btrfs_search_slot(trans, root, cpu_key, path, total_size, 1);
3594 if (ret == 0)
3595 return -EEXIST;
3596 if (ret < 0)
3597 goto out;
3598
3599 leaf = path->nodes[0];
3600 slot = path->slots[0];
3601 BUG_ON(slot < 0);
3602
3603 ret = setup_items_for_insert(trans, root, path, cpu_key, data_size,
3604 total_data, total_size, nr);
3605
3563out: 3606out:
3564 btrfs_unlock_up_safe(path, 1);
3565 return ret; 3607 return ret;
3566} 3608}
3567 3609
@@ -3749,7 +3791,8 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3749 } 3791 }
3750 3792
3751 /* delete the leaf if it is mostly empty */ 3793 /* delete the leaf if it is mostly empty */
3752 if (used < BTRFS_LEAF_DATA_SIZE(root) / 4) { 3794 if (used < BTRFS_LEAF_DATA_SIZE(root) / 4 &&
3795 !trans->transaction->delayed_refs.flushing) {
3753 /* push_leaf_left fixes the path. 3796 /* push_leaf_left fixes the path.
3754 * make sure the path still points to our leaf 3797 * make sure the path still points to our leaf
3755 * for possible call to del_ptr below 3798 * for possible call to del_ptr below
@@ -3757,6 +3800,7 @@ int btrfs_del_items(struct btrfs_trans_handle *trans, struct btrfs_root *root,
3757 slot = path->slots[1]; 3800 slot = path->slots[1];
3758 extent_buffer_get(leaf); 3801 extent_buffer_get(leaf);
3759 3802
3803 btrfs_set_path_blocking(path);
3760 wret = push_leaf_left(trans, root, path, 1, 1); 3804 wret = push_leaf_left(trans, root, path, 1, 1);
3761 if (wret < 0 && wret != -ENOSPC) 3805 if (wret < 0 && wret != -ENOSPC)
3762 ret = wret; 3806 ret = wret;