1 files changed, 226 insertions, 8 deletions

diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c
index 3b6e35aafc9e..3af777357acb 100644
--- a/fs/btrfs/ctree.c
+++ b/fs/btrfs/ctree.c
@@ -54,6 +54,31 @@ struct btrfs_path *btrfs_alloc_path(void)
         return path;
 }
 
+/*
+ * set all locked nodes in the path to blocking locks.  This should
+ * be done before scheduling
+ */
+noinline void btrfs_set_path_blocking(struct btrfs_path *p)
+{
+        int i;
+        for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
+                if (p->nodes[i] && p->locks[i])
+                        btrfs_set_lock_blocking(p->nodes[i]);
+        }
+}
+
+/*
+ * reset all the locked nodes in the patch to spinning locks.
+ */
+noinline void btrfs_clear_path_blocking(struct btrfs_path *p)
+{
+        int i;
+        for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
+                if (p->nodes[i] && p->locks[i])
+                        btrfs_clear_lock_blocking(p->nodes[i]);
+        }
+}
+
 /* this also releases the path */
 void btrfs_free_path(struct btrfs_path *p)
 {
@@ -272,6 +297,8 @@ static noinline int __btrfs_cow_block(struct btrfs_trans_handle *trans,
         if (IS_ERR(cow))
                 return PTR_ERR(cow);
 
+        /* cow is set to blocking by btrfs_init_new_buffer */
+
         copy_extent_buffer(cow, buf, 0, 0, cow->len);
         btrfs_set_header_bytenr(cow, cow->start);
         btrfs_set_header_generation(cow, trans->transid);
@@ -397,6 +424,11 @@ noinline int btrfs_cow_block(struct btrfs_trans_handle *trans,
         }
 
         search_start = buf->start & ~((u64)(1024 * 1024 * 1024) - 1);
+
+        if (parent)
+                btrfs_set_lock_blocking(parent);
+        btrfs_set_lock_blocking(buf);
+
         ret = __btrfs_cow_block(trans, root, buf, parent,
                                  parent_slot, cow_ret, search_start, 0,
                                  prealloc_dest);
@@ -502,6 +534,8 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans,
         if (parent_nritems == 1)
                 return 0;
 
+        btrfs_set_lock_blocking(parent);
+
         for (i = start_slot; i < end_slot; i++) {
                 int close = 1;
 
@@ -562,6 +596,7 @@ int btrfs_realloc_node(struct btrfs_trans_handle *trans,
                         search_start = last_block;
 
                 btrfs_tree_lock(cur);
+                btrfs_set_lock_blocking(cur);
                 err = __btrfs_cow_block(trans, root, cur, parent, i,
                                         &cur, search_start,
                                         min(16 * blocksize,
@@ -860,6 +895,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
                 return 0;
 
         mid = path->nodes[level];
+
         WARN_ON(!path->locks[level]);
         WARN_ON(btrfs_header_generation(mid) != trans->transid);
 
@@ -882,6 +918,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
                 /* promote the child to a root */
                 child = read_node_slot(root, mid, 0);
                 btrfs_tree_lock(child);
+                btrfs_set_lock_blocking(child);
                 BUG_ON(!child);
                 ret = btrfs_cow_block(trans, root, child, mid, 0, &child, 0);
                 BUG_ON(ret);
@@ -898,6 +935,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
 
                 add_root_to_dirty_list(root);
                 btrfs_tree_unlock(child);
+
                 path->locks[level] = 0;
                 path->nodes[level] = NULL;
                 clean_tree_block(trans, root, mid);
@@ -922,6 +960,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
         left = read_node_slot(root, parent, pslot - 1);
         if (left) {
                 btrfs_tree_lock(left);
+                btrfs_set_lock_blocking(left);
                 wret = btrfs_cow_block(trans, root, left,
                                        parent, pslot - 1, &left, 0);
                 if (wret) {
@@ -932,6 +971,7 @@ static noinline int balance_level(struct btrfs_trans_handle *trans,
         right = read_node_slot(root, parent, pslot + 1);
         if (right) {
                 btrfs_tree_lock(right);
+                btrfs_set_lock_blocking(right);
                 wret = btrfs_cow_block(trans, root, right,
                                        parent, pslot + 1, &right, 0);
                 if (wret) {
@@ -1107,6 +1147,8 @@ static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
                 u32 left_nr;
 
                 btrfs_tree_lock(left);
+                btrfs_set_lock_blocking(left);
+
                 left_nr = btrfs_header_nritems(left);
                 if (left_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
                         wret = 1;
@@ -1153,7 +1195,10 @@ static noinline int push_nodes_for_insert(struct btrfs_trans_handle *trans,
          */
         if (right) {
                 u32 right_nr;
+
                 btrfs_tree_lock(right);
+                btrfs_set_lock_blocking(right);
+
                 right_nr = btrfs_header_nritems(right);
                 if (right_nr >= BTRFS_NODEPTRS_PER_BLOCK(root) - 1) {
                         wret = 1;
@@ -1265,6 +1310,68 @@ static noinline void reada_for_search(struct btrfs_root *root,
 }
 
 /*
+ * returns -EAGAIN if it had to drop the path, or zero if everything was in
+ * cache
+ */
+static noinline int reada_for_balance(struct btrfs_root *root,
+                                      struct btrfs_path *path, int level)
+{
+        int slot;
+        int nritems;
+        struct extent_buffer *parent;
+        struct extent_buffer *eb;
+        u64 gen;
+        u64 block1 = 0;
+        u64 block2 = 0;
+        int ret = 0;
+        int blocksize;
+
+        parent = path->nodes[level - 1];
+        if (!parent)
+                return 0;
+
+        nritems = btrfs_header_nritems(parent);
+        slot = path->slots[level];
+        blocksize = btrfs_level_size(root, level);
+
+        if (slot > 0) {
+                block1 = btrfs_node_blockptr(parent, slot - 1);
+                gen = btrfs_node_ptr_generation(parent, slot - 1);
+                eb = btrfs_find_tree_block(root, block1, blocksize);
+                if (eb && btrfs_buffer_uptodate(eb, gen))
+                        block1 = 0;
+                free_extent_buffer(eb);
+        }
+        if (slot < nritems) {
+                block2 = btrfs_node_blockptr(parent, slot + 1);
+                gen = btrfs_node_ptr_generation(parent, slot + 1);
+                eb = btrfs_find_tree_block(root, block2, blocksize);
+                if (eb && btrfs_buffer_uptodate(eb, gen))
+                        block2 = 0;
+                free_extent_buffer(eb);
+        }
+        if (block1 || block2) {
+                ret = -EAGAIN;
+                btrfs_release_path(root, path);
+                if (block1)
+                        readahead_tree_block(root, block1, blocksize, 0);
+                if (block2)
+                        readahead_tree_block(root, block2, blocksize, 0);
+
+                if (block1) {
+                        eb = read_tree_block(root, block1, blocksize, 0);
+                        free_extent_buffer(eb);
+                }
+                if (block1) {
+                        eb = read_tree_block(root, block2, blocksize, 0);
+                        free_extent_buffer(eb);
+                }
+        }
+        return ret;
+}
+
+
+/*
  * when we walk down the tree, it is usually safe to unlock the higher layers
  * in the tree.  The exceptions are when our path goes through slot 0, because
  * operations on the tree might require changing key pointers higher up in the
@@ -1315,6 +1422,32 @@ static noinline void unlock_up(struct btrfs_path *path, int level,
 }
 
 /*
+ * This releases any locks held in the path starting at level and
+ * going all the way up to the root.
+ *
+ * btrfs_search_slot will keep the lock held on higher nodes in a few
+ * corner cases, such as COW of the block at slot zero in the node.  This
+ * ignores those rules, and it should only be called when there are no
+ * more updates to be done higher up in the tree.
+ */
+noinline void btrfs_unlock_up_safe(struct btrfs_path *path, int level)
+{
+        int i;
+
+        if (path->keep_locks || path->lowest_level)
+                return;
+
+        for (i = level; i < BTRFS_MAX_LEVEL; i++) {
+                if (!path->nodes[i])
+                        break;
+                if (!path->locks[i])
+                        break;
+                btrfs_tree_unlock(path->nodes[i]);
+                path->locks[i] = 0;
+        }
+}
+
+/*
  * look for key in the tree.  path is filled in with nodes along the way
  * if key is found, we return zero and you can find the item in the leaf
  * level of the path (level 0)
@@ -1385,6 +1518,7 @@ again:
                          */
                         if (prealloc_block.objectid &&
                             prealloc_block.offset != b->len) {
+                                btrfs_set_path_blocking(p);
                                 btrfs_free_reserved_extent(root,
                                            prealloc_block.objectid,
                                            prealloc_block.offset);
@@ -1409,6 +1543,8 @@ again:
                                 goto again;
                         }
 
+                        btrfs_set_path_blocking(p);
+
                         wret = btrfs_cow_block(trans, root, b,
                                                p->nodes[level + 1],
                                                p->slots[level + 1],
@@ -1430,6 +1566,22 @@ cow_done:
                 if (!p->skip_locking)
                         p->locks[level] = 1;
 
+                btrfs_clear_path_blocking(p);
+
+                /*
+                 * we have a lock on b and as long as we aren't changing
+                 * the tree, there is no way to for the items in b to change.
+                 * It is safe to drop the lock on our parent before we
+                 * go through the expensive btree search on b.
+                 *
+                 * If cow is true, then we might be changing slot zero,
+                 * which may require changing the parent.  So, we can't
+                 * drop the lock until after we know which slot we're
+                 * operating on.
+                 */
+                if (!cow)
+                        btrfs_unlock_up_safe(p, level + 1);
+
                 ret = check_block(root, p, level);
                 if (ret) {
                         ret = -1;
@@ -1437,6 +1589,7 @@ cow_done:
                 }
 
                 ret = bin_search(b, key, level, &slot);
+
                 if (level != 0) {
                         if (ret && slot > 0)
                                 slot -= 1;
@@ -1444,7 +1597,16 @@ cow_done:
                         if ((p->search_for_split || ins_len > 0) &&
                             btrfs_header_nritems(b) >=
                             BTRFS_NODEPTRS_PER_BLOCK(root) - 3) {
-                                int sret = split_node(trans, root, p, level);
+                                int sret;
+
+                                sret = reada_for_balance(root, p, level);
+                                if (sret)
+                                        goto again;
+
+                                btrfs_set_path_blocking(p);
+                                sret = split_node(trans, root, p, level);
+                                btrfs_clear_path_blocking(p);
+
                                 BUG_ON(sret > 0);
                                 if (sret) {
                                         ret = sret;
@@ -1453,8 +1615,16 @@ cow_done:
                                 b = p->nodes[level];
                                 slot = p->slots[level];
                         } else if (ins_len < 0) {
-                                int sret = balance_level(trans, root, p,
-                                                         level);
+                                int sret;
+
+                                sret = reada_for_balance(root, p, level);
+                                if (sret)
+                                        goto again;
+
+                                btrfs_set_path_blocking(p);
+                                sret = balance_level(trans, root, p, level);
+                                btrfs_clear_path_blocking(p);
+
                                 if (sret) {
                                         ret = sret;
                                         goto done;
@@ -1488,7 +1658,7 @@ cow_done:
                                  * of the btree by dropping locks before
                                  * we read.
                                  */
-                                if (level > 1) {
+                                if (level > 0) {
                                         btrfs_release_path(NULL, p);
                                         if (tmp)
                                                 free_extent_buffer(tmp);
@@ -1503,6 +1673,7 @@ cow_done:
                                                 free_extent_buffer(tmp);
                                         goto again;
                                 } else {
+                                        btrfs_set_path_blocking(p);
                                         if (tmp)
                                                 free_extent_buffer(tmp);
                                         if (should_reada)
@@ -1512,14 +1683,29 @@ cow_done:
                                         b = read_node_slot(root, b, slot);
                                 }
                         }
-                        if (!p->skip_locking)
-                                btrfs_tree_lock(b);
+                        if (!p->skip_locking) {
+                                int lret;
+
+                                btrfs_clear_path_blocking(p);
+                                lret = btrfs_try_spin_lock(b);
+
+                                if (!lret) {
+                                        btrfs_set_path_blocking(p);
+                                        btrfs_tree_lock(b);
+                                        btrfs_clear_path_blocking(p);
+                                }
+                        }
                 } else {
                         p->slots[level] = slot;
                         if (ins_len > 0 &&
                             btrfs_leaf_free_space(root, b) < ins_len) {
-                                int sret = split_leaf(trans, root, key,
+                                int sret;
+
+                                btrfs_set_path_blocking(p);
+                                sret = split_leaf(trans, root, key,
                                                       p, ins_len, ret == 0);
+                                btrfs_clear_path_blocking(p);
+
                                 BUG_ON(sret > 0);
                                 if (sret) {
                                         ret = sret;
@@ -1533,12 +1719,16 @@ cow_done:
         }
         ret = 1;
 done:
+        /*
+         * we don't really know what they plan on doing with the path
+         * from here on, so for now just mark it as blocking
+         */
+        btrfs_set_path_blocking(p);
         if (prealloc_block.objectid) {
                 btrfs_free_reserved_extent(root,
                            prealloc_block.objectid,
                            prealloc_block.offset);
         }
-
         return ret;
 }
 
@@ -1562,6 +1752,8 @@ int btrfs_merge_path(struct btrfs_trans_handle *trans,
         ret = btrfs_cow_block(trans, root, eb, NULL, 0, &eb, 0);
         BUG_ON(ret);
 
+        btrfs_set_lock_blocking(eb);
+
         parent = eb;
         while (1) {
                 level = btrfs_header_level(parent);
@@ -1586,6 +1778,7 @@ int btrfs_merge_path(struct btrfs_trans_handle *trans,
                         eb = read_tree_block(root, bytenr, blocksize,
                                              generation);
                         btrfs_tree_lock(eb);
+                        btrfs_set_lock_blocking(eb);
                 }
 
                 /*
@@ -1610,6 +1803,7 @@ int btrfs_merge_path(struct btrfs_trans_handle *trans,
                                 eb = read_tree_block(root, bytenr, blocksize,
                                                 generation);
                                 btrfs_tree_lock(eb);
+                                btrfs_set_lock_blocking(eb);
                         }
 
                         ret = btrfs_cow_block(trans, root, eb, parent, slot,
@@ -2156,6 +2350,8 @@ static int push_leaf_right(struct btrfs_trans_handle *trans, struct btrfs_root
 
         right = read_node_slot(root, upper, slot + 1);
         btrfs_tree_lock(right);
+        btrfs_set_lock_blocking(right);
+
         free_space = btrfs_leaf_free_space(root, right);
         if (free_space < data_size)
                 goto out_unlock;
@@ -2351,6 +2547,8 @@ static int push_leaf_left(struct btrfs_trans_handle *trans, struct btrfs_root
 
         left = read_node_slot(root, path->nodes[1], slot - 1);
         btrfs_tree_lock(left);
+        btrfs_set_lock_blocking(left);
+
         free_space = btrfs_leaf_free_space(root, left);
         if (free_space < data_size) {
                 ret = 1;
@@ -2809,6 +3007,12 @@ int btrfs_split_item(struct btrfs_trans_handle *trans,
         path->keep_locks = 0;
         BUG_ON(ret);
 
+        /*
+         * make sure any changes to the path from split_leaf leave it
+         * in a blocking state
+         */
+        btrfs_set_path_blocking(path);
+
         leaf = path->nodes[0];
         BUG_ON(btrfs_leaf_free_space(root, leaf) < sizeof(struct btrfs_item));
 
@@ -3338,6 +3542,7 @@ int btrfs_insert_empty_items(struct btrfs_trans_handle *trans,
                 BUG();
         }
 out:
+        btrfs_unlock_up_safe(path, 1);
         return ret;
 }
 
@@ -3705,12 +3910,14 @@ find_next_key:
                  */
                 if (slot >= nritems) {
                         path->slots[level] = slot;
+                        btrfs_set_path_blocking(path);
                         sret = btrfs_find_next_key(root, path, min_key, level,
                                                   cache_only, min_trans);
                         if (sret == 0) {
                                 btrfs_release_path(root, path);
                                 goto again;
                         } else {
+                                btrfs_clear_path_blocking(path);
                                 goto out;
                         }
                 }
@@ -3722,16 +3929,20 @@ find_next_key:
                         unlock_up(path, level, 1);
                         goto out;
                 }
+                btrfs_set_path_blocking(path);
                 cur = read_node_slot(root, cur, slot);
 
                 btrfs_tree_lock(cur);
+
                 path->locks[level - 1] = 1;
                 path->nodes[level - 1] = cur;
                 unlock_up(path, level, 1);
+                btrfs_clear_path_blocking(path);
         }
 out:
         if (ret == 0)
                 memcpy(min_key, &found_key, sizeof(found_key));
+        btrfs_set_path_blocking(path);
         return ret;
 }
 
@@ -3827,6 +4038,7 @@ int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
         if (ret < 0)
                 return ret;
 
+        btrfs_set_path_blocking(path);
         nritems = btrfs_header_nritems(path->nodes[0]);
         /*
          * by releasing the path above we dropped all our locks.  A balance
@@ -3857,6 +4069,7 @@ int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
                         free_extent_buffer(next);
                 }
 
+                /* the path was set to blocking above */
                 if (level == 1 && (path->locks[1] || path->skip_locking) &&
                     path->reada)
                         reada_for_search(root, path, level, slot, 0);
@@ -3865,6 +4078,7 @@ int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
                 if (!path->skip_locking) {
                         WARN_ON(!btrfs_tree_locked(c));
                         btrfs_tree_lock(next);
+                        btrfs_set_lock_blocking(next);
                 }
                 break;
         }
@@ -3881,12 +4095,15 @@ int btrfs_next_leaf(struct btrfs_root *root, struct btrfs_path *path)
                         path->locks[level] = 1;
                 if (!level)
                         break;
+
+                btrfs_set_path_blocking(path);
                 if (level == 1 && path->locks[1] && path->reada)
                         reada_for_search(root, path, level, slot, 0);
                 next = read_node_slot(root, next, 0);
                 if (!path->skip_locking) {
                         WARN_ON(!btrfs_tree_locked(path->nodes[level]));
                         btrfs_tree_lock(next);
+                        btrfs_set_lock_blocking(next);
                 }
         }
 done:
@@ -3911,6 +4128,7 @@ int btrfs_previous_item(struct btrfs_root *root,
 
         while (1) {
                 if (path->slots[0] == 0) {
+                        btrfs_set_path_blocking(path);
                         ret = btrfs_prev_leaf(root, path);
                         if (ret != 0)
                                 return ret;