raid5: reduce chance release_stripe() taking device_lock

release_stripe() is a place conf->device_lock is heavily contended. We take the
lock even stripe count isn't 1, which isn't required.

Signed-off-by: Shaohua Li <shli@fusionio.com>
Signed-off-by: NeilBrown <neilb@suse.de>

1 files changed, 41 insertions, 34 deletions

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 04348d76bb30..848034666342 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -190,49 +190,56 @@ static int stripe_operations_active(struct stripe_head *sh)
                test_bit(STRIPE_COMPUTE_RUN, &sh->state);
 }
 
-static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
-{
-        if (atomic_dec_and_test(&sh->count)) {
-                BUG_ON(!list_empty(&sh->lru));
-                BUG_ON(atomic_read(&conf->active_stripes)==0);
-                if (test_bit(STRIPE_HANDLE, &sh->state)) {
-                        if (test_bit(STRIPE_DELAYED, &sh->state) &&
-                            !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-                                list_add_tail(&sh->lru, &conf->delayed_list);
-                        else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
-                                   sh->bm_seq - conf->seq_write > 0)
-                                list_add_tail(&sh->lru, &conf->bitmap_list);
-                        else {
-                                clear_bit(STRIPE_DELAYED, &sh->state);
-                                clear_bit(STRIPE_BIT_DELAY, &sh->state);
-                                list_add_tail(&sh->lru, &conf->handle_list);
-                        }
-                        md_wakeup_thread(conf->mddev->thread);
-                } else {
-                        BUG_ON(stripe_operations_active(sh));
-                        if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
-                                if (atomic_dec_return(&conf->preread_active_stripes)
-                                    < IO_THRESHOLD)
-                                        md_wakeup_thread(conf->mddev->thread);
-                        atomic_dec(&conf->active_stripes);
-                        if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
-                                list_add_tail(&sh->lru, &conf->inactive_list);
-                                wake_up(&conf->wait_for_stripe);
-                                if (conf->retry_read_aligned)
-                                        md_wakeup_thread(conf->mddev->thread);
-                        }
+static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh)
+{
+        BUG_ON(!list_empty(&sh->lru));
+        BUG_ON(atomic_read(&conf->active_stripes)==0);
+        if (test_bit(STRIPE_HANDLE, &sh->state)) {
+                if (test_bit(STRIPE_DELAYED, &sh->state) &&
+                    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                        list_add_tail(&sh->lru, &conf->delayed_list);
+                else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
+                           sh->bm_seq - conf->seq_write > 0)
+                        list_add_tail(&sh->lru, &conf->bitmap_list);
+                else {
+                        clear_bit(STRIPE_DELAYED, &sh->state);
+                        clear_bit(STRIPE_BIT_DELAY, &sh->state);
+                        list_add_tail(&sh->lru, &conf->handle_list);
+                }
+                md_wakeup_thread(conf->mddev->thread);
+        } else {
+                BUG_ON(stripe_operations_active(sh));
+                if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
+                        if (atomic_dec_return(&conf->preread_active_stripes)
+                            < IO_THRESHOLD)
+                                md_wakeup_thread(conf->mddev->thread);
+                atomic_dec(&conf->active_stripes);
+                if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
+                        list_add_tail(&sh->lru, &conf->inactive_list);
+                        wake_up(&conf->wait_for_stripe);
+                        if (conf->retry_read_aligned)
+                                md_wakeup_thread(conf->mddev->thread);
                 }
         }
 }
 
+static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
+{
+        if (atomic_dec_and_test(&sh->count))
+                do_release_stripe(conf, sh);
+}
+
 static void release_stripe(struct stripe_head *sh)
 {
         struct r5conf *conf = sh->raid_conf;
         unsigned long flags;
 
-        spin_lock_irqsave(&conf->device_lock, flags);
-        __release_stripe(conf, sh);
-        spin_unlock_irqrestore(&conf->device_lock, flags);
+        local_irq_save(flags);
+        if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
+                do_release_stripe(conf, sh);
+                spin_unlock(&conf->device_lock);
+        }
+        local_irq_restore(flags);
 }
 
 static inline void remove_hash(struct stripe_head *sh)