raid5: lockless access raid5 overrided bi_phys_segments

Raid5 overrides bio->bi_phys_segments, accessing it is with device_lock hold,
which is unnecessary, We can make it lockless actually.

Signed-off-by: Shaohua Li <shli@fusionio.com>
Signed-off-by: NeilBrown <neilb@suse.de>

1 files changed, 32 insertions, 30 deletions

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 848034666342..6ef1eeb68f7c 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -99,34 +99,40 @@ static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
  * We maintain a biased count of active stripes in the bottom 16 bits of
  * bi_phys_segments, and a count of processed stripes in the upper 16 bits
  */
-static inline int raid5_bi_phys_segments(struct bio *bio)
+static inline int raid5_bi_processed_stripes(struct bio *bio)
 {
-        return bio->bi_phys_segments & 0xffff;
+        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+        return (atomic_read(segments) >> 16) & 0xffff;
 }
 
-static inline int raid5_bi_hw_segments(struct bio *bio)
+static inline int raid5_dec_bi_active_stripes(struct bio *bio)
 {
-        return (bio->bi_phys_segments >> 16) & 0xffff;
+        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+        return atomic_sub_return(1, segments) & 0xffff;
 }
 
-static inline int raid5_dec_bi_phys_segments(struct bio *bio)
+static inline void raid5_inc_bi_active_stripes(struct bio *bio)
 {
-        --bio->bi_phys_segments;
-        return raid5_bi_phys_segments(bio);
+        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+        atomic_inc(segments);
 }
 
-static inline int raid5_dec_bi_hw_segments(struct bio *bio)
+static inline void raid5_set_bi_processed_stripes(struct bio *bio,
+        unsigned int cnt)
 {
-        unsigned short val = raid5_bi_hw_segments(bio);
+        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+        int old, new;
 
-        --val;
-        bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
-        return val;
+        do {
+                old = atomic_read(segments);
+                new = (old & 0xffff) | (cnt << 16);
+        } while (atomic_cmpxchg(segments, old, new) != old);
 }
 
-static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
+static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
 {
-        bio->bi_phys_segments = raid5_bi_phys_segments(bio) | (cnt << 16);
+        atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
+        atomic_set(segments, cnt);
 }
 
 /* Find first data disk in a raid6 stripe */
@@ -781,7 +787,7 @@ static void ops_complete_biofill(void *stripe_head_ref)
                         while (rbi && rbi->bi_sector <
                                 dev->sector + STRIPE_SECTORS) {
                                 rbi2 = r5_next_bio(rbi, dev->sector);
-                                if (!raid5_dec_bi_phys_segments(rbi)) {
+                                if (!raid5_dec_bi_active_stripes(rbi)) {
                                         rbi->bi_next = return_bi;
                                         return_bi = rbi;
                                 }
@@ -2367,7 +2373,7 @@ static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, in
         if (*bip)
                 bi->bi_next = *bip;
         *bip = bi;
-        bi->bi_phys_segments++;
+        raid5_inc_bi_active_stripes(bi);
 
         if (forwrite) {
                 /* check if page is covered */
@@ -2464,7 +2470,7 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
                         sh->dev[i].sector + STRIPE_SECTORS) {
                         struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
-                        if (!raid5_dec_bi_phys_segments(bi)) {
+                        if (!raid5_dec_bi_active_stripes(bi)) {
                                 md_write_end(conf->mddev);
                                 bi->bi_next = *return_bi;
                                 *return_bi = bi;
@@ -2479,7 +2485,7 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
                        sh->dev[i].sector + STRIPE_SECTORS) {
                         struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
                         clear_bit(BIO_UPTODATE, &bi->bi_flags);
-                        if (!raid5_dec_bi_phys_segments(bi)) {
+                        if (!raid5_dec_bi_active_stripes(bi)) {
                                 md_write_end(conf->mddev);
                                 bi->bi_next = *return_bi;
                                 *return_bi = bi;
@@ -2503,7 +2509,7 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
                                 struct bio *nextbi =
                                         r5_next_bio(bi, sh->dev[i].sector);
                                 clear_bit(BIO_UPTODATE, &bi->bi_flags);
-                                if (!raid5_dec_bi_phys_segments(bi)) {
+                                if (!raid5_dec_bi_active_stripes(bi)) {
                                         bi->bi_next = *return_bi;
                                         *return_bi = bi;
                                 }
@@ -2722,7 +2728,7 @@ static void handle_stripe_clean_event(struct r5conf *conf,
                                 while (wbi && wbi->bi_sector <
                                         dev->sector + STRIPE_SECTORS) {
                                         wbi2 = r5_next_bio(wbi, dev->sector);
-                                        if (!raid5_dec_bi_phys_segments(wbi)) {
+                                        if (!raid5_dec_bi_active_stripes(wbi)) {
                                                 md_write_end(conf->mddev);
                                                 wbi->bi_next = *return_bi;
                                                 *return_bi = wbi;
@@ -3798,7 +3804,7 @@ static struct bio *remove_bio_from_retry(struct r5conf *conf)
                  * this sets the active strip count to 1 and the processed
                  * strip count to zero (upper 8 bits)
                  */
-                bi->bi_phys_segments = 1; /* biased count of active stripes */
+                raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
         }
 
         return bi;
@@ -4133,9 +4139,7 @@ static void make_request(struct mddev *mddev, struct bio * bi)
                 }
         }
 
-        spin_lock_irq(&conf->device_lock);
-        remaining = raid5_dec_bi_phys_segments(bi);
-        spin_unlock_irq(&conf->device_lock);
+        remaining = raid5_dec_bi_active_stripes(bi);
         if (remaining == 0) {
 
                 if ( rw == WRITE )
@@ -4491,7 +4495,7 @@ static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
                      sector += STRIPE_SECTORS,
                      scnt++) {
 
-                if (scnt < raid5_bi_hw_segments(raid_bio))
+                if (scnt < raid5_bi_processed_stripes(raid_bio))
                         /* already done this stripe */
                         continue;
 
@@ -4499,14 +4503,14 @@ static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
 
                 if (!sh) {
                         /* failed to get a stripe - must wait */
-                        raid5_set_bi_hw_segments(raid_bio, scnt);
+                        raid5_set_bi_processed_stripes(raid_bio, scnt);
                         conf->retry_read_aligned = raid_bio;
                         return handled;
                 }
 
                 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
                         release_stripe(sh);
-                        raid5_set_bi_hw_segments(raid_bio, scnt);
+                        raid5_set_bi_processed_stripes(raid_bio, scnt);
                         conf->retry_read_aligned = raid_bio;
                         return handled;
                 }
@@ -4515,9 +4519,7 @@ static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
                 release_stripe(sh);
                 handled++;
         }
-        spin_lock_irq(&conf->device_lock);
-        remaining = raid5_dec_bi_phys_segments(raid_bio);
-        spin_unlock_irq(&conf->device_lock);
+        remaining = raid5_dec_bi_active_stripes(raid_bio);
         if (remaining == 0)
                 bio_endio(raid_bio, 0);
         if (atomic_dec_and_test(&conf->active_aligned_reads))