md/raid5: activate raid6 rmw feature

Glue it altogehter. The raid6 rmw path should work the same as the
already existing raid5 logic. So emulate the prexor handling/flags
and split functions as needed.

1) Enable xor_syndrome() in the async layer.

2) Split ops_run_prexor() into RAID4/5 and RAID6 logic. Xor the syndrome
at the start of a rmw run as we did it before for the single parity.

3) Take care of rmw run in ops_run_reconstruct6(). Again process only
the changed pages to get syndrome back into sync.

4) Enhance set_syndrome_sources() to fill NULL pages if we are in a rmw
run. The lower layers will calculate start & end pages from that and
call the xor_syndrome() correspondingly.

5) Adapt the several places where we ignored Q handling up to now.

Performance numbers for a single E5630 system with a mix of 10 7200k
desktop/server disks. 300 seconds random write with 8 threads onto a
3,2TB (10*400GB) RAID6 64K chunk without spare (group_thread_cnt=4)

bsize   rmw_level=1   rmw_level=0   rmw_level=1   rmw_level=0
        skip_copy=1   skip_copy=1   skip_copy=0   skip_copy=0
   4K      115 KB/s      141 KB/s      165 KB/s      140 KB/s
   8K      225 KB/s      275 KB/s      324 KB/s      274 KB/s
  16K      434 KB/s      536 KB/s      640 KB/s      534 KB/s
  32K      751 KB/s    1,051 KB/s    1,234 KB/s    1,045 KB/s
  64K    1,339 KB/s    1,958 KB/s    2,282 KB/s    1,962 KB/s
 128K    2,673 KB/s    3,862 KB/s    4,113 KB/s    3,898 KB/s
 256K    7,685 KB/s    7,539 KB/s    7,557 KB/s    7,638 KB/s
 512K   19,556 KB/s   19,558 KB/s   19,652 KB/s   19,688 Kb/s

Signed-off-by: Markus Stockhausen <stockhausen@collogia.de>
Signed-off-by: NeilBrown <neilb@suse.de>

4 files changed, 115 insertions, 30 deletions

diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index d05327caf69d..5d355e0c2633 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -124,6 +124,7 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
 {
         void **srcs;
         int i;
+        int start = -1, stop = disks - 3;
 
         if (submit->scribble)
                 srcs = submit->scribble;
@@ -134,10 +135,21 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
                 if (blocks[i] == NULL) {
                         BUG_ON(i > disks - 3); /* P or Q can't be zero */
                         srcs[i] = (void*)raid6_empty_zero_page;
-                } else
+                } else {
                         srcs[i] = page_address(blocks[i]) + offset;
+                        if (i < disks - 2) {
+                                stop = i;
+                                if (start == -1)
+                                        start = i;
+                        }
+                }
         }
-        raid6_call.gen_syndrome(disks, len, srcs);
+        if (submit->flags & ASYNC_TX_PQ_XOR_DST) {
+                BUG_ON(!raid6_call.xor_syndrome);
+                if (start >= 0)
+                        raid6_call.xor_syndrome(disks, start, stop, len, srcs);
+        } else
+                raid6_call.gen_syndrome(disks, len, srcs);
         async_tx_sync_epilog(submit);
 }
 
@@ -178,7 +190,8 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks,
         if (device)
                 unmap = dmaengine_get_unmap_data(device->dev, disks, GFP_NOIO);
 
-        if (unmap &&
+        /* XORing P/Q is only implemented in software */
+        if (unmap && !(submit->flags & ASYNC_TX_PQ_XOR_DST) &&
             (src_cnt <= dma_maxpq(device, 0) ||
              dma_maxpq(device, DMA_PREP_CONTINUE) > 0) &&
             is_dma_pq_aligned(device, offset, 0, len)) {
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 3ae097d50b51..c82ce1fd8723 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -1317,7 +1317,9 @@ ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
  * destination buffer is recorded in srcs[count] and the Q destination
  * is recorded in srcs[count+1]].
  */
-static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
+static int set_syndrome_sources(struct page **srcs,
+                                struct stripe_head *sh,
+                                int srctype)
 {
         int disks = sh->disks;
         int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
@@ -1332,8 +1334,15 @@ static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
         i = d0_idx;
         do {
                 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
+                struct r5dev *dev = &sh->dev[i];
 
-                srcs[slot] = sh->dev[i].page;
+                if (i == sh->qd_idx || i == sh->pd_idx ||
+                    (srctype == SYNDROME_SRC_ALL) ||
+                    (srctype == SYNDROME_SRC_WANT_DRAIN &&
+                     test_bit(R5_Wantdrain, &dev->flags)) ||
+                    (srctype == SYNDROME_SRC_WRITTEN &&
+                     dev->written))
+                        srcs[slot] = sh->dev[i].page;
                 i = raid6_next_disk(i, disks);
         } while (i != d0_idx);
 
@@ -1373,7 +1382,7 @@ ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
         atomic_inc(&sh->count);
 
         if (target == qd_idx) {
-                count = set_syndrome_sources(blocks, sh);
+                count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
                 blocks[count] = NULL; /* regenerating p is not necessary */
                 BUG_ON(blocks[count+1] != dest); /* q should already be set */
                 init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
@@ -1481,7 +1490,7 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
                         tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
                                        &submit);
 
-                        count = set_syndrome_sources(blocks, sh);
+                        count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
                         init_async_submit(&submit, ASYNC_TX_FENCE, tx,
                                           ops_complete_compute, sh,
                                           to_addr_conv(sh, percpu, 0));
@@ -1515,8 +1524,8 @@ static void ops_complete_prexor(void *stripe_head_ref)
 }
 
 static struct dma_async_tx_descriptor *
-ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
-               struct dma_async_tx_descriptor *tx)
+ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
+                struct dma_async_tx_descriptor *tx)
 {
         int disks = sh->disks;
         struct page **xor_srcs = to_addr_page(percpu, 0);
@@ -1545,6 +1554,26 @@ ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
 }
 
 static struct dma_async_tx_descriptor *
+ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
+                struct dma_async_tx_descriptor *tx)
+{
+        struct page **blocks = to_addr_page(percpu, 0);
+        int count;
+        struct async_submit_ctl submit;
+
+        pr_debug("%s: stripe %llu\n", __func__,
+                (unsigned long long)sh->sector);
+
+        count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);
+
+        init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
+                          ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
+        tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
+
+        return tx;
+}
+
+static struct dma_async_tx_descriptor *
 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
 {
         int disks = sh->disks;
@@ -1746,6 +1775,8 @@ ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
         int count, i, j = 0;
         struct stripe_head *head_sh = sh;
         int last_stripe;
+        int synflags;
+        unsigned long txflags;
 
         pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
 
@@ -1765,14 +1796,23 @@ ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
 
 again:
         blocks = to_addr_page(percpu, j);
-        count = set_syndrome_sources(blocks, sh);
+
+        if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
+                synflags = SYNDROME_SRC_WRITTEN;
+                txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
+        } else {
+                synflags = SYNDROME_SRC_ALL;
+                txflags = ASYNC_TX_ACK;
+        }
+
+        count = set_syndrome_sources(blocks, sh, synflags);
         last_stripe = !head_sh->batch_head ||
                 list_first_entry(&sh->batch_list,
                                  struct stripe_head, batch_list) == head_sh;