aoe: improve handling of misbehaving network paths

An AoE target can have multiple network ports used for AoE, and in the
aoe driver, those are tracked by the aoetgt struct.  These changes allow
the aoe driver to handle network paths, or aoetgts, that are not working
well, compared to the others.

Paths that do not get responses despite the retransmission of AoE
commands are marked as "tainted", and non-tainted paths are preferred.

Meanwhile, the aoe driver attempts to "probe" the tainted path in the
background by issuing reads of LBA 0 that are padded out to full
(possibly jumbo-frame) size.  If the probes get responses, then the path
is "redeemed", and its taint is removed.

This mechanism has been shown to be helpful in transparently handling
and recovering from real-world network "brown outs" in ways that the
earlier "shoot the help-needing target in the head" mechanism could not.

Signed-off-by: Ed Cashin <ecashin@coraid.com>
Cc: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

3 files changed, 268 insertions, 121 deletions

diff --git a/drivers/block/aoe/aoe.h b/drivers/block/aoe/aoe.h
index bfd765cf0eb7..b6d2b16358be 100644
--- a/drivers/block/aoe/aoe.h
+++ b/drivers/block/aoe/aoe.h
@@ -91,6 +91,9 @@ enum {
         RTTDSCALE = 3,
         RTTAVG_INIT = USEC_PER_SEC / 4 << RTTSCALE,
         RTTDEV_INIT = RTTAVG_INIT / 4,
+
+        HARD_SCORN_SECS = 10,   /* try another remote port after this */
+        MAX_TAINT = 1000,       /* cap on aoetgt taint */
 };
 
 struct buf {
@@ -103,6 +106,10 @@ struct buf {
         struct request *rq;
 };
 
+enum frame_flags {
+        FFL_PROBE = 1,
+};
+
 struct frame {
         struct list_head head;
         u32 tag;
@@ -118,6 +125,7 @@ struct frame {
         struct bio_vec *bv;
         ulong bcnt;
         ulong bv_off;
+        char flags;
 };
 
 struct aoeif {
@@ -138,8 +146,10 @@ struct aoetgt {
         ushort next_cwnd;       /* incr maxout after decrementing to zero */
         ushort ssthresh;        /* slow start threshold */
         ulong falloc;           /* number of allocated frames */
+        int taint;              /* how much we want to avoid this aoetgt */
         int minbcnt;
         int wpkts, rpkts;
+        char nout_probes;
 };
 
 struct aoedev {
@@ -174,7 +184,6 @@ struct aoedev {
         struct list_head rexmitq; /* deferred retransmissions */
         struct aoetgt *targets[NTARGETS];
         struct aoetgt **tgt;    /* target in use when working */
-        struct aoetgt *htgt;    /* target needing rexmit assistance */
         ulong ntargets;
         ulong kicked;
         char ident[512];
diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 391dd8ee2009..000f7fb48841 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -22,6 +22,7 @@
 #define MAXIOC (8192)   /* default meant to avoid most soft lockups */
 
 static void ktcomplete(struct frame *, struct sk_buff *);
+static int count_targets(struct aoedev *d, int *untainted);
 
 static struct buf *nextbuf(struct aoedev *);
 
@@ -43,6 +44,8 @@ static struct {
         spinlock_t lock;
 } iocq;
 
+static struct page *empty_page;
+
 static struct sk_buff *
 new_skb(ulong len)
 {
@@ -179,8 +182,10 @@ aoe_freetframe(struct frame *f)
 
         t = f->t;
         f->buf = NULL;
+        f->lba = 0;
         f->bv = NULL;
         f->r_skb = NULL;
+        f->flags = 0;
         list_add(&f->head, &t->ffree);
 }
 
@@ -234,20 +239,25 @@ newframe(struct aoedev *d)
         struct frame *f;
         struct aoetgt *t, **tt;
         int totout = 0;
+        int use_tainted;
+        int has_untainted;
 
         if (d->targets[0] == NULL) {    /* shouldn't happen, but I'm paranoid */
                 printk(KERN_ERR "aoe: NULL TARGETS!\n");
                 return NULL;
         }
         tt = d->tgt;    /* last used target */
-        for (;;) {
+        for (use_tainted = 0, has_untainted = 0;;) {
                 tt++;
                 if (tt >= &d->targets[NTARGETS] || !*tt)
                         tt = d->targets;
                 t = *tt;
-                totout += t->nout;
+                if (!t->taint) {
+                        has_untainted = 1;
+                        totout += t->nout;
+                }
                 if (t->nout < t->maxout
-                && t != d->htgt
+                && (use_tainted || !t->taint)
                 && t->ifp->nd) {
                         f = newtframe(d, t);
                         if (f) {
@@ -256,8 +266,12 @@ newframe(struct aoedev *d)
                                 return f;
                         }
                 }
-                if (tt == d->tgt)       /* we've looped and found nada */
-                        break;
+                if (tt == d->tgt) {     /* we've looped and found nada */
+                        if (!use_tainted && !has_untainted)
+                                use_tainted = 1;
+                        else
+                                break;
+                }
         }
         if (totout == 0) {
                 d->kicked++;
@@ -294,21 +308,68 @@ fhash(struct frame *f)
         list_add_tail(&f->head, &d->factive[n]);
 }
 
+static void
+ata_rw_frameinit(struct frame *f)
+{
+        struct aoetgt *t;
+        struct aoe_hdr *h;
+        struct aoe_atahdr *ah;
+        struct sk_buff *skb;
+        char writebit, extbit;
+
+        skb = f->skb;
+        h = (struct aoe_hdr *) skb_mac_header(skb);
+        ah = (struct aoe_atahdr *) (h + 1);
+        skb_put(skb, sizeof(*h) + sizeof(*ah));
+        memset(h, 0, skb->len);
+
+        writebit = 0x10;
+        extbit = 0x4;
+
+        t = f->t;
+        f->tag = aoehdr_atainit(t->d, t, h);
+        fhash(f);
+        t->nout++;
+        f->waited = 0;
+        f->waited_total = 0;
+        if (f->buf)
+                f->lba = f->buf->sector;
+
+        /* set up ata header */
+        ah->scnt = f->bcnt >> 9;
+        put_lba(ah, f->lba);