1 files changed, 471 insertions, 261 deletions

diff --git a/drivers/block/aoe/aoecmd.c b/drivers/block/aoe/aoecmd.c
index 9a58242290c0..59b333c902a6 100644
--- a/drivers/block/aoe/aoecmd.c
+++ b/drivers/block/aoe/aoecmd.c
@@ -12,10 +12,17 @@
 #include <linux/netdevice.h>
 #include <linux/genhd.h>
 #include <linux/moduleparam.h>
+#include <linux/workqueue.h>
+#include <linux/kthread.h>
 #include <net/net_namespace.h>
 #include <asm/unaligned.h>
+#include <linux/uio.h>
 #include "aoe.h"
 
+#define MAXIOC (8192)   /* default meant to avoid most soft lockups */
+
+static void ktcomplete(struct frame *, struct sk_buff *);
+
 static int aoe_deadsecs = 60 * 3;
 module_param(aoe_deadsecs, int, 0644);
 MODULE_PARM_DESC(aoe_deadsecs, "After aoe_deadsecs seconds, give up and fail dev.");
@@ -25,6 +32,15 @@ module_param(aoe_maxout, int, 0644);
 MODULE_PARM_DESC(aoe_maxout,
         "Only aoe_maxout outstanding packets for every MAC on eX.Y.");
 
+static wait_queue_head_t ktiowq;
+static struct ktstate kts;
+
+/* io completion queue */
+static struct {
+        struct list_head head;
+        spinlock_t lock;
+} iocq;
+
 static struct sk_buff *
 new_skb(ulong len)
 {
@@ -41,15 +57,21 @@ new_skb(ulong len)
 }
 
 static struct frame *
-getframe(struct aoetgt *t, int tag)
+getframe(struct aoetgt *t, u32 tag)
 {
-        struct frame *f, *e;
+        struct frame *f;
+        struct list_head *head, *pos, *nx;
+        u32 n;
 
-        f = t->frames;
-        e = f + t->nframes;
-        for (; f<e; f++)
-                if (f->tag == tag)
+        n = tag % NFACTIVE;
+        head = &t->factive[n];
+        list_for_each_safe(pos, nx, head) {
+                f = list_entry(pos, struct frame, head);
+                if (f->tag == tag) {
+                        list_del(pos);
                         return f;
+                }
+        }
         return NULL;
 }
 
@@ -67,7 +89,7 @@ newtag(struct aoetgt *t)
         return n |= (++t->lasttag & 0x7fff) << 16;
 }
 
-static int
+static u32
 aoehdr_atainit(struct aoedev *d, struct aoetgt *t, struct aoe_hdr *h)
 {
         u32 host_tag = newtag(t);
@@ -129,75 +151,96 @@ skb_pool_get(struct aoedev *d)
         return NULL;
 }
 
-/* freeframe is where we do our load balancing so it's a little hairy. */
+void
+aoe_freetframe(struct frame *f)
+{
+        struct aoetgt *t;
+
+        t = f->t;
+        f->buf = NULL;
+        f->bv = NULL;
+        f->r_skb = NULL;
+        list_add(&f->head, &t->ffree);
+}
+
 static struct frame *
-freeframe(struct aoedev *d)
+newtframe(struct aoedev *d, struct aoetgt *t)
 {
-        struct frame *f, *e, *rf;
-        struct aoetgt **t;
+        struct frame *f;
         struct sk_buff *skb;
+        struct list_head *pos;
+
+        if (list_empty(&t->ffree)) {
+                if (t->falloc >= NSKBPOOLMAX*2)
+                        return NULL;
+                f = kcalloc(1, sizeof(*f), GFP_ATOMIC);
+                if (f == NULL)
+                        return NULL;
+                t->falloc++;
+                f->t = t;
+        } else {
+                pos = t->ffree.next;
+                list_del(pos);
+                f = list_entry(pos, struct frame, head);
+        }
+
+        skb = f->skb;
+        if (skb == NULL) {
+                f->skb = skb = new_skb(ETH_ZLEN);
+                if (!skb) {
+bail:                   aoe_freetframe(f);
+                        return NULL;
+                }
+        }
+
+        if (atomic_read(&skb_shinfo(skb)->dataref) != 1) {
+                skb = skb_pool_get(d);
+                if (skb == NULL)
+                        goto bail;
+                skb_pool_put(d, f->skb);
+                f->skb = skb;
+        }
+
+        skb->truesize -= skb->data_len;
+        skb_shinfo(skb)->nr_frags = skb->data_len = 0;
+        skb_trim(skb, 0);
+        return f;
+}
+
+static struct frame *
+newframe(struct aoedev *d)
+{
+        struct frame *f;
+        struct aoetgt *t, **tt;
+        int totout = 0;
 
         if (d->targets[0] == NULL) {    /* shouldn't happen, but I'm paranoid */
                 printk(KERN_ERR "aoe: NULL TARGETS!\n");
                 return NULL;
         }
-        t = d->tgt;
-        t++;
-        if (t >= &d->targets[NTARGETS] || !*t)
-                t = d->targets;
+        tt = d->tgt;    /* last used target */
         for (;;) {
-                if ((*t)->nout < (*t)->maxout
+                tt++;
+                if (tt >= &d->targets[NTARGETS] || !*tt)
+                        tt = d->targets;
+                t = *tt;
+                totout += t->nout;
+                if (t->nout < t->maxout
                 && t != d->htgt
-                && (*t)->ifp->nd) {
-                        rf = NULL;
-                        f = (*t)->frames;
-                        e = f + (*t)->nframes;
-                        for (; f < e; f++) {
-                                if (f->tag != FREETAG)
-                                        continue;
-                                skb = f->skb;
-                                if (!skb
-                                && !(f->skb = skb = new_skb(ETH_ZLEN)))
-                                        continue;
-                                if (atomic_read(&skb_shinfo(skb)->dataref)
-                                        != 1) {
-                                        if (!rf)
-                                                rf = f;
-                                        continue;
-                                }
-gotone:                         skb->truesize -= skb->data_len;
-                                skb_shinfo(skb)->nr_frags = skb->data_len = 0;
-                                skb_trim(skb, 0);
-                                d->tgt = t;
-                                ifrotate(*t);
+                && t->ifp->nd) {
+                        f = newtframe(d, t);
+                        if (f) {
+                                d->tgt = tt;
+                                ifrotate(t);
                                 return f;
                         }
-                        /* Work can be done, but the network layer is
-                           holding our precious packets.  Try to grab
-                           one from the pool. */
-                        f = rf;
-                        if (f == NULL) {        /* more paranoia */
-                                printk(KERN_ERR
-                                        "aoe: freeframe: %s.\n",
-                                        "unexpected null rf");
-                                d->flags |= DEVFL_KICKME;
-                                return NULL;
-                        }
-                        skb = skb_pool_get(d);
-                        if (skb) {
-                                skb_pool_put(d, f->skb);
-                                f->skb = skb;
-                                goto gotone;
-                        }
-                        (*t)->dataref++;
-                        if ((*t)->nout == 0)
-                                d->flags |= DEVFL_KICKME;
                 }
-                if (t == d->tgt)        /* we've looped and found nada */
+                if (tt == d->tgt)       /* we've looped and found nada */
                         break;
-                t++;
-                if (t >= &d->targets[NTARGETS] || !*t)
-                        t = d->targets;
+        }
+        if (totout == 0) {
+                d->kicked++;
+                d->flags |= DEVFL_KICKME;
         }
         return NULL;
 }
@@ -220,6 +263,16 @@ loop:
         goto loop;
 }
 
+static void
+fhash(struct frame *f)
+{
+        struct aoetgt *t = f->t;
+        u32 n;
+
+        n = f->tag % NFACTIVE;
+        list_add_tail(&f->head, &t->factive[n]);
+}
+
 static int
 aoecmd_ata_rw(struct aoedev *d)
 {
@@ -236,7 +289,7 @@ aoecmd_ata_rw(struct aoedev *d)
         writebit = 0x10;
         extbit = 0x4;
 
-        f = freeframe(d);
+        f = newframe(d);
         if (f == NULL)
                 return 0;
         t = *d->tgt;
@@ -274,6 +327,7 @@ aoecmd_ata_rw(struct aoedev *d)
         skb_put(skb, sizeof *h + sizeof *ah);
         memset(h, 0, skb->len);
         f->tag = aoehdr_atainit(d, t, h);
+        fhash(f);
         t->nout++;
         f->waited = 0;
         f->buf = buf;
@@ -358,14 +412,16 @@ cont:
 }
 
 static void
-resend(struct aoedev *d, struct aoetgt *t, struct frame *f)
+resend(struct aoedev *d, struct frame *f)
 {
         struct sk_buff *skb;
         struct aoe_hdr *h;
         struct aoe_atahdr *ah;
+        struct aoetgt *t;
         char buf[128];
         u32 n;
 
+        t = f->t;
         ifrotate(t);
         n = newtag(t);
         skb = f->skb;
@@ -379,28 +435,11 @@ resend(struct aoedev *d, struct aoetgt *t, struct frame *f)
         aoechr_error(buf);
 
         f->tag = n;
+        fhash(f);
         h->tag = cpu_to_be32(n);
         memcpy(h->dst, t->addr, sizeof h->dst);
         memcpy(h->src, t->ifp->nd->dev_addr, sizeof h->src);
 
-        switch (ah->cmdstat) {
-        default:
-                break;
-        case ATA_CMD_PIO_READ:
-        case ATA_CMD_PIO_READ_EXT:
-        case ATA_CMD_PIO_WRITE:
-        case ATA_CMD_PIO_WRITE_EXT:
-                put_lba(ah, f->lba);
-
-                n = f->bcnt;
-                ah->scnt = n >> 9;
-                if (ah->aflags & AOEAFL_WRITE) {
-                        skb_fillup(skb, f->bv, f->bv_off, n);
-                        skb->len = sizeof *h + sizeof *ah + n;
-                        skb->data_len = n;
-                        skb->truesize += n;
-                }
-        }
         skb->dev = t->ifp->nd;
         skb = skb_clone(skb, GFP_ATOMIC);
         if (skb == NULL)
@@ -409,7 +448,7 @@ resend(struct aoedev *d, struct aoetgt *t, struct frame *f)
 }
 
 static int
-tsince(int tag)
+tsince(u32 tag)
 {
         int n;
 
@@ -463,26 +502,38 @@ ejectif(struct aoetgt *t, struct aoeif *ifp)
 static int
 sthtith(struct aoedev *d)
 {
-        struct frame *f, *e, *nf;
+        struct frame *f, *nf;
+        struct list_head *nx, *pos, *head;
         struct sk_buff *skb;
-        struct aoetgt *ht = *d->htgt;
-
-        f = ht->frames;
-        e = f + ht->nframes;
-        for (; f < e; f++) {
-                if (f->tag == FREETAG)
-                        continue;
-                nf = freeframe(d);
-                if (!nf)
-                        return 0;
-                skb = nf->skb;
-                *nf = *f;
-                f->skb = skb;
-                f->tag = FREETAG;
-                nf->waited = 0;
-                ht->nout--;
-                (*d->tgt)->nout++;
-                resend(d, *d->tgt, nf);
+        struct aoetgt *ht = d->htgt;
+        int i;
+
+        for (i = 0; i < NFACTIVE; i++) {
+                head = &ht->factive[i];
+                list_for_each_safe(pos, nx, head) {
+                        f = list_entry(pos, struct frame, head);
+                        nf = newframe(d);
+                        if (!nf)
+                                return 0;
+
+                        /* remove frame from active list */
+                        list_del(pos);
+
+                        /* reassign all pertinent bits to new outbound frame */
+                        skb = nf->skb;
+                        nf->skb = f->skb;
+                        nf->buf = f->buf;
+                        nf->bcnt = f->bcnt;
+                        nf->lba = f->lba;
+                        nf->bv = f->bv;
+                        nf->bv_off = f->bv_off;
+                        nf->waited = 0;
+                        f->skb = skb;
+                        aoe_freetframe(f);
+                        ht->nout--;
+                        nf->t->nout++;
+                        resend(d, nf);
+                }
         }
         /* he's clean, he's useless.  take away his interfaces */
         memset(ht->ifs, 0, sizeof ht->ifs);
@@ -507,9 +558,12 @@ rexmit_timer(ulong vp)
         struct aoedev *d;
         struct aoetgt *t, **tt, **te;
         struct aoeif *ifp;
-        struct frame *f, *e;
+        struct frame *f;
+        struct list_head *head, *pos, *nx;
+        LIST_HEAD(flist);
         register long timeout;
         ulong flags, n;
+        int i;
 
         d = (struct aoedev *) vp;
 
@@ -523,41 +577,21 @@ rexmit_timer(ulong vp)
                 spin_unlock_irqrestore(&d->lock, flags);
                 return;
         }
+
+        /* collect all frames to rexmit into flist */
         tt = d->targets;
         te = tt + NTARGETS;
         for (; tt < te && *tt; tt++) {
                 t = *tt;
-                f = t->frames;
-                e = f + t->nframes;
-                for (; f < e; f++) {
-                        if (f->tag == FREETAG
-                        || tsince(f->tag) < timeout)
-                                continue;
-                        n = f->waited += timeout;
-                        n /= HZ;
-                        if (n > aoe_deadsecs) {
-                                /* waited too long.  device failure. */
-                                aoedev_downdev(d);
-                                break;
-                        }
-
-                        if (n > HELPWAIT /* see if another target can help */
-                        && (tt != d->targets || d->targets[1]))
-                                d->htgt = tt;
-
-                        if (t->nout == t->maxout) {
-                                if (t->maxout > 1)
-                                        t->maxout--;
-                                t->lastwadj = jiffies;
-                        }
-
-                        ifp = getif(t, f->skb->dev);
-                        if (ifp && ++ifp->lost > (t->nframes << 1)
-                        && (ifp != t->ifs || t->ifs[1].nd)) {
-                                ejectif(t, ifp);
-                                ifp = NULL;
+                for (i = 0; i < NFACTIVE; i++) {
+                        head = &t->factive[i];
+                        list_for_each_safe(pos, nx, head) {
+                                f = list_entry(pos, struct frame, head);
+                                if (tsince(f->tag) < timeout)
+                                        continue;
+                                /* move to flist for later processing */
+                                list_move_tail(pos, &flist);
                         }
-                        resend(d, t, f);
                 }
 
                 /* window check */
@@ -569,6 +603,44 @@ rexmit_timer(ulong vp)
                 }
         }
 
+        /* process expired frames */
+        while (!list_empty(&flist)) {
+                pos = flist.next;
+                f = list_entry(pos, struct frame, head);
+                n = f->waited += timeout;
+                n /= HZ;
+                if (n > aoe_deadsecs) {
+                        /* Waited too long.  Device failure.
+                         * Hang all frames on first hash bucket for downdev
+                         * to clean up.
+                         */
+                        list_splice(&flist, &f->t->factive[0]);
+                        aoedev_downdev(d);
+                        break;
+                }
+                list_del(pos);
+
+                t = f->t;
+                if (n > HELPWAIT) {
+                        /* see if another target can help */
+                        if (d->ntargets > 1)
+                                d->htgt = t;
+                }
+                if (t->nout == t->maxout) {
+                        if (t->maxout > 1)
+                                t->maxout--;
+                        t->lastwadj = jiffies;
+                }
+
+                ifp = getif(t, f->skb->dev);
+                if (ifp && ++ifp->lost > (t->nframes << 1)
+                && (ifp != t->ifs || t->ifs[1].nd)) {
+                        ejectif(t, ifp);
+                        ifp = NULL;
+                }
+                resend(d, f);
+        }
+
         if (!skb_queue_empty(&d->sendq)) {
                 n = d->rttavg <<= 1;
                 if (n > MAXTIMER)
@@ -750,7 +822,7 @@ diskstats(struct gendisk *disk, struct bio *bio, ulong duration, sector_t sector
 }
 
 static void
-bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, ulong cnt)
+bvcpy(struct bio_vec *bv, ulong off, struct sk_buff *skb, long cnt)
 {
         ulong fcnt;
         char *p;
@@ -771,60 +843,225 @@ loop:
 }
 
 static void
-fadvance(struct frame *f, ulong cnt)
+ktiocomplete(struct frame *f)
 {
-        ulong fcnt;
+        struct aoe_hdr *hin, *hout;
+        struct aoe_atahdr *ahin, *ahout;
+        struct buf *buf;
+        struct sk_buff *skb;
+        struct aoetgt *t;
+        struct aoeif *ifp;
+        struct aoedev *d;
+        long n;
 
-        f->lba += cnt >> 9;
-loop:
-        fcnt = f->bv->bv_len - (f->bv_off - f->bv->bv_offset);
-        if (fcnt > cnt) {
-                f->bv_off += cnt;
+        if (f == NULL)
                 return;
+
+        t = f->t;
+        d = t->d;
+
+        hout = (struct aoe_hdr *) skb_mac_header(f->skb);
+        ahout = (struct aoe_atahdr *) (hout+1);
+        buf = f->buf;
+        skb = f->r_skb;
+        if (skb == NULL)
+                goto noskb;     /* just fail the buf. */
+
+        hin = (struct aoe_hdr *) skb->data;
+        skb_pull(skb, sizeof(*hin));
+        ahin = (struct aoe_atahdr *) skb->data;
+        skb_pull(skb, sizeof(*ahin));
+        if (ahin->cmdstat & 0xa9) {     /* these bits cleared on success */
+                pr_err("aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
+                        ahout->cmdstat, ahin->cmdstat,
+                        d->aoemajor, d->aoeminor);
+noskb:  if (buf)
+                        buf->flags |= BUFFL_FAIL;
+                goto badrsp;
         }
-        cnt -= fcnt;
-        f->bv++;
-        f->bv_off = f->bv->bv_offset;
-        goto loop;
+
+        n = ahout->scnt << 9;
+        switch (ahout->cmdstat) {
+        case ATA_CMD_PIO_READ:
+        case ATA_CMD_PIO_READ_EXT:
+                if (skb->len < n) {
+                        pr_err("aoe: runt data size in read.  skb->len=%d need=%ld\n",
+                                skb->len, n);
+                        buf->flags |= BUFFL_FAIL;
+                        break;
+                }
+                bvcpy(f->bv, f->bv_off, skb, n);
+        case ATA_CMD_PIO_WRITE:
+        case ATA_CMD_PIO_WRITE_EXT:
+                spin_lock_irq(&d->lock);
+                ifp = getif(t, skb->dev);
+                if (ifp) {
+                        ifp->lost = 0;
+                        if (n > DEFAULTBCNT)
+                                ifp->lostjumbo = 0;
+                }
+                if (d->htgt == t) /* I'll help myself, thank you. */
+                        d->htgt = NULL;
+                spin_unlock_irq(&d->lock);
+                break;
+        case ATA_CMD_ID_ATA:
+                if (skb->len < 512) {
+                        pr_info("aoe: runt data size in ataid.  skb->len=%d\n",
+                                skb->len);
+                        break;
+                }
+                if (skb_linearize(skb))
+                        break;
+                spin_lock_irq(&d->lock);
+                ataid_complete(d, t, skb->data);
+                spin_unlock_irq(&d->lock);
+                break;
+        default:
+                pr_info("aoe: unrecognized ata command %2.2Xh for %d.%d\n",
+                        ahout->cmdstat,
+                        be16_to_cpu(get_unaligned(&hin->major)),
+                        hin->minor);
+        }
+badrsp:
+        spin_lock_irq(&d->lock);
+
+        aoe_freetframe(f);
+
+        if (buf && --buf->nframesout == 0 && buf->resid == 0) {
+                struct bio *bio = buf->bio;
+
+                diskstats(d->gd, bio, jiffies - buf->stime, buf->sector);
+                n = (buf->flags & BUFFL_FAIL) ? -EIO : 0;
+                mempool_free(buf, d->bufpool);
+                spin_unlock_irq(&d->lock);
+                if (n != -EIO)
+                        bio_flush_dcache_pages(buf->bio);
+                bio_endio(bio, n);
+        } else
+                spin_unlock_irq(&d->lock);
+        dev_kfree_skb(skb);
 }
 
-void
+/* Enters with iocq.lock held.
+ * Returns true iff responses needing processing remain.
+ */
+static int
+ktio(void)
+{
+        struct frame *f;
+        struct list_head *pos;
+        int i;
+
+        for (i = 0; ; ++i) {
+                if (i == MAXIOC)
+                        return 1;
+                if (list_empty(&iocq.head))
+                        return 0;
+                pos = iocq.head.next;
+                list_del(pos);
+                spin_unlock_irq(&iocq.lock);
+                f = list_entry(pos, struct frame, head);
+                ktiocomplete(f);
+                spin_lock_irq(&iocq.lock);
+        }
+}
+
+static int
+kthread(void *vp)
+{
+        struct ktstate *k;
+        DECLARE_WAITQUEUE(wait, current);
+        int more;
+
+        k = vp;
+        current->flags |= PF_NOFREEZE;
+        set_user_nice(current, -10);
+        complete(&k->rendez);   /* tell spawner we're running */
+        do {
+                spin_lock_irq(k->lock);
+                more = k->fn();
+                if (!more) {
+                        add_wait_queue(k->waitq, &wait);
+                        __set_current_state(TASK_INTERRUPTIBLE);
+                }
+                spin_unlock_irq(k->lock);
+                if (!more) {
+                        schedule();
+                        remove_wait_queue(k->waitq, &wait);
+                } else
+                        cond_resched();
+        } while (!kthread_should_stop());
+        complete(&k->rendez);   /* tell spawner we're stopping */
+        return 0;
+}
+
+static void
+aoe_ktstop(struct ktstate *k)
+{
+        kthread_stop(k->task);
+        wait_for_completion(&k->rendez);
+}
+
+static int
+aoe_ktstart(struct ktstate *k)
+{
+        struct task_struct *task;
+
+        init_completion(&k->rendez);
+        task = kthread_run(kthread, k, k->name);
+        if (task == NULL || IS_ERR(task))
+                return -ENOMEM;
+        k->task = task;
+        wait_for_completion(&k->rendez); /* allow kthread to start */
+        init_completion(&k->rendez);    /* for waiting for exit later */
+        return 0;
+}
+
+/* pass it off to kthreads for processing */
+static void
+ktcomplete(struct frame *f, struct sk_buff *skb)
+{
+        ulong flags;
+
+        f->r_skb = skb;
+        spin_lock_irqsave(&iocq.lock, flags);
+        list_add_tail(&f->head, &iocq.head);
+        spin_unlock_irqrestore(&iocq.lock, flags);
+        wake_up(&ktiowq);
+}
+
+struct sk_buff *
 aoecmd_ata_rsp(struct sk_buff *skb)
 {
-        struct sk_buff_head queue;
         struct aoedev *d;
-        struct aoe_hdr *hin, *hout;
-        struct aoe_atahdr *ahin, *ahout;
+        struct aoe_hdr *h;
         struct frame *f;
-        struct buf *buf;
         struct aoetgt *t;
-        struct aoeif *ifp;
-        register long n;
+        u32 n;
         ulong flags;
         char ebuf[128];
         u16 aoemajor;
 
-        hin = (struct aoe_hdr *) skb_mac_header(skb);
-        skb_pull(skb, sizeof(*hin));
-        aoemajor = get_unaligned_be16(&hin->major);
-        d = aoedev_by_aoeaddr(aoemajor, hin->minor);
+        h = (struct aoe_hdr *) skb->data;
+        aoemajor = be16_to_cpu(get_unaligned(&h->major));
+        d = aoedev_by_aoeaddr(aoemajor, h->minor);
         if (d == NULL) {
                 snprintf(ebuf, sizeof ebuf, "aoecmd_ata_rsp: ata response "
                         "for unknown device %d.%d\n",
-                         aoemajor, hin->minor);
+                        aoemajor, h->minor);
                 aoechr_error(ebuf);
-                return;
+                return skb;
         }
 
         spin_lock_irqsave(&d->lock, flags);
 
-        n = get_unaligned_be32(&hin->tag);
-        t = gettgt(d, hin->src);
+        n = be32_to_cpu(get_unaligned(&h->tag));
+        t = gettgt(d, h->src);
         if (t == NULL) {
                 printk(KERN_INFO "aoe: can't find target e%ld.%d:%pm\n",
-                        d->aoemajor, d->aoeminor, hin->src);
+                       d->aoemajor, d->aoeminor, h->src);
                 spin_unlock_irqrestore(&d->lock, flags);
-                return;
+                return skb;
         }
         f = getframe(t, n);
         if (f == NULL) {
@@ -833,102 +1070,26 @@ aoecmd_ata_rsp(struct sk_buff *skb)
                 snprintf(ebuf, sizeof ebuf,
                         "%15s e%d.%d    tag=%08x@%08lx\n",
                         "unexpected rsp",
-                        get_unaligned_be16(&hin->major),
-                        hin->minor,
-                        get_unaligned_be32(&hin->tag),
+                        get_unaligned_be16(&h->major),
+                        h->minor,
+                        get_unaligned_be32(&h->tag),
                         jiffies);
                 aoechr_error(ebuf);
-                return;
+                return skb;
         }
-
         calc_rttavg(d, tsince(f->tag));
-
-        ahin = (struct aoe_atahdr *) skb->data;
-        skb_pull(skb, sizeof(*ahin));
-        hout = (struct aoe_hdr *) skb_mac_header(f->skb);
-        ahout = (struct aoe_atahdr *) (hout+1);
-        buf = f->buf;
-
-        if (ahin->cmdstat & 0xa9) {     /* these bits cleared on success */
-                printk(KERN_ERR
-                        "aoe: ata error cmd=%2.2Xh stat=%2.2Xh from e%ld.%d\n",
-                        ahout->cmdstat, ahin->cmdstat,
-                        d->aoemajor, d->aoeminor);
-                if (buf)
-                        buf->flags |= BUFFL_FAIL;
-        } else {
-                if (d->htgt && t == *d->htgt) /* I'll help myself, thank you. */
-                        d->htgt = NULL;
-                n = ahout->scnt << 9;
-                switch (ahout->cmdstat) {
-                case ATA_CMD_PIO_READ:
-                case ATA_CMD_PIO_READ_EXT:
-                        if (skb->len < n) {
-                                printk(KERN_ERR
-                                        "aoe: %s.  skb->len=%d need=%ld\n",
-                                        "runt data size in read", skb->len, n);
-                                /* fail frame f?  just returning will rexmit. */
-                                spin_unlock_irqrestore(&d->lock, flags);
-                                return;
-                        }
-                        bvcpy(f->bv, f->bv_off, skb, n);
-                case ATA_CMD_PIO_WRITE:
-                case ATA_CMD_PIO_WRITE_EXT:
-                        ifp = getif(t, skb->dev);
-                        if (ifp) {
-                                ifp->lost = 0;
-                                if (n > DEFAULTBCNT)
-                                        ifp->lostjumbo = 0;
-                        }
-                        if (f->bcnt -= n) {
-                                fadvance(f, n);
-                                resend(d, t, f);
-                                goto xmit;
-                        }
-                        break;
-                case ATA_CMD_ID_ATA:
-                        if (skb->len < 512) {
-                                printk(KERN_INFO
-                                        "aoe: runt data size in ataid.  skb->len=%d\n",
-                                        skb->len);
-                                spin_unlock_irqrestore(&d->lock, flags);
-                                return;
-                        }
-                        if (skb_linearize(skb))
-                                break;
-                        ataid_complete(d, t, skb->data);
-                        break;
-                default:
-                        printk(KERN_INFO
-                                "aoe: unrecognized ata command %2.2Xh for %d.%d\n",
-                                ahout->cmdstat,
-                                get_unaligned_be16(&hin->major),
-                                hin->minor);
-                }
-        }
-
-        if (buf && --buf->nframesout == 0 && buf->resid == 0) {
-                diskstats(d->gd, buf->bio, jiffies - buf->stime, buf->sector);
-                if (buf->flags & BUFFL_FAIL)
-                        bio_endio(buf->bio, -EIO);
-                else {
-                        bio_flush_dcache_pages(buf->bio);
-                        bio_endio(buf->bio, 0);
-                }
-                mempool_free(buf, d->bufpool);
-        }
-
-        f->buf = NULL;
-        f->tag = FREETAG;
         t->nout--;
-
         aoecmd_work(d);
-xmit:
-        __skb_queue_head_init(&queue);
-        skb_queue_splice_init(&d->sendq, &queue);
 
         spin_unlock_irqrestore(&d->lock, flags);
-        aoenet_xmit(&queue);
+
+        ktcomplete(f, skb);
+
+        /*
+         * Note here that we do not perform an aoedev_put, as we are
+         * leaving this reference for the ktio to release.
+         */
+        return NULL;
 }
 
 void
@@ -950,7 +1111,7 @@ aoecmd_ata_id(struct aoedev *d)
         struct sk_buff *skb;
         struct aoetgt *t;
 
-        f = freeframe(d);
+        f = newframe(d);
         if (f == NULL)
                 return NULL;
 
@@ -963,6 +1124,7 @@ aoecmd_ata_id(struct aoedev *d)
         skb_put(skb, sizeof *h + sizeof *ah);
         memset(h, 0, skb->len);
         f->tag = aoehdr_atainit(d, t, h);
+        fhash(f);
         t->nout++;
         f->waited = 0;
 
@@ -983,7 +1145,7 @@ static struct aoetgt *
 addtgt(struct aoedev *d, char *addr, ulong nframes)
 {
         struct aoetgt *t, **tt, **te;
-        struct frame *f, *e;
+        int i;
 
         tt = d->targets;
         te = tt + NTARGETS;
@@ -995,23 +1157,21 @@ addtgt(struct aoedev *d, char *addr, ulong nframes)
                         "aoe: device addtgt failure; too many targets\n");
                 return NULL;
         }
-        t = kcalloc(1, sizeof *t, GFP_ATOMIC);
-        f = kcalloc(nframes, sizeof *f, GFP_ATOMIC);
-        if (!t || !f) {
-                kfree(f);
-                kfree(t);
+        t = kzalloc(sizeof(*t), GFP_ATOMIC);
+        if (!t) {
                 printk(KERN_INFO "aoe: cannot allocate memory to add target\n");
                 return NULL;
         }
 
+        d->ntargets++;
         t->nframes = nframes;
-        t->frames = f;
-        e = f + nframes;
-        for (; f < e; f++)
-                f->tag = FREETAG;
+        t->d = d;
         memcpy(t->addr, addr, sizeof t->addr);
         t->ifp = t->ifs;
         t->maxout = t->nframes;
+        INIT_LIST_HEAD(&t->ffree);
+        for (i = 0; i < NFACTIVE; ++i)
+                INIT_LIST_HEAD(&t->factive[i]);
         return *tt = t;
 }
 
@@ -1136,3 +1296,53 @@ aoecmd_cleanslate(struct aoedev *d)
                 }
         }
 }
+
+static void
+flush_iocq(void)
+{
+        struct frame *f;
+        struct aoedev *d;
+        LIST_HEAD(flist);
+        struct list_head *pos;
+        struct sk_buff *skb;
+        ulong flags;
+
+        spin_lock_irqsave(&iocq.lock, flags);
+        list_splice_init(&iocq.head, &flist);
+        spin_unlock_irqrestore(&iocq.lock, flags);
+        while (!list_empty(&flist)) {
+                pos = flist.next;
+                list_del(pos);
+                f = list_entry(pos, struct frame, head);
+                d = f->t->d;
+                skb = f->r_skb;
+                spin_lock_irqsave(&d->lock, flags);
+                if (f->buf) {
+                        f->buf->nframesout--;
+                        aoe_failbuf(d, f->buf);
+                }
+                aoe_freetframe(f);
+                spin_unlock_irqrestore(&d->lock, flags);
+                dev_kfree_skb(skb);
+        }
+}
+
+int __init
+aoecmd_init(void)
+{
+        INIT_LIST_HEAD(&iocq.head);
+        spin_lock_init(&iocq.lock);
+        init_waitqueue_head(&ktiowq);
+        kts.name = "aoe_ktio";
+        kts.fn = ktio;
+        kts.waitq = &ktiowq;
+        kts.lock = &iocq.lock;
+        return aoe_ktstart(&kts);
+}
+
+void
+aoecmd_exit(void)
+{
+        aoe_ktstop(&kts);
+        flush_iocq();
+}