5 files changed, 179 insertions, 107 deletions

diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index de1f319f7bd7..dd5b2fed97e9 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -164,7 +164,7 @@ static void make_response(struct xen_blkif *blkif, u64 id,
 
 #define foreach_grant_safe(pos, n, rbtree, node) \
         for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \
-             (n) = rb_next(&(pos)->node); \
+             (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \
              &(pos)->node != NULL; \
              (pos) = container_of(n, typeof(*(pos)), node), \
              (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL)
@@ -381,8 +381,8 @@ irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
 
 static void print_stats(struct xen_blkif *blkif)
 {
-        pr_info("xen-blkback (%s): oo %3d  |  rd %4d  |  wr %4d  |  f %4d"
-                 "  |  ds %4d\n",
+        pr_info("xen-blkback (%s): oo %3llu  |  rd %4llu  |  wr %4llu  |  f %4llu"
+                 "  |  ds %4llu\n",
                  current->comm, blkif->st_oo_req,
                  blkif->st_rd_req, blkif->st_wr_req,
                  blkif->st_f_req, blkif->st_ds_req);
@@ -442,7 +442,7 @@ int xen_blkif_schedule(void *arg)
 }
 
 struct seg_buf {
-        unsigned long buf;
+        unsigned int offset;
         unsigned int nsec;
 };
 /*
@@ -621,30 +621,21 @@ static int xen_blkbk_map(struct blkif_request *req,
                                  * If this is a new persistent grant
                                  * save the handler
                                  */
-                                persistent_gnts[i]->handle = map[j].handle;
-                                persistent_gnts[i]->dev_bus_addr =
-                                        map[j++].dev_bus_addr;
+                                persistent_gnts[i]->handle = map[j++].handle;
                         }
                         pending_handle(pending_req, i) =
                                 persistent_gnts[i]->handle;
 
                         if (ret)
                                 continue;
-
-                        seg[i].buf = persistent_gnts[i]->dev_bus_addr |
-                                (req->u.rw.seg[i].first_sect << 9);
                 } else {
-                        pending_handle(pending_req, i) = map[j].handle;
+                        pending_handle(pending_req, i) = map[j++].handle;
                         bitmap_set(pending_req->unmap_seg, i, 1);
 
-                        if (ret) {
-                                j++;
+                        if (ret)
                                 continue;
-                        }
-
-                        seg[i].buf = map[j++].dev_bus_addr |
-                                (req->u.rw.seg[i].first_sect << 9);
                 }
+                seg[i].offset = (req->u.rw.seg[i].first_sect << 9);
         }
         return ret;
 }
@@ -679,6 +670,16 @@ static int dispatch_discard_io(struct xen_blkif *blkif,
         return err;
 }
 
+static int dispatch_other_io(struct xen_blkif *blkif,
+                             struct blkif_request *req,
+                             struct pending_req *pending_req)
+{
+        free_req(pending_req);
+        make_response(blkif, req->u.other.id, req->operation,
+                      BLKIF_RSP_EOPNOTSUPP);
+        return -EIO;
+}
+
 static void xen_blk_drain_io(struct xen_blkif *blkif)
 {
         atomic_set(&blkif->drain, 1);
@@ -800,17 +801,30 @@ __do_block_io_op(struct xen_blkif *blkif)
 
                 /* Apply all sanity checks to /private copy/ of request. */
                 barrier();
-                if (unlikely(req.operation == BLKIF_OP_DISCARD)) {
+
+                switch (req.operation) {
+                case BLKIF_OP_READ:
+                case BLKIF_OP_WRITE:
+                case BLKIF_OP_WRITE_BARRIER:
+                case BLKIF_OP_FLUSH_DISKCACHE:
+                        if (dispatch_rw_block_io(blkif, &req, pending_req))
+                                goto done;
+                        break;
+                case BLKIF_OP_DISCARD:
                         free_req(pending_req);
                         if (dispatch_discard_io(blkif, &req))
-                                break;
-                } else if (dispatch_rw_block_io(blkif, &req, pending_req))
+                                goto done;
                         break;
+                default:
+                        if (dispatch_other_io(blkif, &req, pending_req))
+                                goto done;
+                        break;
+                }
 
                 /* Yield point for this unbounded loop. */
                 cond_resched();
         }
-
+done:
         return more_to_do;
 }
 
@@ -904,7 +918,8 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
                 pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n",
                          operation == READ ? "read" : "write",
                          preq.sector_number,
-                         preq.sector_number + preq.nr_sects, preq.dev);
+                         preq.sector_number + preq.nr_sects,
+                         blkif->vbd.pdevice);
                 goto fail_response;
         }
 
@@ -947,7 +962,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
                        (bio_add_page(bio,
                                      pages[i],
                                      seg[i].nsec << 9,
-                                     seg[i].buf & ~PAGE_MASK) == 0)) {
+                                     seg[i].offset) == 0)) {
 
                         bio = bio_alloc(GFP_KERNEL, nseg-i);
                         if (unlikely(bio == NULL))
@@ -977,13 +992,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
                 bio->bi_end_io  = end_block_io_op;
         }
 
-        /*
-         * We set it one so that the last submit_bio does not have to call
-         * atomic_inc.
-         */
         atomic_set(&pending_req->pendcnt, nbio);
-
-        /* Get a reference count for the disk queue and start sending I/O */
         blk_start_plug(&plug);
 
         for (i = 0; i < nbio; i++)
@@ -1011,6 +1020,7 @@ static int dispatch_rw_block_io(struct xen_blkif *blkif,
  fail_put_bio:
         for (i = 0; i < nbio; i++)
                 bio_put(biolist[i]);
+        atomic_set(&pending_req->pendcnt, 1);
         __end_block_io_op(pending_req, -EINVAL);
         msleep(1); /* back off a bit */
         return -EIO;
diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h
index 6072390c7f57..60103e2517ba 100644
--- a/drivers/block/xen-blkback/common.h
+++ b/drivers/block/xen-blkback/common.h
@@ -77,11 +77,18 @@ struct blkif_x86_32_request_discard {
         uint64_t       nr_sectors;
 } __attribute__((__packed__));
 
+struct blkif_x86_32_request_other {
+        uint8_t        _pad1;
+        blkif_vdev_t   _pad2;
+        uint64_t       id;           /* private guest value, echoed in resp  */
+} __attribute__((__packed__));
+
 struct blkif_x86_32_request {
         uint8_t        operation;    /* BLKIF_OP_???                         */
         union {
                 struct blkif_x86_32_request_rw rw;
                 struct blkif_x86_32_request_discard discard;
+                struct blkif_x86_32_request_other other;
         } u;
 } __attribute__((__packed__));
 
@@ -113,11 +120,19 @@ struct blkif_x86_64_request_discard {
         uint64_t       nr_sectors;
 } __attribute__((__packed__));
 
+struct blkif_x86_64_request_other {
+        uint8_t        _pad1;
+        blkif_vdev_t   _pad2;
+        uint32_t       _pad3;        /* offsetof(blkif_..,u.discard.id)==8   */
+        uint64_t       id;           /* private guest value, echoed in resp  */
+} __attribute__((__packed__));
+
 struct blkif_x86_64_request {
         uint8_t        operation;    /* BLKIF_OP_???                         */
         union {
                 struct blkif_x86_64_request_rw rw;
                 struct blkif_x86_64_request_discard discard;
+                struct blkif_x86_64_request_other other;
         } u;
 } __attribute__((__packed__));
 
@@ -172,7 +187,6 @@ struct persistent_gnt {
         struct page *page;
         grant_ref_t gnt;
         grant_handle_t handle;
-        uint64_t dev_bus_addr;
         struct rb_node node;
 };
 
@@ -208,13 +222,13 @@ struct xen_blkif {
 
         /* statistics */
         unsigned long           st_print;
-        int                     st_rd_req;
-        int                     st_wr_req;
-        int                     st_oo_req;
-        int                     st_f_req;
-        int                     st_ds_req;
-        int                     st_rd_sect;
-        int                     st_wr_sect;
+        unsigned long long                      st_rd_req;
+        unsigned long long                      st_wr_req;
+        unsigned long long                      st_oo_req;
+        unsigned long long                      st_f_req;
+        unsigned long long                      st_ds_req;
+        unsigned long long                      st_rd_sect;
+        unsigned long long                      st_wr_sect;
 
         wait_queue_head_t       waiting_to_free;
 };
@@ -278,6 +292,11 @@ static inline void blkif_get_x86_32_req(struct blkif_request *dst,
                 dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
                 break;
         default:
+                /*
+                 * Don't know how to translate this op. Only get the
+                 * ID so failure can be reported to the frontend.
+                 */
+                dst->u.other.id = src->u.other.id;
                 break;
         }
 }
@@ -309,6 +328,11 @@ static inline void blkif_get_x86_64_req(struct blkif_request *dst,
                 dst->u.discard.nr_sectors = src->u.discard.nr_sectors;
                 break;
         default:
+                /*
+                 * Don't know how to translate this op. Only get the
+                 * ID so failure can be reported to the frontend.
+                 */
+                dst->u.other.id = src->u.other.id;
                 break;
         }
 }
diff --git a/drivers/block/xen-blkback/xenbus.c b/drivers/block/xen-blkback/xenbus.c
index 5e237f630c47..8bfd1bcf95ec 100644
--- a/drivers/block/xen-blkback/xenbus.c
+++ b/drivers/block/xen-blkback/xenbus.c
@@ -230,13 +230,13 @@ int __init xen_blkif_interface_init(void)
         }                                                               \
         static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
 
-VBD_SHOW(oo_req,  "%d\n", be->blkif->st_oo_req);
-VBD_SHOW(rd_req,  "%d\n", be->blkif->st_rd_req);
-VBD_SHOW(wr_req,  "%d\n", be->blkif->st_wr_req);
-VBD_SHOW(f_req,  "%d\n", be->blkif->st_f_req);
-VBD_SHOW(ds_req,  "%d\n", be->blkif->st_ds_req);
-VBD_SHOW(rd_sect, "%d\n", be->blkif->st_rd_sect);
-VBD_SHOW(wr_sect, "%d\n", be->blkif->st_wr_sect);
+VBD_SHOW(oo_req,  "%llu\n", be->blkif->st_oo_req);
+VBD_SHOW(rd_req,  "%llu\n", be->blkif->st_rd_req);
+VBD_SHOW(wr_req,  "%llu\n", be->blkif->st_wr_req);
+VBD_SHOW(f_req,  "%llu\n", be->blkif->st_f_req);
+VBD_SHOW(ds_req,  "%llu\n", be->blkif->st_ds_req);
+VBD_SHOW(rd_sect, "%llu\n", be->blkif->st_rd_sect);
+VBD_SHOW(wr_sect, "%llu\n", be->blkif->st_wr_sect);
 
 static struct attribute *xen_vbdstat_attrs[] = {
         &dev_attr_oo_req.attr,
diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c
index c3dae2e0f290..a894f88762d8 100644
--- a/drivers/block/xen-blkfront.c
+++ b/drivers/block/xen-blkfront.c
@@ -44,7 +44,7 @@
 #include <linux/mutex.h>
 #include <linux/scatterlist.h>
 #include <linux/bitmap.h>
-#include <linux/llist.h>
+#include <linux/list.h>
 
 #include <xen/xen.h>
 #include <xen/xenbus.h>
@@ -68,13 +68,12 @@ enum blkif_state {
 struct grant {
         grant_ref_t gref;
         unsigned long pfn;
-        struct llist_node node;
+        struct list_head node;
 };
 
 struct blk_shadow {
         struct blkif_request req;
         struct request *request;
-        unsigned long frame[BLKIF_MAX_SEGMENTS_PER_REQUEST];
         struct grant *grants_used[BLKIF_MAX_SEGMENTS_PER_REQUEST];
 };
 
@@ -105,7 +104,7 @@ struct blkfront_info
         struct work_struct work;
         struct gnttab_free_callback callback;
         struct blk_shadow shadow[BLK_RING_SIZE];
-        struct llist_head persistent_gnts;
+        struct list_head persistent_gnts;
         unsigned int persistent_gnts_c;
         unsigned long shadow_free;
         unsigned int feature_flush;
@@ -165,6 +164,69 @@ static int add_id_to_freelist(struct blkfront_info *info,
         return 0;
 }
 
+static int fill_grant_buffer(struct blkfront_info *info, int num)
+{
+        struct page *granted_page;
+        struct grant *gnt_list_entry, *n;
+        int i = 0;
+
+        while(i < num) {
+                gnt_list_entry = kzalloc(sizeof(struct grant), GFP_NOIO);
+                if (!gnt_list_entry)
+                        goto out_of_memory;
+
+                granted_page = alloc_page(GFP_NOIO);
+                if (!granted_page) {
+                        kfree(gnt_list_entry);
+                        goto out_of_memory;
+                }
+
+                gnt_list_entry->pfn = page_to_pfn(granted_page);
+                gnt_list_entry->gref = GRANT_INVALID_REF;
+                list_add(&gnt_list_entry->node, &info->persistent_gnts);
+                i++;
+        }
+
+        return 0;
+
+out_of_memory:
+        list_for_each_entry_safe(gnt_list_entry, n,
+                                 &info->persistent_gnts, node) {
+                list_del(&gnt_list_entry->node);
+                __free_page(pfn_to_page(gnt_list_entry->pfn));
+                kfree(gnt_list_entry);
+                i--;
+        }
+        BUG_ON(i != 0);
+        return -ENOMEM;
+}
+
+static struct grant *get_grant(grant_ref_t *gref_head,
+                               struct blkfront_info *info)
+{
+        struct grant *gnt_list_entry;
+        unsigned long buffer_mfn;
+
+        BUG_ON(list_empty(&info->persistent_gnts));
+        gnt_list_entry = list_first_entry(&info->persistent_gnts, struct grant,
+                                          node);
+        list_del(&gnt_list_entry->node);
+
+        if (gnt_list_entry->gref != GRANT_INVALID_REF) {
+                info->persistent_gnts_c--;
+                return gnt_list_entry;
+        }
+
+        /* Assign a gref to this page */
+        gnt_list_entry->gref = gnttab_claim_grant_reference(gref_head);
+        BUG_ON(gnt_list_entry->gref == -ENOSPC);
+        buffer_mfn = pfn_to_mfn(gnt_list_entry->pfn);
+        gnttab_grant_foreign_access_ref(gnt_list_entry->gref,
+                                        info->xbdev->otherend_id,
+                                        buffer_mfn, 0);
+        return gnt_list_entry;
+}
+
 static const char *op_name(int op)
 {
         static const char *const names[] = {
@@ -293,7 +355,6 @@ static int blkif_ioctl(struct block_device *bdev, fmode_t mode,
 static int blkif_queue_request(struct request *req)
 {
         struct blkfront_info *info = req->rq_disk->private_data;
-        unsigned long buffer_mfn;
         struct blkif_request *ring_req;
         unsigned long id;
         unsigned int fsect, lsect;
@@ -306,7 +367,6 @@ static int blkif_queue_request(struct request *req)
          */
         bool new_persistent_gnts;
         grant_ref_t gref_head;
-        struct page *granted_page;
         struct grant *gnt_list_entry = NULL;
         struct scatterlist *sg;
 
@@ -370,41 +430,8 @@ static int blkif_queue_request(struct request *req)
                         fsect = sg->offset >> 9;
                         lsect = fsect + (sg->length >> 9) - 1;
 
-                        if (info->persistent_gnts_c) {
-                                BUG_ON(llist_empty(&info->persistent_gnts));
-                                gnt_list_entry = llist_entry(
-                                        llist_del_first(&info->persistent_gnts),
-                                        struct grant, node);
-
-                                ref = gnt_list_entry->gref;
-                                buffer_mfn = pfn_to_mfn(gnt_list_entry->pfn);
-                                info->persistent_gnts_c--;
-                        } else {
-                                ref = gnttab_claim_grant_reference(&gref_head);
-                                BUG_ON(ref == -ENOSPC);
-
-                                gnt_list_entry =
-                                        kmalloc(sizeof(struct grant),
-                                                         GFP_ATOMIC);
-                                if (!gnt_list_entry)
-                                        return -ENOMEM;
-
-                                granted_page = alloc_page(GFP_ATOMIC);
-                                if (!granted_page) {
-                                        kfree(gnt_list_entry);
-                                        return -ENOMEM;
-                                }
-
-                                gnt_list_entry->pfn =
-                                        page_to_pfn(granted_page);
-                                gnt_list_entry->gref = ref;
-
-                                buffer_mfn = pfn_to_mfn(page_to_pfn(
-                                                                granted_page));
-                                gnttab_grant_foreign_access_ref(ref,
-                                        info->xbdev->otherend_id,
-                                        buffer_mfn, 0);
-                        }
+                        gnt_list_entry = get_grant(&gref_head, info);
+                        ref = gnt_list_entry->gref;
 
                         info->shadow[id].grants_used[i] = gnt_list_entry;
 
@@ -435,7 +462,6 @@ static int blkif_queue_request(struct request *req)
                                 kunmap_atomic(shared_data);
                         }
 
-                        info->shadow[id].frame[i] = mfn_to_pfn(buffer_mfn);
                         ring_req->u.rw.seg[i] =
                                         (struct blkif_request_segment) {
                                                 .gref       = ref,
@@ -790,9 +816,8 @@ static void blkif_restart_queue(struct work_struct *work)
 
 static void blkif_free(struct blkfront_info *info, int suspend)
 {
-        struct llist_node *all_gnts;
-        struct grant *persistent_gnt, *tmp;
-        struct llist_node *n;
+        struct grant *persistent_gnt;
+        struct grant *n;
 
         /* Prevent new requests being issued until we fix things up. */
         spin_lock_irq(&info->io_lock);
@@ -803,22 +828,20 @@ static void blkif_free(struct blkfront_info *info, int suspend)
                 blk_stop_queue(info->rq);
 
         /* Remove all persistent grants */
-        if (info->persistent_gnts_c) {
-                all_gnts = llist_del_all(&info->persistent_gnts);
-                persistent_gnt = llist_entry(all_gnts, typeof(*(persistent_gnt)), node);
-                while (persistent_gnt) {
-                        gnttab_end_foreign_access(persistent_gnt->gref, 0, 0UL);
+        if (!list_empty(&info->persistent_gnts)) {
+                list_for_each_entry_safe(persistent_gnt, n,
+                                         &info->persistent_gnts, node) {
+                        list_del(&persistent_gnt->node);
+                        if (persistent_gnt->gref != GRANT_INVALID_REF) {
+                                gnttab_end_foreign_access(persistent_gnt->gref,
+                                                          0, 0UL);
+                                info->persistent_gnts_c--;
+                        }
                         __free_page(pfn_to_page(persistent_gnt->pfn));
-                        tmp = persistent_gnt;
-                        n = persistent_gnt->node.next;
-                        if (n)
-                                persistent_gnt = llist_entry(n, typeof(*(persistent_gnt)), node);
-                        else
-                                persistent_gnt = NULL;
-                        kfree(tmp);
+                        kfree(persistent_gnt);
                 }
-                info->persistent_gnts_c = 0;
         }
+        BUG_ON(info->persistent_gnts_c != 0);
 
         /* No more gnttab callback work. */
         gnttab_cancel_free_callback(&info->callback);
@@ -875,7 +898,7 @@ static void blkif_completion(struct blk_shadow *s, struct blkfront_info *info,
         }
         /* Add the persistent grant into the list of free grants */
         for (i = 0; i < s->req.u.rw.nr_segments; i++) {
-                llist_add(&s->grants_used[i]->node, &info->persistent_gnts);
+                list_add(&s->grants_used[i]->node, &info->persistent_gnts);
                 info->persistent_gnts_c++;
         }
 }
@@ -1013,6 +1036,12 @@ static int setup_blkring(struct xenbus_device *dev,
 
         sg_init_table(info->sg, BLKIF_MAX_SEGMENTS_PER_REQUEST);
 
+        /* Allocate memory for grants */
+        err = fill_grant_buffer(info, BLK_RING_SIZE *
+                                      BLKIF_MAX_SEGMENTS_PER_REQUEST);
+        if (err)
+                goto fail;
+
         err = xenbus_grant_ring(dev, virt_to_mfn(info->ring.sring));
         if (err < 0) {
                 free_page((unsigned long)sring);
@@ -1171,7 +1200,7 @@ static int blkfront_probe(struct xenbus_device *dev,
         spin_lock_init(&info->io_lock);
         info->xbdev = dev;
         info->vdevice = vdevice;
-        init_llist_head(&info->persistent_gnts);
+        INIT_LIST_HEAD(&info->persistent_gnts);
         info->persistent_gnts_c = 0;
         info->connected = BLKIF_STATE_DISCONNECTED;
         INIT_WORK(&info->work, blkif_restart_queue);
@@ -1203,11 +1232,10 @@ static int blkif_recover(struct blkfront_info *info)
         int j;
 
         /* Stage 1: Make a safe copy of the shadow state. */
-        copy = kmalloc(sizeof(info->shadow),
+        copy = kmemdup(info->shadow, sizeof(info->shadow),
                        GFP_NOIO | __GFP_REPEAT | __GFP_HIGH);
         if (!copy)
                 return -ENOMEM;
-        memcpy(copy, info->shadow, sizeof(info->shadow));
 
         /* Stage 2: Set up free list. */
         memset(&info->shadow, 0, sizeof(info->shadow));
@@ -1236,7 +1264,7 @@ static int blkif_recover(struct blkfront_info *info)
                                 gnttab_grant_foreign_access_ref(
                                         req->u.rw.seg[j].gref,
                                         info->xbdev->otherend_id,
-                                        pfn_to_mfn(info->shadow[req->u.rw.id].frame[j]),
+                                        pfn_to_mfn(copy[i].grants_used[j]->pfn),
                                         0);
                 }
                 info->shadow[req->u.rw.id].req = *req;
diff --git a/include/xen/interface/io/blkif.h b/include/xen/interface/io/blkif.h
index 01c3d62436ef..ffd4652de91c 100644
--- a/include/xen/interface/io/blkif.h
+++ b/include/xen/interface/io/blkif.h
@@ -138,11 +138,21 @@ struct blkif_request_discard {
         uint8_t        _pad3;
 } __attribute__((__packed__));
 
+struct blkif_request_other {
+        uint8_t      _pad1;
+        blkif_vdev_t _pad2;        /* only for read/write requests         */
+#ifdef CONFIG_X86_64
+        uint32_t     _pad3;        /* offsetof(blkif_req..,u.other.id)==8*/
+#endif
+        uint64_t     id;           /* private guest value, echoed in resp  */
+} __attribute__((__packed__));
+
 struct blkif_request {
         uint8_t        operation;    /* BLKIF_OP_???                         */
         union {
                 struct blkif_request_rw rw;
                 struct blkif_request_discard discard;
+                struct blkif_request_other other;
         } u;
 } __attribute__((__packed__));