16 files changed, 135 insertions, 95 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 151070541e21..2f4002f79a24 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -64,13 +64,27 @@ static void drive_stat_acct(struct request *rq, int new_io)
                 return;
 
         cpu = part_stat_lock();
-        part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
 
-        if (!new_io)
+        if (!new_io) {
+                part = rq->part;
                 part_stat_inc(cpu, part, merges[rw]);
-        else {
+        } else {
+                part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
+                if (!hd_struct_try_get(part)) {
+                        /*
+                         * The partition is already being removed,
+                         * the request will be accounted on the disk only
+                         *
+                         * We take a reference on disk->part0 although that
+                         * partition will never be deleted, so we can treat
+                         * it as any other partition.
+                         */
+                        part = &rq->rq_disk->part0;
+                        hd_struct_get(part);
+                }
                 part_round_stats(cpu, part);
                 part_inc_in_flight(part, rw);
+                rq->part = part;
         }
 
         part_stat_unlock();
@@ -128,6 +142,7 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
         rq->ref_count = 1;
         rq->start_time = jiffies;
         set_start_time_ns(rq);
+        rq->part = NULL;
 }
 EXPORT_SYMBOL(blk_rq_init);
 
@@ -1776,7 +1791,7 @@ static void blk_account_io_completion(struct request *req, unsigned int bytes)
                 int cpu;
 
                 cpu = part_stat_lock();
-                part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
+                part = req->part;
                 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
                 part_stat_unlock();
         }
@@ -1796,13 +1811,14 @@ static void blk_account_io_done(struct request *req)
                 int cpu;
 
                 cpu = part_stat_lock();
-                part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
+                part = req->part;
 
                 part_stat_inc(cpu, part, ios[rw]);
                 part_stat_add(cpu, part, ticks[rw], duration);
                 part_round_stats(cpu, part);
                 part_dec_in_flight(part, rw);
 
+                hd_struct_put(part);
                 part_stat_unlock();
         }
 }
@@ -2606,7 +2622,9 @@ int __init blk_dev_init(void)
         BUILD_BUG_ON(__REQ_NR_BITS > 8 *
                         sizeof(((struct request *)0)->cmd_flags));
 
-        kblockd_workqueue = create_workqueue("kblockd");
+        /* used for unplugging and affects IO latency/throughput - HIGHPRI */
+        kblockd_workqueue = alloc_workqueue("kblockd",
+                                            WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
         if (!kblockd_workqueue)
                 panic("Failed to create kblockd\n");
 
diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index 3c7a339fe381..b791022beef3 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -64,7 +64,7 @@ static void cfq_exit(struct io_context *ioc)
         rcu_read_unlock();
 }
 
-/* Called by the exitting task */
+/* Called by the exiting task */
 void exit_io_context(struct task_struct *task)
 {
         struct io_context *ioc;
@@ -74,10 +74,9 @@ void exit_io_context(struct task_struct *task)
         task->io_context = NULL;
         task_unlock(task);
 
-        if (atomic_dec_and_test(&ioc->nr_tasks)) {
+        if (atomic_dec_and_test(&ioc->nr_tasks))
                 cfq_exit(ioc);
 
-        }
         put_io_context(ioc);
 }
 
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 77b7c26df6b5..00b7d31b38a2 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -351,11 +351,12 @@ static void blk_account_io_merge(struct request *req)
                 int cpu;
 
                 cpu = part_stat_lock();
-                part = disk_map_sector_rcu(req->rq_disk, blk_rq_pos(req));
+                part = req->part;
 
                 part_round_stats(cpu, part);
                 part_dec_in_flight(part, rq_data_dir(req));
 
+                hd_struct_put(part);
                 part_stat_unlock();
         }
 }
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 9b186fd6bf47..f083bda30546 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -96,7 +96,7 @@ struct cfq_rb_root {
  */
 struct cfq_queue {
         /* reference count */
-        atomic_t ref;
+        int ref;
         /* various state flags, see below */
         unsigned int flags;
         /* parent cfq_data */
@@ -207,7 +207,7 @@ struct cfq_group {
         struct blkio_group blkg;
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
         struct hlist_node cfqd_node;
-        atomic_t ref;
+        int ref;
 #endif
         /* number of requests that are on the dispatch list or inside driver */
         int dispatched;
@@ -1014,7 +1014,7 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
          * elevator which will be dropped by either elevator exit
          * or cgroup deletion path depending on who is exiting first.
          */
-        atomic_set(&cfqg->ref, 1);
+        cfqg->ref = 1;
 
         /*
          * Add group onto cgroup list. It might happen that bdi->dev is
@@ -1059,7 +1059,7 @@ static struct cfq_group *cfq_get_cfqg(struct cfq_data *cfqd, int create)
 
 static inline struct cfq_group *cfq_ref_get_cfqg(struct cfq_group *cfqg)
 {
-        atomic_inc(&cfqg->ref);
+        cfqg->ref++;
         return cfqg;
 }
 
@@ -1071,7 +1071,7 @@ static void cfq_link_cfqq_cfqg(struct cfq_queue *cfqq, struct cfq_group *cfqg)
 
         cfqq->cfqg = cfqg;
         /* cfqq reference on cfqg */
-        atomic_inc(&cfqq->cfqg->ref);
+        cfqq->cfqg->ref++;
 }
 
 static void cfq_put_cfqg(struct cfq_group *cfqg)
@@ -1079,8 +1079,9 @@ static void cfq_put_cfqg(struct cfq_group *cfqg)
         struct cfq_rb_root *st;
         int i, j;
 
-        BUG_ON(atomic_read(&cfqg->ref) <= 0);
-        if (!atomic_dec_and_test(&cfqg->ref))
+        BUG_ON(cfqg->ref <= 0);
+        cfqg->ref--;
+        if (cfqg->ref)
                 return;
         for_each_cfqg_st(cfqg, i, j, st)
                 BUG_ON(!RB_EMPTY_ROOT(&st->rb));
@@ -1188,7 +1189,7 @@ static void cfq_service_tree_add(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                         cfq_group_service_tree_del(cfqd, cfqq->cfqg);
                 cfqq->orig_cfqg = cfqq->cfqg;
                 cfqq->cfqg = &cfqd->root_group;
-                atomic_inc(&cfqd->root_group.ref);
+                cfqd->root_group.ref++;
                 group_changed = 1;
         } else if (!cfqd->cfq_group_isolation
                    && cfqq_type(cfqq) == SYNC_WORKLOAD && cfqq->orig_cfqg) {
@@ -2025,7 +2026,7 @@ static int cfqq_process_refs(struct cfq_queue *cfqq)
         int process_refs, io_refs;
 
         io_refs = cfqq->allocated[READ] + cfqq->allocated[WRITE];
-        process_refs = atomic_read(&cfqq->ref) - io_refs;
+        process_refs = cfqq->ref - io_refs;
         BUG_ON(process_refs < 0);
         return process_refs;
 }
@@ -2065,10 +2066,10 @@ static void cfq_setup_merge(struct cfq_queue *cfqq, struct cfq_queue *new_cfqq)
          */
         if (new_process_refs >= process_refs) {
                 cfqq->new_cfqq = new_cfqq;
-                atomic_add(process_refs, &new_cfqq->ref);
+                new_cfqq->ref += process_refs;
         } else {
                 new_cfqq->new_cfqq = cfqq;
-                atomic_add(new_process_refs, &cfqq->ref);
+                cfqq->ref += new_process_refs;
         }
 }
 
@@ -2103,12 +2104,6 @@ static void choose_service_tree(struct cfq_data *cfqd, struct cfq_group *cfqg)
         unsigned group_slice;
         enum wl_prio_t original_prio = cfqd->serving_prio;
 
-        if (!cfqg) {
-                cfqd->serving_prio = IDLE_WORKLOAD;
-                cfqd->workload_expires = jiffies + 1;
-                return;
-        }
-
         /* Choose next priority. RT > BE > IDLE */
         if (cfq_group_busy_queues_wl(RT_WORKLOAD, cfqd, cfqg))
                 cfqd->serving_prio = RT_WORKLOAD;
@@ -2538,9 +2533,10 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
         struct cfq_data *cfqd = cfqq->cfqd;
         struct cfq_group *cfqg, *orig_cfqg;
 
-        BUG_ON(atomic_read(&cfqq->ref) <= 0);
+        BUG_ON(cfqq->ref <= 0);
 
-        if (!atomic_dec_and_test(&cfqq->ref))
+        cfqq->ref--;
+        if (cfqq->ref)
                 return;
 
         cfq_log_cfqq(cfqd, cfqq, "put_queue");
@@ -2843,7 +2839,7 @@ static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
         RB_CLEAR_NODE(&cfqq->p_node);
         INIT_LIST_HEAD(&cfqq->fifo);
 
-        atomic_set(&cfqq->ref, 0);
+        cfqq->ref = 0;
         cfqq->cfqd = cfqd;
 
         cfq_mark_cfqq_prio_changed(cfqq);
@@ -2979,11 +2975,11 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct io_context *ioc,
          * pin the queue now that it's allocated, scheduler exit will prune it
          */
         if (!is_sync && !(*async_cfqq)) {
-                atomic_inc(&cfqq->ref);
+                cfqq->ref++;
                 *async_cfqq = cfqq;
         }
 
-        atomic_inc(&cfqq->ref);
+        cfqq->ref++;
         return cfqq;
 }
 
@@ -3685,13 +3681,13 @@ new_queue:
         }
 
         cfqq->allocated[rw]++;
-        atomic_inc(&cfqq->ref);
-
-        spin_unlock_irqrestore(q->queue_lock, flags);
-
+        cfqq->ref++;
         rq->elevator_private = cic;
         rq->elevator_private2 = cfqq;
         rq->elevator_private3 = cfq_ref_get_cfqg(cfqq->cfqg);
+
+        spin_unlock_irqrestore(q->queue_lock, flags);
+
         return 0;
 
 queue_fail:
@@ -3866,6 +3862,10 @@ static void *cfq_init_queue(struct request_queue *q)
         if (!cfqd)
                 return NULL;
 
+        /*
+         * Don't need take queue_lock in the routine, since we are
+         * initializing the ioscheduler, and nobody is using cfqd
+         */
         cfqd->cic_index = i;
 
         /* Init root service tree */
@@ -3885,7 +3885,7 @@ static void *cfq_init_queue(struct request_queue *q)
          * Take a reference to root group which we never drop. This is just
          * to make sure that cfq_put_cfqg() does not try to kfree root group
          */
-        atomic_set(&cfqg->ref, 1);
+        cfqg->ref = 1;
         rcu_read_lock();
         cfq_blkiocg_add_blkio_group(&blkio_root_cgroup, &cfqg->blkg,
                                         (void *)cfqd, 0);
@@ -3905,7 +3905,7 @@ static void *cfq_init_queue(struct request_queue *q)
          * will not attempt to free it.
          */
         cfq_init_cfqq(cfqd, &cfqd->oom_cfqq, 1, 0);
-        atomic_inc(&cfqd->oom_cfqq.ref);
+        cfqd->oom_cfqq.ref++;
         cfq_link_cfqq_cfqg(&cfqd->oom_cfqq, &cfqd->root_group);
 
         INIT_LIST_HEAD(&cfqd->cic_list);
diff --git a/block/genhd.c b/block/genhd.c
index 5465a824d489..6a5b772aa201 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -244,7 +244,7 @@ static struct blk_major_name {
 } *major_names[BLKDEV_MAJOR_HASH_SIZE];
 
 /* index in the above - for now: assume no multimajor ranges */
-static inline int major_to_index(int major)
+static inline int major_to_index(unsigned major)
 {
         return major % BLKDEV_MAJOR_HASH_SIZE;
 }
@@ -828,7 +828,7 @@ static void *show_partition_start(struct seq_file *seqf, loff_t *pos)
         static void *p;
 
         p = disk_seqf_start(seqf, pos);
-        if (!IS_ERR(p) && p && !*pos)
+        if (!IS_ERR_OR_NULL(p) && !*pos)
                 seq_puts(seqf, "major minor  #blocks  name\n\n");
         return p;
 }
@@ -1264,6 +1264,8 @@ struct gendisk *alloc_disk_node(int minors, int node_id)
                 }
                 disk->part_tbl->part[0] = &disk->part0;
 
+                hd_ref_init(&disk->part0);
+
                 disk->minors = minors;
                 rand_initialize_disk(disk);
                 disk_to_dev(disk)->class = &block_class;
diff --git a/drivers/block/loop.c b/drivers/block/loop.c
index 7ea0bea2f7e3..44e18c073c44 100644
--- a/drivers/block/loop.c
+++ b/drivers/block/loop.c
@@ -395,11 +395,7 @@ lo_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
         struct loop_device *lo = p->lo;
         struct page *page = buf->page;
         sector_t IV;
-        int size, ret;
-
-        ret = buf->ops->confirm(pipe, buf);
-        if (unlikely(ret))
-                return ret;
+        int size;
 
         IV = ((sector_t) page->index << (PAGE_CACHE_SHIFT - 9)) +
                                                         (buf->offset >> 9);
diff --git a/drivers/md/dm.c b/drivers/md/dm.c
index 0a2b5516bc21..f48a2f359ac4 100644
--- a/drivers/md/dm.c
+++ b/drivers/md/dm.c
@@ -630,7 +630,7 @@ static void dec_pending(struct dm_io *io, int error)
                         queue_io(md, bio);
                 } else {
                         /* done with normal IO or empty flush */
-                        trace_block_bio_complete(md->queue, bio);
+                        trace_block_bio_complete(md->queue, bio, io_error);
                         bio_endio(bio, io_error);
                 }
         }
diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c
index 4d0ff5ee27b8..e49cce234c65 100644
--- a/fs/bio-integrity.c
+++ b/fs/bio-integrity.c
@@ -782,7 +782,12 @@ void __init bio_integrity_init(void)
 {
         unsigned int i;
 
-        kintegrityd_wq = create_workqueue("kintegrityd");
+        /*
+         * kintegrityd won't block much but may burn a lot of CPU cycles.
+         * Make it highpri CPU intensive wq with max concurrency of 1.
+         */
+        kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
+                                         WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
         if (!kintegrityd_wq)
                 panic("Failed to create kintegrityd\n");
 
diff --git a/fs/char_dev.c b/fs/char_dev.c
index e5b9df993b93..143f0207c7eb 100644
--- a/fs/char_dev.c
+++ b/fs/char_dev.c
@@ -59,7 +59,7 @@ static struct char_device_struct {
 } *chrdevs[CHRDEV_MAJOR_HASH_SIZE];
 
 /* index in the above */
-static inline int major_to_index(int major)
+static inline int major_to_index(unsigned major)
 {
         return major % CHRDEV_MAJOR_HASH_SIZE;
 }
diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c
index 184938fcff04..106ed482f119 100644
--- a/fs/nfsd/vfs.c
+++ b/fs/nfsd/vfs.c
@@ -845,11 +845,6 @@ nfsd_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
         struct page **pp = rqstp->rq_respages + rqstp->rq_resused;
         struct page *page = buf->page;
         size_t size;
-        int ret;
-
-        ret = buf->ops->confirm(pipe, buf);
-        if (unlikely(ret))
-                return ret;
 
         size = sd->len;
 
diff --git a/fs/partitions/check.c b/fs/partitions/check.c
index 9a48d65d9855..9c21119512b9 100644
--- a/fs/partitions/check.c
+++ b/fs/partitions/check.c
@@ -381,6 +381,11 @@ static void delete_partition_rcu_cb(struct rcu_head *head)
         put_device(part_to_dev(part));
 }
 
+void __delete_partition(struct hd_struct *part)
+{
+        call_rcu(&part->rcu_head, delete_partition_rcu_cb);
+}
+
 void delete_partition(struct gendisk *disk, int partno)
 {
         struct disk_part_tbl *ptbl = disk->part_tbl;
@@ -399,7 +404,7 @@ void delete_partition(struct gendisk *disk, int partno)
         kobject_put(part->holder_dir);
         device_del(part_to_dev(part));
 
-        call_rcu(&part->rcu_head, delete_partition_rcu_cb);
+        hd_struct_put(part);
 }
 
 static ssize_t whole_disk_show(struct device *dev,
@@ -498,6 +503,7 @@ struct hd_struct *add_partition(struct gendisk *disk, int partno,
         if (!dev_get_uevent_suppress(ddev))
                 kobject_uevent(&pdev->kobj, KOBJ_ADD);
 
+        hd_ref_init(p);
         return p;
 
 out_free_info:
diff --git a/fs/splice.c b/fs/splice.c
index 8f1dfaecc8f0..d2026382ac3e 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -682,19 +682,14 @@ static int pipe_to_sendpage(struct pipe_inode_info *pipe,
 {
         struct file *file = sd->u.file;
         loff_t pos = sd->pos;
-        int ret, more;
-
-        ret = buf->ops->confirm(pipe, buf);
-        if (!ret) {
-                more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
-                if (file->f_op && file->f_op->sendpage)
-                        ret = file->f_op->sendpage(file, buf->page, buf->offset,
-                                                   sd->len, &pos, more);
-                else
-                        ret = -EINVAL;
-        }
+        int more;
 
-        return ret;
+        if (!likely(file->f_op && file->f_op->sendpage))
+                return -EINVAL;
+
+        more = (sd->flags & SPLICE_F_MORE) || sd->len < sd->total_len;
+        return file->f_op->sendpage(file, buf->page, buf->offset,
+                                    sd->len, &pos, more);
 }
 
 /*
@@ -727,13 +722,6 @@ int pipe_to_file(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
         void *fsdata;
         int ret;
 
-        /*
-         * make sure the data in this buffer is uptodate
-         */
-        ret = buf->ops->confirm(pipe, buf);
-        if (unlikely(ret))
-                return ret;
-
         offset = sd->pos & ~PAGE_CACHE_MASK;
 
         this_len = sd->len;
@@ -805,12 +793,17 @@ int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
                 if (sd->len > sd->total_len)
                         sd->len = sd->total_len;
 
-                ret = actor(pipe, buf, sd);
-                if (ret <= 0) {
+                ret = buf->ops->confirm(pipe, buf);
+                if (unlikely(ret)) {
                         if (ret == -ENODATA)
                                 ret = 0;
                         return ret;
                 }
+
+                ret = actor(pipe, buf, sd);
+                if (ret <= 0)
+                        return ret;
+
                 buf->offset += ret;
                 buf->len -= ret;
 
@@ -1044,10 +1037,6 @@ static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
         int ret;
         void *data;
 
-        ret = buf->ops->confirm(pipe, buf);
-        if (ret)
-                return ret;
-
         data = buf->ops->map(pipe, buf, 0);
         ret = kernel_write(sd->u.file, data + buf->offset, sd->len, sd->pos);
         buf->ops->unmap(pipe, buf, data);
@@ -1507,10 +1496,6 @@ static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
         char *src;
         int ret;
 
-        ret = buf->ops->confirm(pipe, buf);
-        if (unlikely(ret))
-                return ret;
-
         /*
          * See if we can use the atomic maps, by prefaulting in the
          * pages and doing an atomic copy
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 05667e6989f1..5730043eb15a 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -115,6 +115,7 @@ struct request {
         void *elevator_private3;
 
         struct gendisk *rq_disk;
+        struct hd_struct *part;
         unsigned long start_time;
 #ifdef CONFIG_BLK_CGROUP
         unsigned long long start_time_ns;
diff --git a/include/linux/genhd.h b/include/linux/genhd.h
index 13893aa2ac9d..c0d5f6945c1e 100644
--- a/include/linux/genhd.h
+++ b/include/linux/genhd.h
@@ -115,6 +115,7 @@ struct hd_struct {
 #else
         struct disk_stats dkstats;
 #endif
+        atomic_t ref;
         struct rcu_head rcu_head;
 };
 
@@ -597,6 +598,7 @@ extern struct hd_struct * __must_check add_partition(struct gendisk *disk,
                                                      sector_t len, int flags,
                                                      struct partition_meta_info
                                                        *info);
+extern void __delete_partition(struct hd_struct *);
 extern void delete_partition(struct gendisk *, int);
 extern void printk_all_partitions(void);
 
@@ -625,6 +627,29 @@ extern ssize_t part_fail_store(struct device *dev,
                                const char *buf, size_t count);
 #endif /* CONFIG_FAIL_MAKE_REQUEST */
 
+static inline void hd_ref_init(struct hd_struct *part)
+{
+        atomic_set(&part->ref, 1);
+        smp_mb();
+}
+
+static inline void hd_struct_get(struct hd_struct *part)
+{
+        atomic_inc(&part->ref);
+        smp_mb__after_atomic_inc();
+}
+
+static inline int hd_struct_try_get(struct hd_struct *part)
+{
+        return atomic_inc_not_zero(&part->ref);
+}
+
+static inline void hd_struct_put(struct hd_struct *part)
+{
+        if (atomic_dec_and_test(&part->ref))
+                __delete_partition(part);
+}
+
 #else /* CONFIG_BLOCK */
 
 static inline void printk_all_partitions(void) { }
diff --git a/include/trace/events/block.h b/include/trace/events/block.h
index b56c65dc105d..aba421d68f6f 100644
--- a/include/trace/events/block.h
+++ b/include/trace/events/block.h
@@ -206,15 +206,16 @@ TRACE_EVENT(block_bio_bounce,
  * block_bio_complete - completed all work on the block operation
  * @q: queue holding the block operation
  * @bio: block operation completed
+ * @error: io error value
  *
  * This tracepoint indicates there is no further work to do on this
  * block IO operation @bio.
  */
 TRACE_EVENT(block_bio_complete,
 
-        TP_PROTO(struct request_queue *q, struct bio *bio),
+        TP_PROTO(struct request_queue *q, struct bio *bio, int error),
 
-        TP_ARGS(q, bio),
+        TP_ARGS(q, bio, error),
 
         TP_STRUCT__entry(
                 __field( dev_t,         dev             )
@@ -228,6 +229,7 @@ TRACE_EVENT(block_bio_complete,
                 __entry->dev            = bio->bi_bdev->bd_dev;
                 __entry->sector         = bio->bi_sector;
                 __entry->nr_sector      = bio->bi_size >> 9;
+                __entry->error          = error;
                 blk_fill_rwbs(__entry->rwbs, bio->bi_rw, bio->bi_size);
         ),
 
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index 2b8e2ee7c0ef..153562d0b93c 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -758,53 +758,58 @@ static void blk_add_trace_rq_complete(void *ignore,
  * @q:          queue the io is for
  * @bio:        the source bio
  * @what:       the action
+ * @error:      error, if any
  *
  * Description:
  *     Records an action against a bio. Will log the bio offset + size.
  *
  **/
 static void blk_add_trace_bio(struct request_queue *q, struct bio *bio,
-                                     u32 what)
+                              u32 what, int error)
 {
         struct blk_trace *bt = q->blk_trace;
 
         if (likely(!bt))
                 return;
 
+        if (!error && !bio_flagged(bio, BIO_UPTODATE))
+                error = EIO;
+
         __blk_add_trace(bt, bio->bi_sector, bio->bi_size, bio->bi_rw, what,
-                        !bio_flagged(bio, BIO_UPTODATE), 0, NULL);
+                        error, 0, NULL);
 }
 
 static void blk_add_trace_bio_bounce(void *ignore,
                                      struct request_queue *q, struct bio *bio)
 {
-        blk_add_trace_bio(q, bio, BLK_TA_BOUNCE);
+        blk_add_trace_bio(q, bio, BLK_TA_BOUNCE, 0);
 }
 
 static void blk_add_trace_bio_complete(void *ignore,
-                                       struct request_queue *q, struct bio *bio)
+                                       struct request_queue *q, struct bio *bio,
+                                       int error)
 {
-        blk_add_trace_bio(q, bio, BLK_TA_COMPLETE);
+        blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, error);
 }
 
 static void blk_add_trace_bio_backmerge(void *ignore,
                                         struct request_queue *q,
                                         struct bio *bio)
 {
-        blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE);
+        blk_add_trace_bio(q, bio, BLK_TA_BACKMERGE, 0);
 }
 
 static void blk_add_trace_bio_frontmerge(void *ignore,
                                          struct request_queue *q,
                                          struct bio *bio)
 {
-        blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE);
+        blk_add_trace_bio(q, bio, BLK_TA_FRONTMERGE, 0);
 }
 
 static void blk_add_trace_bio_queue(void *ignore,
                                     struct request_queue *q, struct bio *bio)
 {
-        blk_add_trace_bio(q, bio, BLK_TA_QUEUE);
+        blk_add_trace_bio(q, bio, BLK_TA_QUEUE, 0);
 }
 
 static void blk_add_trace_getrq(void *ignore,
@@ -812,7 +817,7 @@ static void blk_add_trace_getrq(void *ignore,
                                 struct bio *bio, int rw)
 {
         if (bio)
-                blk_add_trace_bio(q, bio, BLK_TA_GETRQ);
+                blk_add_trace_bio(q, bio, BLK_TA_GETRQ, 0);
         else {
                 struct blk_trace *bt = q->blk_trace;
 
@@ -827,7 +832,7 @@ static void blk_add_trace_sleeprq(void *ignore,
                                   struct bio *bio, int rw)
 {
         if (bio)
-                blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ);
+                blk_add_trace_bio(q, bio, BLK_TA_SLEEPRQ, 0);
         else {
                 struct blk_trace *bt = q->blk_trace;