Merge branch 'for-3.9/core' of git://git.kernel.dk/linux-block

Pull block IO core bits from Jens Axboe:
 "Below are the core block IO bits for 3.9.  It was delayed a few days
  since my workstation kept crashing every 2-8h after pulling it into
  current -git, but turns out it is a bug in the new pstate code (divide
  by zero, will report separately).  In any case, it contains:

   - The big cfq/blkcg update from Tejun and and Vivek.

   - Additional block and writeback tracepoints from Tejun.

   - Improvement of the should sort (based on queues) logic in the plug
     flushing.

   - _io() variants of the wait_for_completion() interface, using
     io_schedule() instead of schedule() to contribute to io wait
     properly.

   - Various little fixes.

  You'll get two trivial merge conflicts, which should be easy enough to
  fix up"

Fix up the trivial conflicts due to hlist traversal cleanups (commit
b67bfe0d42ca: "hlist: drop the node parameter from iterators").

* 'for-3.9/core' of git://git.kernel.dk/linux-block: (39 commits)
  block: remove redundant check to bd_openers()
  block: use i_size_write() in bd_set_size()
  cfq: fix lock imbalance with failed allocations
  drivers/block/swim3.c: fix null pointer dereference
  block: don't select PERCPU_RWSEM
  block: account iowait time when waiting for completion of IO request
  sched: add wait_for_completion_io[_timeout]
  writeback: add more tracepoints
  block: add block_{touch|dirty}_buffer tracepoint
  buffer: make touch_buffer() an exported function
  block: add @req to bio_{front|back}_merge tracepoints
  block: add missing block_bio_complete() tracepoint
  block: Remove should_sort judgement when flush blk_plug
  block,elevator: use new hashtable implementation
  cfq-iosched: add hierarchical cfq_group statistics
  cfq-iosched: collect stats from dead cfqgs
  cfq-iosched: separate out cfqg_stats_reset() from cfq_pd_reset_stats()
  blkcg: make blkcg_print_blkgs() grab q locks instead of blkcg lock
  block: RCU free request_queue
  blkcg: implement blkg_[rw]stat_recursive_sum() and blkg_[rw]stat_merge()
  ...

7 files changed, 216 insertions, 18 deletions

diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index f94bc83011ed..78feda9bbae2 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -19,6 +19,7 @@
 #include <linux/gfp.h>
 #include <linux/bsg.h>
 #include <linux/smp.h>
+#include <linux/rcupdate.h>
 
 #include <asm/scatterlist.h>
 
@@ -437,6 +438,7 @@ struct request_queue {
         /* Throttle data */
         struct throtl_data *td;
 #endif
+        struct rcu_head         rcu_head;
 };
 
 #define QUEUE_FLAG_QUEUED       1       /* uses generic tag queueing */
@@ -974,7 +976,6 @@ struct blk_plug {
         unsigned long magic; /* detect uninitialized use-cases */
         struct list_head list; /* requests */
         struct list_head cb_list; /* md requires an unplug callback */
-        unsigned int should_sort; /* list to be sorted before flushing? */
 };
 #define BLK_MAX_REQUEST_COUNT 16
 
diff --git a/include/linux/blktrace_api.h b/include/linux/blktrace_api.h
index 7c2e030e72f1..0ea61e07a91c 100644
--- a/include/linux/blktrace_api.h
+++ b/include/linux/blktrace_api.h
@@ -12,6 +12,7 @@
 
 struct blk_trace {
         int trace_state;
+        bool rq_based;
         struct rchan *rchan;
         unsigned long __percpu *sequence;
         unsigned char __percpu *msg_data;
diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
index 458f497738a4..5afc4f94d110 100644
--- a/include/linux/buffer_head.h
+++ b/include/linux/buffer_head.h
@@ -126,7 +126,6 @@ BUFFER_FNS(Write_EIO, write_io_error)
 BUFFER_FNS(Unwritten, unwritten)
 
 #define bh_offset(bh)           ((unsigned long)(bh)->b_data & ~PAGE_MASK)
-#define touch_buffer(bh)        mark_page_accessed(bh->b_page)
 
 /* If we *know* page->private refers to buffer_heads */
 #define page_buffers(page)                                      \
@@ -142,6 +141,7 @@ BUFFER_FNS(Unwritten, unwritten)
 
 void mark_buffer_dirty(struct buffer_head *bh);
 void init_buffer(struct buffer_head *, bh_end_io_t *, void *);
+void touch_buffer(struct buffer_head *bh);
 void set_bh_page(struct buffer_head *bh,
                 struct page *page, unsigned long offset);
 int try_to_free_buffers(struct page *);
diff --git a/include/linux/completion.h b/include/linux/completion.h
index 51494e6b5548..33f0280fd533 100644
--- a/include/linux/completion.h
+++ b/include/linux/completion.h
@@ -77,10 +77,13 @@ static inline void init_completion(struct completion *x)
 }
 
 extern void wait_for_completion(struct completion *);
+extern void wait_for_completion_io(struct completion *);
 extern int wait_for_completion_interruptible(struct completion *x);
 extern int wait_for_completion_killable(struct completion *x);
 extern unsigned long wait_for_completion_timeout(struct completion *x,
                                                    unsigned long timeout);
+extern unsigned long wait_for_completion_io_timeout(struct completion *x,
+                                                    unsigned long timeout);
 extern long wait_for_completion_interruptible_timeout(
         struct completion *x, unsigned long timeout);
 extern long wait_for_completion_killable_timeout(
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index 186620631750..acd0312d46fb 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -2,6 +2,7 @@
 #define _LINUX_ELEVATOR_H
 
 #include <linux/percpu.h>
+#include <linux/hashtable.h>
 
 #ifdef CONFIG_BLOCK
 
@@ -96,6 +97,8 @@ struct elevator_type
         struct list_head list;
 };
 
+#define ELV_HASH_BITS 6
+
 /*
  * each queue has an elevator_queue associated with it
  */
@@ -105,8 +108,8 @@ struct elevator_queue
         void *elevator_data;
         struct kobject kobj;
         struct mutex sysfs_lock;
-        struct hlist_head *hash;
         unsigned int registered:1;
+        DECLARE_HASHTABLE(hash, ELV_HASH_BITS);
 };
 
 /*
diff --git a/include/trace/events/block.h b/include/trace/events/block.h
index 05c5e61f0a7c..9961726523d0 100644
--- a/include/trace/events/block.h
+++ b/include/trace/events/block.h
@@ -6,10 +6,61 @@
 
 #include <linux/blktrace_api.h>
 #include <linux/blkdev.h>
+#include <linux/buffer_head.h>
 #include <linux/tracepoint.h>
 
 #define RWBS_LEN        8
 
+DECLARE_EVENT_CLASS(block_buffer,
+
+        TP_PROTO(struct buffer_head *bh),
+
+        TP_ARGS(bh),
+
+        TP_STRUCT__entry (
+                __field(  dev_t,        dev                     )
+                __field(  sector_t,     sector                  )
+                __field(  size_t,       size                    )
+        ),
+
+        TP_fast_assign(
+                __entry->dev            = bh->b_bdev->bd_dev;
+                __entry->sector         = bh->b_blocknr;
+                __entry->size           = bh->b_size;
+        ),
+
+        TP_printk("%d,%d sector=%llu size=%zu",
+                MAJOR(__entry->dev), MINOR(__entry->dev),
+                (unsigned long long)__entry->sector, __entry->size
+        )
+);
+
+/**
+ * block_touch_buffer - mark a buffer accessed
+ * @bh: buffer_head being touched
+ *
+ * Called from touch_buffer().
+ */
+DEFINE_EVENT(block_buffer, block_touch_buffer,
+
+        TP_PROTO(struct buffer_head *bh),
+
+        TP_ARGS(bh)
+);
+
+/**
+ * block_dirty_buffer - mark a buffer dirty
+ * @bh: buffer_head being dirtied
+ *
+ * Called from mark_buffer_dirty().
+ */
+DEFINE_EVENT(block_buffer, block_dirty_buffer,
+
+        TP_PROTO(struct buffer_head *bh),
+
+        TP_ARGS(bh)
+);
+
 DECLARE_EVENT_CLASS(block_rq_with_error,
 
         TP_PROTO(struct request_queue *q, struct request *rq),
@@ -206,7 +257,6 @@ TRACE_EVENT(block_bio_bounce,
 
 /**
  * block_bio_complete - completed all work on the block operation
- * @q: queue holding the block operation
  * @bio: block operation completed
  * @error: io error value
  *
@@ -215,9 +265,9 @@ TRACE_EVENT(block_bio_bounce,
  */
 TRACE_EVENT(block_bio_complete,
 
-        TP_PROTO(struct request_queue *q, struct bio *bio, int error),
+        TP_PROTO(struct bio *bio, int error),
 
-        TP_ARGS(q, bio, error),
+        TP_ARGS(bio, error),
 
         TP_STRUCT__entry(
                 __field( dev_t,         dev             )
@@ -228,7 +278,8 @@ TRACE_EVENT(block_bio_complete,
         ),
 
         TP_fast_assign(
-                __entry->dev            = bio->bi_bdev->bd_dev;
+                __entry->dev            = bio->bi_bdev ?
+                                          bio->bi_bdev->bd_dev : 0;
                 __entry->sector         = bio->bi_sector;
                 __entry->nr_sector      = bio->bi_size >> 9;
                 __entry->error          = error;
@@ -241,11 +292,11 @@ TRACE_EVENT(block_bio_complete,
                   __entry->nr_sector, __entry->error)
 );
 
-DECLARE_EVENT_CLASS(block_bio,
+DECLARE_EVENT_CLASS(block_bio_merge,
 
-        TP_PROTO(struct request_queue *q, struct bio *bio),
+        TP_PROTO(struct request_queue *q, struct request *rq, struct bio *bio),
 
-        TP_ARGS(q, bio),
+        TP_ARGS(q, rq, bio),
 
         TP_STRUCT__entry(
                 __field( dev_t,         dev                     )
@@ -272,31 +323,33 @@ DECLARE_EVENT_CLASS(block_bio,
 /**
  * block_bio_backmerge - merging block operation to the end of an existing operation
  * @q: queue holding operation
+ * @rq: request bio is being merged into
  * @bio: new block operation to merge
  *
  * Merging block request @bio to the end of an existing block request
  * in queue @q.
  */
-DEFINE_EVENT(block_bio, block_bio_backmerge,
+DEFINE_EVENT(block_bio_merge, block_bio_backmerge,
 
-        TP_PROTO(struct request_queue *q, struct bio *bio),
+        TP_PROTO(struct request_queue *q, struct request *rq, struct bio *bio),
 
-        TP_ARGS(q, bio)
+        TP_ARGS(q, rq, bio)
 );
 
 /**
  * block_bio_frontmerge - merging block operation to the beginning of an existing operation
  * @q: queue holding operation
+ * @rq: request bio is being merged into
  * @bio: new block operation to merge
  *
  * Merging block IO operation @bio to the beginning of an existing block
  * operation in queue @q.
  */
-DEFINE_EVENT(block_bio, block_bio_frontmerge,
+DEFINE_EVENT(block_bio_merge, block_bio_frontmerge,
 
-        TP_PROTO(struct request_queue *q, struct bio *bio),
+        TP_PROTO(struct request_queue *q, struct request *rq, struct bio *bio),
 
-        TP_ARGS(q, bio)
+        TP_ARGS(q, rq, bio)
 );
 
 /**
@@ -306,11 +359,32 @@ DEFINE_EVENT(block_bio, block_bio_frontmerge,
  *
  * About to place the block IO operation @bio into queue @q.
  */
-DEFINE_EVENT(block_bio, block_bio_queue,
+TRACE_EVENT(block_bio_queue,
 
         TP_PROTO(struct request_queue *q, struct bio *bio),
 
-        TP_ARGS(q, bio)
+        TP_ARGS(q, bio),
+
+        TP_STRUCT__entry(
+                __field( dev_t,         dev                     )
+                __field( sector_t,      sector                  )
+                __field( unsigned int,  nr_sector               )
+                __array( char,          rwbs,   RWBS_LEN        )
+                __array( char,          comm,   TASK_COMM_LEN   )
+        ),
+
+        TP_fast_assign(
+                __entry->dev            = bio->bi_bdev->bd_dev;
+                __entry->sector         = bio->bi_sector;
+                __entry->nr_sector      = bio->bi_size >> 9;
+                blk_fill_rwbs(__entry->rwbs, bio->bi_rw, bio->bi_size);
+                memcpy(__entry->comm, current->comm, TASK_COMM_LEN);
+        ),
+
+        TP_printk("%d,%d %s %llu + %u [%s]",
+                  MAJOR(__entry->dev), MINOR(__entry->dev), __entry->rwbs,
+                  (unsigned long long)__entry->sector,
+                  __entry->nr_sector, __entry->comm)
 );
 
 DECLARE_EVENT_CLASS(block_get_rq,
diff --git a/include/trace/events/writeback.h b/include/trace/events/writeback.h
index b453d92c2253..6a16fd2e70ed 100644
--- a/include/trace/events/writeback.h
+++ b/include/trace/events/writeback.h
@@ -32,6 +32,115 @@
 
 struct wb_writeback_work;
 
+TRACE_EVENT(writeback_dirty_page,
+
+        TP_PROTO(struct page *page, struct address_space *mapping),
+
+        TP_ARGS(page, mapping),
+
+        TP_STRUCT__entry (
+                __array(char, name, 32)
+                __field(unsigned long, ino)
+                __field(pgoff_t, index)
+        ),
+
+        TP_fast_assign(
+                strncpy(__entry->name,
+                        mapping ? dev_name(mapping->backing_dev_info->dev) : "(unknown)", 32);
+                __entry->ino = mapping ? mapping->host->i_ino : 0;
+                __entry->index = page->index;
+        ),
+
+        TP_printk("bdi %s: ino=%lu index=%lu",
+                __entry->name,
+                __entry->ino,
+                __entry->index
+        )
+);
+
+DECLARE_EVENT_CLASS(writeback_dirty_inode_template,
+
+        TP_PROTO(struct inode *inode, int flags),
+
+        TP_ARGS(inode, flags),
+
+        TP_STRUCT__entry (
+                __array(char, name, 32)
+                __field(unsigned long, ino)
+                __field(unsigned long, flags)
+        ),
+
+        TP_fast_assign(
+                struct backing_dev_info *bdi = inode->i_mapping->backing_dev_info;
+
+                /* may be called for files on pseudo FSes w/ unregistered bdi */
+                strncpy(__entry->name,
+                        bdi->dev ? dev_name(bdi->dev) : "(unknown)", 32);
+                __entry->ino            = inode->i_ino;
+                __entry->flags          = flags;
+        ),
+
+        TP_printk("bdi %s: ino=%lu flags=%s",
+                __entry->name,
+                __entry->ino,
+                show_inode_state(__entry->flags)
+        )
+);
+
+DEFINE_EVENT(writeback_dirty_inode_template, writeback_dirty_inode_start,
+
+        TP_PROTO(struct inode *inode, int flags),
+
+        TP_ARGS(inode, flags)
+);
+
+DEFINE_EVENT(writeback_dirty_inode_template, writeback_dirty_inode,
+
+        TP_PROTO(struct inode *inode, int flags),
+
+        TP_ARGS(inode, flags)
+);
+
+DECLARE_EVENT_CLASS(writeback_write_inode_template,
+
+        TP_PROTO(struct inode *inode, struct writeback_control *wbc),
+
+        TP_ARGS(inode, wbc),
+
+        TP_STRUCT__entry (
+                __array(char, name, 32)
+                __field(unsigned long, ino)
+                __field(int, sync_mode)
+        ),
+
+        TP_fast_assign(
+                strncpy(__entry->name,
+                        dev_name(inode->i_mapping->backing_dev_info->dev), 32);
+                __entry->ino            = inode->i_ino;
+                __entry->sync_mode      = wbc->sync_mode;
+        ),
+
+        TP_printk("bdi %s: ino=%lu sync_mode=%d",
+                __entry->name,
+                __entry->ino,
+                __entry->sync_mode
+        )
+);
+
+DEFINE_EVENT(writeback_write_inode_template, writeback_write_inode_start,
+
+        TP_PROTO(struct inode *inode, struct writeback_control *wbc),
+
+        TP_ARGS(inode, wbc)
+);
+
+DEFINE_EVENT(writeback_write_inode_template, writeback_write_inode,
+
+        TP_PROTO(struct inode *inode, struct writeback_control *wbc),
+
+        TP_ARGS(inode, wbc)
+);
+
 DECLARE_EVENT_CLASS(writeback_work_class,
         TP_PROTO(struct backing_dev_info *bdi, struct wb_writeback_work *work),
         TP_ARGS(bdi, work),
@@ -479,6 +588,13 @@ DECLARE_EVENT_CLASS(writeback_single_inode_template,
         )
 );
 
+DEFINE_EVENT(writeback_single_inode_template, writeback_single_inode_start,
+        TP_PROTO(struct inode *inode,
+                 struct writeback_control *wbc,
+                 unsigned long nr_to_write),
+        TP_ARGS(inode, wbc, nr_to_write)
+);
+
 DEFINE_EVENT(writeback_single_inode_template, writeback_single_inode,
         TP_PROTO(struct inode *inode,
                  struct writeback_control *wbc,