Merge branch 'for-3.6/core' of git://git.kernel.dk/linux-block

Pull core block IO bits from Jens Axboe:
 "The most complicated part if this is the request allocation rework by
  Tejun, which has been queued up for a long time and has been in
  for-next ditto as well.

  There are a few commits from yesterday and today, mostly trivial and
  obvious fixes.  So I'm pretty confident that it is sound.  It's also
  smaller than usual."

* 'for-3.6/core' of git://git.kernel.dk/linux-block:
  block: remove dead func declaration
  block: add partition resize function to blkpg ioctl
  block: uninitialized ioc->nr_tasks triggers WARN_ON
  block: do not artificially constrain max_sectors for stacking drivers
  blkcg: implement per-blkg request allocation
  block: prepare for multiple request_lists
  block: add q->nr_rqs[] and move q->rq.elvpriv to q->nr_rqs_elvpriv
  blkcg: inline bio_blkcg() and friends
  block: allocate io_context upfront
  block: refactor get_request[_wait]()
  block: drop custom queue draining used by scsi_transport_{iscsi|fc}
  mempool: add @gfp_mask to mempool_create_node()
  blkcg: make root blkcg allocation use %GFP_KERNEL
  blkcg: __blkg_lookup_create() doesn't need radix preload

21 files changed, 530 insertions, 301 deletions

diff --git a/Documentation/block/queue-sysfs.txt b/Documentation/block/queue-sysfs.txt
index d8147b336c35..6518a55273e7 100644
--- a/Documentation/block/queue-sysfs.txt
+++ b/Documentation/block/queue-sysfs.txt
@@ -38,6 +38,13 @@ read or write requests. Note that the total allocated number may be twice
 this amount, since it applies only to reads or writes (not the accumulated
 sum).
 
+To avoid priority inversion through request starvation, a request
+queue maintains a separate request pool per each cgroup when
+CONFIG_BLK_CGROUP is enabled, and this parameter applies to each such
+per-block-cgroup request pool.  IOW, if there are N block cgroups,
+each request queue may have upto N request pools, each independently
+regulated by nr_requests.
+
 read_ahead_kb (RW)
 ------------------
 Maximum number of kilobytes to read-ahead for filesystems on this block
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index e7dee617358e..f3b44a65fc7a 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -31,27 +31,6 @@ EXPORT_SYMBOL_GPL(blkcg_root);
 
 static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
 
-struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
-{
-        return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
-                            struct blkcg, css);
-}
-EXPORT_SYMBOL_GPL(cgroup_to_blkcg);
-
-static struct blkcg *task_blkcg(struct task_struct *tsk)
-{
-        return container_of(task_subsys_state(tsk, blkio_subsys_id),
-                            struct blkcg, css);
-}
-
-struct blkcg *bio_blkcg(struct bio *bio)
-{
-        if (bio && bio->bi_css)
-                return container_of(bio->bi_css, struct blkcg, css);
-        return task_blkcg(current);
-}
-EXPORT_SYMBOL_GPL(bio_blkcg);
-
 static bool blkcg_policy_enabled(struct request_queue *q,
                                  const struct blkcg_policy *pol)
 {
@@ -84,6 +63,7 @@ static void blkg_free(struct blkcg_gq *blkg)
                 kfree(pd);
         }
 
+        blk_exit_rl(&blkg->rl);
         kfree(blkg);
 }
 
@@ -91,16 +71,18 @@ static void blkg_free(struct blkcg_gq *blkg)
  * blkg_alloc - allocate a blkg
  * @blkcg: block cgroup the new blkg is associated with
  * @q: request_queue the new blkg is associated with
+ * @gfp_mask: allocation mask to use
  *
  * Allocate a new blkg assocating @blkcg and @q.
  */
-static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
+static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
+                                   gfp_t gfp_mask)
 {
         struct blkcg_gq *blkg;
         int i;
 
         /* alloc and init base part */
-        blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
+        blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
         if (!blkg)
                 return NULL;
 
@@ -109,6 +91,13 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
         blkg->blkcg = blkcg;
         blkg->refcnt = 1;
 
+        /* root blkg uses @q->root_rl, init rl only for !root blkgs */
+        if (blkcg != &blkcg_root) {
+                if (blk_init_rl(&blkg->rl, q, gfp_mask))
+                        goto err_free;
+                blkg->rl.blkg = blkg;
+        }
+
         for (i = 0; i < BLKCG_MAX_POLS; i++) {
                 struct blkcg_policy *pol = blkcg_policy[i];
                 struct blkg_policy_data *pd;
@@ -117,11 +106,9 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
                         continue;
 
                 /* alloc per-policy data and attach it to blkg */
-                pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node);
-                if (!pd) {
-                        blkg_free(blkg);
-                        return NULL;
-                }
+                pd = kzalloc_node(pol->pd_size, gfp_mask, q->node);
+                if (!pd)
+                        goto err_free;
 
                 blkg->pd[i] = pd;
                 pd->blkg = blkg;
@@ -132,6 +119,10 @@ static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
         }
 
         return blkg;
+
+err_free:
+        blkg_free(blkg);
+        return NULL;
 }
 
 static struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
@@ -175,9 +166,13 @@ struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q)
 }
 EXPORT_SYMBOL_GPL(blkg_lookup);
 
+/*
+ * If @new_blkg is %NULL, this function tries to allocate a new one as
+ * necessary using %GFP_ATOMIC.  @new_blkg is always consumed on return.
+ */
 static struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
-                                             struct request_queue *q)
-        __releases(q->queue_lock) __acquires(q->queue_lock)
+                                             struct request_queue *q,
+                                             struct blkcg_gq *new_blkg)
 {
         struct blkcg_gq *blkg;
         int ret;
@@ -189,24 +184,26 @@ static struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
         blkg = __blkg_lookup(blkcg, q);
         if (blkg) {
                 rcu_assign_pointer(blkcg->blkg_hint, blkg);
-                return blkg;
+                goto out_free;
         }