4 files changed, 71 insertions, 54 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index fd4749391e17..6804fdf27eff 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -771,9 +771,12 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
 {
         struct request *rq = NULL;
         struct request_list *rl = &q->rq;
-        struct io_context *ioc = NULL;
+        struct io_context *ioc;
         const bool is_sync = rw_is_sync(rw_flags) != 0;
+        bool retried = false;
         int may_queue;
+retry:
+        ioc = current->io_context;
 
         if (unlikely(blk_queue_dead(q)))
                 return NULL;
@@ -784,7 +787,20 @@ static struct request *get_request(struct request_queue *q, int rw_flags,
 
         if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
                 if (rl->count[is_sync]+1 >= q->nr_requests) {
-                        ioc = current_io_context(GFP_ATOMIC, q->node);
+                        /*
+                         * We want ioc to record batching state.  If it's
+                         * not already there, creating a new one requires
+                         * dropping queue_lock, which in turn requires
+                         * retesting conditions to avoid queue hang.
+                         */
+                        if (!ioc && !retried) {
+                                spin_unlock_irq(q->queue_lock);
+                                create_io_context(current, gfp_mask, q->node);
+                                spin_lock_irq(q->queue_lock);
+                                retried = true;
+                                goto retry;
+                        }
+
                         /*
                          * The queue will fill after this allocation, so set
                          * it as full, and mark this process as "batching".
@@ -892,7 +908,6 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,
         rq = get_request(q, rw_flags, bio, GFP_NOIO);
         while (!rq) {
                 DEFINE_WAIT(wait);
-                struct io_context *ioc;
                 struct request_list *rl = &q->rq;
 
                 if (unlikely(blk_queue_dead(q)))
@@ -912,8 +927,8 @@ static struct request *get_request_wait(struct request_queue *q, int rw_flags,
                  * up to a big batch of them for a small period time.
                  * See ioc_batching, ioc_set_batching
                  */
-                ioc = current_io_context(GFP_NOIO, q->node);
-                ioc_set_batching(q, ioc);
+                create_io_context(current, GFP_NOIO, q->node);
+                ioc_set_batching(q, current->io_context);
 
                 spin_lock_irq(q->queue_lock);
                 finish_wait(&rl->wait[is_sync], &wait);
diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index fb23965595da..e23c797b4685 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -205,16 +205,15 @@ void exit_io_context(struct task_struct *task)
         put_io_context(ioc, NULL);
 }
 
-static struct io_context *create_task_io_context(struct task_struct *task,
-                                                 gfp_t gfp_flags, int node,
-                                                 bool take_ref)
+void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_flags,
+                                int node)
 {
         struct io_context *ioc;
 
         ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
                                     node);
         if (unlikely(!ioc))
-                return NULL;
+                return;
 
         /* initialize */
         atomic_long_set(&ioc->refcount, 1);
@@ -226,42 +225,13 @@ static struct io_context *create_task_io_context(struct task_struct *task,
 
         /* try to install, somebody might already have beaten us to it */
         task_lock(task);
-
-        if (!task->io_context && !(task->flags & PF_EXITING)) {
+        if (!task->io_context && !(task->flags & PF_EXITING))
                 task->io_context = ioc;
-        } else {
+        else
                 kmem_cache_free(iocontext_cachep, ioc);
-                ioc = task->io_context;
-        }
-
-        if (ioc && take_ref)
-                get_io_context(ioc);
-
         task_unlock(task);
-        return ioc;
 }
-
-/**
- * current_io_context - get io_context of %current
- * @gfp_flags: allocation flags, used if allocation is necessary
- * @node: allocation node, used if allocation is necessary
- *
- * Return io_context of %current.  If it doesn't exist, it is created with
- * @gfp_flags and @node.  The returned io_context does NOT have its
- * reference count incremented.  Because io_context is exited only on task
- * exit, %current can be sure that the returned io_context is valid and
- * alive as long as it is executing.
- */
-struct io_context *current_io_context(gfp_t gfp_flags, int node)
-{
-        might_sleep_if(gfp_flags & __GFP_WAIT);
-
-        if (current->io_context)
-                return current->io_context;
-
-        return create_task_io_context(current, gfp_flags, node, false);
-}
-EXPORT_SYMBOL(current_io_context);
+EXPORT_SYMBOL(create_io_context_slowpath);
 
 /**
  * get_task_io_context - get io_context of a task
@@ -274,7 +244,7 @@ EXPORT_SYMBOL(current_io_context);
  * incremented.
  *
  * This function always goes through task_lock() and it's better to use
- * current_io_context() + get_io_context() for %current.
+ * %current->io_context + get_io_context() for %current.
  */
 struct io_context *get_task_io_context(struct task_struct *task,
                                        gfp_t gfp_flags, int node)
@@ -283,16 +253,18 @@ struct io_context *get_task_io_context(struct task_struct *task,
 
         might_sleep_if(gfp_flags & __GFP_WAIT);
 
-        task_lock(task);
-        ioc = task->io_context;
-        if (likely(ioc)) {
-                get_io_context(ioc);
+        do {
+                task_lock(task);
+                ioc = task->io_context;
+                if (likely(ioc)) {
+                        get_io_context(ioc);
+                        task_unlock(task);
+                        return ioc;
+                }
                 task_unlock(task);
-                return ioc;
-        }
-        task_unlock(task);
+        } while (create_io_context(task, gfp_flags, node));
 
-        return create_task_io_context(task, gfp_flags, node, true);
+        return NULL;
 }
 EXPORT_SYMBOL(get_task_io_context);
 
diff --git a/block/blk.h b/block/blk.h
index 8d421156fefb..5bca2668e1bf 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -127,9 +127,6 @@ static inline int blk_should_fake_timeout(struct request_queue *q)
 }
 #endif
 
-void get_io_context(struct io_context *ioc);
-struct io_context *current_io_context(gfp_t gfp_flags, int node);
-
 int ll_back_merge_fn(struct request_queue *q, struct request *req,
                      struct bio *bio);
 int ll_front_merge_fn(struct request_queue *q, struct request *req, 
@@ -198,6 +195,39 @@ static inline int blk_do_io_stat(struct request *rq)
                 (rq->cmd_flags & REQ_DISCARD));
 }
 
+/*
+ * Internal io_context interface
+ */
+void get_io_context(struct io_context *ioc);
+
+void create_io_context_slowpath(struct task_struct *task, gfp_t gfp_mask,
+                                int node);
+
+/**
+ * create_io_context - try to create task->io_context
+ * @task: target task
+ * @gfp_mask: allocation mask
+ * @node: allocation node
+ *
+ * If @task->io_context is %NULL, allocate a new io_context and install it.
+ * Returns the current @task->io_context which may be %NULL if allocation
+ * failed.
+ *
+ * Note that this function can't be called with IRQ disabled because
+ * task_lock which protects @task->io_context is IRQ-unsafe.
+ */
+static inline struct io_context *create_io_context(struct task_struct *task,
+                                                   gfp_t gfp_mask, int node)
+{
+        WARN_ON_ONCE(irqs_disabled());
+        if (unlikely(!task->io_context))
+                create_io_context_slowpath(task, gfp_mask, node);
+        return task->io_context;
+}
+
+/*
+ * Internal throttling interface
+ */
 #ifdef CONFIG_BLK_DEV_THROTTLING
 extern bool blk_throtl_bio(struct request_queue *q, struct bio *bio);
 extern void blk_throtl_drain(struct request_queue *q);
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 3b07ce168780..5f7e4d161404 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -3012,7 +3012,7 @@ static int cfq_create_cic(struct cfq_data *cfqd, gfp_t gfp_mask)
         might_sleep_if(gfp_mask & __GFP_WAIT);
 
         /* allocate stuff */
-        ioc = current_io_context(gfp_mask, q->node);
+        ioc = create_io_context(current, gfp_mask, q->node);
         if (!ioc)
                 goto out;