1 files changed, 94 insertions, 84 deletions

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 833ec18eaa63..fd7080ed7935 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -71,6 +71,51 @@ struct cfq_rb_root {
 #define CFQ_RB_ROOT     (struct cfq_rb_root) { RB_ROOT, NULL, }
 
 /*
+ * Per process-grouping structure
+ */
+struct cfq_queue {
+        /* reference count */
+        atomic_t ref;
+        /* various state flags, see below */
+        unsigned int flags;
+        /* parent cfq_data */
+        struct cfq_data *cfqd;
+        /* service_tree member */
+        struct rb_node rb_node;
+        /* service_tree key */
+        unsigned long rb_key;
+        /* prio tree member */
+        struct rb_node p_node;
+        /* prio tree root we belong to, if any */
+        struct rb_root *p_root;
+        /* sorted list of pending requests */
+        struct rb_root sort_list;
+        /* if fifo isn't expired, next request to serve */
+        struct request *next_rq;
+        /* requests queued in sort_list */
+        int queued[2];
+        /* currently allocated requests */
+        int allocated[2];
+        /* fifo list of requests in sort_list */
+        struct list_head fifo;
+
+        unsigned long slice_end;
+        long slice_resid;
+        unsigned int slice_dispatch;
+
+        /* pending metadata requests */
+        int meta_pending;
+        /* number of requests that are on the dispatch list or inside driver */
+        int dispatched;
+
+        /* io prio of this group */
+        unsigned short ioprio, org_ioprio;
+        unsigned short ioprio_class, org_ioprio_class;
+
+        pid_t pid;
+};
+
+/*
  * Per block device queue structure
  */
 struct cfq_data {
@@ -135,51 +180,11 @@ struct cfq_data {
         unsigned int cfq_slice_idle;
 
         struct list_head cic_list;
-};
 
-/*
- * Per process-grouping structure
- */
-struct cfq_queue {
-        /* reference count */
-        atomic_t ref;
-        /* various state flags, see below */
-        unsigned int flags;
-        /* parent cfq_data */
-        struct cfq_data *cfqd;
-        /* service_tree member */
-        struct rb_node rb_node;
-        /* service_tree key */
-        unsigned long rb_key;
-        /* prio tree member */
-        struct rb_node p_node;
-        /* prio tree root we belong to, if any */
-        struct rb_root *p_root;
-        /* sorted list of pending requests */
-        struct rb_root sort_list;
-        /* if fifo isn't expired, next request to serve */
-        struct request *next_rq;
-        /* requests queued in sort_list */
-        int queued[2];
-        /* currently allocated requests */
-        int allocated[2];
-        /* fifo list of requests in sort_list */
-        struct list_head fifo;
-
-        unsigned long slice_end;
-        long slice_resid;
-        unsigned int slice_dispatch;
-
-        /* pending metadata requests */
-        int meta_pending;
-        /* number of requests that are on the dispatch list or inside driver */
-        int dispatched;
-
-        /* io prio of this group */
-        unsigned short ioprio, org_ioprio;
-        unsigned short ioprio_class, org_ioprio_class;
-
-        pid_t pid;
+        /*
+         * Fallback dummy cfqq for extreme OOM conditions
+         */
+        struct cfq_queue oom_cfqq;
 };
 
 enum cfqq_state_flags {
@@ -1641,6 +1646,26 @@ static void cfq_ioc_set_ioprio(struct io_context *ioc)
         ioc->ioprio_changed = 0;
 }
 
+static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
+                          pid_t pid, int is_sync)
+{
+        RB_CLEAR_NODE(&cfqq->rb_node);
+        RB_CLEAR_NODE(&cfqq->p_node);
+        INIT_LIST_HEAD(&cfqq->fifo);
+
+        atomic_set(&cfqq->ref, 0);
+        cfqq->cfqd = cfqd;
+
+        cfq_mark_cfqq_prio_changed(cfqq);
+
+        if (is_sync) {
+                if (!cfq_class_idle(cfqq))
+                        cfq_mark_cfqq_idle_window(cfqq);
+                cfq_mark_cfqq_sync(cfqq);
+        }
+        cfqq->pid = pid;
+}
+
 static struct cfq_queue *
 cfq_find_alloc_queue(struct cfq_data *cfqd, int is_sync,
                      struct io_context *ioc, gfp_t gfp_mask)
@@ -1653,56 +1678,40 @@ retry:
         /* cic always exists here */
         cfqq = cic_to_cfqq(cic, is_sync);
 
-        if (!cfqq) {
+        /*
+         * Always try a new alloc if we fell back to the OOM cfqq
+         * originally, since it should just be a temporary situation.
+         */
+        if (!cfqq || cfqq == &cfqd->oom_cfqq) {
+                cfqq = NULL;
                 if (new_cfqq) {
                         cfqq = new_cfqq;
                         new_cfqq = NULL;
                 } else if (gfp_mask & __GFP_WAIT) {
-                        /*
-                         * Inform the allocator of the fact that we will
-                         * just repeat this allocation if it fails, to allow
-                         * the allocator to do whatever it needs to attempt to
-                         * free memory.
-                         */
                         spin_unlock_irq(cfqd->queue->queue_lock);
                         new_cfqq = kmem_cache_alloc_node(cfq_pool,
-                                        gfp_mask | __GFP_NOFAIL | __GFP_ZERO,
+                                        gfp_mask | __GFP_ZERO,
                                         cfqd->queue->node);
                         spin_lock_irq(cfqd->queue->queue_lock);
-                        goto retry;
+                        if (new_cfqq)
+                                goto retry;
                 } else {
                         cfqq = kmem_cache_alloc_node(cfq_pool,
                                         gfp_mask | __GFP_ZERO,
                                         cfqd->queue->node);
-                        if (!cfqq)
-                                goto out;
                 }
 
-                RB_CLEAR_NODE(&cfqq->rb_node);
-                RB_CLEAR_NODE(&cfqq->p_node);
-                INIT_LIST_HEAD(&cfqq->fifo);
-
-                atomic_set(&cfqq->ref, 0);
-                cfqq->cfqd = cfqd;
-
-                cfq_mark_cfqq_prio_changed(cfqq);
-
-                cfq_init_prio_data(cfqq, ioc);
-
-                if (is_sync) {
-                        if (!cfq_class_idle(cfqq))
-                                cfq_mark_cfqq_idle_window(cfqq);
-                        cfq_mark_cfqq_sync(cfqq);
-                }
-                cfqq->pid = current->pid;
-                cfq_log_cfqq(cfqd, cfqq, "alloced");
+                if (cfqq) {
+                        cfq_init_cfqq(cfqd, cfqq, current->pid, is_sync);
+                        cfq_init_prio_data(cfqq, ioc);
+                        cfq_log_cfqq(cfqd, cfqq, "alloced");
+                } else
+                        cfqq = &cfqd->oom_cfqq;
         }
 
         if (new_cfqq)
                 kmem_cache_free(cfq_pool, new_cfqq);
 
-out:
-        WARN_ON((gfp_mask & __GFP_WAIT) && !cfqq);
         return cfqq;
 }
 
@@ -1735,11 +1744,8 @@ cfq_get_queue(struct cfq_data *cfqd, int is_sync, struct io_context *ioc,
                 cfqq = *async_cfqq;
         }
 
-        if (!cfqq) {
+        if (!cfqq)
                 cfqq = cfq_find_alloc_queue(cfqd, is_sync, ioc, gfp_mask);
-                if (!cfqq)
-                        return NULL;
-        }
 
         /*
          * pin the queue now that it's allocated, scheduler exit will prune it
@@ -2305,12 +2311,8 @@ cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
                 goto queue_fail;
 
         cfqq = cic_to_cfqq(cic, is_sync);
-        if (!cfqq) {
+        if (!cfqq || cfqq == &cfqd->oom_cfqq) {
                 cfqq = cfq_get_queue(cfqd, is_sync, cic->ioc, gfp_mask);
-
-                if (!cfqq)
-                        goto queue_fail;
-
                 cic_set_cfqq(cic, cfqq, is_sync);
         }
 
@@ -2465,6 +2467,14 @@ static void *cfq_init_queue(struct request_queue *q)
         for (i = 0; i < CFQ_PRIO_LISTS; i++)
                 cfqd->prio_trees[i] = RB_ROOT;
 
+        /*
+         * Our fallback cfqq if cfq_find_alloc_queue() runs into OOM issues.
+         * Grab a permanent reference to it, so that the normal code flow
+         * will not attempt to free it.
+         */
+        cfq_init_cfqq(cfqd, &cfqd->oom_cfqq, 1, 0);
+        atomic_inc(&cfqd->oom_cfqq.ref);
+
         INIT_LIST_HEAD(&cfqd->cic_list);
 
         cfqd->queue = q;