[PATCH] make cfq_exit_queue() prune the cfq_io_context for that queue

Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>

2 files changed, 60 insertions, 1 deletions

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 3bacf4bb7dd2..3fc6e505e9c8 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -177,6 +177,8 @@ struct cfq_data {
         unsigned int cfq_slice_async_rq;
         unsigned int cfq_slice_idle;
         unsigned int cfq_max_depth;
+
+        struct list_head cic_list;
 };
 
 /*
@@ -1215,7 +1217,12 @@ static void cfq_free_io_context(struct cfq_io_context *cic)
 static void cfq_exit_single_io_context(struct cfq_io_context *cic)
 {
         struct cfq_data *cfqd = cic->key;
-        request_queue_t *q = cfqd->queue;
+        request_queue_t *q;
+
+        if (!cfqd)
+                return;
+
+        q = cfqd->queue;
 
         WARN_ON(!irqs_disabled());
 
@@ -1236,6 +1243,7 @@ static void cfq_exit_single_io_context(struct cfq_io_context *cic)
         }
 
         cic->key = NULL;
+        list_del_init(&cic->queue_list);
         spin_unlock(q->queue_lock);
 }
 
@@ -1254,12 +1262,14 @@ static void cfq_exit_io_context(struct cfq_io_context *cic)
         /*
          * put the reference this task is holding to the various queues
          */
+        read_lock(&cfq_exit_lock);
         list_for_each(entry, &cic->list) {
                 __cic = list_entry(entry, struct cfq_io_context, list);
                 cfq_exit_single_io_context(__cic);
         }
 
         cfq_exit_single_io_context(cic);
+        read_unlock(&cfq_exit_lock);
         local_irq_restore(flags);
 }
 
@@ -1279,6 +1289,7 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
                 cic->ttime_mean = 0;
                 cic->dtor = cfq_free_io_context;
                 cic->exit = cfq_exit_io_context;
+                INIT_LIST_HEAD(&cic->queue_list);
         }
 
         return cic;
@@ -1446,6 +1457,7 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
         if (!ioc)
                 return NULL;
 
+restart:
         if ((cic = ioc->cic) == NULL) {
                 cic = cfq_alloc_io_context(cfqd, gfp_mask);
 
@@ -1461,6 +1473,7 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
                 read_lock(&cfq_exit_lock);
                 ioc->set_ioprio = cfq_ioc_set_ioprio;
                 ioc->cic = cic;
+                list_add(&cic->queue_list, &cfqd->cic_list);
                 read_unlock(&cfq_exit_lock);
         } else {
                 struct cfq_io_context *__cic;
@@ -1471,6 +1484,19 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
                 if (cic->key == cfqd)
                         goto out;
 
+                if (unlikely(!cic->key)) {
+                        read_lock(&cfq_exit_lock);
+                        if (list_empty(&cic->list))
+                                ioc->cic = NULL;
+                        else
+                                ioc->cic = list_entry(cic->list.next,
+                                                      struct cfq_io_context,
+                                                      list);
+                        read_unlock(&cfq_exit_lock);
+                        kmem_cache_free(cfq_ioc_pool, cic);
+                        goto restart;
+                }
+
                 /*
                  * cic exists, check if we already are there. linear search
                  * should be ok here, the list will usually not be more than
@@ -1485,6 +1511,13 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
                                 cic = __cic;
                                 goto out;
                         }
+                        if (unlikely(!__cic->key)) {
+                                read_lock(&cfq_exit_lock);
+                                list_del(&__cic->list);
+                                read_unlock(&cfq_exit_lock);
+                                kmem_cache_free(cfq_ioc_pool, __cic);
+                                goto restart;
+                        }
                 }
 
                 /*
@@ -1499,6 +1532,7 @@ cfq_get_io_context(struct cfq_data *cfqd, pid_t pid, gfp_t gfp_mask)
                 __cic->key = cfqd;
                 read_lock(&cfq_exit_lock);
                 list_add(&__cic->list, &cic->list);
+                list_add(&__cic->queue_list, &cfqd->cic_list);
                 read_unlock(&cfq_exit_lock);
                 cic = __cic;
         }
@@ -2104,8 +2138,30 @@ static void cfq_put_cfqd(struct cfq_data *cfqd)
 static void cfq_exit_queue(elevator_t *e)
 {
         struct cfq_data *cfqd = e->elevator_data;
+        request_queue_t *q = cfqd->queue;
 
         cfq_shutdown_timer_wq(cfqd);
+        write_lock(&cfq_exit_lock);
+        spin_lock_irq(q->queue_lock);
+        if (cfqd->active_queue)
+                __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
+        while(!list_empty(&cfqd->cic_list)) {
+                struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
+                                                        struct cfq_io_context,
+                                                        queue_list);
+                if (cic->cfqq[ASYNC]) {
+                        cfq_put_queue(cic->cfqq[ASYNC]);
+                        cic->cfqq[ASYNC] = NULL;
+                }
+                if (cic->cfqq[SYNC]) {
+                        cfq_put_queue(cic->cfqq[SYNC]);
+                        cic->cfqq[SYNC] = NULL;
+                }
+                cic->key = NULL;
+                list_del_init(&cic->queue_list);
+        }
+        spin_unlock_irq(q->queue_lock);
+        write_unlock(&cfq_exit_lock);
         cfq_put_cfqd(cfqd);
 }
 
@@ -2127,6 +2183,7 @@ static int cfq_init_queue(request_queue_t *q, elevator_t *e)
         INIT_LIST_HEAD(&cfqd->cur_rr);
         INIT_LIST_HEAD(&cfqd->idle_rr);
         INIT_LIST_HEAD(&cfqd->empty_list);
+        INIT_LIST_HEAD(&cfqd->cic_list);
 
         cfqd->crq_hash = kmalloc(sizeof(struct hlist_head) * CFQ_MHASH_ENTRIES, GFP_KERNEL);
         if (!cfqd->crq_hash)
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index e19cb631084e..80518f703538 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -69,6 +69,8 @@ struct cfq_io_context {
         unsigned long ttime_samples;
         unsigned long ttime_mean;
 
+        struct list_head queue_list;
+
         void (*dtor)(struct cfq_io_context *);
         void (*exit)(struct cfq_io_context *);
 };