1 files changed, 79 insertions, 22 deletions

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index ed897b5ef315..5ff4f4850e71 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -64,6 +64,9 @@ static DEFINE_PER_CPU(unsigned long, cfq_ioc_count);
 static struct completion *ioc_gone;
 static DEFINE_SPINLOCK(ioc_gone_lock);
 
+static DEFINE_SPINLOCK(cic_index_lock);
+static DEFINE_IDA(cic_index_ida);
+
 #define CFQ_PRIO_LISTS          IOPRIO_BE_NR
 #define cfq_class_idle(cfqq)    ((cfqq)->ioprio_class == IOPRIO_CLASS_IDLE)
 #define cfq_class_rt(cfqq)      ((cfqq)->ioprio_class == IOPRIO_CLASS_RT)
@@ -271,6 +274,7 @@ struct cfq_data {
         unsigned int cfq_latency;
         unsigned int cfq_group_isolation;
 
+        unsigned int cic_index;
         struct list_head cic_list;
 
         /*
@@ -430,6 +434,24 @@ static inline void cic_set_cfqq(struct cfq_io_context *cic,
         cic->cfqq[is_sync] = cfqq;
 }
 
+#define CIC_DEAD_KEY    1ul
+#define CIC_DEAD_INDEX_SHIFT    1
+
+static inline void *cfqd_dead_key(struct cfq_data *cfqd)
+{
+        return (void *)(cfqd->cic_index << CIC_DEAD_INDEX_SHIFT | CIC_DEAD_KEY);
+}
+
+static inline struct cfq_data *cic_to_cfqd(struct cfq_io_context *cic)
+{
+        struct cfq_data *cfqd = cic->key;
+
+        if (unlikely((unsigned long) cfqd & CIC_DEAD_KEY))
+                return NULL;
+
+        return cfqd;
+}
+
 /*
  * We regard a request as SYNC, if it's either a read or has the SYNC bit
  * set (in which case it could also be direct WRITE).
@@ -2510,11 +2532,12 @@ static void cfq_cic_free(struct cfq_io_context *cic)
 static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic)
 {
         unsigned long flags;
+        unsigned long dead_key = (unsigned long) cic->key;
 
-        BUG_ON(!cic->dead_key);
+        BUG_ON(!(dead_key & CIC_DEAD_KEY));
 
         spin_lock_irqsave(&ioc->lock, flags);
-        radix_tree_delete(&ioc->radix_root, cic->dead_key);
+        radix_tree_delete(&ioc->radix_root, dead_key >> CIC_DEAD_INDEX_SHIFT);
         hlist_del_rcu(&cic->cic_list);
         spin_unlock_irqrestore(&ioc->lock, flags);
 
@@ -2537,15 +2560,10 @@ static void cfq_free_io_context(struct io_context *ioc)
         __call_for_each_cic(ioc, cic_free_func);
 }
 
-static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
+static void cfq_put_cooperator(struct cfq_queue *cfqq)
 {
         struct cfq_queue *__cfqq, *next;
 
-        if (unlikely(cfqq == cfqd->active_queue)) {
-                __cfq_slice_expired(cfqd, cfqq, 0);
-                cfq_schedule_dispatch(cfqd);
-        }
-
         /*
          * If this queue was scheduled to merge with another queue, be
          * sure to drop the reference taken on that queue (and others in
@@ -2561,6 +2579,16 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
                 cfq_put_queue(__cfqq);
                 __cfqq = next;
         }
+}
+
+static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
+{
+        if (unlikely(cfqq == cfqd->active_queue)) {
+                __cfq_slice_expired(cfqd, cfqq, 0);
+                cfq_schedule_dispatch(cfqd);
+        }
+
+        cfq_put_cooperator(cfqq);
 
         cfq_put_queue(cfqq);
 }
@@ -2573,11 +2601,10 @@ static void __cfq_exit_single_io_context(struct cfq_data *cfqd,
         list_del_init(&cic->queue_list);
 
         /*
-         * Make sure key == NULL is seen for dead queues
+         * Make sure dead mark is seen for dead queues
          */
         smp_wmb();
-        cic->dead_key = (unsigned long) cic->key;
-        cic->key = NULL;
+        cic->key = cfqd_dead_key(cfqd);
 
         if (ioc->ioc_data == cic)
                 rcu_assign_pointer(ioc->ioc_data, NULL);
@@ -2596,7 +2623,7 @@ static void __cfq_exit_single_io_context(struct cfq_data *cfqd,
 static void cfq_exit_single_io_context(struct io_context *ioc,
                                        struct cfq_io_context *cic)
 {
-        struct cfq_data *cfqd = cic->key;
+        struct cfq_data *cfqd = cic_to_cfqd(cic);
 
         if (cfqd) {
                 struct request_queue *q = cfqd->queue;
@@ -2609,7 +2636,7 @@ static void cfq_exit_single_io_context(struct io_context *ioc,
                  * race between exiting task and queue
                  */
                 smp_read_barrier_depends();
-                if (cic->key)
+                if (cic->key == cfqd)
                         __cfq_exit_single_io_context(cfqd, cic);
 
                 spin_unlock_irqrestore(q->queue_lock, flags);
@@ -2689,7 +2716,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc)
 
 static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic)
 {
-        struct cfq_data *cfqd = cic->key;
+        struct cfq_data *cfqd = cic_to_cfqd(cic);
         struct cfq_queue *cfqq;
         unsigned long flags;
 
@@ -2746,7 +2773,7 @@ static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq,
 static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic)
 {
         struct cfq_queue *sync_cfqq = cic_to_cfqq(cic, 1);
-        struct cfq_data *cfqd = cic->key;
+        struct cfq_data *cfqd = cic_to_cfqd(cic);
         unsigned long flags;
         struct request_queue *q;
 
@@ -2883,12 +2910,13 @@ cfq_drop_dead_cic(struct cfq_data *cfqd, struct io_context *ioc,
         unsigned long flags;
 
         WARN_ON(!list_empty(&cic->queue_list));
+        BUG_ON(cic->key != cfqd_dead_key(cfqd));
 
         spin_lock_irqsave(&ioc->lock, flags);
 
         BUG_ON(ioc->ioc_data == cic);
 
-        radix_tree_delete(&ioc->radix_root, (unsigned long) cfqd);
+        radix_tree_delete(&ioc->radix_root, cfqd->cic_index);
         hlist_del_rcu(&cic->cic_list);
         spin_unlock_irqrestore(&ioc->lock, flags);
 
@@ -2900,7 +2928,6 @@ cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc)
 {
         struct cfq_io_context *cic;
         unsigned long flags;
-        void *k;
 
         if (unlikely(!ioc))
                 return NULL;
@@ -2917,13 +2944,11 @@ cfq_cic_lookup(struct cfq_data *cfqd, struct io_context *ioc)
         }
 
         do {
-                cic = radix_tree_lookup(&ioc->radix_root, (unsigned long) cfqd);
+                cic = radix_tree_lookup(&ioc->radix_root, cfqd->cic_index);
                 rcu_read_unlock();
                 if (!cic)
                         break;
-                /* ->key must be copied to avoid race with cfq_exit_queue() */
-                k = cic->key;
-                if (unlikely(!k)) {
+                if (unlikely(cic->key != cfqd)) {
                         cfq_drop_dead_cic(cfqd, ioc, cic);
                         rcu_read_lock();
                         continue;
@@ -2956,7 +2981,7 @@ static int cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc,
 
                 spin_lock_irqsave(&ioc->lock, flags);
                 ret = radix_tree_insert(&ioc->radix_root,
-                                                (unsigned long) cfqd, cic);
+                                                cfqd->cic_index, cic);
                 if (!ret)
                         hlist_add_head_rcu(&cic->cic_list, &ioc->cic_list);
                 spin_unlock_irqrestore(&ioc->lock, flags);
@@ -3516,6 +3541,9 @@ split_cfqq(struct cfq_io_context *cic, struct cfq_queue *cfqq)
         }
 
         cic_set_cfqq(cic, NULL, 1);
+
+        cfq_put_cooperator(cfqq);
+
         cfq_put_queue(cfqq);
         return NULL;
 }
@@ -3708,10 +3736,32 @@ static void cfq_exit_queue(struct elevator_queue *e)
 
         cfq_shutdown_timer_wq(cfqd);
 
+        spin_lock(&cic_index_lock);
+        ida_remove(&cic_index_ida, cfqd->cic_index);
+        spin_unlock(&cic_index_lock);
+
         /* Wait for cfqg->blkg->key accessors to exit their grace periods. */
         call_rcu(&cfqd->rcu, cfq_cfqd_free);
 }
 
+static int cfq_alloc_cic_index(void)
+{
+        int index, error;
+
+        do {
+                if (!ida_pre_get(&cic_index_ida, GFP_KERNEL))
+                        return -ENOMEM;
+
+                spin_lock(&cic_index_lock);
+                error = ida_get_new(&cic_index_ida, &index);
+                spin_unlock(&cic_index_lock);
+                if (error && error != -EAGAIN)
+                        return error;
+        } while (error);
+
+        return index;
+}
+
 static void *cfq_init_queue(struct request_queue *q)
 {
         struct cfq_data *cfqd;
@@ -3719,10 +3769,16 @@ static void *cfq_init_queue(struct request_queue *q)
         struct cfq_group *cfqg;
         struct cfq_rb_root *st;
 
+        i = cfq_alloc_cic_index();
+        if (i < 0)
+                return NULL;
+
         cfqd = kmalloc_node(sizeof(*cfqd), GFP_KERNEL | __GFP_ZERO, q->node);
         if (!cfqd)
                 return NULL;
 
+        cfqd->cic_index = i;
+
         /* Init root service tree */
         cfqd->grp_service_tree = CFQ_RB_ROOT;
 
@@ -3984,6 +4040,7 @@ static void __exit cfq_exit(void)
          */
         if (elv_ioc_count_read(cfq_ioc_count))
                 wait_for_completion(&all_gone);
+        ida_destroy(&cic_index_ida);
         cfq_slab_kill();
 }