summaryrefslogtreecommitdiffstats
path: root/block/cfq-iosched.c
diff options
context:
space:
mode:
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r--block/cfq-iosched.c135
1 files changed, 11 insertions, 124 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 11f49d036845..f3b44c394e6d 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -2935,117 +2935,6 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct io_context *ioc,
2935 return cfqq; 2935 return cfqq;
2936} 2936}
2937 2937
2938/**
2939 * ioc_create_icq - create and link io_cq
2940 * @q: request_queue of interest
2941 * @gfp_mask: allocation mask
2942 *
2943 * Make sure io_cq linking %current->io_context and @q exists. If either
2944 * io_context and/or icq don't exist, they will be created using @gfp_mask.
2945 *
2946 * The caller is responsible for ensuring @ioc won't go away and @q is
2947 * alive and will stay alive until this function returns.
2948 */
2949static struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask)
2950{
2951 struct elevator_type *et = q->elevator->type;
2952 struct io_context *ioc;
2953 struct io_cq *icq;
2954
2955 /* allocate stuff */
2956 ioc = create_io_context(current, gfp_mask, q->node);
2957 if (!ioc)
2958 return NULL;
2959
2960 icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
2961 q->node);
2962 if (!icq)
2963 return NULL;
2964
2965 if (radix_tree_preload(gfp_mask) < 0) {
2966 kmem_cache_free(et->icq_cache, icq);
2967 return NULL;
2968 }
2969
2970 icq->ioc = ioc;
2971 icq->q = q;
2972 INIT_LIST_HEAD(&icq->q_node);
2973 INIT_HLIST_NODE(&icq->ioc_node);
2974
2975 /* lock both q and ioc and try to link @icq */
2976 spin_lock_irq(q->queue_lock);
2977 spin_lock(&ioc->lock);
2978
2979 if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
2980 hlist_add_head(&icq->ioc_node, &ioc->icq_list);
2981 list_add(&icq->q_node, &q->icq_list);
2982 if (et->ops.elevator_init_icq_fn)
2983 et->ops.elevator_init_icq_fn(icq);
2984 } else {
2985 kmem_cache_free(et->icq_cache, icq);
2986 icq = ioc_lookup_icq(ioc, q);
2987 if (!icq)
2988 printk(KERN_ERR "cfq: icq link failed!\n");
2989 }
2990
2991 spin_unlock(&ioc->lock);
2992 spin_unlock_irq(q->queue_lock);
2993 radix_tree_preload_end();
2994 return icq;
2995}
2996
2997/**
2998 * cfq_get_cic - acquire cfq_io_cq and bump refcnt on io_context
2999 * @cfqd: cfqd to setup cic for
3000 * @gfp_mask: allocation mask
3001 *
3002 * Return cfq_io_cq associating @cfqd and %current->io_context and
3003 * bump refcnt on io_context. If ioc or cic doesn't exist, they're created
3004 * using @gfp_mask.
3005 *
3006 * Must be called under queue_lock which may be released and re-acquired.
3007 * This function also may sleep depending on @gfp_mask.
3008 */
3009static struct cfq_io_cq *cfq_get_cic(struct cfq_data *cfqd, gfp_t gfp_mask)
3010{
3011 struct request_queue *q = cfqd->queue;
3012 struct cfq_io_cq *cic = NULL;
3013 struct io_context *ioc;
3014
3015 lockdep_assert_held(q->queue_lock);
3016
3017 while (true) {
3018 /* fast path */
3019 ioc = current->io_context;
3020 if (likely(ioc)) {
3021 cic = cfq_cic_lookup(cfqd, ioc);
3022 if (likely(cic))
3023 break;
3024 }
3025
3026 /* slow path - unlock, create missing ones and retry */
3027 spin_unlock_irq(q->queue_lock);
3028 cic = icq_to_cic(ioc_create_icq(q, gfp_mask));
3029 spin_lock_irq(q->queue_lock);
3030 if (!cic)
3031 return NULL;
3032 }
3033
3034 /* bump @ioc's refcnt and handle changed notifications */
3035 get_io_context(ioc);
3036
3037 if (unlikely(cic->icq.changed)) {
3038 if (test_and_clear_bit(ICQ_IOPRIO_CHANGED, &cic->icq.changed))
3039 changed_ioprio(cic);
3040#ifdef CONFIG_CFQ_GROUP_IOSCHED
3041 if (test_and_clear_bit(ICQ_CGROUP_CHANGED, &cic->icq.changed))
3042 changed_cgroup(cic);
3043#endif
3044 }
3045
3046 return cic;
3047}
3048
3049static void 2938static void
3050__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle) 2939__cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle)
3051{ 2940{
@@ -3524,8 +3413,6 @@ static void cfq_put_request(struct request *rq)
3524 BUG_ON(!cfqq->allocated[rw]); 3413 BUG_ON(!cfqq->allocated[rw]);
3525 cfqq->allocated[rw]--; 3414 cfqq->allocated[rw]--;
3526 3415
3527 put_io_context(RQ_CIC(rq)->icq.ioc, cfqq->cfqd->queue);
3528
3529 /* Put down rq reference on cfqg */ 3416 /* Put down rq reference on cfqg */
3530 cfq_put_cfqg(RQ_CFQG(rq)); 3417 cfq_put_cfqg(RQ_CFQG(rq));
3531 rq->elv.priv[0] = NULL; 3418 rq->elv.priv[0] = NULL;
@@ -3574,7 +3461,7 @@ static int
3574cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) 3461cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
3575{ 3462{
3576 struct cfq_data *cfqd = q->elevator->elevator_data; 3463 struct cfq_data *cfqd = q->elevator->elevator_data;
3577 struct cfq_io_cq *cic; 3464 struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq);
3578 const int rw = rq_data_dir(rq); 3465 const int rw = rq_data_dir(rq);
3579 const bool is_sync = rq_is_sync(rq); 3466 const bool is_sync = rq_is_sync(rq);
3580 struct cfq_queue *cfqq; 3467 struct cfq_queue *cfqq;
@@ -3582,9 +3469,16 @@ cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask)
3582 might_sleep_if(gfp_mask & __GFP_WAIT); 3469 might_sleep_if(gfp_mask & __GFP_WAIT);
3583 3470
3584 spin_lock_irq(q->queue_lock); 3471 spin_lock_irq(q->queue_lock);
3585 cic = cfq_get_cic(cfqd, gfp_mask); 3472
3586 if (!cic) 3473 /* handle changed notifications */
3587 goto queue_fail; 3474 if (unlikely(cic->icq.changed)) {
3475 if (test_and_clear_bit(ICQ_IOPRIO_CHANGED, &cic->icq.changed))
3476 changed_ioprio(cic);
3477#ifdef CONFIG_CFQ_GROUP_IOSCHED
3478 if (test_and_clear_bit(ICQ_CGROUP_CHANGED, &cic->icq.changed))
3479 changed_cgroup(cic);
3480#endif
3481 }
3588 3482
3589new_queue: 3483new_queue:
3590 cfqq = cic_to_cfqq(cic, is_sync); 3484 cfqq = cic_to_cfqq(cic, is_sync);
@@ -3615,17 +3509,10 @@ new_queue:
3615 cfqq->allocated[rw]++; 3509 cfqq->allocated[rw]++;
3616 3510
3617 cfqq->ref++; 3511 cfqq->ref++;
3618 rq->elv.icq = &cic->icq;
3619 rq->elv.priv[0] = cfqq; 3512 rq->elv.priv[0] = cfqq;
3620 rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg); 3513 rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg);
3621 spin_unlock_irq(q->queue_lock); 3514 spin_unlock_irq(q->queue_lock);
3622 return 0; 3515 return 0;
3623
3624queue_fail:
3625 cfq_schedule_dispatch(cfqd);
3626 spin_unlock_irq(q->queue_lock);
3627 cfq_log(cfqd, "set_request fail");
3628 return 1;
3629} 3516}
3630 3517
3631static void cfq_kick_queue(struct work_struct *work) 3518static void cfq_kick_queue(struct work_struct *work)