diff options
Diffstat (limited to 'block/cfq-iosched.c')
-rw-r--r-- | block/cfq-iosched.c | 135 |
1 files changed, 11 insertions, 124 deletions
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 11f49d036845..f3b44c394e6d 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c | |||
@@ -2935,117 +2935,6 @@ cfq_get_queue(struct cfq_data *cfqd, bool is_sync, struct io_context *ioc, | |||
2935 | return cfqq; | 2935 | return cfqq; |
2936 | } | 2936 | } |
2937 | 2937 | ||
2938 | /** | ||
2939 | * ioc_create_icq - create and link io_cq | ||
2940 | * @q: request_queue of interest | ||
2941 | * @gfp_mask: allocation mask | ||
2942 | * | ||
2943 | * Make sure io_cq linking %current->io_context and @q exists. If either | ||
2944 | * io_context and/or icq don't exist, they will be created using @gfp_mask. | ||
2945 | * | ||
2946 | * The caller is responsible for ensuring @ioc won't go away and @q is | ||
2947 | * alive and will stay alive until this function returns. | ||
2948 | */ | ||
2949 | static struct io_cq *ioc_create_icq(struct request_queue *q, gfp_t gfp_mask) | ||
2950 | { | ||
2951 | struct elevator_type *et = q->elevator->type; | ||
2952 | struct io_context *ioc; | ||
2953 | struct io_cq *icq; | ||
2954 | |||
2955 | /* allocate stuff */ | ||
2956 | ioc = create_io_context(current, gfp_mask, q->node); | ||
2957 | if (!ioc) | ||
2958 | return NULL; | ||
2959 | |||
2960 | icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO, | ||
2961 | q->node); | ||
2962 | if (!icq) | ||
2963 | return NULL; | ||
2964 | |||
2965 | if (radix_tree_preload(gfp_mask) < 0) { | ||
2966 | kmem_cache_free(et->icq_cache, icq); | ||
2967 | return NULL; | ||
2968 | } | ||
2969 | |||
2970 | icq->ioc = ioc; | ||
2971 | icq->q = q; | ||
2972 | INIT_LIST_HEAD(&icq->q_node); | ||
2973 | INIT_HLIST_NODE(&icq->ioc_node); | ||
2974 | |||
2975 | /* lock both q and ioc and try to link @icq */ | ||
2976 | spin_lock_irq(q->queue_lock); | ||
2977 | spin_lock(&ioc->lock); | ||
2978 | |||
2979 | if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) { | ||
2980 | hlist_add_head(&icq->ioc_node, &ioc->icq_list); | ||
2981 | list_add(&icq->q_node, &q->icq_list); | ||
2982 | if (et->ops.elevator_init_icq_fn) | ||
2983 | et->ops.elevator_init_icq_fn(icq); | ||
2984 | } else { | ||
2985 | kmem_cache_free(et->icq_cache, icq); | ||
2986 | icq = ioc_lookup_icq(ioc, q); | ||
2987 | if (!icq) | ||
2988 | printk(KERN_ERR "cfq: icq link failed!\n"); | ||
2989 | } | ||
2990 | |||
2991 | spin_unlock(&ioc->lock); | ||
2992 | spin_unlock_irq(q->queue_lock); | ||
2993 | radix_tree_preload_end(); | ||
2994 | return icq; | ||
2995 | } | ||
2996 | |||
2997 | /** | ||
2998 | * cfq_get_cic - acquire cfq_io_cq and bump refcnt on io_context | ||
2999 | * @cfqd: cfqd to setup cic for | ||
3000 | * @gfp_mask: allocation mask | ||
3001 | * | ||
3002 | * Return cfq_io_cq associating @cfqd and %current->io_context and | ||
3003 | * bump refcnt on io_context. If ioc or cic doesn't exist, they're created | ||
3004 | * using @gfp_mask. | ||
3005 | * | ||
3006 | * Must be called under queue_lock which may be released and re-acquired. | ||
3007 | * This function also may sleep depending on @gfp_mask. | ||
3008 | */ | ||
3009 | static struct cfq_io_cq *cfq_get_cic(struct cfq_data *cfqd, gfp_t gfp_mask) | ||
3010 | { | ||
3011 | struct request_queue *q = cfqd->queue; | ||
3012 | struct cfq_io_cq *cic = NULL; | ||
3013 | struct io_context *ioc; | ||
3014 | |||
3015 | lockdep_assert_held(q->queue_lock); | ||
3016 | |||
3017 | while (true) { | ||
3018 | /* fast path */ | ||
3019 | ioc = current->io_context; | ||
3020 | if (likely(ioc)) { | ||
3021 | cic = cfq_cic_lookup(cfqd, ioc); | ||
3022 | if (likely(cic)) | ||
3023 | break; | ||
3024 | } | ||
3025 | |||
3026 | /* slow path - unlock, create missing ones and retry */ | ||
3027 | spin_unlock_irq(q->queue_lock); | ||
3028 | cic = icq_to_cic(ioc_create_icq(q, gfp_mask)); | ||
3029 | spin_lock_irq(q->queue_lock); | ||
3030 | if (!cic) | ||
3031 | return NULL; | ||
3032 | } | ||
3033 | |||
3034 | /* bump @ioc's refcnt and handle changed notifications */ | ||
3035 | get_io_context(ioc); | ||
3036 | |||
3037 | if (unlikely(cic->icq.changed)) { | ||
3038 | if (test_and_clear_bit(ICQ_IOPRIO_CHANGED, &cic->icq.changed)) | ||
3039 | changed_ioprio(cic); | ||
3040 | #ifdef CONFIG_CFQ_GROUP_IOSCHED | ||
3041 | if (test_and_clear_bit(ICQ_CGROUP_CHANGED, &cic->icq.changed)) | ||
3042 | changed_cgroup(cic); | ||
3043 | #endif | ||
3044 | } | ||
3045 | |||
3046 | return cic; | ||
3047 | } | ||
3048 | |||
3049 | static void | 2938 | static void |
3050 | __cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle) | 2939 | __cfq_update_io_thinktime(struct cfq_ttime *ttime, unsigned long slice_idle) |
3051 | { | 2940 | { |
@@ -3524,8 +3413,6 @@ static void cfq_put_request(struct request *rq) | |||
3524 | BUG_ON(!cfqq->allocated[rw]); | 3413 | BUG_ON(!cfqq->allocated[rw]); |
3525 | cfqq->allocated[rw]--; | 3414 | cfqq->allocated[rw]--; |
3526 | 3415 | ||
3527 | put_io_context(RQ_CIC(rq)->icq.ioc, cfqq->cfqd->queue); | ||
3528 | |||
3529 | /* Put down rq reference on cfqg */ | 3416 | /* Put down rq reference on cfqg */ |
3530 | cfq_put_cfqg(RQ_CFQG(rq)); | 3417 | cfq_put_cfqg(RQ_CFQG(rq)); |
3531 | rq->elv.priv[0] = NULL; | 3418 | rq->elv.priv[0] = NULL; |
@@ -3574,7 +3461,7 @@ static int | |||
3574 | cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) | 3461 | cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) |
3575 | { | 3462 | { |
3576 | struct cfq_data *cfqd = q->elevator->elevator_data; | 3463 | struct cfq_data *cfqd = q->elevator->elevator_data; |
3577 | struct cfq_io_cq *cic; | 3464 | struct cfq_io_cq *cic = icq_to_cic(rq->elv.icq); |
3578 | const int rw = rq_data_dir(rq); | 3465 | const int rw = rq_data_dir(rq); |
3579 | const bool is_sync = rq_is_sync(rq); | 3466 | const bool is_sync = rq_is_sync(rq); |
3580 | struct cfq_queue *cfqq; | 3467 | struct cfq_queue *cfqq; |
@@ -3582,9 +3469,16 @@ cfq_set_request(struct request_queue *q, struct request *rq, gfp_t gfp_mask) | |||
3582 | might_sleep_if(gfp_mask & __GFP_WAIT); | 3469 | might_sleep_if(gfp_mask & __GFP_WAIT); |
3583 | 3470 | ||
3584 | spin_lock_irq(q->queue_lock); | 3471 | spin_lock_irq(q->queue_lock); |
3585 | cic = cfq_get_cic(cfqd, gfp_mask); | 3472 | |
3586 | if (!cic) | 3473 | /* handle changed notifications */ |
3587 | goto queue_fail; | 3474 | if (unlikely(cic->icq.changed)) { |
3475 | if (test_and_clear_bit(ICQ_IOPRIO_CHANGED, &cic->icq.changed)) | ||
3476 | changed_ioprio(cic); | ||
3477 | #ifdef CONFIG_CFQ_GROUP_IOSCHED | ||
3478 | if (test_and_clear_bit(ICQ_CGROUP_CHANGED, &cic->icq.changed)) | ||
3479 | changed_cgroup(cic); | ||
3480 | #endif | ||
3481 | } | ||
3588 | 3482 | ||
3589 | new_queue: | 3483 | new_queue: |
3590 | cfqq = cic_to_cfqq(cic, is_sync); | 3484 | cfqq = cic_to_cfqq(cic, is_sync); |
@@ -3615,17 +3509,10 @@ new_queue: | |||
3615 | cfqq->allocated[rw]++; | 3509 | cfqq->allocated[rw]++; |
3616 | 3510 | ||
3617 | cfqq->ref++; | 3511 | cfqq->ref++; |
3618 | rq->elv.icq = &cic->icq; | ||
3619 | rq->elv.priv[0] = cfqq; | 3512 | rq->elv.priv[0] = cfqq; |
3620 | rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg); | 3513 | rq->elv.priv[1] = cfq_ref_get_cfqg(cfqq->cfqg); |
3621 | spin_unlock_irq(q->queue_lock); | 3514 | spin_unlock_irq(q->queue_lock); |
3622 | return 0; | 3515 | return 0; |
3623 | |||
3624 | queue_fail: | ||
3625 | cfq_schedule_dispatch(cfqd); | ||
3626 | spin_unlock_irq(q->queue_lock); | ||
3627 | cfq_log(cfqd, "set_request fail"); | ||
3628 | return 1; | ||
3629 | } | 3516 | } |
3630 | 3517 | ||
3631 | static void cfq_kick_queue(struct work_struct *work) | 3518 | static void cfq_kick_queue(struct work_struct *work) |