diff options
-rw-r--r-- | block/blk-ioc.c | 35 | ||||
-rw-r--r-- | block/cfq-iosched.c | 38 | ||||
-rw-r--r-- | include/linux/iocontext.h | 2 |
3 files changed, 29 insertions, 46 deletions
diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 4ae0929c6e38..e34df7c9fc36 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c | |||
@@ -17,17 +17,13 @@ static struct kmem_cache *iocontext_cachep; | |||
17 | 17 | ||
18 | static void cfq_dtor(struct io_context *ioc) | 18 | static void cfq_dtor(struct io_context *ioc) |
19 | { | 19 | { |
20 | struct cfq_io_context *cic[1]; | 20 | if (!hlist_empty(&ioc->cic_list)) { |
21 | int r; | 21 | struct cfq_io_context *cic; |
22 | 22 | ||
23 | /* | 23 | cic = list_entry(ioc->cic_list.first, struct cfq_io_context, |
24 | * We don't have a specific key to lookup with, so use the gang | 24 | cic_list); |
25 | * lookup to just retrieve the first item stored. The cfq exit | 25 | cic->dtor(ioc); |
26 | * function will iterate the full tree, so any member will do. | 26 | } |
27 | */ | ||
28 | r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); | ||
29 | if (r > 0) | ||
30 | cic[0]->dtor(ioc); | ||
31 | } | 27 | } |
32 | 28 | ||
33 | /* | 29 | /* |
@@ -57,18 +53,16 @@ EXPORT_SYMBOL(put_io_context); | |||
57 | 53 | ||
58 | static void cfq_exit(struct io_context *ioc) | 54 | static void cfq_exit(struct io_context *ioc) |
59 | { | 55 | { |
60 | struct cfq_io_context *cic[1]; | ||
61 | int r; | ||
62 | |||
63 | rcu_read_lock(); | 56 | rcu_read_lock(); |
64 | /* | ||
65 | * See comment for cfq_dtor() | ||
66 | */ | ||
67 | r = radix_tree_gang_lookup(&ioc->radix_root, (void **) cic, 0, 1); | ||
68 | rcu_read_unlock(); | ||
69 | 57 | ||
70 | if (r > 0) | 58 | if (!hlist_empty(&ioc->cic_list)) { |
71 | cic[0]->exit(ioc); | 59 | struct cfq_io_context *cic; |
60 | |||
61 | cic = list_entry(ioc->cic_list.first, struct cfq_io_context, | ||
62 | cic_list); | ||
63 | cic->exit(ioc); | ||
64 | } | ||
65 | rcu_read_unlock(); | ||
72 | } | 66 | } |
73 | 67 | ||
74 | /* Called by the exitting task */ | 68 | /* Called by the exitting task */ |
@@ -105,6 +99,7 @@ struct io_context *alloc_io_context(gfp_t gfp_flags, int node) | |||
105 | ret->nr_batch_requests = 0; /* because this is 0 */ | 99 | ret->nr_batch_requests = 0; /* because this is 0 */ |
106 | ret->aic = NULL; | 100 | ret->aic = NULL; |
107 | INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH); | 101 | INIT_RADIX_TREE(&ret->radix_root, GFP_ATOMIC | __GFP_HIGH); |
102 | INIT_HLIST_HEAD(&ret->cic_list); | ||
108 | ret->ioc_data = NULL; | 103 | ret->ioc_data = NULL; |
109 | } | 104 | } |
110 | 105 | ||
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index ca198e61fa65..0f962ecae91f 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c | |||
@@ -1145,38 +1145,19 @@ static void cfq_put_queue(struct cfq_queue *cfqq) | |||
1145 | /* | 1145 | /* |
1146 | * Call func for each cic attached to this ioc. Returns number of cic's seen. | 1146 | * Call func for each cic attached to this ioc. Returns number of cic's seen. |
1147 | */ | 1147 | */ |
1148 | #define CIC_GANG_NR 16 | ||
1149 | static unsigned int | 1148 | static unsigned int |
1150 | call_for_each_cic(struct io_context *ioc, | 1149 | call_for_each_cic(struct io_context *ioc, |
1151 | void (*func)(struct io_context *, struct cfq_io_context *)) | 1150 | void (*func)(struct io_context *, struct cfq_io_context *)) |
1152 | { | 1151 | { |
1153 | struct cfq_io_context *cics[CIC_GANG_NR]; | 1152 | struct cfq_io_context *cic; |
1154 | unsigned long index = 0; | 1153 | struct hlist_node *n; |
1155 | unsigned int called = 0; | 1154 | int called = 0; |
1156 | int nr; | ||
1157 | 1155 | ||
1158 | rcu_read_lock(); | 1156 | rcu_read_lock(); |
1159 | 1157 | hlist_for_each_entry_rcu(cic, n, &ioc->cic_list, cic_list) { | |
1160 | do { | 1158 | func(ioc, cic); |
1161 | int i; | 1159 | called++; |
1162 | 1160 | } | |
1163 | /* | ||
1164 | * Perhaps there's a better way - this just gang lookups from | ||
1165 | * 0 to the end, restarting after each CIC_GANG_NR from the | ||
1166 | * last key + 1. | ||
1167 | */ | ||
1168 | nr = radix_tree_gang_lookup(&ioc->radix_root, (void **) cics, | ||
1169 | index, CIC_GANG_NR); | ||
1170 | if (!nr) | ||
1171 | break; | ||
1172 | |||
1173 | called += nr; | ||
1174 | index = 1 + (unsigned long) cics[nr - 1]->key; | ||
1175 | |||
1176 | for (i = 0; i < nr; i++) | ||
1177 | func(ioc, cics[i]); | ||
1178 | } while (nr == CIC_GANG_NR); | ||
1179 | |||
1180 | rcu_read_unlock(); | 1161 | rcu_read_unlock(); |
1181 | 1162 | ||
1182 | return called; | 1163 | return called; |
@@ -1190,6 +1171,7 @@ static void cic_free_func(struct io_context *ioc, struct cfq_io_context *cic) | |||
1190 | 1171 | ||
1191 | spin_lock_irqsave(&ioc->lock, flags); | 1172 | spin_lock_irqsave(&ioc->lock, flags); |
1192 | radix_tree_delete(&ioc->radix_root, cic->dead_key); | 1173 | radix_tree_delete(&ioc->radix_root, cic->dead_key); |
1174 | hlist_del_rcu(&cic->cic_list); | ||
1193 | spin_unlock_irqrestore(&ioc->lock, flags); | 1175 | spin_unlock_irqrestore(&ioc->lock, flags); |
1194 | 1176 | ||
1195 | kmem_cache_free(cfq_ioc_pool, cic); | 1177 | kmem_cache_free(cfq_ioc_pool, cic); |
@@ -1280,6 +1262,7 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask) | |||
1280 | if (cic) { | 1262 | if (cic) { |
1281 | cic->last_end_request = jiffies; | 1263 | cic->last_end_request = jiffies; |
1282 | INIT_LIST_HEAD(&cic->queue_list); | 1264 | INIT_LIST_HEAD(&cic->queue_list); |
1265 | INIT_HLIST_NODE(&cic->cic_list); | ||
1283 | cic->dtor = cfq_free_io_context; | 1266 | cic->dtor = cfq_free_io_context; |
1284 | cic->exit = cfq_exit_io_context; | 1267 | cic->exit = cfq_exit_io_context; |
1285 | elv_ioc_count_inc(ioc_count); | 1268 | elv_ioc_count_inc(ioc_count); |
@@ -1501,6 +1484,7 @@ cfq_drop_dead_cic(struct cfq_data *cfqd, struct io_context *ioc, | |||
1501 | rcu_assign_pointer(ioc->ioc_data, NULL); | 1484 | rcu_assign_pointer(ioc->ioc_data, NULL); |
1502 | 1485 | ||
1503 | radix_tree_delete(&ioc->radix_root, (unsigned long) cfqd); | 1486 | radix_tree_delete(&ioc->radix_root, (unsigned long) cfqd); |
1487 | hlist_del_rcu(&cic->cic_list); | ||
1504 | spin_unlock_irqrestore(&ioc->lock, flags); | 1488 | spin_unlock_irqrestore(&ioc->lock, flags); |
1505 | 1489 | ||
1506 | cfq_cic_free(cic); | 1490 | cfq_cic_free(cic); |
@@ -1561,6 +1545,8 @@ static int cfq_cic_link(struct cfq_data *cfqd, struct io_context *ioc, | |||
1561 | spin_lock_irqsave(&ioc->lock, flags); | 1545 | spin_lock_irqsave(&ioc->lock, flags); |
1562 | ret = radix_tree_insert(&ioc->radix_root, | 1546 | ret = radix_tree_insert(&ioc->radix_root, |
1563 | (unsigned long) cfqd, cic); | 1547 | (unsigned long) cfqd, cic); |
1548 | if (!ret) | ||
1549 | hlist_add_head_rcu(&cic->cic_list, &ioc->cic_list); | ||
1564 | spin_unlock_irqrestore(&ioc->lock, flags); | 1550 | spin_unlock_irqrestore(&ioc->lock, flags); |
1565 | 1551 | ||
1566 | radix_tree_preload_end(); | 1552 | radix_tree_preload_end(); |
diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 593b222d9dcc..1b4ccf25b4d2 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h | |||
@@ -50,6 +50,7 @@ struct cfq_io_context { | |||
50 | sector_t seek_mean; | 50 | sector_t seek_mean; |
51 | 51 | ||
52 | struct list_head queue_list; | 52 | struct list_head queue_list; |
53 | struct hlist_node cic_list; | ||
53 | 54 | ||
54 | void (*dtor)(struct io_context *); /* destructor */ | 55 | void (*dtor)(struct io_context *); /* destructor */ |
55 | void (*exit)(struct io_context *); /* called on task exit */ | 56 | void (*exit)(struct io_context *); /* called on task exit */ |
@@ -77,6 +78,7 @@ struct io_context { | |||
77 | 78 | ||
78 | struct as_io_context *aic; | 79 | struct as_io_context *aic; |
79 | struct radix_tree_root radix_root; | 80 | struct radix_tree_root radix_root; |
81 | struct hlist_head cic_list; | ||
80 | void *ioc_data; | 82 | void *ioc_data; |
81 | }; | 83 | }; |
82 | 84 | ||