aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--block/blk-cgroup.c2
-rw-r--r--block/blk-core.c2
-rw-r--r--block/blk-ioc.c2
-rw-r--r--block/blk-iopoll.c3
-rw-r--r--block/blk-mq-cpumap.c10
-rw-r--r--block/blk-mq-sysfs.c31
-rw-r--r--block/blk-mq.c70
-rw-r--r--block/blk-mq.h2
-rw-r--r--block/blk-softirq.c19
-rw-r--r--block/cfq-iosched.c8
-rw-r--r--block/deadline-iosched.c8
-rw-r--r--block/partitions/atari.h4
-rw-r--r--block/partitions/efi.h9
-rw-r--r--block/partitions/karma.c3
-rw-r--r--drivers/block/virtio_blk.c3
-rw-r--r--drivers/cpuidle/coupled.c2
-rw-r--r--drivers/scsi/be2iscsi/be_main.c206
-rw-r--r--drivers/scsi/ipr.c15
-rw-r--r--fs/bio-integrity.c84
-rw-r--r--fs/bio.c1
-rw-r--r--include/linux/blk-iopoll.h2
-rw-r--r--include/linux/blk-mq.h5
-rw-r--r--include/linux/blkdev.h1
-rw-r--r--include/linux/elevator.h11
-rw-r--r--include/linux/smp.h8
-rw-r--r--include/trace/events/block.h33
-rw-r--r--kernel/sched/core.c2
-rw-r--r--kernel/smp.c139
-rw-r--r--kernel/sysctl.c12
-rw-r--r--kernel/trace/blktrace.c20
-rw-r--r--kernel/up.c6
-rw-r--r--kernel/watchdog.c3
-rw-r--r--net/core/dev.c4
33 files changed, 352 insertions, 378 deletions
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 4e491d9b5292..b6e95b5e262f 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -336,7 +336,7 @@ static void blkg_destroy(struct blkcg_gq *blkg)
336 * under queue_lock. If it's not pointing to @blkg now, it never 336 * under queue_lock. If it's not pointing to @blkg now, it never
337 * will. Hint assignment itself can race safely. 337 * will. Hint assignment itself can race safely.
338 */ 338 */
339 if (rcu_dereference_raw(blkcg->blkg_hint) == blkg) 339 if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
340 rcu_assign_pointer(blkcg->blkg_hint, NULL); 340 rcu_assign_pointer(blkcg->blkg_hint, NULL);
341 341
342 /* 342 /*
diff --git a/block/blk-core.c b/block/blk-core.c
index bfe16d5af9f9..e45b321cf6a0 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -2353,7 +2353,7 @@ bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
2353 if (!req->bio) 2353 if (!req->bio)
2354 return false; 2354 return false;
2355 2355
2356 trace_block_rq_complete(req->q, req); 2356 trace_block_rq_complete(req->q, req, nr_bytes);
2357 2357
2358 /* 2358 /*
2359 * For fs requests, rq is just carrier of independent bio's 2359 * For fs requests, rq is just carrier of independent bio's
diff --git a/block/blk-ioc.c b/block/blk-ioc.c
index 242df01413f6..1a27f45ec776 100644
--- a/block/blk-ioc.c
+++ b/block/blk-ioc.c
@@ -68,7 +68,7 @@ static void ioc_destroy_icq(struct io_cq *icq)
68 * under queue_lock. If it's not pointing to @icq now, it never 68 * under queue_lock. If it's not pointing to @icq now, it never
69 * will. Hint assignment itself can race safely. 69 * will. Hint assignment itself can race safely.
70 */ 70 */
71 if (rcu_dereference_raw(ioc->icq_hint) == icq) 71 if (rcu_access_pointer(ioc->icq_hint) == icq)
72 rcu_assign_pointer(ioc->icq_hint, NULL); 72 rcu_assign_pointer(ioc->icq_hint, NULL);
73 73
74 ioc_exit_icq(icq); 74 ioc_exit_icq(icq);
diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c
index 1855bf51edb0..c11d24e379e2 100644
--- a/block/blk-iopoll.c
+++ b/block/blk-iopoll.c
@@ -14,9 +14,6 @@
14 14
15#include "blk.h" 15#include "blk.h"
16 16
17int blk_iopoll_enabled = 1;
18EXPORT_SYMBOL(blk_iopoll_enabled);
19
20static unsigned int blk_iopoll_budget __read_mostly = 256; 17static unsigned int blk_iopoll_budget __read_mostly = 256;
21 18
22static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); 19static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c
index f8721278601c..097921329619 100644
--- a/block/blk-mq-cpumap.c
+++ b/block/blk-mq-cpumap.c
@@ -9,15 +9,6 @@
9#include "blk.h" 9#include "blk.h"
10#include "blk-mq.h" 10#include "blk-mq.h"
11 11
12static void show_map(unsigned int *map, unsigned int nr)
13{
14 int i;
15
16 pr_info("blk-mq: CPU -> queue map\n");
17 for_each_online_cpu(i)
18 pr_info(" CPU%2u -> Queue %u\n", i, map[i]);
19}
20
21static int cpu_to_queue_index(unsigned int nr_cpus, unsigned int nr_queues, 12static int cpu_to_queue_index(unsigned int nr_cpus, unsigned int nr_queues,
22 const int cpu) 13 const int cpu)
23{ 14{
@@ -85,7 +76,6 @@ int blk_mq_update_queue_map(unsigned int *map, unsigned int nr_queues)
85 map[i] = map[first_sibling]; 76 map[i] = map[first_sibling];
86 } 77 }
87 78
88 show_map(map, nr_cpus);
89 free_cpumask_var(cpus); 79 free_cpumask_var(cpus);
90 return 0; 80 return 0;
91} 81}
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index b91ce75bd35d..b0ba264b0522 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -244,6 +244,32 @@ static ssize_t blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx *hctx, char *page)
244 return blk_mq_tag_sysfs_show(hctx->tags, page); 244 return blk_mq_tag_sysfs_show(hctx->tags, page);
245} 245}
246 246
247static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
248{
249 unsigned int i, queue_num, first = 1;
250 ssize_t ret = 0;
251
252 blk_mq_disable_hotplug();
253
254 for_each_online_cpu(i) {
255 queue_num = hctx->queue->mq_map[i];
256 if (queue_num != hctx->queue_num)
257 continue;
258
259 if (first)
260 ret += sprintf(ret + page, "%u", i);
261 else
262 ret += sprintf(ret + page, ", %u", i);
263
264 first = 0;
265 }
266
267 blk_mq_enable_hotplug();
268
269 ret += sprintf(ret + page, "\n");
270 return ret;
271}
272
247static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = { 273static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched = {
248 .attr = {.name = "dispatched", .mode = S_IRUGO }, 274 .attr = {.name = "dispatched", .mode = S_IRUGO },
249 .show = blk_mq_sysfs_dispatched_show, 275 .show = blk_mq_sysfs_dispatched_show,
@@ -294,6 +320,10 @@ static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags = {
294 .attr = {.name = "tags", .mode = S_IRUGO }, 320 .attr = {.name = "tags", .mode = S_IRUGO },
295 .show = blk_mq_hw_sysfs_tags_show, 321 .show = blk_mq_hw_sysfs_tags_show,
296}; 322};
323static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
324 .attr = {.name = "cpu_list", .mode = S_IRUGO },
325 .show = blk_mq_hw_sysfs_cpus_show,
326};
297 327
298static struct attribute *default_hw_ctx_attrs[] = { 328static struct attribute *default_hw_ctx_attrs[] = {
299 &blk_mq_hw_sysfs_queued.attr, 329 &blk_mq_hw_sysfs_queued.attr,
@@ -302,6 +332,7 @@ static struct attribute *default_hw_ctx_attrs[] = {
302 &blk_mq_hw_sysfs_pending.attr, 332 &blk_mq_hw_sysfs_pending.attr,
303 &blk_mq_hw_sysfs_ipi.attr, 333 &blk_mq_hw_sysfs_ipi.attr,
304 &blk_mq_hw_sysfs_tags.attr, 334 &blk_mq_hw_sysfs_tags.attr,
335 &blk_mq_hw_sysfs_cpus.attr,
305 NULL, 336 NULL,
306}; 337};
307 338
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 883f72089015..b1bcc619d0ea 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -320,7 +320,7 @@ void __blk_mq_complete_request(struct request *rq)
320 rq->csd.func = __blk_mq_complete_request_remote; 320 rq->csd.func = __blk_mq_complete_request_remote;
321 rq->csd.info = rq; 321 rq->csd.info = rq;
322 rq->csd.flags = 0; 322 rq->csd.flags = 0;
323 __smp_call_function_single(ctx->cpu, &rq->csd, 0); 323 smp_call_function_single_async(ctx->cpu, &rq->csd);
324 } else { 324 } else {
325 rq->q->softirq_done_fn(rq); 325 rq->q->softirq_done_fn(rq);
326 } 326 }
@@ -514,7 +514,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
514 LIST_HEAD(rq_list); 514 LIST_HEAD(rq_list);
515 int bit, queued; 515 int bit, queued;
516 516
517 if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->flags))) 517 if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
518 return; 518 return;
519 519
520 hctx->run++; 520 hctx->run++;
@@ -603,7 +603,7 @@ static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
603 603
604void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async) 604void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
605{ 605{
606 if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->flags))) 606 if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state)))
607 return; 607 return;
608 608
609 if (!async) 609 if (!async)
@@ -623,7 +623,7 @@ void blk_mq_run_queues(struct request_queue *q, bool async)
623 queue_for_each_hw_ctx(q, hctx, i) { 623 queue_for_each_hw_ctx(q, hctx, i) {
624 if ((!blk_mq_hctx_has_pending(hctx) && 624 if ((!blk_mq_hctx_has_pending(hctx) &&
625 list_empty_careful(&hctx->dispatch)) || 625 list_empty_careful(&hctx->dispatch)) ||
626 test_bit(BLK_MQ_S_STOPPED, &hctx->flags)) 626 test_bit(BLK_MQ_S_STOPPED, &hctx->state))
627 continue; 627 continue;
628 628
629 blk_mq_run_hw_queue(hctx, async); 629 blk_mq_run_hw_queue(hctx, async);
@@ -994,8 +994,46 @@ static void blk_mq_hctx_notify(void *data, unsigned long action,
994 blk_mq_put_ctx(ctx); 994 blk_mq_put_ctx(ctx);
995} 995}
996 996
997static void blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx, 997static int blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx,
998 void (*init)(void *, struct blk_mq_hw_ctx *, 998 int (*init)(void *, struct blk_mq_hw_ctx *,
999 struct request *, unsigned int),
1000 void *data)
1001{
1002 unsigned int i;
1003 int ret = 0;
1004
1005 for (i = 0; i < hctx->queue_depth; i++) {
1006 struct request *rq = hctx->rqs[i];
1007
1008 ret = init(data, hctx, rq, i);
1009 if (ret)
1010 break;
1011 }
1012
1013 return ret;
1014}
1015
1016int blk_mq_init_commands(struct request_queue *q,
1017 int (*init)(void *, struct blk_mq_hw_ctx *,
1018 struct request *, unsigned int),
1019 void *data)
1020{
1021 struct blk_mq_hw_ctx *hctx;
1022 unsigned int i;
1023 int ret = 0;
1024
1025 queue_for_each_hw_ctx(q, hctx, i) {
1026 ret = blk_mq_init_hw_commands(hctx, init, data);
1027 if (ret)
1028 break;
1029 }
1030
1031 return ret;
1032}
1033EXPORT_SYMBOL(blk_mq_init_commands);
1034
1035static void blk_mq_free_hw_commands(struct blk_mq_hw_ctx *hctx,
1036 void (*free)(void *, struct blk_mq_hw_ctx *,
999 struct request *, unsigned int), 1037 struct request *, unsigned int),
1000 void *data) 1038 void *data)
1001{ 1039{
@@ -1004,12 +1042,12 @@ static void blk_mq_init_hw_commands(struct blk_mq_hw_ctx *hctx,
1004 for (i = 0; i < hctx->queue_depth; i++) { 1042 for (i = 0; i < hctx->queue_depth; i++) {
1005 struct request *rq = hctx->rqs[i]; 1043 struct request *rq = hctx->rqs[i];
1006 1044
1007 init(data, hctx, rq, i); 1045 free(data, hctx, rq, i);
1008 } 1046 }
1009} 1047}
1010 1048
1011void blk_mq_init_commands(struct request_queue *q, 1049void blk_mq_free_commands(struct request_queue *q,
1012 void (*init)(void *, struct blk_mq_hw_ctx *, 1050 void (*free)(void *, struct blk_mq_hw_ctx *,
1013 struct request *, unsigned int), 1051 struct request *, unsigned int),
1014 void *data) 1052 void *data)
1015{ 1053{
@@ -1017,9 +1055,9 @@ void blk_mq_init_commands(struct request_queue *q,
1017 unsigned int i; 1055 unsigned int i;
1018 1056
1019 queue_for_each_hw_ctx(q, hctx, i) 1057 queue_for_each_hw_ctx(q, hctx, i)
1020 blk_mq_init_hw_commands(hctx, init, data); 1058 blk_mq_free_hw_commands(hctx, free, data);
1021} 1059}
1022EXPORT_SYMBOL(blk_mq_init_commands); 1060EXPORT_SYMBOL(blk_mq_free_commands);
1023 1061
1024static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx) 1062static void blk_mq_free_rq_map(struct blk_mq_hw_ctx *hctx)
1025{ 1063{
@@ -1430,6 +1468,16 @@ static int blk_mq_queue_reinit_notify(struct notifier_block *nb,
1430 return NOTIFY_OK; 1468 return NOTIFY_OK;
1431} 1469}
1432 1470
1471void blk_mq_disable_hotplug(void)
1472{
1473 mutex_lock(&all_q_mutex);
1474}
1475
1476void blk_mq_enable_hotplug(void)
1477{
1478 mutex_unlock(&all_q_mutex);
1479}
1480
1433static int __init blk_mq_init(void) 1481static int __init blk_mq_init(void)
1434{ 1482{
1435 blk_mq_cpu_init(); 1483 blk_mq_cpu_init();
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 72beba1f9d55..ebbe6bac9d61 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -39,6 +39,8 @@ void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier,
39void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier); 39void blk_mq_register_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
40void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier); 40void blk_mq_unregister_cpu_notifier(struct blk_mq_cpu_notifier *notifier);
41void blk_mq_cpu_init(void); 41void blk_mq_cpu_init(void);
42void blk_mq_enable_hotplug(void);
43void blk_mq_disable_hotplug(void);
42 44
43/* 45/*
44 * CPU -> queue mappings 46 * CPU -> queue mappings
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index 57790c1a97eb..ebd6b6f1bdeb 100644
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -30,8 +30,8 @@ static void blk_done_softirq(struct softirq_action *h)
30 while (!list_empty(&local_list)) { 30 while (!list_empty(&local_list)) {
31 struct request *rq; 31 struct request *rq;
32 32
33 rq = list_entry(local_list.next, struct request, csd.list); 33 rq = list_entry(local_list.next, struct request, queuelist);
34 list_del_init(&rq->csd.list); 34 list_del_init(&rq->queuelist);
35 rq->q->softirq_done_fn(rq); 35 rq->q->softirq_done_fn(rq);
36 } 36 }
37} 37}
@@ -45,9 +45,14 @@ static void trigger_softirq(void *data)
45 45
46 local_irq_save(flags); 46 local_irq_save(flags);
47 list = this_cpu_ptr(&blk_cpu_done); 47 list = this_cpu_ptr(&blk_cpu_done);
48 list_add_tail(&rq->csd.list, list); 48 /*
49 * We reuse queuelist for a list of requests to process. Since the
50 * queuelist is used by the block layer only for requests waiting to be
51 * submitted to the device it is unused now.
52 */
53 list_add_tail(&rq->queuelist, list);
49 54
50 if (list->next == &rq->csd.list) 55 if (list->next == &rq->queuelist)
51 raise_softirq_irqoff(BLOCK_SOFTIRQ); 56 raise_softirq_irqoff(BLOCK_SOFTIRQ);
52 57
53 local_irq_restore(flags); 58 local_irq_restore(flags);
@@ -65,7 +70,7 @@ static int raise_blk_irq(int cpu, struct request *rq)
65 data->info = rq; 70 data->info = rq;
66 data->flags = 0; 71 data->flags = 0;
67 72
68 __smp_call_function_single(cpu, data, 0); 73 smp_call_function_single_async(cpu, data);
69 return 0; 74 return 0;
70 } 75 }
71 76
@@ -136,7 +141,7 @@ void __blk_complete_request(struct request *req)
136 struct list_head *list; 141 struct list_head *list;
137do_local: 142do_local:
138 list = this_cpu_ptr(&blk_cpu_done); 143 list = this_cpu_ptr(&blk_cpu_done);
139 list_add_tail(&req->csd.list, list); 144 list_add_tail(&req->queuelist, list);
140 145
141 /* 146 /*
142 * if the list only contains our just added request, 147 * if the list only contains our just added request,
@@ -144,7 +149,7 @@ do_local:
144 * entries there, someone already raised the irq but it 149 * entries there, someone already raised the irq but it
145 * hasn't run yet. 150 * hasn't run yet.
146 */ 151 */
147 if (list->next == &req->csd.list) 152 if (list->next == &req->queuelist)
148 raise_softirq_irqoff(BLOCK_SOFTIRQ); 153 raise_softirq_irqoff(BLOCK_SOFTIRQ);
149 } else if (raise_blk_irq(ccpu, req)) 154 } else if (raise_blk_irq(ccpu, req))
150 goto do_local; 155 goto do_local;
diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 744833b630c6..5873e4ada9eb 100644
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -2367,10 +2367,10 @@ cfq_merged_requests(struct request_queue *q, struct request *rq,
2367 * reposition in fifo if next is older than rq 2367 * reposition in fifo if next is older than rq
2368 */ 2368 */
2369 if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) && 2369 if (!list_empty(&rq->queuelist) && !list_empty(&next->queuelist) &&
2370 time_before(rq_fifo_time(next), rq_fifo_time(rq)) && 2370 time_before(next->fifo_time, rq->fifo_time) &&
2371 cfqq == RQ_CFQQ(next)) { 2371 cfqq == RQ_CFQQ(next)) {
2372 list_move(&rq->queuelist, &next->queuelist); 2372 list_move(&rq->queuelist, &next->queuelist);
2373 rq_set_fifo_time(rq, rq_fifo_time(next)); 2373 rq->fifo_time = next->fifo_time;
2374 } 2374 }
2375 2375
2376 if (cfqq->next_rq == next) 2376 if (cfqq->next_rq == next)
@@ -2814,7 +2814,7 @@ static struct request *cfq_check_fifo(struct cfq_queue *cfqq)
2814 return NULL; 2814 return NULL;
2815 2815
2816 rq = rq_entry_fifo(cfqq->fifo.next); 2816 rq = rq_entry_fifo(cfqq->fifo.next);
2817 if (time_before(jiffies, rq_fifo_time(rq))) 2817 if (time_before(jiffies, rq->fifo_time))
2818 rq = NULL; 2818 rq = NULL;
2819 2819
2820 cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq); 2820 cfq_log_cfqq(cfqq->cfqd, cfqq, "fifo=%p", rq);
@@ -3927,7 +3927,7 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)
3927 cfq_log_cfqq(cfqd, cfqq, "insert_request"); 3927 cfq_log_cfqq(cfqd, cfqq, "insert_request");
3928 cfq_init_prio_data(cfqq, RQ_CIC(rq)); 3928 cfq_init_prio_data(cfqq, RQ_CIC(rq));
3929 3929
3930 rq_set_fifo_time(rq, jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)]); 3930 rq->fifo_time = jiffies + cfqd->cfq_fifo_expire[rq_is_sync(rq)];
3931 list_add_tail(&rq->queuelist, &cfqq->fifo); 3931 list_add_tail(&rq->queuelist, &cfqq->fifo);
3932 cfq_add_rq_rb(rq); 3932 cfq_add_rq_rb(rq);
3933 cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group, 3933 cfqg_stats_update_io_add(RQ_CFQG(rq), cfqd->serving_group,
diff --git a/block/deadline-iosched.c b/block/deadline-iosched.c
index 9ef66406c625..a753df2b3fc2 100644
--- a/block/deadline-iosched.c
+++ b/block/deadline-iosched.c
@@ -106,7 +106,7 @@ deadline_add_request(struct request_queue *q, struct request *rq)
106 /* 106 /*
107 * set expire time and add to fifo list 107 * set expire time and add to fifo list
108 */ 108 */
109 rq_set_fifo_time(rq, jiffies + dd->fifo_expire[data_dir]); 109 rq->fifo_time = jiffies + dd->fifo_expire[data_dir];
110 list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]); 110 list_add_tail(&rq->queuelist, &dd->fifo_list[data_dir]);
111} 111}
112 112
@@ -174,9 +174,9 @@ deadline_merged_requests(struct request_queue *q, struct request *req,
174 * and move into next position (next will be deleted) in fifo 174 * and move into next position (next will be deleted) in fifo
175 */ 175 */
176 if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) { 176 if (!list_empty(&req->queuelist) && !list_empty(&next->queuelist)) {
177 if (time_before(rq_fifo_time(next), rq_fifo_time(req))) { 177 if (time_before(next->fifo_time, req->fifo_time)) {
178 list_move(&req->queuelist, &next->queuelist); 178 list_move(&req->queuelist, &next->queuelist);
179 rq_set_fifo_time(req, rq_fifo_time(next)); 179 req->fifo_time = next->fifo_time;
180 } 180 }
181 } 181 }
182 182
@@ -230,7 +230,7 @@ static inline int deadline_check_fifo(struct deadline_data *dd, int ddir)
230 /* 230 /*
231 * rq is expired! 231 * rq is expired!
232 */ 232 */
233 if (time_after_eq(jiffies, rq_fifo_time(rq))) 233 if (time_after_eq(jiffies, rq->fifo_time))
234 return 1; 234 return 1;
235 235
236 return 0; 236 return 0;
diff --git a/block/partitions/atari.h b/block/partitions/atari.h
index fe2d32a89f36..f2ec43bfeec1 100644
--- a/block/partitions/atari.h
+++ b/block/partitions/atari.h
@@ -11,6 +11,8 @@
11 * by Guenther Kelleter (guenther@pool.informatik.rwth-aachen.de) 11 * by Guenther Kelleter (guenther@pool.informatik.rwth-aachen.de)
12 */ 12 */
13 13
14#include <linux/compiler.h>
15
14struct partition_info 16struct partition_info
15{ 17{
16 u8 flg; /* bit 0: active; bit 7: bootable */ 18 u8 flg; /* bit 0: active; bit 7: bootable */
@@ -29,6 +31,6 @@ struct rootsector
29 u32 bsl_st; /* start of bad sector list */ 31 u32 bsl_st; /* start of bad sector list */
30 u32 bsl_cnt; /* length of bad sector list */ 32 u32 bsl_cnt; /* length of bad sector list */
31 u16 checksum; /* checksum for bootable disks */ 33 u16 checksum; /* checksum for bootable disks */
32} __attribute__((__packed__)); 34} __packed;
33 35
34int atari_partition(struct parsed_partitions *state); 36int atari_partition(struct parsed_partitions *state);
diff --git a/block/partitions/efi.h b/block/partitions/efi.h
index 4efcafba7e64..abd0b19288a6 100644
--- a/block/partitions/efi.h
+++ b/block/partitions/efi.h
@@ -32,6 +32,7 @@
32#include <linux/major.h> 32#include <linux/major.h>
33#include <linux/string.h> 33#include <linux/string.h>
34#include <linux/efi.h> 34#include <linux/efi.h>
35#include <linux/compiler.h>
35 36
36#define MSDOS_MBR_SIGNATURE 0xaa55 37#define MSDOS_MBR_SIGNATURE 0xaa55
37#define EFI_PMBR_OSTYPE_EFI 0xEF 38#define EFI_PMBR_OSTYPE_EFI 0xEF
@@ -87,13 +88,13 @@ typedef struct _gpt_header {
87 * 88 *
88 * uint8_t reserved2[ BlockSize - 92 ]; 89 * uint8_t reserved2[ BlockSize - 92 ];
89 */ 90 */
90} __attribute__ ((packed)) gpt_header; 91} __packed gpt_header;
91 92
92typedef struct _gpt_entry_attributes { 93typedef struct _gpt_entry_attributes {
93 u64 required_to_function:1; 94 u64 required_to_function:1;
94 u64 reserved:47; 95 u64 reserved:47;
95 u64 type_guid_specific:16; 96 u64 type_guid_specific:16;
96} __attribute__ ((packed)) gpt_entry_attributes; 97} __packed gpt_entry_attributes;
97 98
98typedef struct _gpt_entry { 99typedef struct _gpt_entry {
99 efi_guid_t partition_type_guid; 100 efi_guid_t partition_type_guid;
@@ -102,7 +103,7 @@ typedef struct _gpt_entry {
102 __le64 ending_lba; 103 __le64 ending_lba;
103 gpt_entry_attributes attributes; 104 gpt_entry_attributes attributes;
104 efi_char16_t partition_name[72 / sizeof (efi_char16_t)]; 105 efi_char16_t partition_name[72 / sizeof (efi_char16_t)];
105} __attribute__ ((packed)) gpt_entry; 106} __packed gpt_entry;
106 107
107typedef struct _gpt_mbr_record { 108typedef struct _gpt_mbr_record {
108 u8 boot_indicator; /* unused by EFI, set to 0x80 for bootable */ 109 u8 boot_indicator; /* unused by EFI, set to 0x80 for bootable */
@@ -124,7 +125,7 @@ typedef struct _legacy_mbr {
124 __le16 unknown; 125 __le16 unknown;
125 gpt_mbr_record partition_record[4]; 126 gpt_mbr_record partition_record[4];
126 __le16 signature; 127 __le16 signature;
127} __attribute__ ((packed)) legacy_mbr; 128} __packed legacy_mbr;
128 129
129/* Functions */ 130/* Functions */
130extern int efi_partition(struct parsed_partitions *state); 131extern int efi_partition(struct parsed_partitions *state);
diff --git a/block/partitions/karma.c b/block/partitions/karma.c
index 0ea19312706b..9721fa589bb1 100644
--- a/block/partitions/karma.c
+++ b/block/partitions/karma.c
@@ -8,6 +8,7 @@
8 8
9#include "check.h" 9#include "check.h"
10#include "karma.h" 10#include "karma.h"
11#include <linux/compiler.h>
11 12
12int karma_partition(struct parsed_partitions *state) 13int karma_partition(struct parsed_partitions *state)
13{ 14{
@@ -26,7 +27,7 @@ int karma_partition(struct parsed_partitions *state)
26 } d_partitions[2]; 27 } d_partitions[2];
27 u8 d_blank[208]; 28 u8 d_blank[208];
28 __le16 d_magic; 29 __le16 d_magic;
29 } __attribute__((packed)) *label; 30 } __packed *label;
30 struct d_partition *p; 31 struct d_partition *p;
31 32
32 data = read_part_sector(state, 0, &sect); 33 data = read_part_sector(state, 0, &sect);
diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c
index b1cb3f4c4db4..0eace43cea11 100644
--- a/drivers/block/virtio_blk.c
+++ b/drivers/block/virtio_blk.c
@@ -490,13 +490,14 @@ static struct blk_mq_reg virtio_mq_reg = {
490 .flags = BLK_MQ_F_SHOULD_MERGE, 490 .flags = BLK_MQ_F_SHOULD_MERGE,
491}; 491};
492 492
493static void virtblk_init_vbr(void *data, struct blk_mq_hw_ctx *hctx, 493static int virtblk_init_vbr(void *data, struct blk_mq_hw_ctx *hctx,
494 struct request *rq, unsigned int nr) 494 struct request *rq, unsigned int nr)
495{ 495{
496 struct virtio_blk *vblk = data; 496 struct virtio_blk *vblk = data;
497 struct virtblk_req *vbr = rq->special; 497 struct virtblk_req *vbr = rq->special;
498 498
499 sg_init_table(vbr->sg, vblk->sg_elems); 499 sg_init_table(vbr->sg, vblk->sg_elems);
500 return 0;
500} 501}
501 502
502static int virtblk_probe(struct virtio_device *vdev) 503static int virtblk_probe(struct virtio_device *vdev)
diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c
index e952936418d0..cb6654bfad77 100644
--- a/drivers/cpuidle/coupled.c
+++ b/drivers/cpuidle/coupled.c
@@ -323,7 +323,7 @@ static void cpuidle_coupled_poke(int cpu)
323 struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu); 323 struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu);
324 324
325 if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending)) 325 if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending))
326 __smp_call_function_single(cpu, csd, 0); 326 smp_call_function_single_async(cpu, csd);
327} 327}
328 328
329/** 329/**
diff --git a/drivers/scsi/be2iscsi/be_main.c b/drivers/scsi/be2iscsi/be_main.c
index 9be818f7b26d..0d822297aa80 100644
--- a/drivers/scsi/be2iscsi/be_main.c
+++ b/drivers/scsi/be2iscsi/be_main.c
@@ -898,7 +898,6 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
898 struct be_queue_info *cq; 898 struct be_queue_info *cq;
899 unsigned int num_eq_processed; 899 unsigned int num_eq_processed;
900 struct be_eq_obj *pbe_eq; 900 struct be_eq_obj *pbe_eq;
901 unsigned long flags;
902 901
903 pbe_eq = dev_id; 902 pbe_eq = dev_id;
904 eq = &pbe_eq->q; 903 eq = &pbe_eq->q;
@@ -907,31 +906,15 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
907 906
908 phba = pbe_eq->phba; 907 phba = pbe_eq->phba;
909 num_eq_processed = 0; 908 num_eq_processed = 0;
910 if (blk_iopoll_enabled) { 909 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
911 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 910 & EQE_VALID_MASK) {
912 & EQE_VALID_MASK) { 911 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
913 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 912 blk_iopoll_sched(&pbe_eq->iopoll);
914 blk_iopoll_sched(&pbe_eq->iopoll);
915
916 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
917 queue_tail_inc(eq);
918 eqe = queue_tail_node(eq);
919 num_eq_processed++;
920 }
921 } else {
922 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
923 & EQE_VALID_MASK) {
924 spin_lock_irqsave(&phba->isr_lock, flags);
925 pbe_eq->todo_cq = true;
926 spin_unlock_irqrestore(&phba->isr_lock, flags);
927 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
928 queue_tail_inc(eq);
929 eqe = queue_tail_node(eq);
930 num_eq_processed++;
931 }
932 913
933 if (pbe_eq->todo_cq) 914 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
934 queue_work(phba->wq, &pbe_eq->work_cqs); 915 queue_tail_inc(eq);
916 eqe = queue_tail_node(eq);
917 num_eq_processed++;
935 } 918 }
936 919
937 if (num_eq_processed) 920 if (num_eq_processed)
@@ -952,7 +935,6 @@ static irqreturn_t be_isr(int irq, void *dev_id)
952 struct hwi_context_memory *phwi_context; 935 struct hwi_context_memory *phwi_context;
953 struct be_eq_entry *eqe = NULL; 936 struct be_eq_entry *eqe = NULL;
954 struct be_queue_info *eq; 937 struct be_queue_info *eq;
955 struct be_queue_info *cq;
956 struct be_queue_info *mcc; 938 struct be_queue_info *mcc;
957 unsigned long flags, index; 939 unsigned long flags, index;
958 unsigned int num_mcceq_processed, num_ioeq_processed; 940 unsigned int num_mcceq_processed, num_ioeq_processed;
@@ -978,72 +960,40 @@ static irqreturn_t be_isr(int irq, void *dev_id)
978 960
979 num_ioeq_processed = 0; 961 num_ioeq_processed = 0;
980 num_mcceq_processed = 0; 962 num_mcceq_processed = 0;
981 if (blk_iopoll_enabled) { 963 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
982 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32] 964 & EQE_VALID_MASK) {
983 & EQE_VALID_MASK) { 965 if (((eqe->dw[offsetof(struct amap_eq_entry,
984 if (((eqe->dw[offsetof(struct amap_eq_entry, 966 resource_id) / 32] &
985 resource_id) / 32] & 967 EQE_RESID_MASK) >> 16) == mcc->id) {
986 EQE_RESID_MASK) >> 16) == mcc->id) { 968 spin_lock_irqsave(&phba->isr_lock, flags);
987 spin_lock_irqsave(&phba->isr_lock, flags); 969 pbe_eq->todo_mcc_cq = true;
988 pbe_eq->todo_mcc_cq = true; 970 spin_unlock_irqrestore(&phba->isr_lock, flags);
989 spin_unlock_irqrestore(&phba->isr_lock, flags); 971 num_mcceq_processed++;
990 num_mcceq_processed++; 972 } else {
991 } else { 973 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
992 if (!blk_iopoll_sched_prep(&pbe_eq->iopoll)) 974 blk_iopoll_sched(&pbe_eq->iopoll);
993 blk_iopoll_sched(&pbe_eq->iopoll);
994 num_ioeq_processed++;
995 }
996 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
997 queue_tail_inc(eq);
998 eqe = queue_tail_node(eq);
999 }
1000 if (num_ioeq_processed || num_mcceq_processed) {
1001 if (pbe_eq->todo_mcc_cq)
1002 queue_work(phba->wq, &pbe_eq->work_cqs);
1003
1004 if ((num_mcceq_processed) && (!num_ioeq_processed))
1005 hwi_ring_eq_db(phba, eq->id, 0,
1006 (num_ioeq_processed +
1007 num_mcceq_processed) , 1, 1);
1008 else
1009 hwi_ring_eq_db(phba, eq->id, 0,
1010 (num_ioeq_processed +
1011 num_mcceq_processed), 0, 1);
1012
1013 return IRQ_HANDLED;
1014 } else
1015 return IRQ_NONE;
1016 } else {
1017 cq = &phwi_context->be_cq[0];
1018 while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
1019 & EQE_VALID_MASK) {
1020
1021 if (((eqe->dw[offsetof(struct amap_eq_entry,
1022 resource_id) / 32] &
1023 EQE_RESID_MASK) >> 16) != cq->id) {
1024 spin_lock_irqsave(&phba->isr_lock, flags);
1025 pbe_eq->todo_mcc_cq = true;
1026 spin_unlock_irqrestore(&phba->isr_lock, flags);
1027 } else {
1028 spin_lock_irqsave(&phba->isr_lock, flags);
1029 pbe_eq->todo_cq = true;
1030 spin_unlock_irqrestore(&phba->isr_lock, flags);
1031 }
1032 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
1033 queue_tail_inc(eq);
1034 eqe = queue_tail_node(eq);
1035 num_ioeq_processed++; 975 num_ioeq_processed++;
1036 } 976 }
1037 if (pbe_eq->todo_cq || pbe_eq->todo_mcc_cq) 977 AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
978 queue_tail_inc(eq);
979 eqe = queue_tail_node(eq);
980 }
981 if (num_ioeq_processed || num_mcceq_processed) {
982 if (pbe_eq->todo_mcc_cq)
1038 queue_work(phba->wq, &pbe_eq->work_cqs); 983 queue_work(phba->wq, &pbe_eq->work_cqs);
1039 984
1040 if (num_ioeq_processed) { 985 if ((num_mcceq_processed) && (!num_ioeq_processed))
1041 hwi_ring_eq_db(phba, eq->id, 0, 986 hwi_ring_eq_db(phba, eq->id, 0,
1042 num_ioeq_processed, 1, 1); 987 (num_ioeq_processed +
1043 return IRQ_HANDLED; 988 num_mcceq_processed) , 1, 1);
1044 } else 989 else
1045 return IRQ_NONE; 990 hwi_ring_eq_db(phba, eq->id, 0,
1046 } 991 (num_ioeq_processed +
992 num_mcceq_processed), 0, 1);
993
994 return IRQ_HANDLED;
995 } else
996 return IRQ_NONE;
1047} 997}
1048 998
1049static int beiscsi_init_irqs(struct beiscsi_hba *phba) 999static int beiscsi_init_irqs(struct beiscsi_hba *phba)
@@ -5263,11 +5213,10 @@ static void beiscsi_quiesce(struct beiscsi_hba *phba,
5263 } 5213 }
5264 pci_disable_msix(phba->pcidev); 5214 pci_disable_msix(phba->pcidev);
5265 5215
5266 if (blk_iopoll_enabled) 5216 for (i = 0; i < phba->num_cpus; i++) {
5267 for (i = 0; i < phba->num_cpus; i++) { 5217 pbe_eq = &phwi_context->be_eq[i];
5268 pbe_eq = &phwi_context->be_eq[i]; 5218 blk_iopoll_disable(&pbe_eq->iopoll);
5269 blk_iopoll_disable(&pbe_eq->iopoll); 5219 }
5270 }
5271 5220
5272 if (unload_state == BEISCSI_CLEAN_UNLOAD) { 5221 if (unload_state == BEISCSI_CLEAN_UNLOAD) {
5273 destroy_workqueue(phba->wq); 5222 destroy_workqueue(phba->wq);
@@ -5478,32 +5427,18 @@ static void beiscsi_eeh_resume(struct pci_dev *pdev)
5478 phwi_ctrlr = phba->phwi_ctrlr; 5427 phwi_ctrlr = phba->phwi_ctrlr;
5479 phwi_context = phwi_ctrlr->phwi_ctxt; 5428 phwi_context = phwi_ctrlr->phwi_ctxt;
5480 5429
5481 if (blk_iopoll_enabled) { 5430 for (i = 0; i < phba->num_cpus; i++) {
5482 for (i = 0; i < phba->num_cpus; i++) {
5483 pbe_eq = &phwi_context->be_eq[i];
5484 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5485 be_iopoll);
5486 blk_iopoll_enable(&pbe_eq->iopoll);
5487 }
5488
5489 i = (phba->msix_enabled) ? i : 0;
5490 /* Work item for MCC handling */
5491 pbe_eq = &phwi_context->be_eq[i]; 5431 pbe_eq = &phwi_context->be_eq[i];
5492 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs); 5432 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5493 } else { 5433 be_iopoll);
5494 if (phba->msix_enabled) { 5434 blk_iopoll_enable(&pbe_eq->iopoll);
5495 for (i = 0; i <= phba->num_cpus; i++) {
5496 pbe_eq = &phwi_context->be_eq[i];
5497 INIT_WORK(&pbe_eq->work_cqs,
5498 beiscsi_process_all_cqs);
5499 }
5500 } else {
5501 pbe_eq = &phwi_context->be_eq[0];
5502 INIT_WORK(&pbe_eq->work_cqs,
5503 beiscsi_process_all_cqs);
5504 }
5505 } 5435 }
5506 5436
5437 i = (phba->msix_enabled) ? i : 0;
5438 /* Work item for MCC handling */
5439 pbe_eq = &phwi_context->be_eq[i];
5440 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5441
5507 ret = beiscsi_init_irqs(phba); 5442 ret = beiscsi_init_irqs(phba);
5508 if (ret < 0) { 5443 if (ret < 0) {
5509 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5444 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
@@ -5665,32 +5600,18 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
5665 phwi_ctrlr = phba->phwi_ctrlr; 5600 phwi_ctrlr = phba->phwi_ctrlr;
5666 phwi_context = phwi_ctrlr->phwi_ctxt; 5601 phwi_context = phwi_ctrlr->phwi_ctxt;
5667 5602
5668 if (blk_iopoll_enabled) { 5603 for (i = 0; i < phba->num_cpus; i++) {
5669 for (i = 0; i < phba->num_cpus; i++) {
5670 pbe_eq = &phwi_context->be_eq[i];
5671 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5672 be_iopoll);
5673 blk_iopoll_enable(&pbe_eq->iopoll);
5674 }
5675
5676 i = (phba->msix_enabled) ? i : 0;
5677 /* Work item for MCC handling */
5678 pbe_eq = &phwi_context->be_eq[i]; 5604 pbe_eq = &phwi_context->be_eq[i];
5679 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs); 5605 blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
5680 } else { 5606 be_iopoll);
5681 if (phba->msix_enabled) { 5607 blk_iopoll_enable(&pbe_eq->iopoll);
5682 for (i = 0; i <= phba->num_cpus; i++) {
5683 pbe_eq = &phwi_context->be_eq[i];
5684 INIT_WORK(&pbe_eq->work_cqs,
5685 beiscsi_process_all_cqs);
5686 }
5687 } else {
5688 pbe_eq = &phwi_context->be_eq[0];
5689 INIT_WORK(&pbe_eq->work_cqs,
5690 beiscsi_process_all_cqs);
5691 }
5692 } 5608 }
5693 5609
5610 i = (phba->msix_enabled) ? i : 0;
5611 /* Work item for MCC handling */
5612 pbe_eq = &phwi_context->be_eq[i];
5613 INIT_WORK(&pbe_eq->work_cqs, beiscsi_process_all_cqs);
5614
5694 ret = beiscsi_init_irqs(phba); 5615 ret = beiscsi_init_irqs(phba);
5695 if (ret < 0) { 5616 if (ret < 0) {
5696 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT, 5617 beiscsi_log(phba, KERN_ERR, BEISCSI_LOG_INIT,
@@ -5719,11 +5640,10 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
5719 5640
5720free_blkenbld: 5641free_blkenbld:
5721 destroy_workqueue(phba->wq); 5642 destroy_workqueue(phba->wq);
5722 if (blk_iopoll_enabled) 5643 for (i = 0; i < phba->num_cpus; i++) {
5723 for (i = 0; i < phba->num_cpus; i++) { 5644 pbe_eq = &phwi_context->be_eq[i];
5724 pbe_eq = &phwi_context->be_eq[i]; 5645 blk_iopoll_disable(&pbe_eq->iopoll);
5725 blk_iopoll_disable(&pbe_eq->iopoll); 5646 }
5726 }
5727free_twq: 5647free_twq:
5728 beiscsi_clean_port(phba); 5648 beiscsi_clean_port(phba);
5729 beiscsi_free_mem(phba); 5649 beiscsi_free_mem(phba);
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 2f8dd8e4225b..924b0ba74dfe 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -3670,16 +3670,14 @@ static ssize_t ipr_store_iopoll_weight(struct device *dev,
3670 return strlen(buf); 3670 return strlen(buf);
3671 } 3671 }
3672 3672
3673 if (blk_iopoll_enabled && ioa_cfg->iopoll_weight && 3673 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
3674 ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
3675 for (i = 1; i < ioa_cfg->hrrq_num; i++) 3674 for (i = 1; i < ioa_cfg->hrrq_num; i++)
3676 blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll); 3675 blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll);
3677 } 3676 }
3678 3677
3679 spin_lock_irqsave(shost->host_lock, lock_flags); 3678 spin_lock_irqsave(shost->host_lock, lock_flags);
3680 ioa_cfg->iopoll_weight = user_iopoll_weight; 3679 ioa_cfg->iopoll_weight = user_iopoll_weight;
3681 if (blk_iopoll_enabled && ioa_cfg->iopoll_weight && 3680 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
3682 ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
3683 for (i = 1; i < ioa_cfg->hrrq_num; i++) { 3681 for (i = 1; i < ioa_cfg->hrrq_num; i++) {
3684 blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll, 3682 blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll,
3685 ioa_cfg->iopoll_weight, ipr_iopoll); 3683 ioa_cfg->iopoll_weight, ipr_iopoll);
@@ -5525,8 +5523,7 @@ static irqreturn_t ipr_isr_mhrrq(int irq, void *devp)
5525 return IRQ_NONE; 5523 return IRQ_NONE;
5526 } 5524 }
5527 5525
5528 if (blk_iopoll_enabled && ioa_cfg->iopoll_weight && 5526 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
5529 ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
5530 if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) == 5527 if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) ==
5531 hrrq->toggle_bit) { 5528 hrrq->toggle_bit) {
5532 if (!blk_iopoll_sched_prep(&hrrq->iopoll)) 5529 if (!blk_iopoll_sched_prep(&hrrq->iopoll))
@@ -9975,8 +9972,7 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
9975 ioa_cfg->host->max_channel = IPR_VSET_BUS; 9972 ioa_cfg->host->max_channel = IPR_VSET_BUS;
9976 ioa_cfg->iopoll_weight = ioa_cfg->chip_cfg->iopoll_weight; 9973 ioa_cfg->iopoll_weight = ioa_cfg->chip_cfg->iopoll_weight;
9977 9974
9978 if (blk_iopoll_enabled && ioa_cfg->iopoll_weight && 9975 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
9979 ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
9980 for (i = 1; i < ioa_cfg->hrrq_num; i++) { 9976 for (i = 1; i < ioa_cfg->hrrq_num; i++) {
9981 blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll, 9977 blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll,
9982 ioa_cfg->iopoll_weight, ipr_iopoll); 9978 ioa_cfg->iopoll_weight, ipr_iopoll);
@@ -10005,8 +10001,7 @@ static void ipr_shutdown(struct pci_dev *pdev)
10005 int i; 10001 int i;
10006 10002
10007 spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags); 10003 spin_lock_irqsave(ioa_cfg->host->host_lock, lock_flags);
10008 if (blk_iopoll_enabled && ioa_cfg->iopoll_weight && 10004 if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
10009 ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
10010 ioa_cfg->iopoll_weight = 0; 10005 ioa_cfg->iopoll_weight = 0;
10011 for (i = 1; i < ioa_cfg->hrrq_num; i++) 10006 for (i = 1; i < ioa_cfg->hrrq_num; i++)
10012 blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll); 10007 blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll);
diff --git a/fs/bio-integrity.c b/fs/bio-integrity.c
index 4f70f383132c..29696b78d1f4 100644
--- a/fs/bio-integrity.c
+++ b/fs/bio-integrity.c
@@ -301,25 +301,25 @@ int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len)
301EXPORT_SYMBOL(bio_integrity_get_tag); 301EXPORT_SYMBOL(bio_integrity_get_tag);
302 302
303/** 303/**
304 * bio_integrity_generate - Generate integrity metadata for a bio 304 * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio
305 * @bio: bio to generate integrity metadata for 305 * @bio: bio to generate/verify integrity metadata for
306 * 306 * @operate: operate number, 1 for generate, 0 for verify
307 * Description: Generates integrity metadata for a bio by calling the
308 * block device's generation callback function. The bio must have a
309 * bip attached with enough room to accommodate the generated
310 * integrity metadata.
311 */ 307 */
312static void bio_integrity_generate(struct bio *bio) 308static int bio_integrity_generate_verify(struct bio *bio, int operate)
313{ 309{
314 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 310 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev);
315 struct blk_integrity_exchg bix; 311 struct blk_integrity_exchg bix;
316 struct bio_vec bv; 312 struct bio_vec bv;
317 struct bvec_iter iter; 313 struct bvec_iter iter;
318 sector_t sector = bio->bi_iter.bi_sector; 314 sector_t sector;
319 unsigned int sectors, total; 315 unsigned int sectors, ret = 0;
320 void *prot_buf = bio->bi_integrity->bip_buf; 316 void *prot_buf = bio->bi_integrity->bip_buf;
321 317
322 total = 0; 318 if (operate)
319 sector = bio->bi_iter.bi_sector;
320 else
321 sector = bio->bi_integrity->bip_iter.bi_sector;
322
323 bix.disk_name = bio->bi_bdev->bd_disk->disk_name; 323 bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
324 bix.sector_size = bi->sector_size; 324 bix.sector_size = bi->sector_size;
325 325
@@ -330,16 +330,37 @@ static void bio_integrity_generate(struct bio *bio)
330 bix.prot_buf = prot_buf; 330 bix.prot_buf = prot_buf;
331 bix.sector = sector; 331 bix.sector = sector;
332 332
333 bi->generate_fn(&bix); 333 if (operate) {
334 bi->generate_fn(&bix);
335 } else {
336 ret = bi->verify_fn(&bix);
337 if (ret) {
338 kunmap_atomic(kaddr);
339 return ret;
340 }
341 }
334 342
335 sectors = bv.bv_len / bi->sector_size; 343 sectors = bv.bv_len / bi->sector_size;
336 sector += sectors; 344 sector += sectors;
337 prot_buf += sectors * bi->tuple_size; 345 prot_buf += sectors * bi->tuple_size;
338 total += sectors * bi->tuple_size;
339 BUG_ON(total > bio->bi_integrity->bip_iter.bi_size);
340 346
341 kunmap_atomic(kaddr); 347 kunmap_atomic(kaddr);
342 } 348 }
349 return ret;
350}
351
352/**
353 * bio_integrity_generate - Generate integrity metadata for a bio
354 * @bio: bio to generate integrity metadata for
355 *
356 * Description: Generates integrity metadata for a bio by calling the
357 * block device's generation callback function. The bio must have a
358 * bip attached with enough room to accommodate the generated
359 * integrity metadata.
360 */
361static void bio_integrity_generate(struct bio *bio)
362{
363 bio_integrity_generate_verify(bio, 1);
343} 364}
344 365
345static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi) 366static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi)
@@ -454,40 +475,7 @@ EXPORT_SYMBOL(bio_integrity_prep);
454 */ 475 */
455static int bio_integrity_verify(struct bio *bio) 476static int bio_integrity_verify(struct bio *bio)
456{ 477{
457 struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); 478 return bio_integrity_generate_verify(bio, 0);
458 struct blk_integrity_exchg bix;
459 struct bio_vec *bv;
460 sector_t sector = bio->bi_integrity->bip_iter.bi_sector;
461 unsigned int sectors, ret = 0;
462 void *prot_buf = bio->bi_integrity->bip_buf;
463 int i;
464
465 bix.disk_name = bio->bi_bdev->bd_disk->disk_name;
466 bix.sector_size = bi->sector_size;
467
468 bio_for_each_segment_all(bv, bio, i) {
469 void *kaddr = kmap_atomic(bv->bv_page);
470
471 bix.data_buf = kaddr + bv->bv_offset;
472 bix.data_size = bv->bv_len;
473 bix.prot_buf = prot_buf;
474 bix.sector = sector;
475
476 ret = bi->verify_fn(&bix);
477
478 if (ret) {
479 kunmap_atomic(kaddr);
480 return ret;
481 }
482
483 sectors = bv->bv_len / bi->sector_size;
484 sector += sectors;
485 prot_buf += sectors * bi->tuple_size;
486
487 kunmap_atomic(kaddr);
488 }
489
490 return ret;
491} 479}
492 480
493/** 481/**
diff --git a/fs/bio.c b/fs/bio.c
index 8754e7b6eb49..b2dd42ed9edd 100644
--- a/fs/bio.c
+++ b/fs/bio.c
@@ -116,7 +116,6 @@ static struct kmem_cache *bio_find_or_create_slab(unsigned int extra_size)
116 if (!slab) 116 if (!slab)
117 goto out_unlock; 117 goto out_unlock;
118 118
119 printk(KERN_INFO "bio: create slab <%s> at %d\n", bslab->name, entry);
120 bslab->slab = slab; 119 bslab->slab = slab;
121 bslab->slab_ref = 1; 120 bslab->slab_ref = 1;
122 bslab->slab_size = sz; 121 bslab->slab_size = sz;
diff --git a/include/linux/blk-iopoll.h b/include/linux/blk-iopoll.h
index 308734d3d4a2..77ae77c0b704 100644
--- a/include/linux/blk-iopoll.h
+++ b/include/linux/blk-iopoll.h
@@ -43,6 +43,4 @@ extern void __blk_iopoll_complete(struct blk_iopoll *);
43extern void blk_iopoll_enable(struct blk_iopoll *); 43extern void blk_iopoll_enable(struct blk_iopoll *);
44extern void blk_iopoll_disable(struct blk_iopoll *); 44extern void blk_iopoll_disable(struct blk_iopoll *);
45 45
46extern int blk_iopoll_enabled;
47
48#endif 46#endif
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 2ff2e8d982be..0120451545d8 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -109,7 +109,7 @@ enum {
109 BLK_MQ_F_SHOULD_SORT = 1 << 1, 109 BLK_MQ_F_SHOULD_SORT = 1 << 1,
110 BLK_MQ_F_SHOULD_IPI = 1 << 2, 110 BLK_MQ_F_SHOULD_IPI = 1 << 2,
111 111
112 BLK_MQ_S_STOPPED = 1 << 0, 112 BLK_MQ_S_STOPPED = 0,
113 113
114 BLK_MQ_MAX_DEPTH = 2048, 114 BLK_MQ_MAX_DEPTH = 2048,
115}; 115};
@@ -117,7 +117,8 @@ enum {
117struct request_queue *blk_mq_init_queue(struct blk_mq_reg *, void *); 117struct request_queue *blk_mq_init_queue(struct blk_mq_reg *, void *);
118int blk_mq_register_disk(struct gendisk *); 118int blk_mq_register_disk(struct gendisk *);
119void blk_mq_unregister_disk(struct gendisk *); 119void blk_mq_unregister_disk(struct gendisk *);
120void blk_mq_init_commands(struct request_queue *, void (*init)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data); 120int blk_mq_init_commands(struct request_queue *, int (*init)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data);
121void blk_mq_free_commands(struct request_queue *, void (*free)(void *data, struct blk_mq_hw_ctx *, struct request *, unsigned int), void *data);
121 122
122void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); 123void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
123 124
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 4afa4f8f6090..1e1fa3f93d5f 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -99,6 +99,7 @@ struct request {
99 union { 99 union {
100 struct call_single_data csd; 100 struct call_single_data csd;
101 struct work_struct mq_flush_work; 101 struct work_struct mq_flush_work;
102 unsigned long fifo_time;
102 }; 103 };
103 104
104 struct request_queue *q; 105 struct request_queue *q;
diff --git a/include/linux/elevator.h b/include/linux/elevator.h
index 306dd8cd0b6f..df63bd3a8cf1 100644
--- a/include/linux/elevator.h
+++ b/include/linux/elevator.h
@@ -202,17 +202,8 @@ enum {
202#define rq_end_sector(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq)) 202#define rq_end_sector(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
203#define rb_entry_rq(node) rb_entry((node), struct request, rb_node) 203#define rb_entry_rq(node) rb_entry((node), struct request, rb_node)
204 204
205/*
206 * Hack to reuse the csd.list list_head as the fifo time holder while
207 * the request is in the io scheduler. Saves an unsigned long in rq.
208 */
209#define rq_fifo_time(rq) ((unsigned long) (rq)->csd.list.next)
210#define rq_set_fifo_time(rq,exp) ((rq)->csd.list.next = (void *) (exp))
211#define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist) 205#define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist)
212#define rq_fifo_clear(rq) do { \ 206#define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist)
213 list_del_init(&(rq)->queuelist); \
214 INIT_LIST_HEAD(&(rq)->csd.list); \
215 } while (0)
216 207
217#else /* CONFIG_BLOCK */ 208#else /* CONFIG_BLOCK */
218 209
diff --git a/include/linux/smp.h b/include/linux/smp.h
index 6ae004e437ea..633f5edd7470 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -17,10 +17,7 @@ extern void cpu_idle(void);
17 17
18typedef void (*smp_call_func_t)(void *info); 18typedef void (*smp_call_func_t)(void *info);
19struct call_single_data { 19struct call_single_data {
20 union { 20 struct llist_node llist;
21 struct list_head list;
22 struct llist_node llist;
23 };
24 smp_call_func_t func; 21 smp_call_func_t func;
25 void *info; 22 void *info;
26 u16 flags; 23 u16 flags;
@@ -53,8 +50,7 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info),
53 smp_call_func_t func, void *info, bool wait, 50 smp_call_func_t func, void *info, bool wait,
54 gfp_t gfp_flags); 51 gfp_t gfp_flags);
55 52
56void __smp_call_function_single(int cpuid, struct call_single_data *data, 53int smp_call_function_single_async(int cpu, struct call_single_data *csd);
57 int wait);
58 54
59#ifdef CONFIG_SMP 55#ifdef CONFIG_SMP
60 56
diff --git a/include/trace/events/block.h b/include/trace/events/block.h
index e76ae19a8d6f..e8a5eca1dbe5 100644
--- a/include/trace/events/block.h
+++ b/include/trace/events/block.h
@@ -132,6 +132,7 @@ DEFINE_EVENT(block_rq_with_error, block_rq_requeue,
132 * block_rq_complete - block IO operation completed by device driver 132 * block_rq_complete - block IO operation completed by device driver
133 * @q: queue containing the block operation request 133 * @q: queue containing the block operation request
134 * @rq: block operations request 134 * @rq: block operations request
135 * @nr_bytes: number of completed bytes
135 * 136 *
136 * The block_rq_complete tracepoint event indicates that some portion 137 * The block_rq_complete tracepoint event indicates that some portion
137 * of operation request has been completed by the device driver. If 138 * of operation request has been completed by the device driver. If
@@ -139,11 +140,37 @@ DEFINE_EVENT(block_rq_with_error, block_rq_requeue,
139 * do for the request. If @rq->bio is non-NULL then there is 140 * do for the request. If @rq->bio is non-NULL then there is
140 * additional work required to complete the request. 141 * additional work required to complete the request.
141 */ 142 */
142DEFINE_EVENT(block_rq_with_error, block_rq_complete, 143TRACE_EVENT(block_rq_complete,
143 144
144 TP_PROTO(struct request_queue *q, struct request *rq), 145 TP_PROTO(struct request_queue *q, struct request *rq,
146 unsigned int nr_bytes),
145 147
146 TP_ARGS(q, rq) 148 TP_ARGS(q, rq, nr_bytes),
149
150 TP_STRUCT__entry(
151 __field( dev_t, dev )
152 __field( sector_t, sector )
153 __field( unsigned int, nr_sector )
154 __field( int, errors )
155 __array( char, rwbs, RWBS_LEN )
156 __dynamic_array( char, cmd, blk_cmd_buf_len(rq) )
157 ),
158
159 TP_fast_assign(
160 __entry->dev = rq->rq_disk ? disk_devt(rq->rq_disk) : 0;
161 __entry->sector = blk_rq_pos(rq);
162 __entry->nr_sector = nr_bytes >> 9;
163 __entry->errors = rq->errors;
164
165 blk_fill_rwbs(__entry->rwbs, rq->cmd_flags, nr_bytes);
166 blk_dump_cmd(__get_str(cmd), rq);
167 ),
168
169 TP_printk("%d,%d %s (%s) %llu + %u [%d]",
170 MAJOR(__entry->dev), MINOR(__entry->dev),
171 __entry->rwbs, __get_str(cmd),
172 (unsigned long long)__entry->sector,
173 __entry->nr_sector, __entry->errors)
147); 174);
148 175
149DECLARE_EVENT_CLASS(block_rq, 176DECLARE_EVENT_CLASS(block_rq,
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3c4d096544ce..9cae286824bb 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -432,7 +432,7 @@ void hrtick_start(struct rq *rq, u64 delay)
432 if (rq == this_rq()) { 432 if (rq == this_rq()) {
433 __hrtick_restart(rq); 433 __hrtick_restart(rq);
434 } else if (!rq->hrtick_csd_pending) { 434 } else if (!rq->hrtick_csd_pending) {
435 __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0); 435 smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd);
436 rq->hrtick_csd_pending = 1; 436 rq->hrtick_csd_pending = 1;
437 } 437 }
438} 438}
diff --git a/kernel/smp.c b/kernel/smp.c
index ffee35bef179..06d574e42c72 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -117,13 +117,43 @@ static void csd_unlock(struct call_single_data *csd)
117 csd->flags &= ~CSD_FLAG_LOCK; 117 csd->flags &= ~CSD_FLAG_LOCK;
118} 118}
119 119
120static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
121
120/* 122/*
121 * Insert a previously allocated call_single_data element 123 * Insert a previously allocated call_single_data element
122 * for execution on the given CPU. data must already have 124 * for execution on the given CPU. data must already have
123 * ->func, ->info, and ->flags set. 125 * ->func, ->info, and ->flags set.
124 */ 126 */
125static void generic_exec_single(int cpu, struct call_single_data *csd, int wait) 127static int generic_exec_single(int cpu, struct call_single_data *csd,
128 smp_call_func_t func, void *info, int wait)
126{ 129{
130 struct call_single_data csd_stack = { .flags = 0 };
131 unsigned long flags;
132
133
134 if (cpu == smp_processor_id()) {
135 local_irq_save(flags);
136 func(info);
137 local_irq_restore(flags);
138 return 0;
139 }
140
141
142 if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu))
143 return -ENXIO;
144
145
146 if (!csd) {
147 csd = &csd_stack;
148 if (!wait)
149 csd = &__get_cpu_var(csd_data);
150 }
151
152 csd_lock(csd);
153
154 csd->func = func;
155 csd->info = info;
156
127 if (wait) 157 if (wait)
128 csd->flags |= CSD_FLAG_WAIT; 158 csd->flags |= CSD_FLAG_WAIT;
129 159
@@ -143,6 +173,8 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
143 173
144 if (wait) 174 if (wait)
145 csd_lock_wait(csd); 175 csd_lock_wait(csd);
176
177 return 0;
146} 178}
147 179
148/* 180/*
@@ -151,7 +183,8 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait)
151 */ 183 */
152void generic_smp_call_function_single_interrupt(void) 184void generic_smp_call_function_single_interrupt(void)
153{ 185{
154 struct llist_node *entry, *next; 186 struct llist_node *entry;
187 struct call_single_data *csd, *csd_next;
155 188
156 /* 189 /*
157 * Shouldn't receive this interrupt on a cpu that is not yet online. 190 * Shouldn't receive this interrupt on a cpu that is not yet online.
@@ -161,21 +194,12 @@ void generic_smp_call_function_single_interrupt(void)
161 entry = llist_del_all(&__get_cpu_var(call_single_queue)); 194 entry = llist_del_all(&__get_cpu_var(call_single_queue));
162 entry = llist_reverse_order(entry); 195 entry = llist_reverse_order(entry);
163 196
164 while (entry) { 197 llist_for_each_entry_safe(csd, csd_next, entry, llist) {
165 struct call_single_data *csd;
166
167 next = entry->next;
168
169 csd = llist_entry(entry, struct call_single_data, llist);
170 csd->func(csd->info); 198 csd->func(csd->info);
171 csd_unlock(csd); 199 csd_unlock(csd);
172
173 entry = next;
174 } 200 }
175} 201}
176 202
177static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
178
179/* 203/*
180 * smp_call_function_single - Run a function on a specific CPU 204 * smp_call_function_single - Run a function on a specific CPU
181 * @func: The function to run. This must be fast and non-blocking. 205 * @func: The function to run. This must be fast and non-blocking.
@@ -187,12 +211,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data);
187int smp_call_function_single(int cpu, smp_call_func_t func, void *info, 211int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
188 int wait) 212 int wait)
189{ 213{
190 struct call_single_data d = {
191 .flags = 0,
192 };
193 unsigned long flags;
194 int this_cpu; 214 int this_cpu;
195 int err = 0; 215 int err;
196 216
197 /* 217 /*
198 * prevent preemption and reschedule on another processor, 218 * prevent preemption and reschedule on another processor,
@@ -209,32 +229,41 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
209 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() 229 WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled()
210 && !oops_in_progress); 230 && !oops_in_progress);
211 231
212 if (cpu == this_cpu) { 232 err = generic_exec_single(cpu, NULL, func, info, wait);
213 local_irq_save(flags);
214 func(info);
215 local_irq_restore(flags);
216 } else {
217 if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) {
218 struct call_single_data *csd = &d;
219 233
220 if (!wait) 234 put_cpu();
221 csd = &__get_cpu_var(csd_data);
222 235
223 csd_lock(csd); 236 return err;
237}
238EXPORT_SYMBOL(smp_call_function_single);
224 239
225 csd->func = func; 240/**
226 csd->info = info; 241 * smp_call_function_single_async(): Run an asynchronous function on a
227 generic_exec_single(cpu, csd, wait); 242 * specific CPU.
228 } else { 243 * @cpu: The CPU to run on.
229 err = -ENXIO; /* CPU not online */ 244 * @csd: Pre-allocated and setup data structure
230 } 245 *
231 } 246 * Like smp_call_function_single(), but the call is asynchonous and
247 * can thus be done from contexts with disabled interrupts.
248 *
249 * The caller passes his own pre-allocated data structure
250 * (ie: embedded in an object) and is responsible for synchronizing it
251 * such that the IPIs performed on the @csd are strictly serialized.
252 *
253 * NOTE: Be careful, there is unfortunately no current debugging facility to
254 * validate the correctness of this serialization.
255 */
256int smp_call_function_single_async(int cpu, struct call_single_data *csd)
257{
258 int err = 0;
232 259
233 put_cpu(); 260 preempt_disable();
261 err = generic_exec_single(cpu, csd, csd->func, csd->info, 0);
262 preempt_enable();
234 263
235 return err; 264 return err;
236} 265}
237EXPORT_SYMBOL(smp_call_function_single); 266EXPORT_SYMBOL_GPL(smp_call_function_single_async);
238 267
239/* 268/*
240 * smp_call_function_any - Run a function on any of the given cpus 269 * smp_call_function_any - Run a function on any of the given cpus
@@ -280,44 +309,6 @@ call:
280EXPORT_SYMBOL_GPL(smp_call_function_any); 309EXPORT_SYMBOL_GPL(smp_call_function_any);
281 310
282/** 311/**
283 * __smp_call_function_single(): Run a function on a specific CPU
284 * @cpu: The CPU to run on.
285 * @data: Pre-allocated and setup data structure
286 * @wait: If true, wait until function has completed on specified CPU.
287 *
288 * Like smp_call_function_single(), but allow caller to pass in a
289 * pre-allocated data structure. Useful for embedding @data inside
290 * other structures, for instance.
291 */
292void __smp_call_function_single(int cpu, struct call_single_data *csd,
293 int wait)
294{
295 unsigned int this_cpu;
296 unsigned long flags;
297
298 this_cpu = get_cpu();
299 /*
300 * Can deadlock when called with interrupts disabled.
301 * We allow cpu's that are not yet online though, as no one else can
302 * send smp call function interrupt to this cpu and as such deadlocks
303 * can't happen.
304 */
305 WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled()
306 && !oops_in_progress);
307
308 if (cpu == this_cpu) {
309 local_irq_save(flags);
310 csd->func(csd->info);
311 local_irq_restore(flags);
312 } else {
313 csd_lock(csd);
314 generic_exec_single(cpu, csd, wait);
315 }
316 put_cpu();
317}
318EXPORT_SYMBOL_GPL(__smp_call_function_single);
319
320/**
321 * smp_call_function_many(): Run a function on a set of other CPUs. 312 * smp_call_function_many(): Run a function on a set of other CPUs.
322 * @mask: The set of cpus to run on (only runs on online subset). 313 * @mask: The set of cpus to run on (only runs on online subset).
323 * @func: The function to run. This must be fast and non-blocking. 314 * @func: The function to run. This must be fast and non-blocking.
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 7754ff16f334..09d2e2413605 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -112,9 +112,6 @@ extern int sysctl_nr_open_min, sysctl_nr_open_max;
112#ifndef CONFIG_MMU 112#ifndef CONFIG_MMU
113extern int sysctl_nr_trim_pages; 113extern int sysctl_nr_trim_pages;
114#endif 114#endif
115#ifdef CONFIG_BLOCK
116extern int blk_iopoll_enabled;
117#endif
118 115
119/* Constants used for minimum and maximum */ 116/* Constants used for minimum and maximum */
120#ifdef CONFIG_LOCKUP_DETECTOR 117#ifdef CONFIG_LOCKUP_DETECTOR
@@ -1087,15 +1084,6 @@ static struct ctl_table kern_table[] = {
1087 .proc_handler = proc_dointvec, 1084 .proc_handler = proc_dointvec,
1088 }, 1085 },
1089#endif 1086#endif
1090#ifdef CONFIG_BLOCK
1091 {
1092 .procname = "blk_iopoll",
1093 .data = &blk_iopoll_enabled,
1094 .maxlen = sizeof(int),
1095 .mode = 0644,
1096 .proc_handler = proc_dointvec,
1097 },
1098#endif
1099 { } 1087 { }
1100}; 1088};
1101 1089
diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c
index b418cb0d7242..4f3a3c03eadb 100644
--- a/kernel/trace/blktrace.c
+++ b/kernel/trace/blktrace.c
@@ -702,6 +702,7 @@ void blk_trace_shutdown(struct request_queue *q)
702 * blk_add_trace_rq - Add a trace for a request oriented action 702 * blk_add_trace_rq - Add a trace for a request oriented action
703 * @q: queue the io is for 703 * @q: queue the io is for
704 * @rq: the source request 704 * @rq: the source request
705 * @nr_bytes: number of completed bytes
705 * @what: the action 706 * @what: the action
706 * 707 *
707 * Description: 708 * Description:
@@ -709,7 +710,7 @@ void blk_trace_shutdown(struct request_queue *q)
709 * 710 *
710 **/ 711 **/
711static void blk_add_trace_rq(struct request_queue *q, struct request *rq, 712static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
712 u32 what) 713 unsigned int nr_bytes, u32 what)
713{ 714{
714 struct blk_trace *bt = q->blk_trace; 715 struct blk_trace *bt = q->blk_trace;
715 716
@@ -718,11 +719,11 @@ static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
718 719
719 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) { 720 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
720 what |= BLK_TC_ACT(BLK_TC_PC); 721 what |= BLK_TC_ACT(BLK_TC_PC);
721 __blk_add_trace(bt, 0, blk_rq_bytes(rq), rq->cmd_flags, 722 __blk_add_trace(bt, 0, nr_bytes, rq->cmd_flags,
722 what, rq->errors, rq->cmd_len, rq->cmd); 723 what, rq->errors, rq->cmd_len, rq->cmd);
723 } else { 724 } else {
724 what |= BLK_TC_ACT(BLK_TC_FS); 725 what |= BLK_TC_ACT(BLK_TC_FS);
725 __blk_add_trace(bt, blk_rq_pos(rq), blk_rq_bytes(rq), 726 __blk_add_trace(bt, blk_rq_pos(rq), nr_bytes,
726 rq->cmd_flags, what, rq->errors, 0, NULL); 727 rq->cmd_flags, what, rq->errors, 0, NULL);
727 } 728 }
728} 729}
@@ -730,33 +731,34 @@ static void blk_add_trace_rq(struct request_queue *q, struct request *rq,
730static void blk_add_trace_rq_abort(void *ignore, 731static void blk_add_trace_rq_abort(void *ignore,
731 struct request_queue *q, struct request *rq) 732 struct request_queue *q, struct request *rq)
732{ 733{
733 blk_add_trace_rq(q, rq, BLK_TA_ABORT); 734 blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_ABORT);
734} 735}
735 736
736static void blk_add_trace_rq_insert(void *ignore, 737static void blk_add_trace_rq_insert(void *ignore,
737 struct request_queue *q, struct request *rq) 738 struct request_queue *q, struct request *rq)
738{ 739{
739 blk_add_trace_rq(q, rq, BLK_TA_INSERT); 740 blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_INSERT);
740} 741}
741 742
742static void blk_add_trace_rq_issue(void *ignore, 743static void blk_add_trace_rq_issue(void *ignore,
743 struct request_queue *q, struct request *rq) 744 struct request_queue *q, struct request *rq)
744{ 745{
745 blk_add_trace_rq(q, rq, BLK_TA_ISSUE); 746 blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_ISSUE);
746} 747}
747 748
748static void blk_add_trace_rq_requeue(void *ignore, 749static void blk_add_trace_rq_requeue(void *ignore,
749 struct request_queue *q, 750 struct request_queue *q,
750 struct request *rq) 751 struct request *rq)
751{ 752{
752 blk_add_trace_rq(q, rq, BLK_TA_REQUEUE); 753 blk_add_trace_rq(q, rq, blk_rq_bytes(rq), BLK_TA_REQUEUE);
753} 754}
754 755
755static void blk_add_trace_rq_complete(void *ignore, 756static void blk_add_trace_rq_complete(void *ignore,
756 struct request_queue *q, 757 struct request_queue *q,
757 struct request *rq) 758 struct request *rq,
759 unsigned int nr_bytes)
758{ 760{
759 blk_add_trace_rq(q, rq, BLK_TA_COMPLETE); 761 blk_add_trace_rq(q, rq, nr_bytes, BLK_TA_COMPLETE);
760} 762}
761 763
762/** 764/**
diff --git a/kernel/up.c b/kernel/up.c
index 509403e3fbc6..1760bf3d1463 100644
--- a/kernel/up.c
+++ b/kernel/up.c
@@ -22,16 +22,16 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info,
22} 22}
23EXPORT_SYMBOL(smp_call_function_single); 23EXPORT_SYMBOL(smp_call_function_single);
24 24
25void __smp_call_function_single(int cpu, struct call_single_data *csd, 25int smp_call_function_single_async(int cpu, struct call_single_data *csd)
26 int wait)
27{ 26{
28 unsigned long flags; 27 unsigned long flags;
29 28
30 local_irq_save(flags); 29 local_irq_save(flags);
31 csd->func(csd->info); 30 csd->func(csd->info);
32 local_irq_restore(flags); 31 local_irq_restore(flags);
32 return 0;
33} 33}
34EXPORT_SYMBOL(__smp_call_function_single); 34EXPORT_SYMBOL(smp_call_function_single_async);
35 35
36int on_each_cpu(smp_call_func_t func, void *info, int wait) 36int on_each_cpu(smp_call_func_t func, void *info, int wait)
37{ 37{
diff --git a/kernel/watchdog.c b/kernel/watchdog.c
index 4431610f049a..01c6f979486f 100644
--- a/kernel/watchdog.c
+++ b/kernel/watchdog.c
@@ -505,7 +505,6 @@ static void restart_watchdog_hrtimer(void *info)
505 505
506static void update_timers(int cpu) 506static void update_timers(int cpu)
507{ 507{
508 struct call_single_data data = {.func = restart_watchdog_hrtimer};
509 /* 508 /*
510 * Make sure that perf event counter will adopt to a new 509 * Make sure that perf event counter will adopt to a new
511 * sampling period. Updating the sampling period directly would 510 * sampling period. Updating the sampling period directly would
@@ -515,7 +514,7 @@ static void update_timers(int cpu)
515 * might be late already so we have to restart the timer as well. 514 * might be late already so we have to restart the timer as well.
516 */ 515 */
517 watchdog_nmi_disable(cpu); 516 watchdog_nmi_disable(cpu);
518 __smp_call_function_single(cpu, &data, 1); 517 smp_call_function_single(cpu, restart_watchdog_hrtimer, NULL, 1);
519 watchdog_nmi_enable(cpu); 518 watchdog_nmi_enable(cpu);
520} 519}
521 520
diff --git a/net/core/dev.c b/net/core/dev.c
index 45fa2f11f84d..bc3c89792b0c 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -4135,8 +4135,8 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd)
4135 struct softnet_data *next = remsd->rps_ipi_next; 4135 struct softnet_data *next = remsd->rps_ipi_next;
4136 4136
4137 if (cpu_online(remsd->cpu)) 4137 if (cpu_online(remsd->cpu))
4138 __smp_call_function_single(remsd->cpu, 4138 smp_call_function_single_async(remsd->cpu,
4139 &remsd->csd, 0); 4139 &remsd->csd);
4140 remsd = next; 4140 remsd = next;
4141 } 4141 }
4142 } else 4142 } else
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-31 10:14:30 -0500