aboutsummaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
Diffstat (limited to 'block')
-rw-r--r--block/blk-softirq.c16
-rw-r--r--block/blk.h16
2 files changed, 8 insertions, 24 deletions
diff --git a/block/blk-softirq.c b/block/blk-softirq.c
index 1366a89d8e66..467c8de88642 100644
--- a/block/blk-softirq.c
+++ b/block/blk-softirq.c
@@ -8,6 +8,7 @@
8#include <linux/blkdev.h> 8#include <linux/blkdev.h>
9#include <linux/interrupt.h> 9#include <linux/interrupt.h>
10#include <linux/cpu.h> 10#include <linux/cpu.h>
11#include <linux/sched.h>
11 12
12#include "blk.h" 13#include "blk.h"
13 14
@@ -103,9 +104,10 @@ static struct notifier_block __cpuinitdata blk_cpu_notifier = {
103 104
104void __blk_complete_request(struct request *req) 105void __blk_complete_request(struct request *req)
105{ 106{
106 int ccpu, cpu, group_cpu = NR_CPUS; 107 int ccpu, cpu;
107 struct request_queue *q = req->q; 108 struct request_queue *q = req->q;
108 unsigned long flags; 109 unsigned long flags;
110 bool shared = false;
109 111
110 BUG_ON(!q->softirq_done_fn); 112 BUG_ON(!q->softirq_done_fn);
111 113
@@ -117,22 +119,20 @@ void __blk_complete_request(struct request *req)
117 */ 119 */
118 if (req->cpu != -1) { 120 if (req->cpu != -1) {
119 ccpu = req->cpu; 121 ccpu = req->cpu;
120 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags)) { 122 if (!test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags))
121 ccpu = blk_cpu_to_group(ccpu); 123 shared = cpus_share_cache(cpu, ccpu);
122 group_cpu = blk_cpu_to_group(cpu);
123 }
124 } else 124 } else
125 ccpu = cpu; 125 ccpu = cpu;
126 126
127 /* 127 /*
128 * If current CPU and requested CPU are in the same group, running 128 * If current CPU and requested CPU share a cache, run the softirq on
129 * softirq in current CPU. One might concern this is just like 129 * the current CPU. One might concern this is just like
130 * QUEUE_FLAG_SAME_FORCE, but actually not. blk_complete_request() is 130 * QUEUE_FLAG_SAME_FORCE, but actually not. blk_complete_request() is
131 * running in interrupt handler, and currently I/O controller doesn't 131 * running in interrupt handler, and currently I/O controller doesn't
132 * support multiple interrupts, so current CPU is unique actually. This 132 * support multiple interrupts, so current CPU is unique actually. This
133 * avoids IPI sending from current CPU to the first CPU of a group. 133 * avoids IPI sending from current CPU to the first CPU of a group.
134 */ 134 */
135 if (ccpu == cpu || ccpu == group_cpu) { 135 if (ccpu == cpu || shared) {
136 struct list_head *list; 136 struct list_head *list;
137do_local: 137do_local:
138 list = &__get_cpu_var(blk_cpu_done); 138 list = &__get_cpu_var(blk_cpu_done);
diff --git a/block/blk.h b/block/blk.h
index 9c12f80882b0..d45be871329e 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -166,22 +166,6 @@ static inline int queue_congestion_off_threshold(struct request_queue *q)
166 return q->nr_congestion_off; 166 return q->nr_congestion_off;
167} 167}
168 168
169static inline int blk_cpu_to_group(int cpu)
170{
171 int group = NR_CPUS;
172#ifdef CONFIG_SCHED_MC
173 const struct cpumask *mask = cpu_coregroup_mask(cpu);
174 group = cpumask_first(mask);
175#elif defined(CONFIG_SCHED_SMT)
176 group = cpumask_first(topology_thread_cpumask(cpu));
177#else
178 return cpu;
179#endif
180 if (likely(group < NR_CPUS))
181 return group;
182 return cpu;
183}
184
185/* 169/*
186 * Contribute to IO statistics IFF: 170 * Contribute to IO statistics IFF:
187 * 171 *