aboutsummaryrefslogtreecommitdiffstats
path: root/kernel/irq/affinity.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2019-03-05 15:21:47 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2019-03-05 15:21:47 -0500
commit78f860135433a8bba406352fbdcea8e8980583bf (patch)
tree0b7a9ba320e38b5d6eb0fb982bc2d9449aaf57f3 /kernel/irq/affinity.c
parent18483190e7a2a6761b67c6824a31adf5b2b7be51 (diff)
parenta324ca9cad4736252c33c1e28cffe1d87f262d03 (diff)
Merge branch 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull irq updates from Thomas Gleixner: "The interrupt departement delivers this time: - New infrastructure to manage NMIs on platforms which have a sane NMI delivery, i.e. identifiable NMI vectors instead of a single lump. - Simplification of the interrupt affinity management so drivers don't have to implement ugly loops around the PCI/MSI enablement. - Speedup for interrupt statistics in /proc/stat - Provide a function to retrieve the default irq domain - A new interrupt controller for the Loongson LS1X platform - Affinity support for the SiFive PLIC - Better support for the iMX irqsteer driver - NUMA aware memory allocations for GICv3 - The usual small fixes, improvements and cleanups all over the place" * 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (36 commits) irqchip/imx-irqsteer: Add multi output interrupts support irqchip/imx-irqsteer: Change to use reg_num instead of irq_group dt-bindings: irq: imx-irqsteer: Add multi output interrupts support dt-binding: irq: imx-irqsteer: Use irq number instead of group number irqchip/brcmstb-l2: Use _irqsave locking variants in non-interrupt code irqchip/gicv3-its: Use NUMA aware memory allocation for ITS tables irqdomain: Allow the default irq domain to be retrieved irqchip/sifive-plic: Implement irq_set_affinity() for SMP host irqchip/sifive-plic: Differentiate between PLIC handler and context irqchip/sifive-plic: Add warning in plic_init() if handler already present irqchip/sifive-plic: Pre-compute context hart base and enable base PCI/MSI: Remove obsolete sanity checks for multiple interrupt sets genirq/affinity: Remove the leftovers of the original set support nvme-pci: Simplify interrupt allocation genirq/affinity: Add new callback for (re)calculating interrupt sets genirq/affinity: Store interrupt sets size in struct irq_affinity genirq/affinity: Code consolidation irqchip/irq-sifive-plic: Check and continue in case of an invalid cpuid. irqchip/i8259: Fix shutdown order by moving syscore_ops registration dt-bindings: interrupt-controller: loongson ls1x intc ...
Diffstat (limited to 'kernel/irq/affinity.c')
-rw-r--r--kernel/irq/affinity.c121
1 files changed, 70 insertions, 51 deletions
diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
index 45b68b4ea48b..f18cd5aa33e8 100644
--- a/kernel/irq/affinity.c
+++ b/kernel/irq/affinity.c
@@ -9,7 +9,7 @@
9#include <linux/cpu.h> 9#include <linux/cpu.h>
10 10
11static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, 11static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk,
12 int cpus_per_vec) 12 unsigned int cpus_per_vec)
13{ 13{
14 const struct cpumask *siblmsk; 14 const struct cpumask *siblmsk;
15 int cpu, sibl; 15 int cpu, sibl;
@@ -95,15 +95,17 @@ static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask,
95} 95}
96 96
97static int __irq_build_affinity_masks(const struct irq_affinity *affd, 97static int __irq_build_affinity_masks(const struct irq_affinity *affd,
98 int startvec, int numvecs, int firstvec, 98 unsigned int startvec,
99 unsigned int numvecs,
100 unsigned int firstvec,
99 cpumask_var_t *node_to_cpumask, 101 cpumask_var_t *node_to_cpumask,
100 const struct cpumask *cpu_mask, 102 const struct cpumask *cpu_mask,
101 struct cpumask *nmsk, 103 struct cpumask *nmsk,
102 struct irq_affinity_desc *masks) 104 struct irq_affinity_desc *masks)
103{ 105{
104 int n, nodes, cpus_per_vec, extra_vecs, done = 0; 106 unsigned int n, nodes, cpus_per_vec, extra_vecs, done = 0;
105 int last_affv = firstvec + numvecs; 107 unsigned int last_affv = firstvec + numvecs;
106 int curvec = startvec; 108 unsigned int curvec = startvec;
107 nodemask_t nodemsk = NODE_MASK_NONE; 109 nodemask_t nodemsk = NODE_MASK_NONE;
108 110
109 if (!cpumask_weight(cpu_mask)) 111 if (!cpumask_weight(cpu_mask))
@@ -117,18 +119,16 @@ static int __irq_build_affinity_masks(const struct irq_affinity *affd,
117 */ 119 */
118 if (numvecs <= nodes) { 120 if (numvecs <= nodes) {
119 for_each_node_mask(n, nodemsk) { 121 for_each_node_mask(n, nodemsk) {
120 cpumask_or(&masks[curvec].mask, 122 cpumask_or(&masks[curvec].mask, &masks[curvec].mask,
121 &masks[curvec].mask, 123 node_to_cpumask[n]);
122 node_to_cpumask[n]);
123 if (++curvec == last_affv) 124 if (++curvec == last_affv)
124 curvec = firstvec; 125 curvec = firstvec;
125 } 126 }
126 done = numvecs; 127 return numvecs;
127 goto out;
128 } 128 }
129 129
130 for_each_node_mask(n, nodemsk) { 130 for_each_node_mask(n, nodemsk) {
131 int ncpus, v, vecs_to_assign, vecs_per_node; 131 unsigned int ncpus, v, vecs_to_assign, vecs_per_node;
132 132
133 /* Spread the vectors per node */ 133 /* Spread the vectors per node */
134 vecs_per_node = (numvecs - (curvec - firstvec)) / nodes; 134 vecs_per_node = (numvecs - (curvec - firstvec)) / nodes;
@@ -163,8 +163,6 @@ static int __irq_build_affinity_masks(const struct irq_affinity *affd,
163 curvec = firstvec; 163 curvec = firstvec;
164 --nodes; 164 --nodes;
165 } 165 }
166
167out:
168 return done; 166 return done;
169} 167}
170 168
@@ -174,19 +172,24 @@ out:
174 * 2) spread other possible CPUs on these vectors 172 * 2) spread other possible CPUs on these vectors
175 */ 173 */
176static int irq_build_affinity_masks(const struct irq_affinity *affd, 174static int irq_build_affinity_masks(const struct irq_affinity *affd,
177 int startvec, int numvecs, int firstvec, 175 unsigned int startvec, unsigned int numvecs,
178 cpumask_var_t *node_to_cpumask, 176 unsigned int firstvec,
179 struct irq_affinity_desc *masks) 177 struct irq_affinity_desc *masks)
180{ 178{
181 int curvec = startvec, nr_present, nr_others; 179 unsigned int curvec = startvec, nr_present, nr_others;
182 int ret = -ENOMEM; 180 cpumask_var_t *node_to_cpumask;
183 cpumask_var_t nmsk, npresmsk; 181 cpumask_var_t nmsk, npresmsk;
182 int ret = -ENOMEM;
184 183
185 if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) 184 if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
186 return ret; 185 return ret;
187 186
188 if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) 187 if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL))
189 goto fail; 188 goto fail_nmsk;
189
190 node_to_cpumask = alloc_node_to_cpumask();
191 if (!node_to_cpumask)
192 goto fail_npresmsk;
190 193
191 ret = 0; 194 ret = 0;
192 /* Stabilize the cpumasks */ 195 /* Stabilize the cpumasks */
@@ -217,13 +220,22 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd,
217 if (nr_present < numvecs) 220 if (nr_present < numvecs)
218 WARN_ON(nr_present + nr_others < numvecs); 221 WARN_ON(nr_present + nr_others < numvecs);
219 222
223 free_node_to_cpumask(node_to_cpumask);
224
225 fail_npresmsk:
220 free_cpumask_var(npresmsk); 226 free_cpumask_var(npresmsk);
221 227
222 fail: 228 fail_nmsk:
223 free_cpumask_var(nmsk); 229 free_cpumask_var(nmsk);
224 return ret; 230 return ret;
225} 231}
226 232
233static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs)
234{
235 affd->nr_sets = 1;
236 affd->set_size[0] = affvecs;
237}
238
227/** 239/**
228 * irq_create_affinity_masks - Create affinity masks for multiqueue spreading 240 * irq_create_affinity_masks - Create affinity masks for multiqueue spreading
229 * @nvecs: The total number of vectors 241 * @nvecs: The total number of vectors
@@ -232,50 +244,62 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd,
232 * Returns the irq_affinity_desc pointer or NULL if allocation failed. 244 * Returns the irq_affinity_desc pointer or NULL if allocation failed.
233 */ 245 */
234struct irq_affinity_desc * 246struct irq_affinity_desc *
235irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) 247irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd)
236{ 248{
237 int affvecs = nvecs - affd->pre_vectors - affd->post_vectors; 249 unsigned int affvecs, curvec, usedvecs, i;
238 int curvec, usedvecs;
239 cpumask_var_t *node_to_cpumask;
240 struct irq_affinity_desc *masks = NULL; 250 struct irq_affinity_desc *masks = NULL;
241 int i, nr_sets;
242 251
243 /* 252 /*
244 * If there aren't any vectors left after applying the pre/post 253 * Determine the number of vectors which need interrupt affinities
245 * vectors don't bother with assigning affinity. 254 * assigned. If the pre/post request exhausts the available vectors
255 * then nothing to do here except for invoking the calc_sets()
256 * callback so the device driver can adjust to the situation. If there
257 * is only a single vector, then managing the queue is pointless as
258 * well.
246 */ 259 */
247 if (nvecs == affd->pre_vectors + affd->post_vectors) 260 if (nvecs > 1 && nvecs > affd->pre_vectors + affd->post_vectors)
261 affvecs = nvecs - affd->pre_vectors - affd->post_vectors;
262 else
263 affvecs = 0;
264
265 /*
266 * Simple invocations do not provide a calc_sets() callback. Install
267 * the generic one.
268 */
269 if (!affd->calc_sets)
270 affd->calc_sets = default_calc_sets;
271
272 /* Recalculate the sets */
273 affd->calc_sets(affd, affvecs);
274
275 if (WARN_ON_ONCE(affd->nr_sets > IRQ_AFFINITY_MAX_SETS))
248 return NULL; 276 return NULL;
249 277
250 node_to_cpumask = alloc_node_to_cpumask(); 278 /* Nothing to assign? */
251 if (!node_to_cpumask) 279 if (!affvecs)
252 return NULL; 280 return NULL;
253 281
254 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); 282 masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL);
255 if (!masks) 283 if (!masks)
256 goto outnodemsk; 284 return NULL;
257 285
258 /* Fill out vectors at the beginning that don't need affinity */ 286 /* Fill out vectors at the beginning that don't need affinity */
259 for (curvec = 0; curvec < affd->pre_vectors; curvec++) 287 for (curvec = 0; curvec < affd->pre_vectors; curvec++)
260 cpumask_copy(&masks[curvec].mask, irq_default_affinity); 288 cpumask_copy(&masks[curvec].mask, irq_default_affinity);
289
261 /* 290 /*
262 * Spread on present CPUs starting from affd->pre_vectors. If we 291 * Spread on present CPUs starting from affd->pre_vectors. If we
263 * have multiple sets, build each sets affinity mask separately. 292 * have multiple sets, build each sets affinity mask separately.
264 */ 293 */
265 nr_sets = affd->nr_sets; 294 for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) {
266 if (!nr_sets) 295 unsigned int this_vecs = affd->set_size[i];
267 nr_sets = 1;
268
269 for (i = 0, usedvecs = 0; i < nr_sets; i++) {
270 int this_vecs = affd->sets ? affd->sets[i] : affvecs;
271 int ret; 296 int ret;
272 297
273 ret = irq_build_affinity_masks(affd, curvec, this_vecs, 298 ret = irq_build_affinity_masks(affd, curvec, this_vecs,
274 curvec, node_to_cpumask, masks); 299 curvec, masks);
275 if (ret) { 300 if (ret) {
276 kfree(masks); 301 kfree(masks);
277 masks = NULL; 302 return NULL;
278 goto outnodemsk;
279 } 303 }
280 curvec += this_vecs; 304 curvec += this_vecs;
281 usedvecs += this_vecs; 305 usedvecs += this_vecs;
@@ -293,8 +317,6 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
293 for (i = affd->pre_vectors; i < nvecs - affd->post_vectors; i++) 317 for (i = affd->pre_vectors; i < nvecs - affd->post_vectors; i++)
294 masks[i].is_managed = 1; 318 masks[i].is_managed = 1;
295 319
296outnodemsk:
297 free_node_to_cpumask(node_to_cpumask);
298 return masks; 320 return masks;
299} 321}
300 322
@@ -304,25 +326,22 @@ outnodemsk:
304 * @maxvec: The maximum number of vectors available 326 * @maxvec: The maximum number of vectors available
305 * @affd: Description of the affinity requirements 327 * @affd: Description of the affinity requirements
306 */ 328 */
307int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd) 329unsigned int irq_calc_affinity_vectors(unsigned int minvec, unsigned int maxvec,
330 const struct irq_affinity *affd)
308{ 331{
309 int resv = affd->pre_vectors + affd->post_vectors; 332 unsigned int resv = affd->pre_vectors + affd->post_vectors;
310 int vecs = maxvec - resv; 333 unsigned int set_vecs;
311 int set_vecs;
312 334
313 if (resv > minvec) 335 if (resv > minvec)
314 return 0; 336 return 0;
315 337
316 if (affd->nr_sets) { 338 if (affd->calc_sets) {
317 int i; 339 set_vecs = maxvec - resv;
318
319 for (i = 0, set_vecs = 0; i < affd->nr_sets; i++)
320 set_vecs += affd->sets[i];
321 } else { 340 } else {
322 get_online_cpus(); 341 get_online_cpus();
323 set_vecs = cpumask_weight(cpu_possible_mask); 342 set_vecs = cpumask_weight(cpu_possible_mask);
324 put_online_cpus(); 343 put_online_cpus();
325 } 344 }
326 345
327 return resv + min(set_vecs, vecs); 346 return resv + min(set_vecs, maxvec - resv);
328} 347}