diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2019-03-05 15:21:47 -0500 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2019-03-05 15:21:47 -0500 |
| commit | 78f860135433a8bba406352fbdcea8e8980583bf (patch) | |
| tree | 0b7a9ba320e38b5d6eb0fb982bc2d9449aaf57f3 /kernel/irq/affinity.c | |
| parent | 18483190e7a2a6761b67c6824a31adf5b2b7be51 (diff) | |
| parent | a324ca9cad4736252c33c1e28cffe1d87f262d03 (diff) | |
Merge branch 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull irq updates from Thomas Gleixner:
"The interrupt departement delivers this time:
- New infrastructure to manage NMIs on platforms which have a sane
NMI delivery, i.e. identifiable NMI vectors instead of a single
lump.
- Simplification of the interrupt affinity management so drivers
don't have to implement ugly loops around the PCI/MSI enablement.
- Speedup for interrupt statistics in /proc/stat
- Provide a function to retrieve the default irq domain
- A new interrupt controller for the Loongson LS1X platform
- Affinity support for the SiFive PLIC
- Better support for the iMX irqsteer driver
- NUMA aware memory allocations for GICv3
- The usual small fixes, improvements and cleanups all over the
place"
* 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (36 commits)
irqchip/imx-irqsteer: Add multi output interrupts support
irqchip/imx-irqsteer: Change to use reg_num instead of irq_group
dt-bindings: irq: imx-irqsteer: Add multi output interrupts support
dt-binding: irq: imx-irqsteer: Use irq number instead of group number
irqchip/brcmstb-l2: Use _irqsave locking variants in non-interrupt code
irqchip/gicv3-its: Use NUMA aware memory allocation for ITS tables
irqdomain: Allow the default irq domain to be retrieved
irqchip/sifive-plic: Implement irq_set_affinity() for SMP host
irqchip/sifive-plic: Differentiate between PLIC handler and context
irqchip/sifive-plic: Add warning in plic_init() if handler already present
irqchip/sifive-plic: Pre-compute context hart base and enable base
PCI/MSI: Remove obsolete sanity checks for multiple interrupt sets
genirq/affinity: Remove the leftovers of the original set support
nvme-pci: Simplify interrupt allocation
genirq/affinity: Add new callback for (re)calculating interrupt sets
genirq/affinity: Store interrupt sets size in struct irq_affinity
genirq/affinity: Code consolidation
irqchip/irq-sifive-plic: Check and continue in case of an invalid cpuid.
irqchip/i8259: Fix shutdown order by moving syscore_ops registration
dt-bindings: interrupt-controller: loongson ls1x intc
...
Diffstat (limited to 'kernel/irq/affinity.c')
| -rw-r--r-- | kernel/irq/affinity.c | 121 |
1 files changed, 70 insertions, 51 deletions
diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c index 45b68b4ea48b..f18cd5aa33e8 100644 --- a/kernel/irq/affinity.c +++ b/kernel/irq/affinity.c | |||
| @@ -9,7 +9,7 @@ | |||
| 9 | #include <linux/cpu.h> | 9 | #include <linux/cpu.h> |
| 10 | 10 | ||
| 11 | static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, | 11 | static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk, |
| 12 | int cpus_per_vec) | 12 | unsigned int cpus_per_vec) |
| 13 | { | 13 | { |
| 14 | const struct cpumask *siblmsk; | 14 | const struct cpumask *siblmsk; |
| 15 | int cpu, sibl; | 15 | int cpu, sibl; |
| @@ -95,15 +95,17 @@ static int get_nodes_in_cpumask(cpumask_var_t *node_to_cpumask, | |||
| 95 | } | 95 | } |
| 96 | 96 | ||
| 97 | static int __irq_build_affinity_masks(const struct irq_affinity *affd, | 97 | static int __irq_build_affinity_masks(const struct irq_affinity *affd, |
| 98 | int startvec, int numvecs, int firstvec, | 98 | unsigned int startvec, |
| 99 | unsigned int numvecs, | ||
| 100 | unsigned int firstvec, | ||
| 99 | cpumask_var_t *node_to_cpumask, | 101 | cpumask_var_t *node_to_cpumask, |
| 100 | const struct cpumask *cpu_mask, | 102 | const struct cpumask *cpu_mask, |
| 101 | struct cpumask *nmsk, | 103 | struct cpumask *nmsk, |
| 102 | struct irq_affinity_desc *masks) | 104 | struct irq_affinity_desc *masks) |
| 103 | { | 105 | { |
| 104 | int n, nodes, cpus_per_vec, extra_vecs, done = 0; | 106 | unsigned int n, nodes, cpus_per_vec, extra_vecs, done = 0; |
| 105 | int last_affv = firstvec + numvecs; | 107 | unsigned int last_affv = firstvec + numvecs; |
| 106 | int curvec = startvec; | 108 | unsigned int curvec = startvec; |
| 107 | nodemask_t nodemsk = NODE_MASK_NONE; | 109 | nodemask_t nodemsk = NODE_MASK_NONE; |
| 108 | 110 | ||
| 109 | if (!cpumask_weight(cpu_mask)) | 111 | if (!cpumask_weight(cpu_mask)) |
| @@ -117,18 +119,16 @@ static int __irq_build_affinity_masks(const struct irq_affinity *affd, | |||
| 117 | */ | 119 | */ |
| 118 | if (numvecs <= nodes) { | 120 | if (numvecs <= nodes) { |
| 119 | for_each_node_mask(n, nodemsk) { | 121 | for_each_node_mask(n, nodemsk) { |
| 120 | cpumask_or(&masks[curvec].mask, | 122 | cpumask_or(&masks[curvec].mask, &masks[curvec].mask, |
| 121 | &masks[curvec].mask, | 123 | node_to_cpumask[n]); |
| 122 | node_to_cpumask[n]); | ||
| 123 | if (++curvec == last_affv) | 124 | if (++curvec == last_affv) |
| 124 | curvec = firstvec; | 125 | curvec = firstvec; |
| 125 | } | 126 | } |
| 126 | done = numvecs; | 127 | return numvecs; |
| 127 | goto out; | ||
| 128 | } | 128 | } |
| 129 | 129 | ||
| 130 | for_each_node_mask(n, nodemsk) { | 130 | for_each_node_mask(n, nodemsk) { |
| 131 | int ncpus, v, vecs_to_assign, vecs_per_node; | 131 | unsigned int ncpus, v, vecs_to_assign, vecs_per_node; |
| 132 | 132 | ||
| 133 | /* Spread the vectors per node */ | 133 | /* Spread the vectors per node */ |
| 134 | vecs_per_node = (numvecs - (curvec - firstvec)) / nodes; | 134 | vecs_per_node = (numvecs - (curvec - firstvec)) / nodes; |
| @@ -163,8 +163,6 @@ static int __irq_build_affinity_masks(const struct irq_affinity *affd, | |||
| 163 | curvec = firstvec; | 163 | curvec = firstvec; |
| 164 | --nodes; | 164 | --nodes; |
| 165 | } | 165 | } |
| 166 | |||
| 167 | out: | ||
| 168 | return done; | 166 | return done; |
| 169 | } | 167 | } |
| 170 | 168 | ||
| @@ -174,19 +172,24 @@ out: | |||
| 174 | * 2) spread other possible CPUs on these vectors | 172 | * 2) spread other possible CPUs on these vectors |
| 175 | */ | 173 | */ |
| 176 | static int irq_build_affinity_masks(const struct irq_affinity *affd, | 174 | static int irq_build_affinity_masks(const struct irq_affinity *affd, |
| 177 | int startvec, int numvecs, int firstvec, | 175 | unsigned int startvec, unsigned int numvecs, |
| 178 | cpumask_var_t *node_to_cpumask, | 176 | unsigned int firstvec, |
| 179 | struct irq_affinity_desc *masks) | 177 | struct irq_affinity_desc *masks) |
| 180 | { | 178 | { |
| 181 | int curvec = startvec, nr_present, nr_others; | 179 | unsigned int curvec = startvec, nr_present, nr_others; |
| 182 | int ret = -ENOMEM; | 180 | cpumask_var_t *node_to_cpumask; |
| 183 | cpumask_var_t nmsk, npresmsk; | 181 | cpumask_var_t nmsk, npresmsk; |
| 182 | int ret = -ENOMEM; | ||
| 184 | 183 | ||
| 185 | if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) | 184 | if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) |
| 186 | return ret; | 185 | return ret; |
| 187 | 186 | ||
| 188 | if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) | 187 | if (!zalloc_cpumask_var(&npresmsk, GFP_KERNEL)) |
| 189 | goto fail; | 188 | goto fail_nmsk; |
| 189 | |||
| 190 | node_to_cpumask = alloc_node_to_cpumask(); | ||
| 191 | if (!node_to_cpumask) | ||
| 192 | goto fail_npresmsk; | ||
| 190 | 193 | ||
| 191 | ret = 0; | 194 | ret = 0; |
| 192 | /* Stabilize the cpumasks */ | 195 | /* Stabilize the cpumasks */ |
| @@ -217,13 +220,22 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, | |||
| 217 | if (nr_present < numvecs) | 220 | if (nr_present < numvecs) |
| 218 | WARN_ON(nr_present + nr_others < numvecs); | 221 | WARN_ON(nr_present + nr_others < numvecs); |
| 219 | 222 | ||
| 223 | free_node_to_cpumask(node_to_cpumask); | ||
| 224 | |||
| 225 | fail_npresmsk: | ||
| 220 | free_cpumask_var(npresmsk); | 226 | free_cpumask_var(npresmsk); |
| 221 | 227 | ||
| 222 | fail: | 228 | fail_nmsk: |
| 223 | free_cpumask_var(nmsk); | 229 | free_cpumask_var(nmsk); |
| 224 | return ret; | 230 | return ret; |
| 225 | } | 231 | } |
| 226 | 232 | ||
| 233 | static void default_calc_sets(struct irq_affinity *affd, unsigned int affvecs) | ||
| 234 | { | ||
| 235 | affd->nr_sets = 1; | ||
| 236 | affd->set_size[0] = affvecs; | ||
| 237 | } | ||
| 238 | |||
| 227 | /** | 239 | /** |
| 228 | * irq_create_affinity_masks - Create affinity masks for multiqueue spreading | 240 | * irq_create_affinity_masks - Create affinity masks for multiqueue spreading |
| 229 | * @nvecs: The total number of vectors | 241 | * @nvecs: The total number of vectors |
| @@ -232,50 +244,62 @@ static int irq_build_affinity_masks(const struct irq_affinity *affd, | |||
| 232 | * Returns the irq_affinity_desc pointer or NULL if allocation failed. | 244 | * Returns the irq_affinity_desc pointer or NULL if allocation failed. |
| 233 | */ | 245 | */ |
| 234 | struct irq_affinity_desc * | 246 | struct irq_affinity_desc * |
| 235 | irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) | 247 | irq_create_affinity_masks(unsigned int nvecs, struct irq_affinity *affd) |
| 236 | { | 248 | { |
| 237 | int affvecs = nvecs - affd->pre_vectors - affd->post_vectors; | 249 | unsigned int affvecs, curvec, usedvecs, i; |
| 238 | int curvec, usedvecs; | ||
| 239 | cpumask_var_t *node_to_cpumask; | ||
| 240 | struct irq_affinity_desc *masks = NULL; | 250 | struct irq_affinity_desc *masks = NULL; |
| 241 | int i, nr_sets; | ||
| 242 | 251 | ||
| 243 | /* | 252 | /* |
| 244 | * If there aren't any vectors left after applying the pre/post | 253 | * Determine the number of vectors which need interrupt affinities |
| 245 | * vectors don't bother with assigning affinity. | 254 | * assigned. If the pre/post request exhausts the available vectors |
| 255 | * then nothing to do here except for invoking the calc_sets() | ||
| 256 | * callback so the device driver can adjust to the situation. If there | ||
| 257 | * is only a single vector, then managing the queue is pointless as | ||
| 258 | * well. | ||
| 246 | */ | 259 | */ |
| 247 | if (nvecs == affd->pre_vectors + affd->post_vectors) | 260 | if (nvecs > 1 && nvecs > affd->pre_vectors + affd->post_vectors) |
| 261 | affvecs = nvecs - affd->pre_vectors - affd->post_vectors; | ||
| 262 | else | ||
| 263 | affvecs = 0; | ||
| 264 | |||
| 265 | /* | ||
| 266 | * Simple invocations do not provide a calc_sets() callback. Install | ||
| 267 | * the generic one. | ||
| 268 | */ | ||
| 269 | if (!affd->calc_sets) | ||
| 270 | affd->calc_sets = default_calc_sets; | ||
| 271 | |||
| 272 | /* Recalculate the sets */ | ||
| 273 | affd->calc_sets(affd, affvecs); | ||
| 274 | |||
| 275 | if (WARN_ON_ONCE(affd->nr_sets > IRQ_AFFINITY_MAX_SETS)) | ||
| 248 | return NULL; | 276 | return NULL; |
| 249 | 277 | ||
| 250 | node_to_cpumask = alloc_node_to_cpumask(); | 278 | /* Nothing to assign? */ |
| 251 | if (!node_to_cpumask) | 279 | if (!affvecs) |
| 252 | return NULL; | 280 | return NULL; |
| 253 | 281 | ||
| 254 | masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); | 282 | masks = kcalloc(nvecs, sizeof(*masks), GFP_KERNEL); |
| 255 | if (!masks) | 283 | if (!masks) |
| 256 | goto outnodemsk; | 284 | return NULL; |
| 257 | 285 | ||
| 258 | /* Fill out vectors at the beginning that don't need affinity */ | 286 | /* Fill out vectors at the beginning that don't need affinity */ |
| 259 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) | 287 | for (curvec = 0; curvec < affd->pre_vectors; curvec++) |
| 260 | cpumask_copy(&masks[curvec].mask, irq_default_affinity); | 288 | cpumask_copy(&masks[curvec].mask, irq_default_affinity); |
| 289 | |||
| 261 | /* | 290 | /* |
| 262 | * Spread on present CPUs starting from affd->pre_vectors. If we | 291 | * Spread on present CPUs starting from affd->pre_vectors. If we |
| 263 | * have multiple sets, build each sets affinity mask separately. | 292 | * have multiple sets, build each sets affinity mask separately. |
| 264 | */ | 293 | */ |
| 265 | nr_sets = affd->nr_sets; | 294 | for (i = 0, usedvecs = 0; i < affd->nr_sets; i++) { |
| 266 | if (!nr_sets) | 295 | unsigned int this_vecs = affd->set_size[i]; |
| 267 | nr_sets = 1; | ||
| 268 | |||
| 269 | for (i = 0, usedvecs = 0; i < nr_sets; i++) { | ||
| 270 | int this_vecs = affd->sets ? affd->sets[i] : affvecs; | ||
| 271 | int ret; | 296 | int ret; |
| 272 | 297 | ||
| 273 | ret = irq_build_affinity_masks(affd, curvec, this_vecs, | 298 | ret = irq_build_affinity_masks(affd, curvec, this_vecs, |
| 274 | curvec, node_to_cpumask, masks); | 299 | curvec, masks); |
| 275 | if (ret) { | 300 | if (ret) { |
| 276 | kfree(masks); | 301 | kfree(masks); |
| 277 | masks = NULL; | 302 | return NULL; |
| 278 | goto outnodemsk; | ||
| 279 | } | 303 | } |
| 280 | curvec += this_vecs; | 304 | curvec += this_vecs; |
| 281 | usedvecs += this_vecs; | 305 | usedvecs += this_vecs; |
| @@ -293,8 +317,6 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd) | |||
| 293 | for (i = affd->pre_vectors; i < nvecs - affd->post_vectors; i++) | 317 | for (i = affd->pre_vectors; i < nvecs - affd->post_vectors; i++) |
| 294 | masks[i].is_managed = 1; | 318 | masks[i].is_managed = 1; |
| 295 | 319 | ||
| 296 | outnodemsk: | ||
| 297 | free_node_to_cpumask(node_to_cpumask); | ||
| 298 | return masks; | 320 | return masks; |
| 299 | } | 321 | } |
| 300 | 322 | ||
| @@ -304,25 +326,22 @@ outnodemsk: | |||
| 304 | * @maxvec: The maximum number of vectors available | 326 | * @maxvec: The maximum number of vectors available |
| 305 | * @affd: Description of the affinity requirements | 327 | * @affd: Description of the affinity requirements |
| 306 | */ | 328 | */ |
| 307 | int irq_calc_affinity_vectors(int minvec, int maxvec, const struct irq_affinity *affd) | 329 | unsigned int irq_calc_affinity_vectors(unsigned int minvec, unsigned int maxvec, |
| 330 | const struct irq_affinity *affd) | ||
| 308 | { | 331 | { |
| 309 | int resv = affd->pre_vectors + affd->post_vectors; | 332 | unsigned int resv = affd->pre_vectors + affd->post_vectors; |
| 310 | int vecs = maxvec - resv; | 333 | unsigned int set_vecs; |
| 311 | int set_vecs; | ||
| 312 | 334 | ||
| 313 | if (resv > minvec) | 335 | if (resv > minvec) |
| 314 | return 0; | 336 | return 0; |
| 315 | 337 | ||
| 316 | if (affd->nr_sets) { | 338 | if (affd->calc_sets) { |
| 317 | int i; | 339 | set_vecs = maxvec - resv; |
| 318 | |||
| 319 | for (i = 0, set_vecs = 0; i < affd->nr_sets; i++) | ||
| 320 | set_vecs += affd->sets[i]; | ||
| 321 | } else { | 340 | } else { |
| 322 | get_online_cpus(); | 341 | get_online_cpus(); |
| 323 | set_vecs = cpumask_weight(cpu_possible_mask); | 342 | set_vecs = cpumask_weight(cpu_possible_mask); |
| 324 | put_online_cpus(); | 343 | put_online_cpus(); |
| 325 | } | 344 | } |
| 326 | 345 | ||
| 327 | return resv + min(set_vecs, vecs); | 346 | return resv + min(set_vecs, maxvec - resv); |
| 328 | } | 347 | } |
