aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2017-07-03 19:50:31 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2017-07-03 19:50:31 -0400
commit03ffbcdd7898c0b5299efeb9f18de927487ec1cf (patch)
tree0569222e4dc9db22049d7d8d15920cc085a194f6
parent1b044f1cfc65a7d90b209dfabd57e16d98b58c5b (diff)
parentf9632de40ee0161e864bea8c1b017d957fd7312c (diff)
Merge branch 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull irq updates from Thomas Gleixner: "The irq department delivers: - Expand the generic infrastructure handling the irq migration on CPU hotplug and convert X86 over to it. (Thomas Gleixner) Aside of consolidating code this is a preparatory change for: - Finalizing the affinity management for multi-queue devices. The main change here is to shut down interrupts which are affine to a outgoing CPU and reenabling them when the CPU comes online again. That avoids moving interrupts pointlessly around and breaking and reestablishing affinities for no value. (Christoph Hellwig) Note: This contains also the BLOCK-MQ and NVME changes which depend on the rework of the irq core infrastructure. Jens acked them and agreed that they should go with the irq changes. - Consolidation of irq domain code (Marc Zyngier) - State tracking consolidation in the core code (Jeffy Chen) - Add debug infrastructure for hierarchical irq domains (Thomas Gleixner) - Infrastructure enhancement for managing generic interrupt chips via devmem (Bartosz Golaszewski) - Constification work all over the place (Tobias Klauser) - Two new interrupt controller drivers for MVEBU (Thomas Petazzoni) - The usual set of fixes, updates and enhancements all over the place" * 'irq-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (112 commits) irqchip/or1k-pic: Fix interrupt acknowledgement irqchip/irq-mvebu-gicp: Allocate enough memory for spi_bitmap irqchip/gic-v3: Fix out-of-bound access in gic_set_affinity nvme: Allocate queues for all possible CPUs blk-mq: Create hctx for each present CPU blk-mq: Include all present CPUs in the default queue mapping genirq: Avoid unnecessary low level irq function calls genirq: Set irq masked state when initializing irq_desc genirq/timings: Add infrastructure for estimating the next interrupt arrival time genirq/timings: Add infrastructure to track the interrupt timings genirq/debugfs: Remove pointless NULL pointer check irqchip/gic-v3-its: Don't assume GICv3 hardware supports 16bit INTID irqchip/gic-v3-its: Add ACPI NUMA node mapping irqchip/gic-v3-its-platform-msi: Make of_device_ids const irqchip/gic-v3-its: Make of_device_ids const irqchip/irq-mvebu-icu: Add new driver for Marvell ICU irqchip/irq-mvebu-gicp: Add new driver for Marvell GICP dt-bindings/interrupt-controller: Add DT binding for the Marvell ICU genirq/irqdomain: Remove auto-recursive hierarchy support irqchip/MSI: Use irq_domain_update_bus_token instead of an open coded access ...
-rw-r--r--Documentation/IRQ-domain.txt41
-rw-r--r--Documentation/devicetree/bindings/interrupt-controller/allwinner,sunxi-nmi.txt7
-rw-r--r--Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-i2c-ic.txt25
-rw-r--r--Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt9
-rw-r--r--Documentation/devicetree/bindings/interrupt-controller/marvell,gicp.txt27
-rw-r--r--Documentation/devicetree/bindings/interrupt-controller/marvell,icu.txt51
-rw-r--r--Documentation/driver-model/devres.txt2
-rw-r--r--arch/x86/Kconfig2
-rw-r--r--arch/x86/include/asm/apic.h36
-rw-r--r--arch/x86/include/asm/irq.h1
-rw-r--r--arch/x86/include/asm/irq_remapping.h3
-rw-r--r--arch/x86/kernel/apic/apic.c35
-rw-r--r--arch/x86/kernel/apic/apic_flat_64.c4
-rw-r--r--arch/x86/kernel/apic/apic_noop.c2
-rw-r--r--arch/x86/kernel/apic/apic_numachip.c4
-rw-r--r--arch/x86/kernel/apic/bigsmp_32.c2
-rw-r--r--arch/x86/kernel/apic/htirq.c19
-rw-r--r--arch/x86/kernel/apic/io_apic.c22
-rw-r--r--arch/x86/kernel/apic/msi.c53
-rw-r--r--arch/x86/kernel/apic/probe_32.c2
-rw-r--r--arch/x86/kernel/apic/vector.c49
-rw-r--r--arch/x86/kernel/apic/x2apic_cluster.c36
-rw-r--r--arch/x86/kernel/apic/x2apic_phys.c2
-rw-r--r--arch/x86/kernel/apic/x2apic_uv_x.c26
-rw-r--r--arch/x86/kernel/irq.c78
-rw-r--r--arch/x86/platform/uv/uv_irq.c18
-rw-r--r--arch/x86/xen/apic.c2
-rw-r--r--block/blk-mq-cpumap.c10
-rw-r--r--block/blk-mq.c120
-rw-r--r--block/blk-mq.h5
-rw-r--r--drivers/base/platform-msi.c2
-rw-r--r--drivers/iommu/amd_iommu.c22
-rw-r--r--drivers/iommu/intel_irq_remapping.c31
-rw-r--r--drivers/irqchip/Kconfig6
-rw-r--r--drivers/irqchip/Makefile4
-rw-r--r--drivers/irqchip/irq-armada-370-xp.c148
-rw-r--r--drivers/irqchip/irq-aspeed-i2c-ic.c115
-rw-r--r--drivers/irqchip/irq-aspeed-vic.c5
-rw-r--r--drivers/irqchip/irq-gic-v2m.c2
-rw-r--r--drivers/irqchip/irq-gic-v3-its-pci-msi.c35
-rw-r--r--drivers/irqchip/irq-gic-v3-its-platform-msi.c2
-rw-r--r--drivers/irqchip/irq-gic-v3-its.c113
-rw-r--r--drivers/irqchip/irq-gic-v3.c3
-rw-r--r--drivers/irqchip/irq-i8259.c2
-rw-r--r--drivers/irqchip/irq-imx-gpcv2.c2
-rw-r--r--drivers/irqchip/irq-mbigen.c2
-rw-r--r--drivers/irqchip/irq-mips-cpu.c2
-rw-r--r--drivers/irqchip/irq-mips-gic.c4
-rw-r--r--drivers/irqchip/irq-mvebu-gicp.c279
-rw-r--r--drivers/irqchip/irq-mvebu-gicp.h11
-rw-r--r--drivers/irqchip/irq-mvebu-icu.c289
-rw-r--r--drivers/irqchip/irq-or1k-pic.c2
-rw-r--r--drivers/irqchip/irq-renesas-h8300h.c2
-rw-r--r--drivers/irqchip/irq-renesas-h8s.c2
-rw-r--r--drivers/irqchip/irq-sunxi-nmi.c68
-rw-r--r--drivers/irqchip/qcom-irq-combiner.c7
-rw-r--r--drivers/nvme/host/pci.c2
-rw-r--r--drivers/pci/host/vmd.c8
-rw-r--r--drivers/pci/msi.c2
-rw-r--r--drivers/staging/fsl-mc/bus/fsl-mc-msi.c2
-rw-r--r--drivers/xen/events/events_base.c6
-rw-r--r--include/dt-bindings/interrupt-controller/mvebu-icu.h15
-rw-r--r--include/linux/cpuhotplug.h2
-rw-r--r--include/linux/interrupt.h6
-rw-r--r--include/linux/irq.h89
-rw-r--r--include/linux/irqdesc.h4
-rw-r--r--include/linux/irqdomain.h43
-rw-r--r--kernel/cpu.c5
-rw-r--r--kernel/irq/Kconfig18
-rw-r--r--kernel/irq/Makefile2
-rw-r--r--kernel/irq/affinity.c76
-rw-r--r--kernel/irq/autoprobe.c4
-rw-r--r--kernel/irq/chip.c195
-rw-r--r--kernel/irq/cpuhotplug.c150
-rw-r--r--kernel/irq/debugfs.c213
-rw-r--r--kernel/irq/devres.c86
-rw-r--r--kernel/irq/generic-chip.c7
-rw-r--r--kernel/irq/handle.c2
-rw-r--r--kernel/irq/internals.h225
-rw-r--r--kernel/irq/irqdesc.c36
-rw-r--r--kernel/irq/irqdomain.c359
-rw-r--r--kernel/irq/manage.c119
-rw-r--r--kernel/irq/migration.c30
-rw-r--r--kernel/irq/msi.c13
-rw-r--r--kernel/irq/proc.c110
-rw-r--r--kernel/irq/timings.c369
86 files changed, 3342 insertions, 709 deletions
diff --git a/Documentation/IRQ-domain.txt b/Documentation/IRQ-domain.txt
index 82001a25a14b..1f246eb25ca5 100644
--- a/Documentation/IRQ-domain.txt
+++ b/Documentation/IRQ-domain.txt
@@ -231,5 +231,42 @@ needs to:
2314) No need to implement irq_domain_ops.map and irq_domain_ops.unmap, 2314) No need to implement irq_domain_ops.map and irq_domain_ops.unmap,
232 they are unused with hierarchy irq_domain. 232 they are unused with hierarchy irq_domain.
233 233
234Hierarchy irq_domain may also be used to support other architectures, 234Hierarchy irq_domain is in no way x86 specific, and is heavily used to
235such as ARM, ARM64 etc. 235support other architectures, such as ARM, ARM64 etc.
236
237=== Debugging ===
238
239If you switch on CONFIG_IRQ_DOMAIN_DEBUG (which depends on
240CONFIG_IRQ_DOMAIN and CONFIG_DEBUG_FS), you will find a new file in
241your debugfs mount point, called irq_domain_mapping. This file
242contains a live snapshot of all the IRQ domains in the system:
243
244 name mapped linear-max direct-max devtree-node
245 pl061 8 8 0 /smb/gpio@e0080000
246 pl061 8 8 0 /smb/gpio@e1050000
247 pMSI 0 0 0 /interrupt-controller@e1101000/v2m@e0080000
248 MSI 37 0 0 /interrupt-controller@e1101000/v2m@e0080000
249 GICv2m 37 0 0 /interrupt-controller@e1101000/v2m@e0080000
250 GICv2 448 448 0 /interrupt-controller@e1101000
251
252it also iterates over the interrupts to display their mapping in the
253domains, and makes the domain stacking visible:
254
255
256irq hwirq chip name chip data active type domain
257 1 0x00019 GICv2 0xffff00000916bfd8 * LINEAR GICv2
258 2 0x0001d GICv2 0xffff00000916bfd8 LINEAR GICv2
259 3 0x0001e GICv2 0xffff00000916bfd8 * LINEAR GICv2
260 4 0x0001b GICv2 0xffff00000916bfd8 * LINEAR GICv2
261 5 0x0001a GICv2 0xffff00000916bfd8 LINEAR GICv2
262[...]
263 96 0x81808 MSI 0x (null) RADIX MSI
264 96+ 0x00063 GICv2m 0xffff8003ee116980 RADIX GICv2m
265 96+ 0x00063 GICv2 0xffff00000916bfd8 LINEAR GICv2
266 97 0x08800 MSI 0x (null) * RADIX MSI
267 97+ 0x00064 GICv2m 0xffff8003ee116980 * RADIX GICv2m
268 97+ 0x00064 GICv2 0xffff00000916bfd8 * LINEAR GICv2
269
270Here, interrupts 1-5 are only using a single domain, while 96 and 97
271are build out of a stack of three domain, each level performing a
272particular function.
diff --git a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sunxi-nmi.txt b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sunxi-nmi.txt
index 81cd3692405e..4ae553eb333d 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/allwinner,sunxi-nmi.txt
+++ b/Documentation/devicetree/bindings/interrupt-controller/allwinner,sunxi-nmi.txt
@@ -3,8 +3,11 @@ Allwinner Sunxi NMI Controller
3 3
4Required properties: 4Required properties:
5 5
6- compatible : should be "allwinner,sun7i-a20-sc-nmi" or 6- compatible : should be one of the following:
7 "allwinner,sun6i-a31-sc-nmi" or "allwinner,sun9i-a80-nmi" 7 - "allwinner,sun7i-a20-sc-nmi"
8 - "allwinner,sun6i-a31-sc-nmi" (deprecated)
9 - "allwinner,sun6i-a31-r-intc"
10 - "allwinner,sun9i-a80-nmi"
8- reg : Specifies base physical address and size of the registers. 11- reg : Specifies base physical address and size of the registers.
9- interrupt-controller : Identifies the node as an interrupt controller 12- interrupt-controller : Identifies the node as an interrupt controller
10- #interrupt-cells : Specifies the number of cells needed to encode an 13- #interrupt-cells : Specifies the number of cells needed to encode an
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-i2c-ic.txt b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-i2c-ic.txt
new file mode 100644
index 000000000000..033cc82e5684
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-i2c-ic.txt
@@ -0,0 +1,25 @@
1Device tree configuration for the I2C Interrupt Controller on the AST24XX and
2AST25XX SoCs.
3
4Required Properties:
5- #address-cells : should be 1
6- #size-cells : should be 1
7- #interrupt-cells : should be 1
8- compatible : should be "aspeed,ast2400-i2c-ic"
9 or "aspeed,ast2500-i2c-ic"
10- reg : address start and range of controller
11- interrupts : interrupt number
12- interrupt-controller : denotes that the controller receives and fires
13 new interrupts for child busses
14
15Example:
16
17i2c_ic: interrupt-controller@0 {
18 #address-cells = <1>;
19 #size-cells = <1>;
20 #interrupt-cells = <1>;
21 compatible = "aspeed,ast2400-i2c-ic";
22 reg = <0x0 0x40>;
23 interrupts = <12>;
24 interrupt-controller;
25};
diff --git a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt
index 6c6e85324b9d..e3fea0758d25 100644
--- a/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt
+++ b/Documentation/devicetree/bindings/interrupt-controller/aspeed,ast2400-vic.txt
@@ -1,12 +1,13 @@
1Aspeed Vectored Interrupt Controller 1Aspeed Vectored Interrupt Controller
2 2
3These bindings are for the Aspeed AST2400 interrupt controller register layout. 3These bindings are for the Aspeed interrupt controller. The AST2400 and
4The SoC has an legacy register layout, but this driver does not support that 4AST2500 SoC families include a legacy register layout before a re-designed
5mode of operation. 5layout, but the bindings do not prescribe the use of one or the other.
6 6
7Required properties: 7Required properties:
8 8
9- compatible : should be "aspeed,ast2400-vic". 9- compatible : "aspeed,ast2400-vic"
10 "aspeed,ast2500-vic"
10 11
11- interrupt-controller : Identifies the node as an interrupt controller 12- interrupt-controller : Identifies the node as an interrupt controller
12- #interrupt-cells : Specifies the number of cells needed to encode an 13- #interrupt-cells : Specifies the number of cells needed to encode an
diff --git a/Documentation/devicetree/bindings/interrupt-controller/marvell,gicp.txt b/Documentation/devicetree/bindings/interrupt-controller/marvell,gicp.txt
new file mode 100644
index 000000000000..64a00ceb7da4
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/marvell,gicp.txt
@@ -0,0 +1,27 @@
1Marvell GICP Controller
2-----------------------
3
4GICP is a Marvell extension of the GIC that allows to trigger GIC SPI
5interrupts by doing a memory transaction. It is used by the ICU
6located in the Marvell CP110 to turn wired interrupts inside the CP
7into GIC SPI interrupts.
8
9Required properties:
10
11- compatible: Must be "marvell,ap806-gicp"
12
13- reg: Must be the address and size of the GICP SPI registers
14
15- marvell,spi-ranges: tuples of GIC SPI interrupts ranges available
16 for this GICP
17
18- msi-controller: indicates that this is an MSI controller
19
20Example:
21
22gicp_spi: gicp-spi@3f0040 {
23 compatible = "marvell,ap806-gicp";
24 reg = <0x3f0040 0x10>;
25 marvell,spi-ranges = <64 64>, <288 64>;
26 msi-controller;
27};
diff --git a/Documentation/devicetree/bindings/interrupt-controller/marvell,icu.txt b/Documentation/devicetree/bindings/interrupt-controller/marvell,icu.txt
new file mode 100644
index 000000000000..aa8bf2ec8905
--- /dev/null
+++ b/Documentation/devicetree/bindings/interrupt-controller/marvell,icu.txt
@@ -0,0 +1,51 @@
1Marvell ICU Interrupt Controller
2--------------------------------
3
4The Marvell ICU (Interrupt Consolidation Unit) controller is
5responsible for collecting all wired-interrupt sources in the CP and
6communicating them to the GIC in the AP, the unit translates interrupt
7requests on input wires to MSG memory mapped transactions to the GIC.
8
9Required properties:
10
11- compatible: Should be "marvell,cp110-icu"
12
13- reg: Should contain ICU registers location and length.
14
15- #interrupt-cells: Specifies the number of cells needed to encode an
16 interrupt source. The value shall be 3.
17
18 The 1st cell is the group type of the ICU interrupt. Possible group
19 types are:
20
21 ICU_GRP_NSR (0x0) : Shared peripheral interrupt, non-secure
22 ICU_GRP_SR (0x1) : Shared peripheral interrupt, secure
23 ICU_GRP_SEI (0x4) : System error interrupt
24 ICU_GRP_REI (0x5) : RAM error interrupt
25
26 The 2nd cell is the index of the interrupt in the ICU unit.
27
28 The 3rd cell is the type of the interrupt. See arm,gic.txt for
29 details.
30
31- interrupt-controller: Identifies the node as an interrupt
32 controller.
33
34- msi-parent: Should point to the GICP controller, the GIC extension
35 that allows to trigger interrupts using MSG memory mapped
36 transactions.
37
38Example:
39
40icu: interrupt-controller@1e0000 {
41 compatible = "marvell,cp110-icu";
42 reg = <0x1e0000 0x10>;
43 #interrupt-cells = <3>;
44 interrupt-controller;
45 msi-parent = <&gicp>;
46};
47
48usb3h0: usb3@500000 {
49 interrupt-parent = <&icu>;
50 interrupts = <ICU_GRP_NSR 106 IRQ_TYPE_LEVEL_HIGH>;
51};
diff --git a/Documentation/driver-model/devres.txt b/Documentation/driver-model/devres.txt
index e72587fe477d..6a6618f34440 100644
--- a/Documentation/driver-model/devres.txt
+++ b/Documentation/driver-model/devres.txt
@@ -311,6 +311,8 @@ IRQ
311 devm_irq_alloc_desc_at() 311 devm_irq_alloc_desc_at()
312 devm_irq_alloc_desc_from() 312 devm_irq_alloc_desc_from()
313 devm_irq_alloc_descs_from() 313 devm_irq_alloc_descs_from()
314 devm_irq_alloc_generic_chip()
315 devm_irq_setup_generic_chip()
314 316
315LED 317LED
316 devm_led_classdev_register() 318 devm_led_classdev_register()
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 737212c0333e..7bdb9022c6e7 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -87,6 +87,8 @@ config X86
87 select GENERIC_EARLY_IOREMAP 87 select GENERIC_EARLY_IOREMAP
88 select GENERIC_FIND_FIRST_BIT 88 select GENERIC_FIND_FIRST_BIT
89 select GENERIC_IOMAP 89 select GENERIC_IOMAP
90 select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP
91 select GENERIC_IRQ_MIGRATION if SMP
90 select GENERIC_IRQ_PROBE 92 select GENERIC_IRQ_PROBE
91 select GENERIC_IRQ_SHOW 93 select GENERIC_IRQ_SHOW
92 select GENERIC_PENDING_IRQ if SMP 94 select GENERIC_PENDING_IRQ if SMP
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index bdffcd9eab2b..5f01671c68f2 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -252,6 +252,8 @@ static inline int x2apic_enabled(void) { return 0; }
252#define x2apic_supported() (0) 252#define x2apic_supported() (0)
253#endif /* !CONFIG_X86_X2APIC */ 253#endif /* !CONFIG_X86_X2APIC */
254 254
255struct irq_data;
256
255/* 257/*
256 * Copyright 2004 James Cleverdon, IBM. 258 * Copyright 2004 James Cleverdon, IBM.
257 * Subject to the GNU Public License, v.2 259 * Subject to the GNU Public License, v.2
@@ -296,9 +298,9 @@ struct apic {
296 /* Can't be NULL on 64-bit */ 298 /* Can't be NULL on 64-bit */
297 unsigned long (*set_apic_id)(unsigned int id); 299 unsigned long (*set_apic_id)(unsigned int id);
298 300
299 int (*cpu_mask_to_apicid_and)(const struct cpumask *cpumask, 301 int (*cpu_mask_to_apicid)(const struct cpumask *cpumask,
300 const struct cpumask *andmask, 302 struct irq_data *irqdata,
301 unsigned int *apicid); 303 unsigned int *apicid);
302 304
303 /* ipi */ 305 /* ipi */
304 void (*send_IPI)(int cpu, int vector); 306 void (*send_IPI)(int cpu, int vector);
@@ -540,28 +542,12 @@ static inline int default_phys_pkg_id(int cpuid_apic, int index_msb)
540 542
541#endif 543#endif
542 544
543static inline int 545extern int flat_cpu_mask_to_apicid(const struct cpumask *cpumask,
544flat_cpu_mask_to_apicid_and(const struct cpumask *cpumask, 546 struct irq_data *irqdata,
545 const struct cpumask *andmask, 547 unsigned int *apicid);
546 unsigned int *apicid) 548extern int default_cpu_mask_to_apicid(const struct cpumask *cpumask,
547{ 549 struct irq_data *irqdata,
548 unsigned long cpu_mask = cpumask_bits(cpumask)[0] & 550 unsigned int *apicid);
549 cpumask_bits(andmask)[0] &
550 cpumask_bits(cpu_online_mask)[0] &
551 APIC_ALL_CPUS;
552
553 if (likely(cpu_mask)) {
554 *apicid = (unsigned int)cpu_mask;
555 return 0;
556 } else {
557 return -EINVAL;
558 }
559}
560
561extern int
562default_cpu_mask_to_apicid_and(const struct cpumask *cpumask,
563 const struct cpumask *andmask,
564 unsigned int *apicid);
565 551
566static inline void 552static inline void
567flat_vector_allocation_domain(int cpu, struct cpumask *retmask, 553flat_vector_allocation_domain(int cpu, struct cpumask *retmask,
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index 16d3fa211962..668cca540025 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -29,7 +29,6 @@ struct irq_desc;
29#include <linux/cpumask.h> 29#include <linux/cpumask.h>
30extern int check_irq_vectors_for_cpu_disable(void); 30extern int check_irq_vectors_for_cpu_disable(void);
31extern void fixup_irqs(void); 31extern void fixup_irqs(void);
32extern void irq_force_complete_move(struct irq_desc *desc);
33#endif 32#endif
34 33
35#ifdef CONFIG_HAVE_KVM 34#ifdef CONFIG_HAVE_KVM
diff --git a/arch/x86/include/asm/irq_remapping.h b/arch/x86/include/asm/irq_remapping.h
index a210eba2727c..023b4a9fc846 100644
--- a/arch/x86/include/asm/irq_remapping.h
+++ b/arch/x86/include/asm/irq_remapping.h
@@ -55,7 +55,8 @@ extern struct irq_domain *
55irq_remapping_get_irq_domain(struct irq_alloc_info *info); 55irq_remapping_get_irq_domain(struct irq_alloc_info *info);
56 56
57/* Create PCI MSI/MSIx irqdomain, use @parent as the parent irqdomain. */ 57/* Create PCI MSI/MSIx irqdomain, use @parent as the parent irqdomain. */
58extern struct irq_domain *arch_create_msi_irq_domain(struct irq_domain *parent); 58extern struct irq_domain *
59arch_create_remap_msi_irq_domain(struct irq_domain *par, const char *n, int id);
59 60
60/* Get parent irqdomain for interrupt remapping irqdomain */ 61/* Get parent irqdomain for interrupt remapping irqdomain */
61static inline struct irq_domain *arch_get_ir_parent_domain(void) 62static inline struct irq_domain *arch_get_ir_parent_domain(void)
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 2d75faf743f2..e7409468efc6 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -2201,23 +2201,32 @@ void default_init_apic_ldr(void)
2201 apic_write(APIC_LDR, val); 2201 apic_write(APIC_LDR, val);
2202} 2202}
2203 2203
2204int default_cpu_mask_to_apicid_and(const struct cpumask *cpumask, 2204int default_cpu_mask_to_apicid(const struct cpumask *mask,
2205 const struct cpumask *andmask, 2205 struct irq_data *irqdata,
2206 unsigned int *apicid) 2206 unsigned int *apicid)
2207{ 2207{
2208 unsigned int cpu; 2208 unsigned int cpu = cpumask_first(mask);
2209 2209
2210 for_each_cpu_and(cpu, cpumask, andmask) { 2210 if (cpu >= nr_cpu_ids)
2211 if (cpumask_test_cpu(cpu, cpu_online_mask)) 2211 return -EINVAL;
2212 break; 2212 *apicid = per_cpu(x86_cpu_to_apicid, cpu);
2213 } 2213 irq_data_update_effective_affinity(irqdata, cpumask_of(cpu));
2214 return 0;
2215}
2214 2216
2215 if (likely(cpu < nr_cpu_ids)) { 2217int flat_cpu_mask_to_apicid(const struct cpumask *mask,
2216 *apicid = per_cpu(x86_cpu_to_apicid, cpu); 2218 struct irq_data *irqdata,
2217 return 0; 2219 unsigned int *apicid)
2218 }
2219 2220
2220 return -EINVAL; 2221{
2222 struct cpumask *effmsk = irq_data_get_effective_affinity_mask(irqdata);
2223 unsigned long cpu_mask = cpumask_bits(mask)[0] & APIC_ALL_CPUS;
2224
2225 if (!cpu_mask)
2226 return -EINVAL;
2227 *apicid = (unsigned int)cpu_mask;
2228 cpumask_bits(effmsk)[0] = cpu_mask;
2229 return 0;
2221} 2230}
2222 2231
2223/* 2232/*
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c
index a4d7ff20ed22..dedd5a41ba48 100644
--- a/arch/x86/kernel/apic/apic_flat_64.c
+++ b/arch/x86/kernel/apic/apic_flat_64.c
@@ -172,7 +172,7 @@ static struct apic apic_flat __ro_after_init = {
172 .get_apic_id = flat_get_apic_id, 172 .get_apic_id = flat_get_apic_id,
173 .set_apic_id = set_apic_id, 173 .set_apic_id = set_apic_id,
174 174
175 .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, 175 .cpu_mask_to_apicid = flat_cpu_mask_to_apicid,
176 176
177 .send_IPI = default_send_IPI_single, 177 .send_IPI = default_send_IPI_single,
178 .send_IPI_mask = flat_send_IPI_mask, 178 .send_IPI_mask = flat_send_IPI_mask,
@@ -268,7 +268,7 @@ static struct apic apic_physflat __ro_after_init = {
268 .get_apic_id = flat_get_apic_id, 268 .get_apic_id = flat_get_apic_id,
269 .set_apic_id = set_apic_id, 269 .set_apic_id = set_apic_id,
270 270
271 .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, 271 .cpu_mask_to_apicid = default_cpu_mask_to_apicid,
272 272
273 .send_IPI = default_send_IPI_single_phys, 273 .send_IPI = default_send_IPI_single_phys,
274 .send_IPI_mask = default_send_IPI_mask_sequence_phys, 274 .send_IPI_mask = default_send_IPI_mask_sequence_phys,
diff --git a/arch/x86/kernel/apic/apic_noop.c b/arch/x86/kernel/apic/apic_noop.c
index 2262eb6df796..6599f437b4ab 100644
--- a/arch/x86/kernel/apic/apic_noop.c
+++ b/arch/x86/kernel/apic/apic_noop.c
@@ -141,7 +141,7 @@ struct apic apic_noop __ro_after_init = {
141 .get_apic_id = noop_get_apic_id, 141 .get_apic_id = noop_get_apic_id,
142 .set_apic_id = NULL, 142 .set_apic_id = NULL,
143 143
144 .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, 144 .cpu_mask_to_apicid = flat_cpu_mask_to_apicid,
145 145
146 .send_IPI = noop_send_IPI, 146 .send_IPI = noop_send_IPI,
147 .send_IPI_mask = noop_send_IPI_mask, 147 .send_IPI_mask = noop_send_IPI_mask,
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index e08fe2c8dd8c..2fda912219a6 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -267,7 +267,7 @@ static const struct apic apic_numachip1 __refconst = {
267 .get_apic_id = numachip1_get_apic_id, 267 .get_apic_id = numachip1_get_apic_id,
268 .set_apic_id = numachip1_set_apic_id, 268 .set_apic_id = numachip1_set_apic_id,
269 269
270 .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, 270 .cpu_mask_to_apicid = default_cpu_mask_to_apicid,
271 271
272 .send_IPI = numachip_send_IPI_one, 272 .send_IPI = numachip_send_IPI_one,
273 .send_IPI_mask = numachip_send_IPI_mask, 273 .send_IPI_mask = numachip_send_IPI_mask,
@@ -318,7 +318,7 @@ static const struct apic apic_numachip2 __refconst = {
318 .get_apic_id = numachip2_get_apic_id, 318 .get_apic_id = numachip2_get_apic_id,
319 .set_apic_id = numachip2_set_apic_id, 319 .set_apic_id = numachip2_set_apic_id,
320 320
321 .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, 321 .cpu_mask_to_apicid = default_cpu_mask_to_apicid,
322 322
323 .send_IPI = numachip_send_IPI_one, 323 .send_IPI = numachip_send_IPI_one,
324 .send_IPI_mask = numachip_send_IPI_mask, 324 .send_IPI_mask = numachip_send_IPI_mask,
diff --git a/arch/x86/kernel/apic/bigsmp_32.c b/arch/x86/kernel/apic/bigsmp_32.c
index 56012010332c..456e45e8bf84 100644
--- a/arch/x86/kernel/apic/bigsmp_32.c
+++ b/arch/x86/kernel/apic/bigsmp_32.c
@@ -172,7 +172,7 @@ static struct apic apic_bigsmp __ro_after_init = {
172 .get_apic_id = bigsmp_get_apic_id, 172 .get_apic_id = bigsmp_get_apic_id,
173 .set_apic_id = NULL, 173 .set_apic_id = NULL,
174 174
175 .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, 175 .cpu_mask_to_apicid = default_cpu_mask_to_apicid,
176 176
177 .send_IPI = default_send_IPI_single_phys, 177 .send_IPI = default_send_IPI_single_phys,
178 .send_IPI_mask = default_send_IPI_mask_sequence_phys, 178 .send_IPI_mask = default_send_IPI_mask_sequence_phys,
diff --git a/arch/x86/kernel/apic/htirq.c b/arch/x86/kernel/apic/htirq.c
index 81ff48905623..56ccf9346b08 100644
--- a/arch/x86/kernel/apic/htirq.c
+++ b/arch/x86/kernel/apic/htirq.c
@@ -152,14 +152,25 @@ static const struct irq_domain_ops htirq_domain_ops = {
152 152
153void __init arch_init_htirq_domain(struct irq_domain *parent) 153void __init arch_init_htirq_domain(struct irq_domain *parent)
154{ 154{
155 struct fwnode_handle *fn;
156
155 if (disable_apic) 157 if (disable_apic)
156 return; 158 return;
157 159
158 htirq_domain = irq_domain_add_tree(NULL, &htirq_domain_ops, NULL); 160 fn = irq_domain_alloc_named_fwnode("PCI-HT");
161 if (!fn)
162 goto warn;
163
164 htirq_domain = irq_domain_create_tree(fn, &htirq_domain_ops, NULL);
165 irq_domain_free_fwnode(fn);
159 if (!htirq_domain) 166 if (!htirq_domain)
160 pr_warn("failed to initialize irqdomain for HTIRQ.\n"); 167 goto warn;
161 else 168
162 htirq_domain->parent = parent; 169 htirq_domain->parent = parent;
170 return;
171
172warn:
173 pr_warn("Failed to initialize irqdomain for HTIRQ.\n");
163} 174}
164 175
165int arch_setup_ht_irq(int idx, int pos, struct pci_dev *dev, 176int arch_setup_ht_irq(int idx, int pos, struct pci_dev *dev,
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 247880fc29f9..b4f5f73febdb 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -2201,6 +2201,8 @@ static int mp_irqdomain_create(int ioapic)
2201 struct ioapic *ip = &ioapics[ioapic]; 2201 struct ioapic *ip = &ioapics[ioapic];
2202 struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg; 2202 struct ioapic_domain_cfg *cfg = &ip->irqdomain_cfg;
2203 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic); 2203 struct mp_ioapic_gsi *gsi_cfg = mp_ioapic_gsi_routing(ioapic);
2204 struct fwnode_handle *fn;
2205 char *name = "IO-APIC";
2204 2206
2205 if (cfg->type == IOAPIC_DOMAIN_INVALID) 2207 if (cfg->type == IOAPIC_DOMAIN_INVALID)
2206 return 0; 2208 return 0;
@@ -2211,9 +2213,25 @@ static int mp_irqdomain_create(int ioapic)
2211 parent = irq_remapping_get_ir_irq_domain(&info); 2213 parent = irq_remapping_get_ir_irq_domain(&info);
2212 if (!parent) 2214 if (!parent)
2213 parent = x86_vector_domain; 2215 parent = x86_vector_domain;
2216 else
2217 name = "IO-APIC-IR";
2218
2219 /* Handle device tree enumerated APICs proper */
2220 if (cfg->dev) {
2221 fn = of_node_to_fwnode(cfg->dev);
2222 } else {
2223 fn = irq_domain_alloc_named_id_fwnode(name, ioapic);
2224 if (!fn)
2225 return -ENOMEM;
2226 }
2227
2228 ip->irqdomain = irq_domain_create_linear(fn, hwirqs, cfg->ops,
2229 (void *)(long)ioapic);
2230
2231 /* Release fw handle if it was allocated above */
2232 if (!cfg->dev)
2233 irq_domain_free_fwnode(fn);
2214 2234
2215 ip->irqdomain = irq_domain_add_linear(cfg->dev, hwirqs, cfg->ops,
2216 (void *)(long)ioapic);
2217 if (!ip->irqdomain) 2235 if (!ip->irqdomain)
2218 return -ENOMEM; 2236 return -ENOMEM;
2219 2237
diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c
index 4c5d1882bec5..9b18be764422 100644
--- a/arch/x86/kernel/apic/msi.c
+++ b/arch/x86/kernel/apic/msi.c
@@ -138,11 +138,18 @@ static struct msi_domain_info pci_msi_domain_info = {
138 138
139void __init arch_init_msi_domain(struct irq_domain *parent) 139void __init arch_init_msi_domain(struct irq_domain *parent)
140{ 140{
141 struct fwnode_handle *fn;
142
141 if (disable_apic) 143 if (disable_apic)
142 return; 144 return;
143 145
144 msi_default_domain = pci_msi_create_irq_domain(NULL, 146 fn = irq_domain_alloc_named_fwnode("PCI-MSI");
145 &pci_msi_domain_info, parent); 147 if (fn) {
148 msi_default_domain =
149 pci_msi_create_irq_domain(fn, &pci_msi_domain_info,
150 parent);
151 irq_domain_free_fwnode(fn);
152 }
146 if (!msi_default_domain) 153 if (!msi_default_domain)
147 pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n"); 154 pr_warn("failed to initialize irqdomain for MSI/MSI-x.\n");
148} 155}
@@ -167,9 +174,18 @@ static struct msi_domain_info pci_msi_ir_domain_info = {
167 .handler_name = "edge", 174 .handler_name = "edge",
168}; 175};
169 176
170struct irq_domain *arch_create_msi_irq_domain(struct irq_domain *parent) 177struct irq_domain *arch_create_remap_msi_irq_domain(struct irq_domain *parent,
178 const char *name, int id)
171{ 179{
172 return pci_msi_create_irq_domain(NULL, &pci_msi_ir_domain_info, parent); 180 struct fwnode_handle *fn;
181 struct irq_domain *d;
182
183 fn = irq_domain_alloc_named_id_fwnode(name, id);
184 if (!fn)
185 return NULL;
186 d = pci_msi_create_irq_domain(fn, &pci_msi_ir_domain_info, parent);
187 irq_domain_free_fwnode(fn);
188 return d;
173} 189}
174#endif 190#endif
175 191
@@ -221,13 +237,20 @@ static struct irq_domain *dmar_get_irq_domain(void)
221{ 237{
222 static struct irq_domain *dmar_domain; 238 static struct irq_domain *dmar_domain;
223 static DEFINE_MUTEX(dmar_lock); 239 static DEFINE_MUTEX(dmar_lock);
240 struct fwnode_handle *fn;
224 241
225 mutex_lock(&dmar_lock); 242 mutex_lock(&dmar_lock);
226 if (dmar_domain == NULL) 243 if (dmar_domain)
227 dmar_domain = msi_create_irq_domain(NULL, &dmar_msi_domain_info, 244 goto out;
245
246 fn = irq_domain_alloc_named_fwnode("DMAR-MSI");
247 if (fn) {
248 dmar_domain = msi_create_irq_domain(fn, &dmar_msi_domain_info,
228 x86_vector_domain); 249 x86_vector_domain);
250 irq_domain_free_fwnode(fn);
251 }
252out:
229 mutex_unlock(&dmar_lock); 253 mutex_unlock(&dmar_lock);
230
231 return dmar_domain; 254 return dmar_domain;
232} 255}
233 256
@@ -317,9 +340,10 @@ static struct msi_domain_info hpet_msi_domain_info = {
317 340
318struct irq_domain *hpet_create_irq_domain(int hpet_id) 341struct irq_domain *hpet_create_irq_domain(int hpet_id)
319{ 342{
320 struct irq_domain *parent;
321 struct irq_alloc_info info;
322 struct msi_domain_info *domain_info; 343 struct msi_domain_info *domain_info;
344 struct irq_domain *parent, *d;
345 struct irq_alloc_info info;
346 struct fwnode_handle *fn;
323 347
324 if (x86_vector_domain == NULL) 348 if (x86_vector_domain == NULL)
325 return NULL; 349 return NULL;
@@ -340,7 +364,16 @@ struct irq_domain *hpet_create_irq_domain(int hpet_id)
340 else 364 else
341 hpet_msi_controller.name = "IR-HPET-MSI"; 365 hpet_msi_controller.name = "IR-HPET-MSI";
342 366
343 return msi_create_irq_domain(NULL, domain_info, parent); 367 fn = irq_domain_alloc_named_id_fwnode(hpet_msi_controller.name,
368 hpet_id);
369 if (!fn) {
370 kfree(domain_info);
371 return NULL;
372 }
373
374 d = msi_create_irq_domain(fn, domain_info, parent);
375 irq_domain_free_fwnode(fn);
376 return d;
344} 377}
345 378
346int hpet_assign_irq(struct irq_domain *domain, struct hpet_dev *dev, 379int hpet_assign_irq(struct irq_domain *domain, struct hpet_dev *dev,
diff --git a/arch/x86/kernel/apic/probe_32.c b/arch/x86/kernel/apic/probe_32.c
index 2e8f7f048f4f..63287659adb6 100644
--- a/arch/x86/kernel/apic/probe_32.c
+++ b/arch/x86/kernel/apic/probe_32.c
@@ -102,7 +102,7 @@ static struct apic apic_default __ro_after_init = {
102 .get_apic_id = default_get_apic_id, 102 .get_apic_id = default_get_apic_id,
103 .set_apic_id = NULL, 103 .set_apic_id = NULL,
104 104
105 .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, 105 .cpu_mask_to_apicid = flat_cpu_mask_to_apicid,
106 106
107 .send_IPI = default_send_IPI_single, 107 .send_IPI = default_send_IPI_single,
108 .send_IPI_mask = default_send_IPI_mask_logical, 108 .send_IPI_mask = default_send_IPI_mask_logical,
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index e66d8e48e456..b3af457ed667 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -103,7 +103,8 @@ static void free_apic_chip_data(struct apic_chip_data *data)
103} 103}
104 104
105static int __assign_irq_vector(int irq, struct apic_chip_data *d, 105static int __assign_irq_vector(int irq, struct apic_chip_data *d,
106 const struct cpumask *mask) 106 const struct cpumask *mask,
107 struct irq_data *irqdata)
107{ 108{
108 /* 109 /*
109 * NOTE! The local APIC isn't very good at handling 110 * NOTE! The local APIC isn't very good at handling
@@ -141,7 +142,7 @@ static int __assign_irq_vector(int irq, struct apic_chip_data *d,
141 /* 142 /*
142 * Clear the offline cpus from @vector_cpumask for searching 143 * Clear the offline cpus from @vector_cpumask for searching
143 * and verify whether the result overlaps with @mask. If true, 144 * and verify whether the result overlaps with @mask. If true,
144 * then the call to apic->cpu_mask_to_apicid_and() will 145 * then the call to apic->cpu_mask_to_apicid() will
145 * succeed as well. If not, no point in trying to find a 146 * succeed as well. If not, no point in trying to find a
146 * vector in this mask. 147 * vector in this mask.
147 */ 148 */
@@ -221,34 +222,40 @@ success:
221 * Cache destination APIC IDs into cfg->dest_apicid. This cannot fail 222 * Cache destination APIC IDs into cfg->dest_apicid. This cannot fail
222 * as we already established, that mask & d->domain & cpu_online_mask 223 * as we already established, that mask & d->domain & cpu_online_mask
223 * is not empty. 224 * is not empty.
225 *
226 * vector_searchmask is a subset of d->domain and has the offline
227 * cpus masked out.
224 */ 228 */
225 BUG_ON(apic->cpu_mask_to_apicid_and(mask, d->domain, 229 cpumask_and(vector_searchmask, vector_searchmask, mask);
226 &d->cfg.dest_apicid)); 230 BUG_ON(apic->cpu_mask_to_apicid(vector_searchmask, irqdata,
231 &d->cfg.dest_apicid));
227 return 0; 232 return 0;
228} 233}
229 234
230static int assign_irq_vector(int irq, struct apic_chip_data *data, 235static int assign_irq_vector(int irq, struct apic_chip_data *data,
231 const struct cpumask *mask) 236 const struct cpumask *mask,
237 struct irq_data *irqdata)
232{ 238{
233 int err; 239 int err;
234 unsigned long flags; 240 unsigned long flags;
235 241
236 raw_spin_lock_irqsave(&vector_lock, flags); 242 raw_spin_lock_irqsave(&vector_lock, flags);
237 err = __assign_irq_vector(irq, data, mask); 243 err = __assign_irq_vector(irq, data, mask, irqdata);
238 raw_spin_unlock_irqrestore(&vector_lock, flags); 244 raw_spin_unlock_irqrestore(&vector_lock, flags);
239 return err; 245 return err;
240} 246}
241 247
242static int assign_irq_vector_policy(int irq, int node, 248static int assign_irq_vector_policy(int irq, int node,
243 struct apic_chip_data *data, 249 struct apic_chip_data *data,
244 struct irq_alloc_info *info) 250 struct irq_alloc_info *info,
251 struct irq_data *irqdata)
245{ 252{
246 if (info && info->mask) 253 if (info && info->mask)
247 return assign_irq_vector(irq, data, info->mask); 254 return assign_irq_vector(irq, data, info->mask, irqdata);
248 if (node != NUMA_NO_NODE && 255 if (node != NUMA_NO_NODE &&
249 assign_irq_vector(irq, data, cpumask_of_node(node)) == 0) 256 assign_irq_vector(irq, data, cpumask_of_node(node), irqdata) == 0)
250 return 0; 257 return 0;
251 return assign_irq_vector(irq, data, apic->target_cpus()); 258 return assign_irq_vector(irq, data, apic->target_cpus(), irqdata);
252} 259}
253 260
254static void clear_irq_vector(int irq, struct apic_chip_data *data) 261static void clear_irq_vector(int irq, struct apic_chip_data *data)
@@ -360,9 +367,17 @@ static int x86_vector_alloc_irqs(struct irq_domain *domain, unsigned int virq,
360 irq_data->chip = &lapic_controller; 367 irq_data->chip = &lapic_controller;
361 irq_data->chip_data = data; 368 irq_data->chip_data = data;
362 irq_data->hwirq = virq + i; 369 irq_data->hwirq = virq + i;
363 err = assign_irq_vector_policy(virq + i, node, data, info); 370 err = assign_irq_vector_policy(virq + i, node, data, info,
371 irq_data);
364 if (err) 372 if (err)
365 goto error; 373 goto error;
374 /*
375 * If the apic destination mode is physical, then the
376 * effective affinity is restricted to a single target
377 * CPU. Mark the interrupt accordingly.
378 */
379 if (!apic->irq_dest_mode)
380 irqd_set_single_target(irq_data);
366 } 381 }
367 382
368 return 0; 383 return 0;
@@ -429,11 +444,16 @@ static inline void init_legacy_irqs(void) { }
429 444
430int __init arch_early_irq_init(void) 445int __init arch_early_irq_init(void)
431{ 446{
447 struct fwnode_handle *fn;
448
432 init_legacy_irqs(); 449 init_legacy_irqs();
433 450
434 x86_vector_domain = irq_domain_add_tree(NULL, &x86_vector_domain_ops, 451 fn = irq_domain_alloc_named_fwnode("VECTOR");
435 NULL); 452 BUG_ON(!fn);
453 x86_vector_domain = irq_domain_create_tree(fn, &x86_vector_domain_ops,
454 NULL);
436 BUG_ON(x86_vector_domain == NULL); 455 BUG_ON(x86_vector_domain == NULL);
456 irq_domain_free_fwnode(fn);
437 irq_set_default_host(x86_vector_domain); 457 irq_set_default_host(x86_vector_domain);
438 458
439 arch_init_msi_domain(x86_vector_domain); 459 arch_init_msi_domain(x86_vector_domain);
@@ -529,11 +549,12 @@ static int apic_set_affinity(struct irq_data *irq_data,
529 if (!cpumask_intersects(dest, cpu_online_mask)) 549 if (!cpumask_intersects(dest, cpu_online_mask))
530 return -EINVAL; 550 return -EINVAL;
531 551
532 err = assign_irq_vector(irq, data, dest); 552 err = assign_irq_vector(irq, data, dest, irq_data);
533 return err ? err : IRQ_SET_MASK_OK; 553 return err ? err : IRQ_SET_MASK_OK;
534} 554}
535 555
536static struct irq_chip lapic_controller = { 556static struct irq_chip lapic_controller = {
557 .name = "APIC",
537 .irq_ack = apic_ack_edge, 558 .irq_ack = apic_ack_edge,
538 .irq_set_affinity = apic_set_affinity, 559 .irq_set_affinity = apic_set_affinity,
539 .irq_retrigger = apic_retrigger_irq, 560 .irq_retrigger = apic_retrigger_irq,
diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c
index 5a35f208ed95..481237cb1544 100644
--- a/arch/x86/kernel/apic/x2apic_cluster.c
+++ b/arch/x86/kernel/apic/x2apic_cluster.c
@@ -4,6 +4,7 @@
4#include <linux/kernel.h> 4#include <linux/kernel.h>
5#include <linux/ctype.h> 5#include <linux/ctype.h>
6#include <linux/dmar.h> 6#include <linux/dmar.h>
7#include <linux/irq.h>
7#include <linux/cpu.h> 8#include <linux/cpu.h>
8 9
9#include <asm/smp.h> 10#include <asm/smp.h>
@@ -104,35 +105,30 @@ static void x2apic_send_IPI_all(int vector)
104} 105}
105 106
106static int 107static int
107x2apic_cpu_mask_to_apicid_and(const struct cpumask *cpumask, 108x2apic_cpu_mask_to_apicid(const struct cpumask *mask, struct irq_data *irqdata,
108 const struct cpumask *andmask, 109 unsigned int *apicid)
109 unsigned int *apicid)
110{ 110{
111 struct cpumask *effmsk = irq_data_get_effective_affinity_mask(irqdata);
112 unsigned int cpu;
111 u32 dest = 0; 113 u32 dest = 0;
112 u16 cluster; 114 u16 cluster;
113 int i;
114
115 for_each_cpu_and(i, cpumask, andmask) {
116 if (!cpumask_test_cpu(i, cpu_online_mask))
117 continue;
118 dest = per_cpu(x86_cpu_to_logical_apicid, i);
119 cluster = x2apic_cluster(i);
120 break;
121 }
122 115
123 if (!dest) 116 cpu = cpumask_first(mask);
117 if (cpu >= nr_cpu_ids)
124 return -EINVAL; 118 return -EINVAL;
125 119
126 for_each_cpu_and(i, cpumask, andmask) { 120 dest = per_cpu(x86_cpu_to_logical_apicid, cpu);
127 if (!cpumask_test_cpu(i, cpu_online_mask)) 121 cluster = x2apic_cluster(cpu);
128 continue; 122
129 if (cluster != x2apic_cluster(i)) 123 cpumask_clear(effmsk);
124 for_each_cpu(cpu, mask) {
125 if (cluster != x2apic_cluster(cpu))
130 continue; 126 continue;
131 dest |= per_cpu(x86_cpu_to_logical_apicid, i); 127 dest |= per_cpu(x86_cpu_to_logical_apicid, cpu);
128 cpumask_set_cpu(cpu, effmsk);
132 } 129 }
133 130
134 *apicid = dest; 131 *apicid = dest;
135
136 return 0; 132 return 0;
137} 133}
138 134
@@ -256,7 +252,7 @@ static struct apic apic_x2apic_cluster __ro_after_init = {
256 .get_apic_id = x2apic_get_apic_id, 252 .get_apic_id = x2apic_get_apic_id,
257 .set_apic_id = x2apic_set_apic_id, 253 .set_apic_id = x2apic_set_apic_id,
258 254
259 .cpu_mask_to_apicid_and = x2apic_cpu_mask_to_apicid_and, 255 .cpu_mask_to_apicid = x2apic_cpu_mask_to_apicid,
260 256
261 .send_IPI = x2apic_send_IPI, 257 .send_IPI = x2apic_send_IPI,
262 .send_IPI_mask = x2apic_send_IPI_mask, 258 .send_IPI_mask = x2apic_send_IPI_mask,
diff --git a/arch/x86/kernel/apic/x2apic_phys.c b/arch/x86/kernel/apic/x2apic_phys.c
index ff111f05a314..3baf0c3dc875 100644
--- a/arch/x86/kernel/apic/x2apic_phys.c
+++ b/arch/x86/kernel/apic/x2apic_phys.c
@@ -127,7 +127,7 @@ static struct apic apic_x2apic_phys __ro_after_init = {
127 .get_apic_id = x2apic_get_apic_id, 127 .get_apic_id = x2apic_get_apic_id,
128 .set_apic_id = x2apic_set_apic_id, 128 .set_apic_id = x2apic_set_apic_id,
129 129
130 .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, 130 .cpu_mask_to_apicid = default_cpu_mask_to_apicid,
131 131
132 .send_IPI = x2apic_send_IPI, 132 .send_IPI = x2apic_send_IPI,
133 .send_IPI_mask = x2apic_send_IPI_mask, 133 .send_IPI_mask = x2apic_send_IPI_mask,
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index b487b3a01615..0d57bb9079c9 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -526,27 +526,15 @@ static void uv_init_apic_ldr(void)
526} 526}
527 527
528static int 528static int
529uv_cpu_mask_to_apicid_and(const struct cpumask *cpumask, 529uv_cpu_mask_to_apicid(const struct cpumask *mask, struct irq_data *irqdata,
530 const struct cpumask *andmask, 530 unsigned int *apicid)
531 unsigned int *apicid)
532{ 531{
533 int unsigned cpu; 532 int ret = default_cpu_mask_to_apicid(mask, irqdata, apicid);
534 533
535 /* 534 if (!ret)
536 * We're using fixed IRQ delivery, can only return one phys APIC ID. 535 *apicid |= uv_apicid_hibits;
537 * May as well be the first.
538 */
539 for_each_cpu_and(cpu, cpumask, andmask) {
540 if (cpumask_test_cpu(cpu, cpu_online_mask))
541 break;
542 }
543
544 if (likely(cpu < nr_cpu_ids)) {
545 *apicid = per_cpu(x86_cpu_to_apicid, cpu) | uv_apicid_hibits;
546 return 0;
547 }
548 536
549 return -EINVAL; 537 return ret;
550} 538}
551 539
552static unsigned int x2apic_get_apic_id(unsigned long x) 540static unsigned int x2apic_get_apic_id(unsigned long x)
@@ -614,7 +602,7 @@ static struct apic apic_x2apic_uv_x __ro_after_init = {
614 .get_apic_id = x2apic_get_apic_id, 602 .get_apic_id = x2apic_get_apic_id,
615 .set_apic_id = set_apic_id, 603 .set_apic_id = set_apic_id,
616 604
617 .cpu_mask_to_apicid_and = uv_cpu_mask_to_apicid_and, 605 .cpu_mask_to_apicid = uv_cpu_mask_to_apicid,
618 606
619 .send_IPI = uv_send_IPI_one, 607 .send_IPI = uv_send_IPI_one,
620 .send_IPI_mask = uv_send_IPI_mask, 608 .send_IPI_mask = uv_send_IPI_mask,
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index f34fe7444836..4aa03c5a14c9 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -432,84 +432,12 @@ int check_irq_vectors_for_cpu_disable(void)
432/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */ 432/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
433void fixup_irqs(void) 433void fixup_irqs(void)
434{ 434{
435 unsigned int irq, vector; 435 unsigned int irr, vector;
436 static int warned;
437 struct irq_desc *desc; 436 struct irq_desc *desc;
438 struct irq_data *data; 437 struct irq_data *data;
439 struct irq_chip *chip; 438 struct irq_chip *chip;
440 int ret;
441 439
442 for_each_irq_desc(irq, desc) { 440 irq_migrate_all_off_this_cpu();
443 int break_affinity = 0;
444 int set_affinity = 1;
445 const struct cpumask *affinity;
446
447 if (!desc)
448 continue;
449 if (irq == 2)
450 continue;
451
452 /* interrupt's are disabled at this point */
453 raw_spin_lock(&desc->lock);
454
455 data = irq_desc_get_irq_data(desc);
456 affinity = irq_data_get_affinity_mask(data);
457 if (!irq_has_action(irq) || irqd_is_per_cpu(data) ||
458 cpumask_subset(affinity, cpu_online_mask)) {
459 raw_spin_unlock(&desc->lock);
460 continue;
461 }
462
463 /*
464 * Complete the irq move. This cpu is going down and for
465 * non intr-remapping case, we can't wait till this interrupt
466 * arrives at this cpu before completing the irq move.
467 */
468 irq_force_complete_move(desc);
469
470 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
471 break_affinity = 1;
472 affinity = cpu_online_mask;
473 }
474
475 chip = irq_data_get_irq_chip(data);
476 /*
477 * The interrupt descriptor might have been cleaned up
478 * already, but it is not yet removed from the radix tree
479 */
480 if (!chip) {
481 raw_spin_unlock(&desc->lock);
482 continue;
483 }
484
485 if (!irqd_can_move_in_process_context(data) && chip->irq_mask)
486 chip->irq_mask(data);
487
488 if (chip->irq_set_affinity) {
489 ret = chip->irq_set_affinity(data, affinity, true);
490 if (ret == -ENOSPC)
491 pr_crit("IRQ %d set affinity failed because there are no available vectors. The device assigned to this IRQ is unstable.\n", irq);
492 } else {
493 if (!(warned++))
494 set_affinity = 0;
495 }
496
497 /*
498 * We unmask if the irq was not marked masked by the
499 * core code. That respects the lazy irq disable
500 * behaviour.
501 */
502 if (!irqd_can_move_in_process_context(data) &&
503 !irqd_irq_masked(data) && chip->irq_unmask)
504 chip->irq_unmask(data);
505
506 raw_spin_unlock(&desc->lock);
507
508 if (break_affinity && set_affinity)
509 pr_notice("Broke affinity for irq %i\n", irq);
510 else if (!set_affinity)
511 pr_notice("Cannot set affinity for irq %i\n", irq);
512 }
513 441
514 /* 442 /*
515 * We can remove mdelay() and then send spuriuous interrupts to 443 * We can remove mdelay() and then send spuriuous interrupts to
@@ -528,8 +456,6 @@ void fixup_irqs(void)
528 * nothing else will touch it. 456 * nothing else will touch it.
529 */ 457 */
530 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { 458 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
531 unsigned int irr;
532
533 if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector]))) 459 if (IS_ERR_OR_NULL(__this_cpu_read(vector_irq[vector])))
534 continue; 460 continue;
535 461
diff --git a/arch/x86/platform/uv/uv_irq.c b/arch/x86/platform/uv/uv_irq.c
index 776c6592136c..03fc397335b7 100644
--- a/arch/x86/platform/uv/uv_irq.c
+++ b/arch/x86/platform/uv/uv_irq.c
@@ -160,13 +160,21 @@ static struct irq_domain *uv_get_irq_domain(void)
160{ 160{
161 static struct irq_domain *uv_domain; 161 static struct irq_domain *uv_domain;
162 static DEFINE_MUTEX(uv_lock); 162 static DEFINE_MUTEX(uv_lock);
163 struct fwnode_handle *fn;
163 164
164 mutex_lock(&uv_lock); 165 mutex_lock(&uv_lock);
165 if (uv_domain == NULL) { 166 if (uv_domain)
166 uv_domain = irq_domain_add_tree(NULL, &uv_domain_ops, NULL); 167 goto out;
167 if (uv_domain) 168
168 uv_domain->parent = x86_vector_domain; 169 fn = irq_domain_alloc_named_fwnode("UV-CORE");
169 } 170 if (!fn)
171 goto out;
172
173 uv_domain = irq_domain_create_tree(fn, &uv_domain_ops, NULL);
174 irq_domain_free_fwnode(fn);
175 if (uv_domain)
176 uv_domain->parent = x86_vector_domain;
177out:
170 mutex_unlock(&uv_lock); 178 mutex_unlock(&uv_lock);
171 179
172 return uv_domain; 180 return uv_domain;
diff --git a/arch/x86/xen/apic.c b/arch/x86/xen/apic.c
index bcea81f36fc5..b5e48da7fbff 100644
--- a/arch/x86/xen/apic.c
+++ b/arch/x86/xen/apic.c
@@ -178,7 +178,7 @@ static struct apic xen_pv_apic = {
178 .get_apic_id = xen_get_apic_id, 178 .get_apic_id = xen_get_apic_id,
179 .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */ 179 .set_apic_id = xen_set_apic_id, /* Can be NULL on 32-bit. */
180 180
181 .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, 181 .cpu_mask_to_apicid = flat_cpu_mask_to_apicid,
182 182
183#ifdef CONFIG_SMP 183#ifdef CONFIG_SMP
184 .send_IPI_mask = xen_send_IPI_mask, 184 .send_IPI_mask = xen_send_IPI_mask,
diff --git a/block/blk-mq-cpumap.c b/block/blk-mq-cpumap.c
index 2cca4fc43f45..4891f042a22f 100644
--- a/block/blk-mq-cpumap.c
+++ b/block/blk-mq-cpumap.c
@@ -14,13 +14,12 @@
14#include "blk.h" 14#include "blk.h"
15#include "blk-mq.h" 15#include "blk-mq.h"
16 16
17static int cpu_to_queue_index(unsigned int nr_queues, const int cpu, 17static int cpu_to_queue_index(unsigned int nr_queues, const int cpu)
18 const struct cpumask *online_mask)
19{ 18{
20 /* 19 /*
21 * Non online CPU will be mapped to queue index 0. 20 * Non online CPU will be mapped to queue index 0.
22 */ 21 */
23 if (!cpumask_test_cpu(cpu, online_mask)) 22 if (!cpu_online(cpu))
24 return 0; 23 return 0;
25 return cpu % nr_queues; 24 return cpu % nr_queues;
26} 25}
@@ -40,7 +39,6 @@ int blk_mq_map_queues(struct blk_mq_tag_set *set)
40{ 39{
41 unsigned int *map = set->mq_map; 40 unsigned int *map = set->mq_map;
42 unsigned int nr_queues = set->nr_hw_queues; 41 unsigned int nr_queues = set->nr_hw_queues;
43 const struct cpumask *online_mask = cpu_online_mask;
44 unsigned int cpu, first_sibling; 42 unsigned int cpu, first_sibling;
45 43
46 for_each_possible_cpu(cpu) { 44 for_each_possible_cpu(cpu) {
@@ -51,11 +49,11 @@ int blk_mq_map_queues(struct blk_mq_tag_set *set)
51 * performace optimizations. 49 * performace optimizations.
52 */ 50 */
53 if (cpu < nr_queues) { 51 if (cpu < nr_queues) {
54 map[cpu] = cpu_to_queue_index(nr_queues, cpu, online_mask); 52 map[cpu] = cpu_to_queue_index(nr_queues, cpu);
55 } else { 53 } else {
56 first_sibling = get_first_sibling(cpu); 54 first_sibling = get_first_sibling(cpu);
57 if (first_sibling == cpu) 55 if (first_sibling == cpu)
58 map[cpu] = cpu_to_queue_index(nr_queues, cpu, online_mask); 56 map[cpu] = cpu_to_queue_index(nr_queues, cpu);
59 else 57 else
60 map[cpu] = map[first_sibling]; 58 map[cpu] = map[first_sibling];
61 } 59 }
diff --git a/block/blk-mq.c b/block/blk-mq.c
index ced2b000ca02..6cef42f419a5 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -37,9 +37,6 @@
37#include "blk-wbt.h" 37#include "blk-wbt.h"
38#include "blk-mq-sched.h" 38#include "blk-mq-sched.h"
39 39
40static DEFINE_MUTEX(all_q_mutex);
41static LIST_HEAD(all_q_list);
42
43static void blk_mq_poll_stats_start(struct request_queue *q); 40static void blk_mq_poll_stats_start(struct request_queue *q);
44static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb); 41static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);
45 42
@@ -1984,8 +1981,8 @@ static void blk_mq_init_cpu_queues(struct request_queue *q,
1984 INIT_LIST_HEAD(&__ctx->rq_list); 1981 INIT_LIST_HEAD(&__ctx->rq_list);
1985 __ctx->queue = q; 1982 __ctx->queue = q;
1986 1983
1987 /* If the cpu isn't online, the cpu is mapped to first hctx */ 1984 /* If the cpu isn't present, the cpu is mapped to first hctx */
1988 if (!cpu_online(i)) 1985 if (!cpu_present(i))
1989 continue; 1986 continue;
1990 1987
1991 hctx = blk_mq_map_queue(q, i); 1988 hctx = blk_mq_map_queue(q, i);
@@ -2028,8 +2025,7 @@ static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
2028 } 2025 }
2029} 2026}
2030 2027
2031static void blk_mq_map_swqueue(struct request_queue *q, 2028static void blk_mq_map_swqueue(struct request_queue *q)
2032 const struct cpumask *online_mask)
2033{ 2029{
2034 unsigned int i, hctx_idx; 2030 unsigned int i, hctx_idx;
2035 struct blk_mq_hw_ctx *hctx; 2031 struct blk_mq_hw_ctx *hctx;
@@ -2047,13 +2043,11 @@ static void blk_mq_map_swqueue(struct request_queue *q,
2047 } 2043 }
2048 2044
2049 /* 2045 /*
2050 * Map software to hardware queues 2046 * Map software to hardware queues.
2047 *
2048 * If the cpu isn't present, the cpu is mapped to first hctx.
2051 */ 2049 */
2052 for_each_possible_cpu(i) { 2050 for_each_present_cpu(i) {
2053 /* If the cpu isn't online, the cpu is mapped to first hctx */
2054 if (!cpumask_test_cpu(i, online_mask))
2055 continue;
2056
2057 hctx_idx = q->mq_map[i]; 2051 hctx_idx = q->mq_map[i];
2058 /* unmapped hw queue can be remapped after CPU topo changed */ 2052 /* unmapped hw queue can be remapped after CPU topo changed */
2059 if (!set->tags[hctx_idx] && 2053 if (!set->tags[hctx_idx] &&
@@ -2363,16 +2357,8 @@ struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
2363 blk_queue_softirq_done(q, set->ops->complete); 2357 blk_queue_softirq_done(q, set->ops->complete);
2364 2358
2365 blk_mq_init_cpu_queues(q, set->nr_hw_queues); 2359 blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2366
2367 get_online_cpus();
2368 mutex_lock(&all_q_mutex);
2369
2370 list_add_tail(&q->all_q_node, &all_q_list);
2371 blk_mq_add_queue_tag_set(set, q); 2360 blk_mq_add_queue_tag_set(set, q);
2372 blk_mq_map_swqueue(q, cpu_online_mask); 2361 blk_mq_map_swqueue(q);
2373
2374 mutex_unlock(&all_q_mutex);
2375 put_online_cpus();
2376 2362
2377 if (!(set->flags & BLK_MQ_F_NO_SCHED)) { 2363 if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
2378 int ret; 2364 int ret;
@@ -2398,18 +2384,12 @@ void blk_mq_free_queue(struct request_queue *q)
2398{ 2384{
2399 struct blk_mq_tag_set *set = q->tag_set; 2385 struct blk_mq_tag_set *set = q->tag_set;
2400 2386
2401 mutex_lock(&all_q_mutex);
2402 list_del_init(&q->all_q_node);
2403 mutex_unlock(&all_q_mutex);
2404
2405 blk_mq_del_queue_tag_set(q); 2387 blk_mq_del_queue_tag_set(q);
2406
2407 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues); 2388 blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
2408} 2389}
2409 2390
2410/* Basically redo blk_mq_init_queue with queue frozen */ 2391/* Basically redo blk_mq_init_queue with queue frozen */
2411static void blk_mq_queue_reinit(struct request_queue *q, 2392static void blk_mq_queue_reinit(struct request_queue *q)
2412 const struct cpumask *online_mask)
2413{ 2393{
2414 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth)); 2394 WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2415 2395
@@ -2422,76 +2402,12 @@ static void blk_mq_queue_reinit(struct request_queue *q,
2422 * involves free and re-allocate memory, worthy doing?) 2402 * involves free and re-allocate memory, worthy doing?)
2423 */ 2403 */
2424 2404
2425 blk_mq_map_swqueue(q, online_mask); 2405 blk_mq_map_swqueue(q);
2426 2406
2427 blk_mq_sysfs_register(q); 2407 blk_mq_sysfs_register(q);
2428 blk_mq_debugfs_register_hctxs(q); 2408 blk_mq_debugfs_register_hctxs(q);
2429} 2409}
2430 2410
2431/*
2432 * New online cpumask which is going to be set in this hotplug event.
2433 * Declare this cpumasks as global as cpu-hotplug operation is invoked
2434 * one-by-one and dynamically allocating this could result in a failure.
2435 */
2436static struct cpumask cpuhp_online_new;
2437
2438static void blk_mq_queue_reinit_work(void)
2439{
2440 struct request_queue *q;
2441
2442 mutex_lock(&all_q_mutex);
2443 /*
2444 * We need to freeze and reinit all existing queues. Freezing
2445 * involves synchronous wait for an RCU grace period and doing it
2446 * one by one may take a long time. Start freezing all queues in
2447 * one swoop and then wait for the completions so that freezing can
2448 * take place in parallel.
2449 */
2450 list_for_each_entry(q, &all_q_list, all_q_node)
2451 blk_freeze_queue_start(q);
2452 list_for_each_entry(q, &all_q_list, all_q_node)
2453 blk_mq_freeze_queue_wait(q);
2454
2455 list_for_each_entry(q, &all_q_list, all_q_node)
2456 blk_mq_queue_reinit(q, &cpuhp_online_new);
2457
2458 list_for_each_entry(q, &all_q_list, all_q_node)
2459 blk_mq_unfreeze_queue(q);
2460
2461 mutex_unlock(&all_q_mutex);
2462}
2463
2464static int blk_mq_queue_reinit_dead(unsigned int cpu)
2465{
2466 cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2467 blk_mq_queue_reinit_work();
2468 return 0;
2469}
2470
2471/*
2472 * Before hotadded cpu starts handling requests, new mappings must be
2473 * established. Otherwise, these requests in hw queue might never be
2474 * dispatched.
2475 *
2476 * For example, there is a single hw queue (hctx) and two CPU queues (ctx0
2477 * for CPU0, and ctx1 for CPU1).
2478 *
2479 * Now CPU1 is just onlined and a request is inserted into ctx1->rq_list
2480 * and set bit0 in pending bitmap as ctx1->index_hw is still zero.
2481 *
2482 * And then while running hw queue, blk_mq_flush_busy_ctxs() finds bit0 is set
2483 * in pending bitmap and tries to retrieve requests in hctx->ctxs[0]->rq_list.
2484 * But htx->ctxs[0] is a pointer to ctx0, so the request in ctx1->rq_list is
2485 * ignored.
2486 */
2487static int blk_mq_queue_reinit_prepare(unsigned int cpu)
2488{
2489 cpumask_copy(&cpuhp_online_new, cpu_online_mask);
2490 cpumask_set_cpu(cpu, &cpuhp_online_new);
2491 blk_mq_queue_reinit_work();
2492 return 0;
2493}
2494
2495static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set) 2411static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
2496{ 2412{
2497 int i; 2413 int i;
@@ -2702,7 +2618,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
2702 blk_mq_update_queue_map(set); 2618 blk_mq_update_queue_map(set);
2703 list_for_each_entry(q, &set->tag_list, tag_set_list) { 2619 list_for_each_entry(q, &set->tag_list, tag_set_list) {
2704 blk_mq_realloc_hw_ctxs(set, q); 2620 blk_mq_realloc_hw_ctxs(set, q);
2705 blk_mq_queue_reinit(q, cpu_online_mask); 2621 blk_mq_queue_reinit(q);
2706 } 2622 }
2707 2623
2708 list_for_each_entry(q, &set->tag_list, tag_set_list) 2624 list_for_each_entry(q, &set->tag_list, tag_set_list)
@@ -2918,24 +2834,10 @@ bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
2918} 2834}
2919EXPORT_SYMBOL_GPL(blk_mq_poll); 2835EXPORT_SYMBOL_GPL(blk_mq_poll);
2920 2836
2921void blk_mq_disable_hotplug(void)
2922{
2923 mutex_lock(&all_q_mutex);
2924}
2925
2926void blk_mq_enable_hotplug(void)
2927{
2928 mutex_unlock(&all_q_mutex);
2929}
2930
2931static int __init blk_mq_init(void) 2837static int __init blk_mq_init(void)
2932{ 2838{
2933 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, 2839 cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
2934 blk_mq_hctx_notify_dead); 2840 blk_mq_hctx_notify_dead);
2935
2936 cpuhp_setup_state_nocalls(CPUHP_BLK_MQ_PREPARE, "block/mq:prepare",
2937 blk_mq_queue_reinit_prepare,
2938 blk_mq_queue_reinit_dead);
2939 return 0; 2841 return 0;
2940} 2842}
2941subsys_initcall(blk_mq_init); 2843subsys_initcall(blk_mq_init);
diff --git a/block/blk-mq.h b/block/blk-mq.h
index 1a06fdf9fd4d..60b01c0309bc 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -56,11 +56,6 @@ void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
56 bool at_head); 56 bool at_head);
57void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx, 57void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
58 struct list_head *list); 58 struct list_head *list);
59/*
60 * CPU hotplug helpers
61 */
62void blk_mq_enable_hotplug(void);
63void blk_mq_disable_hotplug(void);
64 59
65/* 60/*
66 * CPU -> queue mappings 61 * CPU -> queue mappings
diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c
index d35e9a20caf7..e5473525e7b2 100644
--- a/drivers/base/platform-msi.c
+++ b/drivers/base/platform-msi.c
@@ -195,7 +195,7 @@ struct irq_domain *platform_msi_create_irq_domain(struct fwnode_handle *fwnode,
195 195
196 domain = msi_create_irq_domain(fwnode, info, parent); 196 domain = msi_create_irq_domain(fwnode, info, parent);
197 if (domain) 197 if (domain)
198 domain->bus_token = DOMAIN_BUS_PLATFORM_MSI; 198 irq_domain_update_bus_token(domain, DOMAIN_BUS_PLATFORM_MSI);
199 199
200 return domain; 200 return domain;
201} 201}
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 0f1219fa8561..5c9759ed22ca 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -4384,21 +4384,29 @@ static void ir_compose_msi_msg(struct irq_data *irq_data, struct msi_msg *msg)
4384} 4384}
4385 4385
4386static struct irq_chip amd_ir_chip = { 4386static struct irq_chip amd_ir_chip = {
4387 .irq_ack = ir_ack_apic_edge, 4387 .name = "AMD-IR",
4388 .irq_set_affinity = amd_ir_set_affinity, 4388 .irq_ack = ir_ack_apic_edge,
4389 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity, 4389 .irq_set_affinity = amd_ir_set_affinity,
4390 .irq_compose_msi_msg = ir_compose_msi_msg, 4390 .irq_set_vcpu_affinity = amd_ir_set_vcpu_affinity,
4391 .irq_compose_msi_msg = ir_compose_msi_msg,
4391}; 4392};
4392 4393
4393int amd_iommu_create_irq_domain(struct amd_iommu *iommu) 4394int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
4394{ 4395{
4395 iommu->ir_domain = irq_domain_add_tree(NULL, &amd_ir_domain_ops, iommu); 4396 struct fwnode_handle *fn;
4397
4398 fn = irq_domain_alloc_named_id_fwnode("AMD-IR", iommu->index);
4399 if (!fn)
4400 return -ENOMEM;
4401 iommu->ir_domain = irq_domain_create_tree(fn, &amd_ir_domain_ops, iommu);
4402 irq_domain_free_fwnode(fn);
4396 if (!iommu->ir_domain) 4403 if (!iommu->ir_domain)
4397 return -ENOMEM; 4404 return -ENOMEM;
4398 4405
4399 iommu->ir_domain->parent = arch_get_ir_parent_domain(); 4406 iommu->ir_domain->parent = arch_get_ir_parent_domain();
4400 iommu->msi_domain = arch_create_msi_irq_domain(iommu->ir_domain); 4407 iommu->msi_domain = arch_create_remap_msi_irq_domain(iommu->ir_domain,
4401 4408 "AMD-IR-MSI",
4409 iommu->index);
4402 return 0; 4410 return 0;
4403} 4411}
4404 4412
diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c
index a190cbd76ef7..8fc641ea2e41 100644
--- a/drivers/iommu/intel_irq_remapping.c
+++ b/drivers/iommu/intel_irq_remapping.c
@@ -500,8 +500,9 @@ static void iommu_enable_irq_remapping(struct intel_iommu *iommu)
500static int intel_setup_irq_remapping(struct intel_iommu *iommu) 500static int intel_setup_irq_remapping(struct intel_iommu *iommu)
501{ 501{
502 struct ir_table *ir_table; 502 struct ir_table *ir_table;
503 struct page *pages; 503 struct fwnode_handle *fn;
504 unsigned long *bitmap; 504 unsigned long *bitmap;
505 struct page *pages;
505 506
506 if (iommu->ir_table) 507 if (iommu->ir_table)
507 return 0; 508 return 0;
@@ -525,15 +526,24 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu)
525 goto out_free_pages; 526 goto out_free_pages;
526 } 527 }
527 528
528 iommu->ir_domain = irq_domain_add_hierarchy(arch_get_ir_parent_domain(), 529 fn = irq_domain_alloc_named_id_fwnode("INTEL-IR", iommu->seq_id);
529 0, INTR_REMAP_TABLE_ENTRIES, 530 if (!fn)
530 NULL, &intel_ir_domain_ops, 531 goto out_free_bitmap;
531 iommu); 532
533 iommu->ir_domain =
534 irq_domain_create_hierarchy(arch_get_ir_parent_domain(),
535 0, INTR_REMAP_TABLE_ENTRIES,
536 fn, &intel_ir_domain_ops,
537 iommu);
538 irq_domain_free_fwnode(fn);
532 if (!iommu->ir_domain) { 539 if (!iommu->ir_domain) {
533 pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id); 540 pr_err("IR%d: failed to allocate irqdomain\n", iommu->seq_id);
534 goto out_free_bitmap; 541 goto out_free_bitmap;
535 } 542 }
536 iommu->ir_msi_domain = arch_create_msi_irq_domain(iommu->ir_domain); 543 iommu->ir_msi_domain =
544 arch_create_remap_msi_irq_domain(iommu->ir_domain,
545 "INTEL-IR-MSI",
546 iommu->seq_id);
537 547
538 ir_table->base = page_address(pages); 548 ir_table->base = page_address(pages);
539 ir_table->bitmap = bitmap; 549 ir_table->bitmap = bitmap;
@@ -1205,10 +1215,11 @@ static int intel_ir_set_vcpu_affinity(struct irq_data *data, void *info)
1205} 1215}
1206 1216
1207static struct irq_chip intel_ir_chip = { 1217static struct irq_chip intel_ir_chip = {
1208 .irq_ack = ir_ack_apic_edge, 1218 .name = "INTEL-IR",
1209 .irq_set_affinity = intel_ir_set_affinity, 1219 .irq_ack = ir_ack_apic_edge,
1210 .irq_compose_msi_msg = intel_ir_compose_msi_msg, 1220 .irq_set_affinity = intel_ir_set_affinity,
1211 .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity, 1221 .irq_compose_msi_msg = intel_ir_compose_msi_msg,
1222 .irq_set_vcpu_affinity = intel_ir_set_vcpu_affinity,
1212}; 1223};
1213 1224
1214static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data, 1225static void intel_irq_remapping_prepare_irte(struct intel_ir_data *data,
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index 478f8ace2664..676232a94f9f 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -268,6 +268,12 @@ config IRQ_MXS
268 select IRQ_DOMAIN 268 select IRQ_DOMAIN
269 select STMP_DEVICE 269 select STMP_DEVICE
270 270
271config MVEBU_GICP
272 bool
273
274config MVEBU_ICU
275 bool
276
271config MVEBU_ODMI 277config MVEBU_ODMI
272 bool 278 bool
273 select GENERIC_MSI_IRQ_DOMAIN 279 select GENERIC_MSI_IRQ_DOMAIN
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index b64c59b838a0..e88d856cc09c 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -69,10 +69,12 @@ obj-$(CONFIG_ARCH_SA1100) += irq-sa11x0.o
69obj-$(CONFIG_INGENIC_IRQ) += irq-ingenic.o 69obj-$(CONFIG_INGENIC_IRQ) += irq-ingenic.o
70obj-$(CONFIG_IMX_GPCV2) += irq-imx-gpcv2.o 70obj-$(CONFIG_IMX_GPCV2) += irq-imx-gpcv2.o
71obj-$(CONFIG_PIC32_EVIC) += irq-pic32-evic.o 71obj-$(CONFIG_PIC32_EVIC) += irq-pic32-evic.o
72obj-$(CONFIG_MVEBU_GICP) += irq-mvebu-gicp.o
73obj-$(CONFIG_MVEBU_ICU) += irq-mvebu-icu.o
72obj-$(CONFIG_MVEBU_ODMI) += irq-mvebu-odmi.o 74obj-$(CONFIG_MVEBU_ODMI) += irq-mvebu-odmi.o
73obj-$(CONFIG_MVEBU_PIC) += irq-mvebu-pic.o 75obj-$(CONFIG_MVEBU_PIC) += irq-mvebu-pic.o
74obj-$(CONFIG_LS_SCFG_MSI) += irq-ls-scfg-msi.o 76obj-$(CONFIG_LS_SCFG_MSI) += irq-ls-scfg-msi.o
75obj-$(CONFIG_EZNPS_GIC) += irq-eznps.o 77obj-$(CONFIG_EZNPS_GIC) += irq-eznps.o
76obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o 78obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o irq-aspeed-i2c-ic.o
77obj-$(CONFIG_STM32_EXTI) += irq-stm32-exti.o 79obj-$(CONFIG_STM32_EXTI) += irq-stm32-exti.o
78obj-$(CONFIG_QCOM_IRQ_COMBINER) += qcom-irq-combiner.o 80obj-$(CONFIG_QCOM_IRQ_COMBINER) += qcom-irq-combiner.o
diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c
index eb0d4d41b156..b207b2c3aa55 100644
--- a/drivers/irqchip/irq-armada-370-xp.c
+++ b/drivers/irqchip/irq-armada-370-xp.c
@@ -34,25 +34,104 @@
34#include <asm/smp_plat.h> 34#include <asm/smp_plat.h>
35#include <asm/mach/irq.h> 35#include <asm/mach/irq.h>
36 36
37/* Interrupt Controller Registers Map */ 37/*
38#define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) 38 * Overall diagram of the Armada XP interrupt controller:
39#define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) 39 *
40#define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54) 40 * To CPU 0 To CPU 1
41#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) 41 *
42 * /\ /\
43 * || ||
44 * +---------------+ +---------------+
45 * | | | |
46 * | per-CPU | | per-CPU |
47 * | mask/unmask | | mask/unmask |
48 * | CPU0 | | CPU1 |
49 * | | | |
50 * +---------------+ +---------------+
51 * /\ /\
52 * || ||
53 * \\_______________________//
54 * ||
55 * +-------------------+
56 * | |
57 * | Global interrupt |
58 * | mask/unmask |
59 * | |
60 * +-------------------+
61 * /\
62 * ||
63 * interrupt from
64 * device
65 *
66 * The "global interrupt mask/unmask" is modified using the
67 * ARMADA_370_XP_INT_SET_ENABLE_OFFS and
68 * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative
69 * to "main_int_base".
70 *
71 * The "per-CPU mask/unmask" is modified using the
72 * ARMADA_370_XP_INT_SET_MASK_OFFS and
73 * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to
74 * "per_cpu_int_base". This base address points to a special address,
75 * which automatically accesses the registers of the current CPU.
76 *
77 * The per-CPU mask/unmask can also be adjusted using the global
78 * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use
79 * to configure interrupt affinity.
80 *
81 * Due to this model, all interrupts need to be mask/unmasked at two
82 * different levels: at the global level and at the per-CPU level.
83 *
84 * This driver takes the following approach to deal with this:
85 *
86 * - For global interrupts:
87 *
88 * At ->map() time, a global interrupt is unmasked at the per-CPU
89 * mask/unmask level. It is therefore unmasked at this level for
90 * the current CPU, running the ->map() code. This allows to have
91 * the interrupt unmasked at this level in non-SMP
92 * configurations. In SMP configurations, the ->set_affinity()
93 * callback is called, which using the
94 * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask
95 * for the interrupt.
96 *
97 * The ->mask() and ->unmask() operations only mask/unmask the
98 * interrupt at the "global" level.
99 *
100 * So, a global interrupt is enabled at the per-CPU level as soon
101 * as it is mapped. At run time, the masking/unmasking takes place
102 * at the global level.
103 *
104 * - For per-CPU interrupts
105 *
106 * At ->map() time, a per-CPU interrupt is unmasked at the global
107 * mask/unmask level.
108 *
109 * The ->mask() and ->unmask() operations mask/unmask the interrupt
110 * at the per-CPU level.
111 *
112 * So, a per-CPU interrupt is enabled at the global level as soon
113 * as it is mapped. At run time, the masking/unmasking takes place
114 * at the per-CPU level.
115 */
42 116
117/* Registers relative to main_int_base */
43#define ARMADA_370_XP_INT_CONTROL (0x00) 118#define ARMADA_370_XP_INT_CONTROL (0x00)
119#define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04)
44#define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) 120#define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30)
45#define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) 121#define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34)
46#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) 122#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4)
47#define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF 123#define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF
48#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) 124#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid)
49 125
50#define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) 126/* Registers relative to per_cpu_int_base */
127#define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08)
128#define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c)
51#define ARMADA_375_PPI_CAUSE (0x10) 129#define ARMADA_375_PPI_CAUSE (0x10)
52 130#define ARMADA_370_XP_CPU_INTACK_OFFS (0x44)
53#define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x4) 131#define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48)
54#define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0xc) 132#define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C)
55#define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x8) 133#define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54)
134#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu)
56 135
57#define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) 136#define ARMADA_370_XP_MAX_PER_CPU_IRQS (28)
58 137
@@ -281,13 +360,11 @@ static int armada_370_xp_mpic_irq_map(struct irq_domain *h,
281 irq_set_percpu_devid(virq); 360 irq_set_percpu_devid(virq);
282 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, 361 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
283 handle_percpu_devid_irq); 362 handle_percpu_devid_irq);
284
285 } else { 363 } else {
286 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, 364 irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip,
287 handle_level_irq); 365 handle_level_irq);
288 } 366 }
289 irq_set_probe(virq); 367 irq_set_probe(virq);
290 irq_clear_status_flags(virq, IRQ_NOAUTOEN);
291 368
292 return 0; 369 return 0;
293} 370}
@@ -345,16 +422,40 @@ static void armada_mpic_send_doorbell(const struct cpumask *mask,
345 ARMADA_370_XP_SW_TRIG_INT_OFFS); 422 ARMADA_370_XP_SW_TRIG_INT_OFFS);
346} 423}
347 424
425static void armada_xp_mpic_reenable_percpu(void)
426{
427 unsigned int irq;
428
429 /* Re-enable per-CPU interrupts that were enabled before suspend */
430 for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) {
431 struct irq_data *data;
432 int virq;
433
434 virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq);
435 if (virq == 0)
436 continue;
437
438 data = irq_get_irq_data(virq);
439
440 if (!irq_percpu_is_enabled(virq))
441 continue;
442
443 armada_370_xp_irq_unmask(data);
444 }
445}
446
348static int armada_xp_mpic_starting_cpu(unsigned int cpu) 447static int armada_xp_mpic_starting_cpu(unsigned int cpu)
349{ 448{
350 armada_xp_mpic_perf_init(); 449 armada_xp_mpic_perf_init();
351 armada_xp_mpic_smp_cpu_init(); 450 armada_xp_mpic_smp_cpu_init();
451 armada_xp_mpic_reenable_percpu();
352 return 0; 452 return 0;
353} 453}
354 454
355static int mpic_cascaded_starting_cpu(unsigned int cpu) 455static int mpic_cascaded_starting_cpu(unsigned int cpu)
356{ 456{
357 armada_xp_mpic_perf_init(); 457 armada_xp_mpic_perf_init();
458 armada_xp_mpic_reenable_percpu();
358 enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); 459 enable_percpu_irq(parent_irq, IRQ_TYPE_NONE);
359 return 0; 460 return 0;
360} 461}
@@ -502,16 +603,27 @@ static void armada_370_xp_mpic_resume(void)
502 if (virq == 0) 603 if (virq == 0)
503 continue; 604 continue;
504 605
505 if (!is_percpu_irq(irq)) 606 data = irq_get_irq_data(virq);
607
608 if (!is_percpu_irq(irq)) {
609 /* Non per-CPU interrupts */
506 writel(irq, per_cpu_int_base + 610 writel(irq, per_cpu_int_base +
507 ARMADA_370_XP_INT_CLEAR_MASK_OFFS); 611 ARMADA_370_XP_INT_CLEAR_MASK_OFFS);
508 else 612 if (!irqd_irq_disabled(data))
613 armada_370_xp_irq_unmask(data);
614 } else {
615 /* Per-CPU interrupts */
509 writel(irq, main_int_base + 616 writel(irq, main_int_base +
510 ARMADA_370_XP_INT_SET_ENABLE_OFFS); 617 ARMADA_370_XP_INT_SET_ENABLE_OFFS);
511 618
512 data = irq_get_irq_data(virq); 619 /*
513 if (!irqd_irq_disabled(data)) 620 * Re-enable on the current CPU,
514 armada_370_xp_irq_unmask(data); 621 * armada_xp_mpic_reenable_percpu() will take
622 * care of secondary CPUs when they come up.
623 */
624 if (irq_percpu_is_enabled(virq))
625 armada_370_xp_irq_unmask(data);
626 }
515 } 627 }
516 628
517 /* Reconfigure doorbells for IPIs and MSIs */ 629 /* Reconfigure doorbells for IPIs and MSIs */
@@ -563,7 +675,7 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node,
563 irq_domain_add_linear(node, nr_irqs, 675 irq_domain_add_linear(node, nr_irqs,
564 &armada_370_xp_mpic_irq_ops, NULL); 676 &armada_370_xp_mpic_irq_ops, NULL);
565 BUG_ON(!armada_370_xp_mpic_domain); 677 BUG_ON(!armada_370_xp_mpic_domain);
566 armada_370_xp_mpic_domain->bus_token = DOMAIN_BUS_WIRED; 678 irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED);
567 679
568 /* Setup for the boot CPU */ 680 /* Setup for the boot CPU */
569 armada_xp_mpic_perf_init(); 681 armada_xp_mpic_perf_init();
diff --git a/drivers/irqchip/irq-aspeed-i2c-ic.c b/drivers/irqchip/irq-aspeed-i2c-ic.c
new file mode 100644
index 000000000000..815b88dd18f2
--- /dev/null
+++ b/drivers/irqchip/irq-aspeed-i2c-ic.c
@@ -0,0 +1,115 @@
1/*
2 * Aspeed 24XX/25XX I2C Interrupt Controller.
3 *
4 * Copyright (C) 2012-2017 ASPEED Technology Inc.
5 * Copyright 2017 IBM Corporation
6 * Copyright 2017 Google, Inc.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13#include <linux/irq.h>
14#include <linux/irqchip.h>
15#include <linux/irqchip/chained_irq.h>
16#include <linux/irqdomain.h>
17#include <linux/of_address.h>
18#include <linux/of_irq.h>
19#include <linux/io.h>
20
21
22#define ASPEED_I2C_IC_NUM_BUS 14
23
24struct aspeed_i2c_ic {
25 void __iomem *base;
26 int parent_irq;
27 struct irq_domain *irq_domain;
28};
29
30/*
31 * The aspeed chip provides a single hardware interrupt for all of the I2C
32 * busses, so we use a dummy interrupt chip to translate this single interrupt
33 * into multiple interrupts, each associated with a single I2C bus.
34 */
35static void aspeed_i2c_ic_irq_handler(struct irq_desc *desc)
36{
37 struct aspeed_i2c_ic *i2c_ic = irq_desc_get_handler_data(desc);
38 struct irq_chip *chip = irq_desc_get_chip(desc);
39 unsigned long bit, status;
40 unsigned int bus_irq;
41
42 chained_irq_enter(chip, desc);
43 status = readl(i2c_ic->base);
44 for_each_set_bit(bit, &status, ASPEED_I2C_IC_NUM_BUS) {
45 bus_irq = irq_find_mapping(i2c_ic->irq_domain, bit);
46 generic_handle_irq(bus_irq);
47 }
48 chained_irq_exit(chip, desc);
49}
50
51/*
52 * Set simple handler and mark IRQ as valid. Nothing interesting to do here
53 * since we are using a dummy interrupt chip.
54 */
55static int aspeed_i2c_ic_map_irq_domain(struct irq_domain *domain,
56 unsigned int irq, irq_hw_number_t hwirq)
57{
58 irq_set_chip_and_handler(irq, &dummy_irq_chip, handle_simple_irq);
59 irq_set_chip_data(irq, domain->host_data);
60
61 return 0;
62}
63
64static const struct irq_domain_ops aspeed_i2c_ic_irq_domain_ops = {
65 .map = aspeed_i2c_ic_map_irq_domain,
66};
67
68static int __init aspeed_i2c_ic_of_init(struct device_node *node,
69 struct device_node *parent)
70{
71 struct aspeed_i2c_ic *i2c_ic;
72 int ret = 0;
73
74 i2c_ic = kzalloc(sizeof(*i2c_ic), GFP_KERNEL);
75 if (!i2c_ic)
76 return -ENOMEM;
77
78 i2c_ic->base = of_iomap(node, 0);
79 if (IS_ERR(i2c_ic->base)) {
80 ret = PTR_ERR(i2c_ic->base);
81 goto err_free_ic;
82 }
83
84 i2c_ic->parent_irq = irq_of_parse_and_map(node, 0);
85 if (i2c_ic->parent_irq < 0) {
86 ret = i2c_ic->parent_irq;
87 goto err_iounmap;
88 }
89
90 i2c_ic->irq_domain = irq_domain_add_linear(node, ASPEED_I2C_IC_NUM_BUS,
91 &aspeed_i2c_ic_irq_domain_ops,
92 NULL);
93 if (!i2c_ic->irq_domain) {
94 ret = -ENOMEM;
95 goto err_iounmap;
96 }
97
98 i2c_ic->irq_domain->name = "aspeed-i2c-domain";
99
100 irq_set_chained_handler_and_data(i2c_ic->parent_irq,
101 aspeed_i2c_ic_irq_handler, i2c_ic);
102
103 pr_info("i2c controller registered, irq %d\n", i2c_ic->parent_irq);
104
105 return 0;
106
107err_iounmap:
108 iounmap(i2c_ic->base);
109err_free_ic:
110 kfree(i2c_ic);
111 return ret;
112}
113
114IRQCHIP_DECLARE(ast2400_i2c_ic, "aspeed,ast2400-i2c-ic", aspeed_i2c_ic_of_init);
115IRQCHIP_DECLARE(ast2500_i2c_ic, "aspeed,ast2500-i2c-ic", aspeed_i2c_ic_of_init);
diff --git a/drivers/irqchip/irq-aspeed-vic.c b/drivers/irqchip/irq-aspeed-vic.c
index d24451d5bf8a..03ba477ea0d0 100644
--- a/drivers/irqchip/irq-aspeed-vic.c
+++ b/drivers/irqchip/irq-aspeed-vic.c
@@ -186,7 +186,7 @@ static int avic_map(struct irq_domain *d, unsigned int irq,
186 return 0; 186 return 0;
187} 187}
188 188
189static struct irq_domain_ops avic_dom_ops = { 189static const struct irq_domain_ops avic_dom_ops = {
190 .map = avic_map, 190 .map = avic_map,
191 .xlate = irq_domain_xlate_onetwocell, 191 .xlate = irq_domain_xlate_onetwocell,
192}; 192};
@@ -227,4 +227,5 @@ static int __init avic_of_init(struct device_node *node,
227 return 0; 227 return 0;
228} 228}
229 229
230IRQCHIP_DECLARE(aspeed_new_vic, "aspeed,ast2400-vic", avic_of_init); 230IRQCHIP_DECLARE(ast2400_vic, "aspeed,ast2400-vic", avic_of_init);
231IRQCHIP_DECLARE(ast2500_vic, "aspeed,ast2500-vic", avic_of_init);
diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c
index 863e073c6f7f..993a8426a453 100644
--- a/drivers/irqchip/irq-gic-v2m.c
+++ b/drivers/irqchip/irq-gic-v2m.c
@@ -280,7 +280,7 @@ static int gicv2m_allocate_domains(struct irq_domain *parent)
280 return -ENOMEM; 280 return -ENOMEM;
281 } 281 }
282 282
283 inner_domain->bus_token = DOMAIN_BUS_NEXUS; 283 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
284 inner_domain->parent = parent; 284 inner_domain->parent = parent;
285 pci_domain = pci_msi_create_irq_domain(v2m->fwnode, 285 pci_domain = pci_msi_create_irq_domain(v2m->fwnode,
286 &gicv2m_msi_domain_info, 286 &gicv2m_msi_domain_info,
diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
index aee1c60d7ab5..77931214d954 100644
--- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-pci-msi.c
@@ -41,27 +41,22 @@ static struct irq_chip its_msi_irq_chip = {
41 .irq_write_msi_msg = pci_msi_domain_write_msg, 41 .irq_write_msi_msg = pci_msi_domain_write_msg,
42}; 42};
43 43
44struct its_pci_alias { 44static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data)
45 struct pci_dev *pdev;
46 u32 count;
47};
48
49static int its_pci_msi_vec_count(struct pci_dev *pdev)
50{ 45{
51 int msi, msix; 46 int msi, msix, *count = data;
52 47
53 msi = max(pci_msi_vec_count(pdev), 0); 48 msi = max(pci_msi_vec_count(pdev), 0);
54 msix = max(pci_msix_vec_count(pdev), 0); 49 msix = max(pci_msix_vec_count(pdev), 0);
50 *count += max(msi, msix);
55 51
56 return max(msi, msix); 52 return 0;
57} 53}
58 54
59static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data) 55static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data)
60{ 56{
61 struct its_pci_alias *dev_alias = data; 57 struct pci_dev **alias_dev = data;
62 58
63 if (pdev != dev_alias->pdev) 59 *alias_dev = pdev;
64 dev_alias->count += its_pci_msi_vec_count(pdev);
65 60
66 return 0; 61 return 0;
67} 62}
@@ -69,9 +64,9 @@ static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data)
69static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev, 64static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev,
70 int nvec, msi_alloc_info_t *info) 65 int nvec, msi_alloc_info_t *info)
71{ 66{
72 struct pci_dev *pdev; 67 struct pci_dev *pdev, *alias_dev;
73 struct its_pci_alias dev_alias;
74 struct msi_domain_info *msi_info; 68 struct msi_domain_info *msi_info;
69 int alias_count = 0;
75 70
76 if (!dev_is_pci(dev)) 71 if (!dev_is_pci(dev))
77 return -EINVAL; 72 return -EINVAL;
@@ -79,16 +74,20 @@ static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev,
79 msi_info = msi_get_domain_info(domain->parent); 74 msi_info = msi_get_domain_info(domain->parent);
80 75
81 pdev = to_pci_dev(dev); 76 pdev = to_pci_dev(dev);
82 dev_alias.pdev = pdev; 77 /*
83 dev_alias.count = nvec; 78 * If pdev is downstream of any aliasing bridges, take an upper
84 79 * bound of how many other vectors could map to the same DevID.
85 pci_for_each_dma_alias(pdev, its_get_pci_alias, &dev_alias); 80 */
81 pci_for_each_dma_alias(pdev, its_get_pci_alias, &alias_dev);
82 if (alias_dev != pdev && alias_dev->subordinate)
83 pci_walk_bus(alias_dev->subordinate, its_pci_msi_vec_count,
84 &alias_count);
86 85
87 /* ITS specific DeviceID, as the core ITS ignores dev. */ 86 /* ITS specific DeviceID, as the core ITS ignores dev. */
88 info->scratchpad[0].ul = pci_msi_domain_get_msi_rid(domain, pdev); 87 info->scratchpad[0].ul = pci_msi_domain_get_msi_rid(domain, pdev);
89 88
90 return msi_info->ops->msi_prepare(domain->parent, 89 return msi_info->ops->msi_prepare(domain->parent,
91 dev, dev_alias.count, info); 90 dev, max(nvec, alias_count), info);
92} 91}
93 92
94static struct msi_domain_ops its_pci_msi_ops = { 93static struct msi_domain_ops its_pci_msi_ops = {
diff --git a/drivers/irqchip/irq-gic-v3-its-platform-msi.c b/drivers/irqchip/irq-gic-v3-its-platform-msi.c
index 9e9dda33eb17..249240d9a425 100644
--- a/drivers/irqchip/irq-gic-v3-its-platform-msi.c
+++ b/drivers/irqchip/irq-gic-v3-its-platform-msi.c
@@ -86,7 +86,7 @@ static struct msi_domain_info its_pmsi_domain_info = {
86 .chip = &its_pmsi_irq_chip, 86 .chip = &its_pmsi_irq_chip,
87}; 87};
88 88
89static struct of_device_id its_device_id[] = { 89static const struct of_device_id its_device_id[] = {
90 { .compatible = "arm,gic-v3-its", }, 90 { .compatible = "arm,gic-v3-its", },
91 {}, 91 {},
92}; 92};
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 45ea193325d2..68932873eebc 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -644,9 +644,12 @@ static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
644 if (cpu >= nr_cpu_ids) 644 if (cpu >= nr_cpu_ids)
645 return -EINVAL; 645 return -EINVAL;
646 646
647 target_col = &its_dev->its->collections[cpu]; 647 /* don't set the affinity when the target cpu is same as current one */
648 its_send_movi(its_dev, target_col, id); 648 if (cpu != its_dev->event_map.col_map[id]) {
649 its_dev->event_map.col_map[id] = cpu; 649 target_col = &its_dev->its->collections[cpu];
650 its_send_movi(its_dev, target_col, id);
651 its_dev->event_map.col_map[id] = cpu;
652 }
650 653
651 return IRQ_SET_MASK_OK_DONE; 654 return IRQ_SET_MASK_OK_DONE;
652} 655}
@@ -688,9 +691,11 @@ static struct irq_chip its_irq_chip = {
688 */ 691 */
689#define IRQS_PER_CHUNK_SHIFT 5 692#define IRQS_PER_CHUNK_SHIFT 5
690#define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT) 693#define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT)
694#define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
691 695
692static unsigned long *lpi_bitmap; 696static unsigned long *lpi_bitmap;
693static u32 lpi_chunks; 697static u32 lpi_chunks;
698static u32 lpi_id_bits;
694static DEFINE_SPINLOCK(lpi_lock); 699static DEFINE_SPINLOCK(lpi_lock);
695 700
696static int its_lpi_to_chunk(int lpi) 701static int its_lpi_to_chunk(int lpi)
@@ -786,17 +791,13 @@ static void its_lpi_free(struct event_lpi_map *map)
786} 791}
787 792
788/* 793/*
789 * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to 794 * We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
790 * deal with (one configuration byte per interrupt). PENDBASE has to 795 * deal with (one configuration byte per interrupt). PENDBASE has to
791 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 796 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
792 */ 797 */
793#define LPI_PROPBASE_SZ SZ_64K 798#define LPI_NRBITS lpi_id_bits
794#define LPI_PENDBASE_SZ (LPI_PROPBASE_SZ / 8 + SZ_1K) 799#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
795 800#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
796/*
797 * This is how many bits of ID we need, including the useless ones.
798 */
799#define LPI_NRBITS ilog2(LPI_PROPBASE_SZ + SZ_8K)
800 801
801#define LPI_PROP_DEFAULT_PRIO 0xa0 802#define LPI_PROP_DEFAULT_PRIO 0xa0
802 803
@@ -804,6 +805,7 @@ static int __init its_alloc_lpi_tables(void)
804{ 805{
805 phys_addr_t paddr; 806 phys_addr_t paddr;
806 807
808 lpi_id_bits = min_t(u32, gic_rdists->id_bits, ITS_MAX_LPI_NRBITS);
807 gic_rdists->prop_page = alloc_pages(GFP_NOWAIT, 809 gic_rdists->prop_page = alloc_pages(GFP_NOWAIT,
808 get_order(LPI_PROPBASE_SZ)); 810 get_order(LPI_PROPBASE_SZ));
809 if (!gic_rdists->prop_page) { 811 if (!gic_rdists->prop_page) {
@@ -822,7 +824,7 @@ static int __init its_alloc_lpi_tables(void)
822 /* Make sure the GIC will observe the written configuration */ 824 /* Make sure the GIC will observe the written configuration */
823 gic_flush_dcache_to_poc(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ); 825 gic_flush_dcache_to_poc(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ);
824 826
825 return 0; 827 return its_lpi_init(lpi_id_bits);
826} 828}
827 829
828static const char *its_base_type_string[] = { 830static const char *its_base_type_string[] = {
@@ -1097,7 +1099,7 @@ static void its_cpu_init_lpis(void)
1097 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. 1099 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
1098 */ 1100 */
1099 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO, 1101 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO,
1100 get_order(max(LPI_PENDBASE_SZ, SZ_64K))); 1102 get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
1101 if (!pend_page) { 1103 if (!pend_page) {
1102 pr_err("Failed to allocate PENDBASE for CPU%d\n", 1104 pr_err("Failed to allocate PENDBASE for CPU%d\n",
1103 smp_processor_id()); 1105 smp_processor_id());
@@ -1661,7 +1663,7 @@ static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
1661 } 1663 }
1662 1664
1663 inner_domain->parent = its_parent; 1665 inner_domain->parent = its_parent;
1664 inner_domain->bus_token = DOMAIN_BUS_NEXUS; 1666 irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
1665 inner_domain->flags |= IRQ_DOMAIN_FLAG_MSI_REMAP; 1667 inner_domain->flags |= IRQ_DOMAIN_FLAG_MSI_REMAP;
1666 info->ops = &its_msi_domain_ops; 1668 info->ops = &its_msi_domain_ops;
1667 info->data = its; 1669 info->data = its;
@@ -1801,7 +1803,7 @@ int its_cpu_init(void)
1801 return 0; 1803 return 0;
1802} 1804}
1803 1805
1804static struct of_device_id its_device_id[] = { 1806static const struct of_device_id its_device_id[] = {
1805 { .compatible = "arm,gic-v3-its", }, 1807 { .compatible = "arm,gic-v3-its", },
1806 {}, 1808 {},
1807}; 1809};
@@ -1833,6 +1835,78 @@ static int __init its_of_probe(struct device_node *node)
1833 1835
1834#define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K) 1836#define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
1835 1837
1838#if defined(CONFIG_ACPI_NUMA) && (ACPI_CA_VERSION >= 0x20170531)
1839struct its_srat_map {
1840 /* numa node id */
1841 u32 numa_node;
1842 /* GIC ITS ID */
1843 u32 its_id;
1844};
1845
1846static struct its_srat_map its_srat_maps[MAX_NUMNODES] __initdata;
1847static int its_in_srat __initdata;
1848
1849static int __init acpi_get_its_numa_node(u32 its_id)
1850{
1851 int i;
1852
1853 for (i = 0; i < its_in_srat; i++) {
1854 if (its_id == its_srat_maps[i].its_id)
1855 return its_srat_maps[i].numa_node;
1856 }
1857 return NUMA_NO_NODE;
1858}
1859
1860static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
1861 const unsigned long end)
1862{
1863 int node;
1864 struct acpi_srat_gic_its_affinity *its_affinity;
1865
1866 its_affinity = (struct acpi_srat_gic_its_affinity *)header;
1867 if (!its_affinity)
1868 return -EINVAL;
1869
1870 if (its_affinity->header.length < sizeof(*its_affinity)) {
1871 pr_err("SRAT: Invalid header length %d in ITS affinity\n",
1872 its_affinity->header.length);
1873 return -EINVAL;
1874 }
1875
1876 if (its_in_srat >= MAX_NUMNODES) {
1877 pr_err("SRAT: ITS affinity exceeding max count[%d]\n",
1878 MAX_NUMNODES);
1879 return -EINVAL;
1880 }
1881
1882 node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
1883
1884 if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
1885 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
1886 return 0;
1887 }
1888
1889 its_srat_maps[its_in_srat].numa_node = node;
1890 its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
1891 its_in_srat++;
1892 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
1893 its_affinity->proximity_domain, its_affinity->its_id, node);
1894
1895 return 0;
1896}
1897
1898static void __init acpi_table_parse_srat_its(void)
1899{
1900 acpi_table_parse_entries(ACPI_SIG_SRAT,
1901 sizeof(struct acpi_table_srat),
1902 ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
1903 gic_acpi_parse_srat_its, 0);
1904}
1905#else
1906static void __init acpi_table_parse_srat_its(void) { }
1907static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
1908#endif
1909
1836static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header, 1910static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
1837 const unsigned long end) 1911 const unsigned long end)
1838{ 1912{
@@ -1861,7 +1935,8 @@ static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
1861 goto dom_err; 1935 goto dom_err;
1862 } 1936 }
1863 1937
1864 err = its_probe_one(&res, dom_handle, NUMA_NO_NODE); 1938 err = its_probe_one(&res, dom_handle,
1939 acpi_get_its_numa_node(its_entry->translation_id));
1865 if (!err) 1940 if (!err)
1866 return 0; 1941 return 0;
1867 1942
@@ -1873,6 +1948,7 @@ dom_err:
1873 1948
1874static void __init its_acpi_probe(void) 1949static void __init its_acpi_probe(void)
1875{ 1950{
1951 acpi_table_parse_srat_its();
1876 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, 1952 acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
1877 gic_acpi_parse_madt_its, 0); 1953 gic_acpi_parse_madt_its, 0);
1878} 1954}
@@ -1898,8 +1974,5 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
1898 } 1974 }
1899 1975
1900 gic_rdists = rdists; 1976 gic_rdists = rdists;
1901 its_alloc_lpi_tables(); 1977 return its_alloc_lpi_tables();
1902 its_lpi_init(rdists->id_bits);
1903
1904 return 0;
1905} 1978}
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index c132f29322cc..dbffb7ab6203 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -645,6 +645,9 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
645 int enabled; 645 int enabled;
646 u64 val; 646 u64 val;
647 647
648 if (cpu >= nr_cpu_ids)
649 return -EINVAL;
650
648 if (gic_irq_in_rdist(d)) 651 if (gic_irq_in_rdist(d))
649 return -EINVAL; 652 return -EINVAL;
650 653
diff --git a/drivers/irqchip/irq-i8259.c b/drivers/irqchip/irq-i8259.c
index 1aec12c6d9ac..7aafbb091b67 100644
--- a/drivers/irqchip/irq-i8259.c
+++ b/drivers/irqchip/irq-i8259.c
@@ -307,7 +307,7 @@ static int i8259A_irq_domain_map(struct irq_domain *d, unsigned int virq,
307 return 0; 307 return 0;
308} 308}
309 309
310static struct irq_domain_ops i8259A_ops = { 310static const struct irq_domain_ops i8259A_ops = {
311 .map = i8259A_irq_domain_map, 311 .map = i8259A_irq_domain_map,
312 .xlate = irq_domain_xlate_onecell, 312 .xlate = irq_domain_xlate_onecell,
313}; 313};
diff --git a/drivers/irqchip/irq-imx-gpcv2.c b/drivers/irqchip/irq-imx-gpcv2.c
index 9463f3557e82..bb36f572e322 100644
--- a/drivers/irqchip/irq-imx-gpcv2.c
+++ b/drivers/irqchip/irq-imx-gpcv2.c
@@ -200,7 +200,7 @@ static int imx_gpcv2_domain_alloc(struct irq_domain *domain,
200 &parent_fwspec); 200 &parent_fwspec);
201} 201}
202 202
203static struct irq_domain_ops gpcv2_irqchip_data_domain_ops = { 203static const struct irq_domain_ops gpcv2_irqchip_data_domain_ops = {
204 .translate = imx_gpcv2_domain_translate, 204 .translate = imx_gpcv2_domain_translate,
205 .alloc = imx_gpcv2_domain_alloc, 205 .alloc = imx_gpcv2_domain_alloc,
206 .free = irq_domain_free_irqs_common, 206 .free = irq_domain_free_irqs_common,
diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c
index 31d6b5a582d2..567b29c47608 100644
--- a/drivers/irqchip/irq-mbigen.c
+++ b/drivers/irqchip/irq-mbigen.c
@@ -228,7 +228,7 @@ static int mbigen_irq_domain_alloc(struct irq_domain *domain,
228 return 0; 228 return 0;
229} 229}
230 230
231static struct irq_domain_ops mbigen_domain_ops = { 231static const struct irq_domain_ops mbigen_domain_ops = {
232 .translate = mbigen_domain_translate, 232 .translate = mbigen_domain_translate,
233 .alloc = mbigen_irq_domain_alloc, 233 .alloc = mbigen_irq_domain_alloc,
234 .free = irq_domain_free_irqs_common, 234 .free = irq_domain_free_irqs_common,
diff --git a/drivers/irqchip/irq-mips-cpu.c b/drivers/irqchip/irq-mips-cpu.c
index b247f3c743ac..0a8ed1c05518 100644
--- a/drivers/irqchip/irq-mips-cpu.c
+++ b/drivers/irqchip/irq-mips-cpu.c
@@ -240,7 +240,7 @@ static void mips_cpu_register_ipi_domain(struct device_node *of_node)
240 ipi_domain_state); 240 ipi_domain_state);
241 if (!ipi_domain) 241 if (!ipi_domain)
242 panic("Failed to add MIPS CPU IPI domain"); 242 panic("Failed to add MIPS CPU IPI domain");
243 ipi_domain->bus_token = DOMAIN_BUS_IPI; 243 irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI);
244} 244}
245 245
246#else /* !CONFIG_GENERIC_IRQ_IPI */ 246#else /* !CONFIG_GENERIC_IRQ_IPI */
diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c
index 929f8558bf1c..832ebf4062f7 100644
--- a/drivers/irqchip/irq-mips-gic.c
+++ b/drivers/irqchip/irq-mips-gic.c
@@ -874,7 +874,7 @@ int gic_ipi_domain_match(struct irq_domain *d, struct device_node *node,
874 } 874 }
875} 875}
876 876
877static struct irq_domain_ops gic_ipi_domain_ops = { 877static const struct irq_domain_ops gic_ipi_domain_ops = {
878 .xlate = gic_ipi_domain_xlate, 878 .xlate = gic_ipi_domain_xlate,
879 .alloc = gic_ipi_domain_alloc, 879 .alloc = gic_ipi_domain_alloc,
880 .free = gic_ipi_domain_free, 880 .free = gic_ipi_domain_free,
@@ -960,7 +960,7 @@ static void __init __gic_init(unsigned long gic_base_addr,
960 panic("Failed to add GIC IPI domain"); 960 panic("Failed to add GIC IPI domain");
961 961
962 gic_ipi_domain->name = "mips-gic-ipi"; 962 gic_ipi_domain->name = "mips-gic-ipi";
963 gic_ipi_domain->bus_token = DOMAIN_BUS_IPI; 963 irq_domain_update_bus_token(gic_ipi_domain, DOMAIN_BUS_IPI);
964 964
965 if (node && 965 if (node &&
966 !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) { 966 !of_property_read_u32_array(node, "mti,reserved-ipi-vectors", v, 2)) {
diff --git a/drivers/irqchip/irq-mvebu-gicp.c b/drivers/irqchip/irq-mvebu-gicp.c
new file mode 100644
index 000000000000..b283fc90be1e
--- /dev/null
+++ b/drivers/irqchip/irq-mvebu-gicp.c
@@ -0,0 +1,279 @@
1/*
2 * Copyright (C) 2017 Marvell
3 *
4 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
5 *
6 * This file is licensed under the terms of the GNU General Public
7 * License version 2. This program is licensed "as is" without any
8 * warranty of any kind, whether express or implied.
9 */
10
11#include <linux/io.h>
12#include <linux/irq.h>
13#include <linux/irqdomain.h>
14#include <linux/msi.h>
15#include <linux/of.h>
16#include <linux/of_irq.h>
17#include <linux/of_platform.h>
18#include <linux/platform_device.h>
19
20#include <dt-bindings/interrupt-controller/arm-gic.h>
21
22#include "irq-mvebu-gicp.h"
23
24#define GICP_SETSPI_NSR_OFFSET 0x0
25#define GICP_CLRSPI_NSR_OFFSET 0x8
26
27struct mvebu_gicp_spi_range {
28 unsigned int start;
29 unsigned int count;
30};
31
32struct mvebu_gicp {
33 struct mvebu_gicp_spi_range *spi_ranges;
34 unsigned int spi_ranges_cnt;
35 unsigned int spi_cnt;
36 unsigned long *spi_bitmap;
37 spinlock_t spi_lock;
38 struct resource *res;
39 struct device *dev;
40};
41
42static int gicp_idx_to_spi(struct mvebu_gicp *gicp, int idx)
43{
44 int i;
45
46 for (i = 0; i < gicp->spi_ranges_cnt; i++) {
47 struct mvebu_gicp_spi_range *r = &gicp->spi_ranges[i];
48
49 if (idx < r->count)
50 return r->start + idx;
51
52 idx -= r->count;
53 }
54
55 return -EINVAL;
56}
57
58int mvebu_gicp_get_doorbells(struct device_node *dn, phys_addr_t *setspi,
59 phys_addr_t *clrspi)
60{
61 struct platform_device *pdev;
62 struct mvebu_gicp *gicp;
63
64 pdev = of_find_device_by_node(dn);
65 if (!pdev)
66 return -ENODEV;
67
68 gicp = platform_get_drvdata(pdev);
69 if (!gicp)
70 return -ENODEV;
71
72 *setspi = gicp->res->start + GICP_SETSPI_NSR_OFFSET;
73 *clrspi = gicp->res->start + GICP_CLRSPI_NSR_OFFSET;
74
75 return 0;
76}
77
78static void gicp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
79{
80 struct mvebu_gicp *gicp = data->chip_data;
81 phys_addr_t setspi = gicp->res->start + GICP_SETSPI_NSR_OFFSET;
82
83 msg->data = data->hwirq;
84 msg->address_lo = lower_32_bits(setspi);
85 msg->address_hi = upper_32_bits(setspi);
86}
87
88static struct irq_chip gicp_irq_chip = {
89 .name = "GICP",
90 .irq_mask = irq_chip_mask_parent,
91 .irq_unmask = irq_chip_unmask_parent,
92 .irq_eoi = irq_chip_eoi_parent,
93 .irq_set_affinity = irq_chip_set_affinity_parent,
94 .irq_set_type = irq_chip_set_type_parent,
95 .irq_compose_msi_msg = gicp_compose_msi_msg,
96};
97
98static int gicp_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
99 unsigned int nr_irqs, void *args)
100{
101 struct mvebu_gicp *gicp = domain->host_data;
102 struct irq_fwspec fwspec;
103 unsigned int hwirq;
104 int ret;
105
106 spin_lock(&gicp->spi_lock);
107 hwirq = find_first_zero_bit(gicp->spi_bitmap, gicp->spi_cnt);
108 if (hwirq == gicp->spi_cnt) {
109 spin_unlock(&gicp->spi_lock);
110 return -ENOSPC;
111 }
112 __set_bit(hwirq, gicp->spi_bitmap);
113 spin_unlock(&gicp->spi_lock);
114
115 fwspec.fwnode = domain->parent->fwnode;
116 fwspec.param_count = 3;
117 fwspec.param[0] = GIC_SPI;
118 fwspec.param[1] = gicp_idx_to_spi(gicp, hwirq) - 32;
119 /*
120 * Assume edge rising for now, it will be properly set when
121 * ->set_type() is called
122 */
123 fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
124
125 ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
126 if (ret) {
127 dev_err(gicp->dev, "Cannot allocate parent IRQ\n");
128 goto free_hwirq;
129 }
130
131 ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
132 &gicp_irq_chip, gicp);
133 if (ret)
134 goto free_irqs_parent;
135
136 return 0;
137
138free_irqs_parent:
139 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
140free_hwirq:
141 spin_lock(&gicp->spi_lock);
142 __clear_bit(hwirq, gicp->spi_bitmap);
143 spin_unlock(&gicp->spi_lock);
144 return ret;
145}
146
147static void gicp_irq_domain_free(struct irq_domain *domain,
148 unsigned int virq, unsigned int nr_irqs)
149{
150 struct mvebu_gicp *gicp = domain->host_data;
151 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
152
153 if (d->hwirq >= gicp->spi_cnt) {
154 dev_err(gicp->dev, "Invalid hwirq %lu\n", d->hwirq);
155 return;
156 }
157
158 irq_domain_free_irqs_parent(domain, virq, nr_irqs);
159
160 spin_lock(&gicp->spi_lock);
161 __clear_bit(d->hwirq, gicp->spi_bitmap);
162 spin_unlock(&gicp->spi_lock);
163}
164
165static const struct irq_domain_ops gicp_domain_ops = {
166 .alloc = gicp_irq_domain_alloc,
167 .free = gicp_irq_domain_free,
168};
169
170static struct irq_chip gicp_msi_irq_chip = {
171 .name = "GICP",
172 .irq_set_type = irq_chip_set_type_parent,
173};
174
175static struct msi_domain_ops gicp_msi_ops = {
176};
177
178static struct msi_domain_info gicp_msi_domain_info = {
179 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS),
180 .ops = &gicp_msi_ops,
181 .chip = &gicp_msi_irq_chip,
182};
183
184static int mvebu_gicp_probe(struct platform_device *pdev)
185{
186 struct mvebu_gicp *gicp;
187 struct irq_domain *inner_domain, *plat_domain, *parent_domain;
188 struct device_node *node = pdev->dev.of_node;
189 struct device_node *irq_parent_dn;
190 int ret, i;
191
192 gicp = devm_kzalloc(&pdev->dev, sizeof(*gicp), GFP_KERNEL);
193 if (!gicp)
194 return -ENOMEM;
195
196 gicp->dev = &pdev->dev;
197
198 gicp->res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
199 if (!gicp->res)
200 return -ENODEV;
201
202 ret = of_property_count_u32_elems(node, "marvell,spi-ranges");
203 if (ret < 0)
204 return ret;
205
206 gicp->spi_ranges_cnt = ret / 2;
207
208 gicp->spi_ranges =
209 devm_kzalloc(&pdev->dev,
210 gicp->spi_ranges_cnt *
211 sizeof(struct mvebu_gicp_spi_range),
212 GFP_KERNEL);
213 if (!gicp->spi_ranges)
214 return -ENOMEM;
215
216 for (i = 0; i < gicp->spi_ranges_cnt; i++) {
217 of_property_read_u32_index(node, "marvell,spi-ranges",
218 i * 2,
219 &gicp->spi_ranges[i].start);
220
221 of_property_read_u32_index(node, "marvell,spi-ranges",
222 i * 2 + 1,
223 &gicp->spi_ranges[i].count);
224
225 gicp->spi_cnt += gicp->spi_ranges[i].count;
226 }
227
228 gicp->spi_bitmap = devm_kzalloc(&pdev->dev,
229 BITS_TO_LONGS(gicp->spi_cnt) * sizeof(long),
230 GFP_KERNEL);
231 if (!gicp->spi_bitmap)
232 return -ENOMEM;
233
234 irq_parent_dn = of_irq_find_parent(node);
235 if (!irq_parent_dn) {
236 dev_err(&pdev->dev, "failed to find parent IRQ node\n");
237 return -ENODEV;
238 }
239
240 parent_domain = irq_find_host(irq_parent_dn);
241 if (!parent_domain) {
242 dev_err(&pdev->dev, "failed to find parent IRQ domain\n");
243 return -ENODEV;
244 }
245
246 inner_domain = irq_domain_create_hierarchy(parent_domain, 0,
247 gicp->spi_cnt,
248 of_node_to_fwnode(node),
249 &gicp_domain_ops, gicp);
250 if (!inner_domain)
251 return -ENOMEM;
252
253
254 plat_domain = platform_msi_create_irq_domain(of_node_to_fwnode(node),
255 &gicp_msi_domain_info,
256 inner_domain);
257 if (!plat_domain) {
258 irq_domain_remove(inner_domain);
259 return -ENOMEM;
260 }
261
262 platform_set_drvdata(pdev, gicp);
263
264 return 0;
265}
266
267static const struct of_device_id mvebu_gicp_of_match[] = {
268 { .compatible = "marvell,ap806-gicp", },
269 {},
270};
271
272static struct platform_driver mvebu_gicp_driver = {
273 .probe = mvebu_gicp_probe,
274 .driver = {
275 .name = "mvebu-gicp",
276 .of_match_table = mvebu_gicp_of_match,
277 },
278};
279builtin_platform_driver(mvebu_gicp_driver);
diff --git a/drivers/irqchip/irq-mvebu-gicp.h b/drivers/irqchip/irq-mvebu-gicp.h
new file mode 100644
index 000000000000..98535e886ea5
--- /dev/null
+++ b/drivers/irqchip/irq-mvebu-gicp.h
@@ -0,0 +1,11 @@
1#ifndef __MVEBU_GICP_H__
2#define __MVEBU_GICP_H__
3
4#include <linux/types.h>
5
6struct device_node;
7
8int mvebu_gicp_get_doorbells(struct device_node *dn, phys_addr_t *setspi,
9 phys_addr_t *clrspi);
10
11#endif /* __MVEBU_GICP_H__ */
diff --git a/drivers/irqchip/irq-mvebu-icu.c b/drivers/irqchip/irq-mvebu-icu.c
new file mode 100644
index 000000000000..e18c48d3a92e
--- /dev/null
+++ b/drivers/irqchip/irq-mvebu-icu.c
@@ -0,0 +1,289 @@
1/*
2 * Copyright (C) 2017 Marvell
3 *
4 * Hanna Hawa <hannah@marvell.com>
5 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com>
6 *
7 * This file is licensed under the terms of the GNU General Public
8 * License version 2. This program is licensed "as is" without any
9 * warranty of any kind, whether express or implied.
10 */
11
12#include <linux/interrupt.h>
13#include <linux/irq.h>
14#include <linux/irqchip.h>
15#include <linux/irqdomain.h>
16#include <linux/kernel.h>
17#include <linux/msi.h>
18#include <linux/of_irq.h>
19#include <linux/of_platform.h>
20#include <linux/platform_device.h>
21
22#include <dt-bindings/interrupt-controller/mvebu-icu.h>
23
24#include "irq-mvebu-gicp.h"
25
26/* ICU registers */
27#define ICU_SETSPI_NSR_AL 0x10
28#define ICU_SETSPI_NSR_AH 0x14
29#define ICU_CLRSPI_NSR_AL 0x18
30#define ICU_CLRSPI_NSR_AH 0x1c
31#define ICU_INT_CFG(x) (0x100 + 4 * (x))
32#define ICU_INT_ENABLE BIT(24)
33#define ICU_IS_EDGE BIT(28)
34#define ICU_GROUP_SHIFT 29
35
36/* ICU definitions */
37#define ICU_MAX_IRQS 207
38#define ICU_SATA0_ICU_ID 109
39#define ICU_SATA1_ICU_ID 107
40
41struct mvebu_icu {
42 struct irq_chip irq_chip;
43 void __iomem *base;
44 struct irq_domain *domain;
45 struct device *dev;
46};
47
48struct mvebu_icu_irq_data {
49 struct mvebu_icu *icu;
50 unsigned int icu_group;
51 unsigned int type;
52};
53
54static void mvebu_icu_write_msg(struct msi_desc *desc, struct msi_msg *msg)
55{
56 struct irq_data *d = irq_get_irq_data(desc->irq);
57 struct mvebu_icu_irq_data *icu_irqd = d->chip_data;
58 struct mvebu_icu *icu = icu_irqd->icu;
59 unsigned int icu_int;
60
61 if (msg->address_lo || msg->address_hi) {
62 /* Configure the ICU with irq number & type */
63 icu_int = msg->data | ICU_INT_ENABLE;
64 if (icu_irqd->type & IRQ_TYPE_EDGE_RISING)
65 icu_int |= ICU_IS_EDGE;
66 icu_int |= icu_irqd->icu_group << ICU_GROUP_SHIFT;
67 } else {
68 /* De-configure the ICU */
69 icu_int = 0;
70 }
71
72 writel_relaxed(icu_int, icu->base + ICU_INT_CFG(d->hwirq));
73
74 /*
75 * The SATA unit has 2 ports, and a dedicated ICU entry per
76 * port. The ahci sata driver supports only one irq interrupt
77 * per SATA unit. To solve this conflict, we configure the 2
78 * SATA wired interrupts in the south bridge into 1 GIC
79 * interrupt in the north bridge. Even if only a single port
80 * is enabled, if sata node is enabled, both interrupts are
81 * configured (regardless of which port is actually in use).
82 */
83 if (d->hwirq == ICU_SATA0_ICU_ID || d->hwirq == ICU_SATA1_ICU_ID) {
84 writel_relaxed(icu_int,
85 icu->base + ICU_INT_CFG(ICU_SATA0_ICU_ID));
86 writel_relaxed(icu_int,
87 icu->base + ICU_INT_CFG(ICU_SATA1_ICU_ID));
88 }
89}
90
91static int
92mvebu_icu_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec,
93 unsigned long *hwirq, unsigned int *type)
94{
95 struct mvebu_icu *icu = d->host_data;
96 unsigned int icu_group;
97
98 /* Check the count of the parameters in dt */
99 if (WARN_ON(fwspec->param_count < 3)) {
100 dev_err(icu->dev, "wrong ICU parameter count %d\n",
101 fwspec->param_count);
102 return -EINVAL;
103 }
104
105 /* Only ICU group type is handled */
106 icu_group = fwspec->param[0];
107 if (icu_group != ICU_GRP_NSR && icu_group != ICU_GRP_SR &&
108 icu_group != ICU_GRP_SEI && icu_group != ICU_GRP_REI) {
109 dev_err(icu->dev, "wrong ICU group type %x\n", icu_group);
110 return -EINVAL;
111 }
112
113 *hwirq = fwspec->param[1];
114 if (*hwirq >= ICU_MAX_IRQS) {
115 dev_err(icu->dev, "invalid interrupt number %ld\n", *hwirq);
116 return -EINVAL;
117 }
118
119 /* Mask the type to prevent wrong DT configuration */
120 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK;
121
122 return 0;
123}
124
125static int
126mvebu_icu_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
127 unsigned int nr_irqs, void *args)
128{
129 int err;
130 unsigned long hwirq;
131 struct irq_fwspec *fwspec = args;
132 struct mvebu_icu *icu = platform_msi_get_host_data(domain);
133 struct mvebu_icu_irq_data *icu_irqd;
134
135 icu_irqd = kmalloc(sizeof(*icu_irqd), GFP_KERNEL);
136 if (!icu_irqd)
137 return -ENOMEM;
138
139 err = mvebu_icu_irq_domain_translate(domain, fwspec, &hwirq,
140 &icu_irqd->type);
141 if (err) {
142 dev_err(icu->dev, "failed to translate ICU parameters\n");
143 goto free_irqd;
144 }
145
146 icu_irqd->icu_group = fwspec->param[0];
147 icu_irqd->icu = icu;
148
149 err = platform_msi_domain_alloc(domain, virq, nr_irqs);
150 if (err) {
151 dev_err(icu->dev, "failed to allocate ICU interrupt in parent domain\n");
152 goto free_irqd;
153 }
154
155 /* Make sure there is no interrupt left pending by the firmware */
156 err = irq_set_irqchip_state(virq, IRQCHIP_STATE_PENDING, false);
157 if (err)
158 goto free_msi;
159
160 err = irq_domain_set_hwirq_and_chip(domain, virq, hwirq,
161 &icu->irq_chip, icu_irqd);
162 if (err) {
163 dev_err(icu->dev, "failed to set the data to IRQ domain\n");
164 goto free_msi;
165 }
166
167 return 0;
168
169free_msi:
170 platform_msi_domain_free(domain, virq, nr_irqs);
171free_irqd:
172 kfree(icu_irqd);
173 return err;
174}
175
176static void
177mvebu_icu_irq_domain_free(struct irq_domain *domain, unsigned int virq,
178 unsigned int nr_irqs)
179{
180 struct irq_data *d = irq_get_irq_data(virq);
181 struct mvebu_icu_irq_data *icu_irqd = d->chip_data;
182
183 kfree(icu_irqd);
184
185 platform_msi_domain_free(domain, virq, nr_irqs);
186}
187
188static const struct irq_domain_ops mvebu_icu_domain_ops = {
189 .translate = mvebu_icu_irq_domain_translate,
190 .alloc = mvebu_icu_irq_domain_alloc,
191 .free = mvebu_icu_irq_domain_free,
192};
193
194static int mvebu_icu_probe(struct platform_device *pdev)
195{
196 struct mvebu_icu *icu;
197 struct device_node *node = pdev->dev.of_node;
198 struct device_node *gicp_dn;
199 struct resource *res;
200 phys_addr_t setspi, clrspi;
201 u32 i, icu_int;
202 int ret;
203
204 icu = devm_kzalloc(&pdev->dev, sizeof(struct mvebu_icu),
205 GFP_KERNEL);
206 if (!icu)
207 return -ENOMEM;
208
209 icu->dev = &pdev->dev;
210
211 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
212 icu->base = devm_ioremap_resource(&pdev->dev, res);
213 if (IS_ERR(icu->base)) {
214 dev_err(&pdev->dev, "Failed to map icu base address.\n");
215 return PTR_ERR(icu->base);
216 }
217
218 icu->irq_chip.name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
219 "ICU.%x",
220 (unsigned int)res->start);
221 if (!icu->irq_chip.name)
222 return -ENOMEM;
223
224 icu->irq_chip.irq_mask = irq_chip_mask_parent;
225 icu->irq_chip.irq_unmask = irq_chip_unmask_parent;
226 icu->irq_chip.irq_eoi = irq_chip_eoi_parent;
227 icu->irq_chip.irq_set_type = irq_chip_set_type_parent;
228#ifdef CONFIG_SMP
229 icu->irq_chip.irq_set_affinity = irq_chip_set_affinity_parent;
230#endif
231
232 /*
233 * We're probed after MSI domains have been resolved, so force
234 * resolution here.
235 */
236 pdev->dev.msi_domain = of_msi_get_domain(&pdev->dev, node,
237 DOMAIN_BUS_PLATFORM_MSI);
238 if (!pdev->dev.msi_domain)
239 return -EPROBE_DEFER;
240
241 gicp_dn = irq_domain_get_of_node(pdev->dev.msi_domain);
242 if (!gicp_dn)
243 return -ENODEV;
244
245 ret = mvebu_gicp_get_doorbells(gicp_dn, &setspi, &clrspi);
246 if (ret)
247 return ret;
248
249 /* Set Clear/Set ICU SPI message address in AP */
250 writel_relaxed(upper_32_bits(setspi), icu->base + ICU_SETSPI_NSR_AH);
251 writel_relaxed(lower_32_bits(setspi), icu->base + ICU_SETSPI_NSR_AL);
252 writel_relaxed(upper_32_bits(clrspi), icu->base + ICU_CLRSPI_NSR_AH);
253 writel_relaxed(lower_32_bits(clrspi), icu->base + ICU_CLRSPI_NSR_AL);
254
255 /*
256 * Clean all ICU interrupts with type SPI_NSR, required to
257 * avoid unpredictable SPI assignments done by firmware.
258 */
259 for (i = 0 ; i < ICU_MAX_IRQS ; i++) {
260 icu_int = readl(icu->base + ICU_INT_CFG(i));
261 if ((icu_int >> ICU_GROUP_SHIFT) == ICU_GRP_NSR)
262 writel_relaxed(0x0, icu->base + ICU_INT_CFG(i));
263 }
264
265 icu->domain =
266 platform_msi_create_device_domain(&pdev->dev, ICU_MAX_IRQS,
267 mvebu_icu_write_msg,
268 &mvebu_icu_domain_ops, icu);
269 if (!icu->domain) {
270 dev_err(&pdev->dev, "Failed to create ICU domain\n");
271 return -ENOMEM;
272 }
273
274 return 0;
275}
276
277static const struct of_device_id mvebu_icu_of_match[] = {
278 { .compatible = "marvell,cp110-icu", },
279 {},
280};
281
282static struct platform_driver mvebu_icu_driver = {
283 .probe = mvebu_icu_probe,
284 .driver = {
285 .name = "mvebu-icu",
286 .of_match_table = mvebu_icu_of_match,
287 },
288};
289builtin_platform_driver(mvebu_icu_driver);
diff --git a/drivers/irqchip/irq-or1k-pic.c b/drivers/irqchip/irq-or1k-pic.c
index 6a9a3e79218b..dd9d5d12fea2 100644
--- a/drivers/irqchip/irq-or1k-pic.c
+++ b/drivers/irqchip/irq-or1k-pic.c
@@ -70,7 +70,7 @@ static struct or1k_pic_dev or1k_pic_level = {
70 .name = "or1k-PIC-level", 70 .name = "or1k-PIC-level",
71 .irq_unmask = or1k_pic_unmask, 71 .irq_unmask = or1k_pic_unmask,
72 .irq_mask = or1k_pic_mask, 72 .irq_mask = or1k_pic_mask,
73 .irq_mask_ack = or1k_pic_mask, 73 .irq_mask_ack = or1k_pic_mask_ack,
74 }, 74 },
75 .handle = handle_level_irq, 75 .handle = handle_level_irq,
76 .flags = IRQ_LEVEL | IRQ_NOPROBE, 76 .flags = IRQ_LEVEL | IRQ_NOPROBE,
diff --git a/drivers/irqchip/irq-renesas-h8300h.c b/drivers/irqchip/irq-renesas-h8300h.c
index c378768d75b3..b8327590ae52 100644
--- a/drivers/irqchip/irq-renesas-h8300h.c
+++ b/drivers/irqchip/irq-renesas-h8300h.c
@@ -67,7 +67,7 @@ static int irq_map(struct irq_domain *h, unsigned int virq,
67 return 0; 67 return 0;
68} 68}
69 69
70static struct irq_domain_ops irq_ops = { 70static const struct irq_domain_ops irq_ops = {
71 .map = irq_map, 71 .map = irq_map,
72 .xlate = irq_domain_xlate_onecell, 72 .xlate = irq_domain_xlate_onecell,
73}; 73};
diff --git a/drivers/irqchip/irq-renesas-h8s.c b/drivers/irqchip/irq-renesas-h8s.c
index af8c6c61c824..71d8139be26c 100644
--- a/drivers/irqchip/irq-renesas-h8s.c
+++ b/drivers/irqchip/irq-renesas-h8s.c
@@ -73,7 +73,7 @@ static __init int irq_map(struct irq_domain *h, unsigned int virq,
73 return 0; 73 return 0;
74} 74}
75 75
76static struct irq_domain_ops irq_ops = { 76static const struct irq_domain_ops irq_ops = {
77 .map = irq_map, 77 .map = irq_map,
78 .xlate = irq_domain_xlate_onecell, 78 .xlate = irq_domain_xlate_onecell,
79}; 79};
diff --git a/drivers/irqchip/irq-sunxi-nmi.c b/drivers/irqchip/irq-sunxi-nmi.c
index 668730c5cb66..a412b5d5d0fa 100644
--- a/drivers/irqchip/irq-sunxi-nmi.c
+++ b/drivers/irqchip/irq-sunxi-nmi.c
@@ -25,6 +25,29 @@
25 25
26#define SUNXI_NMI_SRC_TYPE_MASK 0x00000003 26#define SUNXI_NMI_SRC_TYPE_MASK 0x00000003
27 27
28#define SUNXI_NMI_IRQ_BIT BIT(0)
29
30#define SUN6I_R_INTC_CTRL 0x0c
31#define SUN6I_R_INTC_PENDING 0x10
32#define SUN6I_R_INTC_ENABLE 0x40
33
34/*
35 * For deprecated sun6i-a31-sc-nmi compatible.
36 * Registers are offset by 0x0c.
37 */
38#define SUN6I_R_INTC_NMI_OFFSET 0x0c
39#define SUN6I_NMI_CTRL (SUN6I_R_INTC_CTRL - SUN6I_R_INTC_NMI_OFFSET)
40#define SUN6I_NMI_PENDING (SUN6I_R_INTC_PENDING - SUN6I_R_INTC_NMI_OFFSET)
41#define SUN6I_NMI_ENABLE (SUN6I_R_INTC_ENABLE - SUN6I_R_INTC_NMI_OFFSET)
42
43#define SUN7I_NMI_CTRL 0x00
44#define SUN7I_NMI_PENDING 0x04
45#define SUN7I_NMI_ENABLE 0x08
46
47#define SUN9I_NMI_CTRL 0x00
48#define SUN9I_NMI_ENABLE 0x04
49#define SUN9I_NMI_PENDING 0x08
50
28enum { 51enum {
29 SUNXI_SRC_TYPE_LEVEL_LOW = 0, 52 SUNXI_SRC_TYPE_LEVEL_LOW = 0,
30 SUNXI_SRC_TYPE_EDGE_FALLING, 53 SUNXI_SRC_TYPE_EDGE_FALLING,
@@ -38,22 +61,28 @@ struct sunxi_sc_nmi_reg_offs {
38 u32 enable; 61 u32 enable;
39}; 62};
40 63
41static struct sunxi_sc_nmi_reg_offs sun7i_reg_offs = { 64static const struct sunxi_sc_nmi_reg_offs sun6i_r_intc_reg_offs __initconst = {
42 .ctrl = 0x00, 65 .ctrl = SUN6I_R_INTC_CTRL,
43 .pend = 0x04, 66 .pend = SUN6I_R_INTC_PENDING,
44 .enable = 0x08, 67 .enable = SUN6I_R_INTC_ENABLE,
45}; 68};
46 69
47static struct sunxi_sc_nmi_reg_offs sun6i_reg_offs = { 70static const struct sunxi_sc_nmi_reg_offs sun6i_reg_offs __initconst = {
48 .ctrl = 0x00, 71 .ctrl = SUN6I_NMI_CTRL,
49 .pend = 0x04, 72 .pend = SUN6I_NMI_PENDING,
50 .enable = 0x34, 73 .enable = SUN6I_NMI_ENABLE,
51}; 74};
52 75
53static struct sunxi_sc_nmi_reg_offs sun9i_reg_offs = { 76static const struct sunxi_sc_nmi_reg_offs sun7i_reg_offs __initconst = {
54 .ctrl = 0x00, 77 .ctrl = SUN7I_NMI_CTRL,
55 .pend = 0x08, 78 .pend = SUN7I_NMI_PENDING,
56 .enable = 0x04, 79 .enable = SUN7I_NMI_ENABLE,
80};
81
82static const struct sunxi_sc_nmi_reg_offs sun9i_reg_offs __initconst = {
83 .ctrl = SUN9I_NMI_CTRL,
84 .pend = SUN9I_NMI_PENDING,
85 .enable = SUN9I_NMI_ENABLE,
57}; 86};
58 87
59static inline void sunxi_sc_nmi_write(struct irq_chip_generic *gc, u32 off, 88static inline void sunxi_sc_nmi_write(struct irq_chip_generic *gc, u32 off,
@@ -128,7 +157,7 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type)
128} 157}
129 158
130static int __init sunxi_sc_nmi_irq_init(struct device_node *node, 159static int __init sunxi_sc_nmi_irq_init(struct device_node *node,
131 struct sunxi_sc_nmi_reg_offs *reg_offs) 160 const struct sunxi_sc_nmi_reg_offs *reg_offs)
132{ 161{
133 struct irq_domain *domain; 162 struct irq_domain *domain;
134 struct irq_chip_generic *gc; 163 struct irq_chip_generic *gc;
@@ -187,8 +216,11 @@ static int __init sunxi_sc_nmi_irq_init(struct device_node *node,
187 gc->chip_types[1].regs.type = reg_offs->ctrl; 216 gc->chip_types[1].regs.type = reg_offs->ctrl;
188 gc->chip_types[1].handler = handle_edge_irq; 217 gc->chip_types[1].handler = handle_edge_irq;
189 218
219 /* Disable any active interrupts */
190 sunxi_sc_nmi_write(gc, reg_offs->enable, 0); 220 sunxi_sc_nmi_write(gc, reg_offs->enable, 0);
191 sunxi_sc_nmi_write(gc, reg_offs->pend, 0x1); 221
222 /* Clear any pending NMI interrupts */
223 sunxi_sc_nmi_write(gc, reg_offs->pend, SUNXI_NMI_IRQ_BIT);
192 224
193 irq_set_chained_handler_and_data(irq, sunxi_sc_nmi_handle_irq, domain); 225 irq_set_chained_handler_and_data(irq, sunxi_sc_nmi_handle_irq, domain);
194 226
@@ -200,6 +232,14 @@ fail_irqd_remove:
200 return ret; 232 return ret;
201} 233}
202 234
235static int __init sun6i_r_intc_irq_init(struct device_node *node,
236 struct device_node *parent)
237{
238 return sunxi_sc_nmi_irq_init(node, &sun6i_r_intc_reg_offs);
239}
240IRQCHIP_DECLARE(sun6i_r_intc, "allwinner,sun6i-a31-r-intc",
241 sun6i_r_intc_irq_init);
242
203static int __init sun6i_sc_nmi_irq_init(struct device_node *node, 243static int __init sun6i_sc_nmi_irq_init(struct device_node *node,
204 struct device_node *parent) 244 struct device_node *parent)
205{ 245{
diff --git a/drivers/irqchip/qcom-irq-combiner.c b/drivers/irqchip/qcom-irq-combiner.c
index 226558698344..6aa3ea479214 100644
--- a/drivers/irqchip/qcom-irq-combiner.c
+++ b/drivers/irqchip/qcom-irq-combiner.c
@@ -288,9 +288,4 @@ static struct platform_driver qcom_irq_combiner_probe = {
288 }, 288 },
289 .probe = combiner_probe, 289 .probe = combiner_probe,
290}; 290};
291 291builtin_platform_driver(qcom_irq_combiner_probe);
292static int __init register_qcom_irq_combiner(void)
293{
294 return platform_driver_register(&qcom_irq_combiner_probe);
295}
296device_initcall(register_qcom_irq_combiner);
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 33c3b9db7d36..5b1ac79fb607 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -1695,7 +1695,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
1695 int result, nr_io_queues; 1695 int result, nr_io_queues;
1696 unsigned long size; 1696 unsigned long size;
1697 1697
1698 nr_io_queues = num_online_cpus(); 1698 nr_io_queues = num_present_cpus();
1699 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues); 1699 result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
1700 if (result < 0) 1700 if (result < 0)
1701 return result; 1701 return result;
diff --git a/drivers/pci/host/vmd.c b/drivers/pci/host/vmd.c
index e27ad2a3bd33..31203d69616d 100644
--- a/drivers/pci/host/vmd.c
+++ b/drivers/pci/host/vmd.c
@@ -554,6 +554,7 @@ static int vmd_find_free_domain(void)
554static int vmd_enable_domain(struct vmd_dev *vmd) 554static int vmd_enable_domain(struct vmd_dev *vmd)
555{ 555{
556 struct pci_sysdata *sd = &vmd->sysdata; 556 struct pci_sysdata *sd = &vmd->sysdata;
557 struct fwnode_handle *fn;
557 struct resource *res; 558 struct resource *res;
558 u32 upper_bits; 559 u32 upper_bits;
559 unsigned long flags; 560 unsigned long flags;
@@ -617,8 +618,13 @@ static int vmd_enable_domain(struct vmd_dev *vmd)
617 618
618 sd->node = pcibus_to_node(vmd->dev->bus); 619 sd->node = pcibus_to_node(vmd->dev->bus);
619 620
620 vmd->irq_domain = pci_msi_create_irq_domain(NULL, &vmd_msi_domain_info, 621 fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);
622 if (!fn)
623 return -ENODEV;
624
625 vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info,
621 x86_vector_domain); 626 x86_vector_domain);
627 irq_domain_free_fwnode(fn);
622 if (!vmd->irq_domain) 628 if (!vmd->irq_domain)
623 return -ENODEV; 629 return -ENODEV;
624 630
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c
index ba44fdfda66b..fbad5dca3219 100644
--- a/drivers/pci/msi.c
+++ b/drivers/pci/msi.c
@@ -1463,7 +1463,7 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
1463 if (!domain) 1463 if (!domain)
1464 return NULL; 1464 return NULL;
1465 1465
1466 domain->bus_token = DOMAIN_BUS_PCI_MSI; 1466 irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI);
1467 return domain; 1467 return domain;
1468} 1468}
1469EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); 1469EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
diff --git a/drivers/staging/fsl-mc/bus/fsl-mc-msi.c b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
index b8b2c86e63d4..17d5cf9f91d4 100644
--- a/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
+++ b/drivers/staging/fsl-mc/bus/fsl-mc-msi.c
@@ -170,7 +170,7 @@ struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode,
170 170
171 domain = msi_create_irq_domain(fwnode, info, parent); 171 domain = msi_create_irq_domain(fwnode, info, parent);
172 if (domain) 172 if (domain)
173 domain->bus_token = DOMAIN_BUS_FSL_MC_MSI; 173 irq_domain_update_bus_token(domain, DOMAIN_BUS_FSL_MC_MSI);
174 174
175 return domain; 175 return domain;
176} 176}
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index b52852f38cff..2e567d8433b3 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -1343,8 +1343,12 @@ static int set_affinity_irq(struct irq_data *data, const struct cpumask *dest,
1343 bool force) 1343 bool force)
1344{ 1344{
1345 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask); 1345 unsigned tcpu = cpumask_first_and(dest, cpu_online_mask);
1346 int ret = rebind_irq_to_cpu(data->irq, tcpu);
1346 1347
1347 return rebind_irq_to_cpu(data->irq, tcpu); 1348 if (!ret)
1349 irq_data_update_effective_affinity(data, cpumask_of(tcpu));
1350
1351 return ret;
1348} 1352}
1349 1353
1350static void enable_dynirq(struct irq_data *data) 1354static void enable_dynirq(struct irq_data *data)
diff --git a/include/dt-bindings/interrupt-controller/mvebu-icu.h b/include/dt-bindings/interrupt-controller/mvebu-icu.h
new file mode 100644
index 000000000000..8249558545c7
--- /dev/null
+++ b/include/dt-bindings/interrupt-controller/mvebu-icu.h
@@ -0,0 +1,15 @@
1/*
2 * This header provides constants for the MVEBU ICU driver.
3 */
4
5#ifndef _DT_BINDINGS_INTERRUPT_CONTROLLER_MVEBU_ICU_H
6#define _DT_BINDINGS_INTERRUPT_CONTROLLER_MVEBU_ICU_H
7
8/* interrupt specifier cell 0 */
9
10#define ICU_GRP_NSR 0x0
11#define ICU_GRP_SR 0x1
12#define ICU_GRP_SEI 0x4
13#define ICU_GRP_REI 0x5
14
15#endif
diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h
index 0f2a80377520..7f815d915977 100644
--- a/include/linux/cpuhotplug.h
+++ b/include/linux/cpuhotplug.h
@@ -58,7 +58,6 @@ enum cpuhp_state {
58 CPUHP_XEN_EVTCHN_PREPARE, 58 CPUHP_XEN_EVTCHN_PREPARE,
59 CPUHP_ARM_SHMOBILE_SCU_PREPARE, 59 CPUHP_ARM_SHMOBILE_SCU_PREPARE,
60 CPUHP_SH_SH3X_PREPARE, 60 CPUHP_SH_SH3X_PREPARE,
61 CPUHP_BLK_MQ_PREPARE,
62 CPUHP_NET_FLOW_PREPARE, 61 CPUHP_NET_FLOW_PREPARE,
63 CPUHP_TOPOLOGY_PREPARE, 62 CPUHP_TOPOLOGY_PREPARE,
64 CPUHP_NET_IUCV_PREPARE, 63 CPUHP_NET_IUCV_PREPARE,
@@ -124,6 +123,7 @@ enum cpuhp_state {
124 CPUHP_AP_ONLINE_IDLE, 123 CPUHP_AP_ONLINE_IDLE,
125 CPUHP_AP_SMPBOOT_THREADS, 124 CPUHP_AP_SMPBOOT_THREADS,
126 CPUHP_AP_X86_VDSO_VMA_ONLINE, 125 CPUHP_AP_X86_VDSO_VMA_ONLINE,
126 CPUHP_AP_IRQ_AFFINITY_ONLINE,
127 CPUHP_AP_PERF_ONLINE, 127 CPUHP_AP_PERF_ONLINE,
128 CPUHP_AP_PERF_X86_ONLINE, 128 CPUHP_AP_PERF_X86_ONLINE,
129 CPUHP_AP_PERF_X86_UNCORE_ONLINE, 129 CPUHP_AP_PERF_X86_UNCORE_ONLINE,
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index a6fba4804672..37f8e354f564 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -703,6 +703,12 @@ static inline void init_irq_proc(void)
703} 703}
704#endif 704#endif
705 705
706#ifdef CONFIG_IRQ_TIMINGS
707void irq_timings_enable(void);
708void irq_timings_disable(void);
709u64 irq_timings_next_event(u64 now);
710#endif
711
706struct seq_file; 712struct seq_file;
707int show_interrupts(struct seq_file *p, void *v); 713int show_interrupts(struct seq_file *p, void *v);
708int arch_show_interrupts(struct seq_file *p, int prec); 714int arch_show_interrupts(struct seq_file *p, int prec);
diff --git a/include/linux/irq.h b/include/linux/irq.h
index f887351aa80e..00db35b61e9e 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -22,6 +22,7 @@
22#include <linux/topology.h> 22#include <linux/topology.h>
23#include <linux/wait.h> 23#include <linux/wait.h>
24#include <linux/io.h> 24#include <linux/io.h>
25#include <linux/slab.h>
25 26
26#include <asm/irq.h> 27#include <asm/irq.h>
27#include <asm/ptrace.h> 28#include <asm/ptrace.h>
@@ -136,6 +137,9 @@ struct irq_domain;
136 * @affinity: IRQ affinity on SMP. If this is an IPI 137 * @affinity: IRQ affinity on SMP. If this is an IPI
137 * related irq, then this is the mask of the 138 * related irq, then this is the mask of the
138 * CPUs to which an IPI can be sent. 139 * CPUs to which an IPI can be sent.
140 * @effective_affinity: The effective IRQ affinity on SMP as some irq
141 * chips do not allow multi CPU destinations.
142 * A subset of @affinity.
139 * @msi_desc: MSI descriptor 143 * @msi_desc: MSI descriptor
140 * @ipi_offset: Offset of first IPI target cpu in @affinity. Optional. 144 * @ipi_offset: Offset of first IPI target cpu in @affinity. Optional.
141 */ 145 */
@@ -147,6 +151,9 @@ struct irq_common_data {
147 void *handler_data; 151 void *handler_data;
148 struct msi_desc *msi_desc; 152 struct msi_desc *msi_desc;
149 cpumask_var_t affinity; 153 cpumask_var_t affinity;
154#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
155 cpumask_var_t effective_affinity;
156#endif
150#ifdef CONFIG_GENERIC_IRQ_IPI 157#ifdef CONFIG_GENERIC_IRQ_IPI
151 unsigned int ipi_offset; 158 unsigned int ipi_offset;
152#endif 159#endif
@@ -199,6 +206,10 @@ struct irq_data {
199 * IRQD_WAKEUP_ARMED - Wakeup mode armed 206 * IRQD_WAKEUP_ARMED - Wakeup mode armed
200 * IRQD_FORWARDED_TO_VCPU - The interrupt is forwarded to a VCPU 207 * IRQD_FORWARDED_TO_VCPU - The interrupt is forwarded to a VCPU
201 * IRQD_AFFINITY_MANAGED - Affinity is auto-managed by the kernel 208 * IRQD_AFFINITY_MANAGED - Affinity is auto-managed by the kernel
209 * IRQD_IRQ_STARTED - Startup state of the interrupt
210 * IRQD_MANAGED_SHUTDOWN - Interrupt was shutdown due to empty affinity
211 * mask. Applies only to affinity managed irqs.
212 * IRQD_SINGLE_TARGET - IRQ allows only a single affinity target
202 */ 213 */
203enum { 214enum {
204 IRQD_TRIGGER_MASK = 0xf, 215 IRQD_TRIGGER_MASK = 0xf,
@@ -216,6 +227,9 @@ enum {
216 IRQD_WAKEUP_ARMED = (1 << 19), 227 IRQD_WAKEUP_ARMED = (1 << 19),
217 IRQD_FORWARDED_TO_VCPU = (1 << 20), 228 IRQD_FORWARDED_TO_VCPU = (1 << 20),
218 IRQD_AFFINITY_MANAGED = (1 << 21), 229 IRQD_AFFINITY_MANAGED = (1 << 21),
230 IRQD_IRQ_STARTED = (1 << 22),
231 IRQD_MANAGED_SHUTDOWN = (1 << 23),
232 IRQD_SINGLE_TARGET = (1 << 24),
219}; 233};
220 234
221#define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) 235#define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
@@ -264,6 +278,20 @@ static inline bool irqd_is_level_type(struct irq_data *d)
264 return __irqd_to_state(d) & IRQD_LEVEL; 278 return __irqd_to_state(d) & IRQD_LEVEL;
265} 279}
266 280
281/*
282 * Must only be called of irqchip.irq_set_affinity() or low level
283 * hieararchy domain allocation functions.
284 */
285static inline void irqd_set_single_target(struct irq_data *d)
286{
287 __irqd_to_state(d) |= IRQD_SINGLE_TARGET;
288}
289
290static inline bool irqd_is_single_target(struct irq_data *d)
291{
292 return __irqd_to_state(d) & IRQD_SINGLE_TARGET;
293}
294
267static inline bool irqd_is_wakeup_set(struct irq_data *d) 295static inline bool irqd_is_wakeup_set(struct irq_data *d)
268{ 296{
269 return __irqd_to_state(d) & IRQD_WAKEUP_STATE; 297 return __irqd_to_state(d) & IRQD_WAKEUP_STATE;
@@ -329,6 +357,16 @@ static inline void irqd_clr_activated(struct irq_data *d)
329 __irqd_to_state(d) &= ~IRQD_ACTIVATED; 357 __irqd_to_state(d) &= ~IRQD_ACTIVATED;
330} 358}
331 359
360static inline bool irqd_is_started(struct irq_data *d)
361{
362 return __irqd_to_state(d) & IRQD_IRQ_STARTED;
363}
364
365static inline bool irqd_is_managed_and_shutdown(struct irq_data *d)
366{
367 return __irqd_to_state(d) & IRQD_MANAGED_SHUTDOWN;
368}
369
332#undef __irqd_to_state 370#undef __irqd_to_state
333 371
334static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d) 372static inline irq_hw_number_t irqd_to_hwirq(struct irq_data *d)
@@ -478,14 +516,21 @@ extern int irq_set_affinity_locked(struct irq_data *data,
478 const struct cpumask *cpumask, bool force); 516 const struct cpumask *cpumask, bool force);
479extern int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info); 517extern int irq_set_vcpu_affinity(unsigned int irq, void *vcpu_info);
480 518
519#if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_IRQ_MIGRATION)
481extern void irq_migrate_all_off_this_cpu(void); 520extern void irq_migrate_all_off_this_cpu(void);
521extern int irq_affinity_online_cpu(unsigned int cpu);
522#else
523# define irq_affinity_online_cpu NULL
524#endif
482 525
483#if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_PENDING_IRQ) 526#if defined(CONFIG_SMP) && defined(CONFIG_GENERIC_PENDING_IRQ)
484void irq_move_irq(struct irq_data *data); 527void irq_move_irq(struct irq_data *data);
485void irq_move_masked_irq(struct irq_data *data); 528void irq_move_masked_irq(struct irq_data *data);
529void irq_force_complete_move(struct irq_desc *desc);
486#else 530#else
487static inline void irq_move_irq(struct irq_data *data) { } 531static inline void irq_move_irq(struct irq_data *data) { }
488static inline void irq_move_masked_irq(struct irq_data *data) { } 532static inline void irq_move_masked_irq(struct irq_data *data) { }
533static inline void irq_force_complete_move(struct irq_desc *desc) { }
489#endif 534#endif
490 535
491extern int no_irq_affinity; 536extern int no_irq_affinity;
@@ -727,6 +772,29 @@ static inline struct cpumask *irq_data_get_affinity_mask(struct irq_data *d)
727 return d->common->affinity; 772 return d->common->affinity;
728} 773}
729 774
775#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
776static inline
777struct cpumask *irq_data_get_effective_affinity_mask(struct irq_data *d)
778{
779 return d->common->effective_affinity;
780}
781static inline void irq_data_update_effective_affinity(struct irq_data *d,
782 const struct cpumask *m)
783{
784 cpumask_copy(d->common->effective_affinity, m);
785}
786#else
787static inline void irq_data_update_effective_affinity(struct irq_data *d,
788 const struct cpumask *m)
789{
790}
791static inline
792struct cpumask *irq_data_get_effective_affinity_mask(struct irq_data *d)
793{
794 return d->common->affinity;
795}
796#endif
797
730unsigned int arch_dynirq_lower_bound(unsigned int from); 798unsigned int arch_dynirq_lower_bound(unsigned int from);
731 799
732int __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node, 800int __irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node,
@@ -951,6 +1019,14 @@ int irq_setup_alt_chip(struct irq_data *d, unsigned int type);
951void irq_remove_generic_chip(struct irq_chip_generic *gc, u32 msk, 1019void irq_remove_generic_chip(struct irq_chip_generic *gc, u32 msk,
952 unsigned int clr, unsigned int set); 1020 unsigned int clr, unsigned int set);
953 1021
1022struct irq_chip_generic *
1023devm_irq_alloc_generic_chip(struct device *dev, const char *name, int num_ct,
1024 unsigned int irq_base, void __iomem *reg_base,
1025 irq_flow_handler_t handler);
1026int devm_irq_setup_generic_chip(struct device *dev, struct irq_chip_generic *gc,
1027 u32 msk, enum irq_gc_flags flags,
1028 unsigned int clr, unsigned int set);
1029
954struct irq_chip_generic *irq_get_domain_generic_chip(struct irq_domain *d, unsigned int hw_irq); 1030struct irq_chip_generic *irq_get_domain_generic_chip(struct irq_domain *d, unsigned int hw_irq);
955 1031
956int __irq_alloc_domain_generic_chips(struct irq_domain *d, int irqs_per_chip, 1032int __irq_alloc_domain_generic_chips(struct irq_domain *d, int irqs_per_chip,
@@ -967,6 +1043,19 @@ int __irq_alloc_domain_generic_chips(struct irq_domain *d, int irqs_per_chip,
967 handler, clr, set, flags); \ 1043 handler, clr, set, flags); \
968}) 1044})
969 1045
1046static inline void irq_free_generic_chip(struct irq_chip_generic *gc)
1047{
1048 kfree(gc);
1049}
1050
1051static inline void irq_destroy_generic_chip(struct irq_chip_generic *gc,
1052 u32 msk, unsigned int clr,
1053 unsigned int set)
1054{
1055 irq_remove_generic_chip(gc, msk, clr, set);
1056 irq_free_generic_chip(gc);
1057}
1058
970static inline struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d) 1059static inline struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d)
971{ 1060{
972 return container_of(d->chip, struct irq_chip_type, chip); 1061 return container_of(d->chip, struct irq_chip_type, chip);
diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
index c9be57931b58..d425a3a09722 100644
--- a/include/linux/irqdesc.h
+++ b/include/linux/irqdesc.h
@@ -46,6 +46,7 @@ struct pt_regs;
46 * @rcu: rcu head for delayed free 46 * @rcu: rcu head for delayed free
47 * @kobj: kobject used to represent this struct in sysfs 47 * @kobj: kobject used to represent this struct in sysfs
48 * @dir: /proc/irq/ procfs entry 48 * @dir: /proc/irq/ procfs entry
49 * @debugfs_file: dentry for the debugfs file
49 * @name: flow handler name for /proc/interrupts output 50 * @name: flow handler name for /proc/interrupts output
50 */ 51 */
51struct irq_desc { 52struct irq_desc {
@@ -88,6 +89,9 @@ struct irq_desc {
88#ifdef CONFIG_PROC_FS 89#ifdef CONFIG_PROC_FS
89 struct proc_dir_entry *dir; 90 struct proc_dir_entry *dir;
90#endif 91#endif
92#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
93 struct dentry *debugfs_file;
94#endif
91#ifdef CONFIG_SPARSE_IRQ 95#ifdef CONFIG_SPARSE_IRQ
92 struct rcu_head rcu; 96 struct rcu_head rcu;
93 struct kobject kobj; 97 struct kobject kobj;
diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h
index 9f3616085423..cac77a5c5555 100644
--- a/include/linux/irqdomain.h
+++ b/include/linux/irqdomain.h
@@ -130,6 +130,7 @@ struct irq_domain_chip_generic;
130 * @host_data: private data pointer for use by owner. Not touched by irq_domain 130 * @host_data: private data pointer for use by owner. Not touched by irq_domain
131 * core code. 131 * core code.
132 * @flags: host per irq_domain flags 132 * @flags: host per irq_domain flags
133 * @mapcount: The number of mapped interrupts
133 * 134 *
134 * Optional elements 135 * Optional elements
135 * @of_node: Pointer to device tree nodes associated with the irq_domain. Used 136 * @of_node: Pointer to device tree nodes associated with the irq_domain. Used
@@ -138,6 +139,7 @@ struct irq_domain_chip_generic;
138 * setting up one or more generic chips for interrupt controllers 139 * setting up one or more generic chips for interrupt controllers
139 * drivers using the generic chip library which uses this pointer. 140 * drivers using the generic chip library which uses this pointer.
140 * @parent: Pointer to parent irq_domain to support hierarchy irq_domains 141 * @parent: Pointer to parent irq_domain to support hierarchy irq_domains
142 * @debugfs_file: dentry for the domain debugfs file
141 * 143 *
142 * Revmap data, used internally by irq_domain 144 * Revmap data, used internally by irq_domain
143 * @revmap_direct_max_irq: The largest hwirq that can be set for controllers that 145 * @revmap_direct_max_irq: The largest hwirq that can be set for controllers that
@@ -152,6 +154,7 @@ struct irq_domain {
152 const struct irq_domain_ops *ops; 154 const struct irq_domain_ops *ops;
153 void *host_data; 155 void *host_data;
154 unsigned int flags; 156 unsigned int flags;
157 unsigned int mapcount;
155 158
156 /* Optional data */ 159 /* Optional data */
157 struct fwnode_handle *fwnode; 160 struct fwnode_handle *fwnode;
@@ -160,6 +163,9 @@ struct irq_domain {
160#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY 163#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
161 struct irq_domain *parent; 164 struct irq_domain *parent;
162#endif 165#endif
166#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
167 struct dentry *debugfs_file;
168#endif
163 169
164 /* reverse map data. The linear map gets appended to the irq_domain */ 170 /* reverse map data. The linear map gets appended to the irq_domain */
165 irq_hw_number_t hwirq_max; 171 irq_hw_number_t hwirq_max;
@@ -174,8 +180,8 @@ enum {
174 /* Irq domain is hierarchical */ 180 /* Irq domain is hierarchical */
175 IRQ_DOMAIN_FLAG_HIERARCHY = (1 << 0), 181 IRQ_DOMAIN_FLAG_HIERARCHY = (1 << 0),
176 182
177 /* Core calls alloc/free recursive through the domain hierarchy. */ 183 /* Irq domain name was allocated in __irq_domain_add() */
178 IRQ_DOMAIN_FLAG_AUTO_RECURSIVE = (1 << 1), 184 IRQ_DOMAIN_NAME_ALLOCATED = (1 << 6),
179 185
180 /* Irq domain is an IPI domain with virq per cpu */ 186 /* Irq domain is an IPI domain with virq per cpu */
181 IRQ_DOMAIN_FLAG_IPI_PER_CPU = (1 << 2), 187 IRQ_DOMAIN_FLAG_IPI_PER_CPU = (1 << 2),
@@ -203,7 +209,33 @@ static inline struct device_node *irq_domain_get_of_node(struct irq_domain *d)
203} 209}
204 210
205#ifdef CONFIG_IRQ_DOMAIN 211#ifdef CONFIG_IRQ_DOMAIN
206struct fwnode_handle *irq_domain_alloc_fwnode(void *data); 212struct fwnode_handle *__irq_domain_alloc_fwnode(unsigned int type, int id,
213 const char *name, void *data);
214
215enum {
216 IRQCHIP_FWNODE_REAL,
217 IRQCHIP_FWNODE_NAMED,
218 IRQCHIP_FWNODE_NAMED_ID,
219};
220
221static inline
222struct fwnode_handle *irq_domain_alloc_named_fwnode(const char *name)
223{
224 return __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_NAMED, 0, name, NULL);
225}
226
227static inline
228struct fwnode_handle *irq_domain_alloc_named_id_fwnode(const char *name, int id)
229{
230 return __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_NAMED_ID, id, name,
231 NULL);
232}
233
234static inline struct fwnode_handle *irq_domain_alloc_fwnode(void *data)
235{
236 return __irq_domain_alloc_fwnode(IRQCHIP_FWNODE_REAL, 0, NULL, data);
237}
238
207void irq_domain_free_fwnode(struct fwnode_handle *fwnode); 239void irq_domain_free_fwnode(struct fwnode_handle *fwnode);
208struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, 240struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size,
209 irq_hw_number_t hwirq_max, int direct_max, 241 irq_hw_number_t hwirq_max, int direct_max,
@@ -238,6 +270,9 @@ static inline bool is_fwnode_irqchip(struct fwnode_handle *fwnode)
238 return fwnode && fwnode->type == FWNODE_IRQCHIP; 270 return fwnode && fwnode->type == FWNODE_IRQCHIP;
239} 271}
240 272
273extern void irq_domain_update_bus_token(struct irq_domain *domain,
274 enum irq_domain_bus_token bus_token);
275
241static inline 276static inline
242struct irq_domain *irq_find_matching_fwnode(struct fwnode_handle *fwnode, 277struct irq_domain *irq_find_matching_fwnode(struct fwnode_handle *fwnode,
243 enum irq_domain_bus_token bus_token) 278 enum irq_domain_bus_token bus_token)
@@ -410,7 +445,7 @@ static inline int irq_domain_alloc_irqs(struct irq_domain *domain,
410 NULL); 445 NULL);
411} 446}
412 447
413extern int irq_domain_alloc_irqs_recursive(struct irq_domain *domain, 448extern int irq_domain_alloc_irqs_hierarchy(struct irq_domain *domain,
414 unsigned int irq_base, 449 unsigned int irq_base,
415 unsigned int nr_irqs, void *arg); 450 unsigned int nr_irqs, void *arg);
416extern int irq_domain_set_hwirq_and_chip(struct irq_domain *domain, 451extern int irq_domain_set_hwirq_and_chip(struct irq_domain *domain,
diff --git a/kernel/cpu.c b/kernel/cpu.c
index cb5103413bd8..b86b32ebb3b2 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -1252,6 +1252,11 @@ static struct cpuhp_step cpuhp_ap_states[] = {
1252 .startup.single = smpboot_unpark_threads, 1252 .startup.single = smpboot_unpark_threads,
1253 .teardown.single = NULL, 1253 .teardown.single = NULL,
1254 }, 1254 },
1255 [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
1256 .name = "irq/affinity:online",
1257 .startup.single = irq_affinity_online_cpu,
1258 .teardown.single = NULL,
1259 },
1255 [CPUHP_AP_PERF_ONLINE] = { 1260 [CPUHP_AP_PERF_ONLINE] = {
1256 .name = "perf:online", 1261 .name = "perf:online",
1257 .startup.single = perf_event_init_cpu, 1262 .startup.single = perf_event_init_cpu,
diff --git a/kernel/irq/Kconfig b/kernel/irq/Kconfig
index 3bbfd6a9c475..27c4e774071c 100644
--- a/kernel/irq/Kconfig
+++ b/kernel/irq/Kconfig
@@ -21,6 +21,10 @@ config GENERIC_IRQ_SHOW
21config GENERIC_IRQ_SHOW_LEVEL 21config GENERIC_IRQ_SHOW_LEVEL
22 bool 22 bool
23 23
24# Supports effective affinity mask
25config GENERIC_IRQ_EFFECTIVE_AFF_MASK
26 bool
27
24# Facility to allocate a hardware interrupt. This is legacy support 28# Facility to allocate a hardware interrupt. This is legacy support
25# and should not be used in new code. Use irq domains instead. 29# and should not be used in new code. Use irq domains instead.
26config GENERIC_IRQ_LEGACY_ALLOC_HWIRQ 30config GENERIC_IRQ_LEGACY_ALLOC_HWIRQ
@@ -81,6 +85,9 @@ config GENERIC_MSI_IRQ_DOMAIN
81config HANDLE_DOMAIN_IRQ 85config HANDLE_DOMAIN_IRQ
82 bool 86 bool
83 87
88config IRQ_TIMINGS
89 bool
90
84config IRQ_DOMAIN_DEBUG 91config IRQ_DOMAIN_DEBUG
85 bool "Expose hardware/virtual IRQ mapping via debugfs" 92 bool "Expose hardware/virtual IRQ mapping via debugfs"
86 depends on IRQ_DOMAIN && DEBUG_FS 93 depends on IRQ_DOMAIN && DEBUG_FS
@@ -108,4 +115,15 @@ config SPARSE_IRQ
108 115
109 If you don't know what to do here, say N. 116 If you don't know what to do here, say N.
110 117
118config GENERIC_IRQ_DEBUGFS
119 bool "Expose irq internals in debugfs"
120 depends on DEBUG_FS
121 default n
122 ---help---
123
124 Exposes internal state information through debugfs. Mostly for
125 developers and debugging of hard to diagnose interrupt problems.
126
127 If you don't know what to do here, say N.
128
111endmenu 129endmenu
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 1d3ee3169202..e4aef7351f2b 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -1,5 +1,6 @@
1 1
2obj-y := irqdesc.o handle.o manage.o spurious.o resend.o chip.o dummychip.o devres.o 2obj-y := irqdesc.o handle.o manage.o spurious.o resend.o chip.o dummychip.o devres.o
3obj-$(CONFIG_IRQ_TIMINGS) += timings.o
3obj-$(CONFIG_GENERIC_IRQ_CHIP) += generic-chip.o 4obj-$(CONFIG_GENERIC_IRQ_CHIP) += generic-chip.o
4obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o 5obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o
5obj-$(CONFIG_IRQ_DOMAIN) += irqdomain.o 6obj-$(CONFIG_IRQ_DOMAIN) += irqdomain.o
@@ -10,3 +11,4 @@ obj-$(CONFIG_PM_SLEEP) += pm.o
10obj-$(CONFIG_GENERIC_MSI_IRQ) += msi.o 11obj-$(CONFIG_GENERIC_MSI_IRQ) += msi.o
11obj-$(CONFIG_GENERIC_IRQ_IPI) += ipi.o 12obj-$(CONFIG_GENERIC_IRQ_IPI) += ipi.o
12obj-$(CONFIG_SMP) += affinity.o 13obj-$(CONFIG_SMP) += affinity.o
14obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o
diff --git a/kernel/irq/affinity.c b/kernel/irq/affinity.c
index e2d356dd7581..d2747f9c5707 100644
--- a/kernel/irq/affinity.c
+++ b/kernel/irq/affinity.c
@@ -1,4 +1,7 @@
1 1/*
2 * Copyright (C) 2016 Thomas Gleixner.
3 * Copyright (C) 2016-2017 Christoph Hellwig.
4 */
2#include <linux/interrupt.h> 5#include <linux/interrupt.h>
3#include <linux/kernel.h> 6#include <linux/kernel.h>
4#include <linux/slab.h> 7#include <linux/slab.h>
@@ -35,13 +38,54 @@ static void irq_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk,
35 } 38 }
36} 39}
37 40
38static int get_nodes_in_cpumask(const struct cpumask *mask, nodemask_t *nodemsk) 41static cpumask_var_t *alloc_node_to_present_cpumask(void)
42{
43 cpumask_var_t *masks;
44 int node;
45
46 masks = kcalloc(nr_node_ids, sizeof(cpumask_var_t), GFP_KERNEL);
47 if (!masks)
48 return NULL;
49
50 for (node = 0; node < nr_node_ids; node++) {
51 if (!zalloc_cpumask_var(&masks[node], GFP_KERNEL))
52 goto out_unwind;
53 }
54
55 return masks;
56
57out_unwind:
58 while (--node >= 0)
59 free_cpumask_var(masks[node]);
60 kfree(masks);
61 return NULL;
62}
63
64static void free_node_to_present_cpumask(cpumask_var_t *masks)
65{
66 int node;
67
68 for (node = 0; node < nr_node_ids; node++)
69 free_cpumask_var(masks[node]);
70 kfree(masks);
71}
72
73static void build_node_to_present_cpumask(cpumask_var_t *masks)
74{
75 int cpu;
76
77 for_each_present_cpu(cpu)
78 cpumask_set_cpu(cpu, masks[cpu_to_node(cpu)]);
79}
80
81static int get_nodes_in_cpumask(cpumask_var_t *node_to_present_cpumask,
82 const struct cpumask *mask, nodemask_t *nodemsk)
39{ 83{
40 int n, nodes = 0; 84 int n, nodes = 0;
41 85
42 /* Calculate the number of nodes in the supplied affinity mask */ 86 /* Calculate the number of nodes in the supplied affinity mask */
43 for_each_online_node(n) { 87 for_each_node(n) {
44 if (cpumask_intersects(mask, cpumask_of_node(n))) { 88 if (cpumask_intersects(mask, node_to_present_cpumask[n])) {
45 node_set(n, *nodemsk); 89 node_set(n, *nodemsk);
46 nodes++; 90 nodes++;
47 } 91 }
@@ -64,7 +108,7 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
64 int last_affv = affv + affd->pre_vectors; 108 int last_affv = affv + affd->pre_vectors;
65 nodemask_t nodemsk = NODE_MASK_NONE; 109 nodemask_t nodemsk = NODE_MASK_NONE;
66 struct cpumask *masks; 110 struct cpumask *masks;
67 cpumask_var_t nmsk; 111 cpumask_var_t nmsk, *node_to_present_cpumask;
68 112
69 if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL)) 113 if (!zalloc_cpumask_var(&nmsk, GFP_KERNEL))
70 return NULL; 114 return NULL;
@@ -73,13 +117,19 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
73 if (!masks) 117 if (!masks)
74 goto out; 118 goto out;
75 119
120 node_to_present_cpumask = alloc_node_to_present_cpumask();
121 if (!node_to_present_cpumask)
122 goto out;
123
76 /* Fill out vectors at the beginning that don't need affinity */ 124 /* Fill out vectors at the beginning that don't need affinity */
77 for (curvec = 0; curvec < affd->pre_vectors; curvec++) 125 for (curvec = 0; curvec < affd->pre_vectors; curvec++)
78 cpumask_copy(masks + curvec, irq_default_affinity); 126 cpumask_copy(masks + curvec, irq_default_affinity);
79 127
80 /* Stabilize the cpumasks */ 128 /* Stabilize the cpumasks */
81 get_online_cpus(); 129 get_online_cpus();
82 nodes = get_nodes_in_cpumask(cpu_online_mask, &nodemsk); 130 build_node_to_present_cpumask(node_to_present_cpumask);
131 nodes = get_nodes_in_cpumask(node_to_present_cpumask, cpu_present_mask,
132 &nodemsk);
83 133
84 /* 134 /*
85 * If the number of nodes in the mask is greater than or equal the 135 * If the number of nodes in the mask is greater than or equal the
@@ -87,7 +137,8 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
87 */ 137 */
88 if (affv <= nodes) { 138 if (affv <= nodes) {
89 for_each_node_mask(n, nodemsk) { 139 for_each_node_mask(n, nodemsk) {
90 cpumask_copy(masks + curvec, cpumask_of_node(n)); 140 cpumask_copy(masks + curvec,
141 node_to_present_cpumask[n]);
91 if (++curvec == last_affv) 142 if (++curvec == last_affv)
92 break; 143 break;
93 } 144 }
@@ -101,7 +152,7 @@ irq_create_affinity_masks(int nvecs, const struct irq_affinity *affd)
101 vecs_per_node = (affv - (curvec - affd->pre_vectors)) / nodes; 152 vecs_per_node = (affv - (curvec - affd->pre_vectors)) / nodes;
102 153
103 /* Get the cpus on this node which are in the mask */ 154 /* Get the cpus on this node which are in the mask */
104 cpumask_and(nmsk, cpu_online_mask, cpumask_of_node(n)); 155 cpumask_and(nmsk, cpu_present_mask, node_to_present_cpumask[n]);
105 156
106 /* Calculate the number of cpus per vector */ 157 /* Calculate the number of cpus per vector */
107 ncpus = cpumask_weight(nmsk); 158 ncpus = cpumask_weight(nmsk);
@@ -133,6 +184,7 @@ done:
133 /* Fill out vectors at the end that don't need affinity */ 184 /* Fill out vectors at the end that don't need affinity */
134 for (; curvec < nvecs; curvec++) 185 for (; curvec < nvecs; curvec++)
135 cpumask_copy(masks + curvec, irq_default_affinity); 186 cpumask_copy(masks + curvec, irq_default_affinity);
187 free_node_to_present_cpumask(node_to_present_cpumask);
136out: 188out:
137 free_cpumask_var(nmsk); 189 free_cpumask_var(nmsk);
138 return masks; 190 return masks;
@@ -147,12 +199,10 @@ int irq_calc_affinity_vectors(int maxvec, const struct irq_affinity *affd)
147{ 199{
148 int resv = affd->pre_vectors + affd->post_vectors; 200 int resv = affd->pre_vectors + affd->post_vectors;
149 int vecs = maxvec - resv; 201 int vecs = maxvec - resv;
150 int cpus; 202 int ret;
151 203
152 /* Stabilize the cpumasks */
153 get_online_cpus(); 204 get_online_cpus();
154 cpus = cpumask_weight(cpu_online_mask); 205 ret = min_t(int, cpumask_weight(cpu_present_mask), vecs) + resv;
155 put_online_cpus(); 206 put_online_cpus();
156 207 return ret;
157 return min(cpus, vecs) + resv;
158} 208}
diff --git a/kernel/irq/autoprobe.c b/kernel/irq/autoprobe.c
index 0119b9d467ae..d30a0dd5cc02 100644
--- a/kernel/irq/autoprobe.c
+++ b/kernel/irq/autoprobe.c
@@ -53,7 +53,7 @@ unsigned long probe_irq_on(void)
53 if (desc->irq_data.chip->irq_set_type) 53 if (desc->irq_data.chip->irq_set_type)
54 desc->irq_data.chip->irq_set_type(&desc->irq_data, 54 desc->irq_data.chip->irq_set_type(&desc->irq_data,
55 IRQ_TYPE_PROBE); 55 IRQ_TYPE_PROBE);
56 irq_startup(desc, false); 56 irq_startup(desc, IRQ_NORESEND, IRQ_START_FORCE);
57 } 57 }
58 raw_spin_unlock_irq(&desc->lock); 58 raw_spin_unlock_irq(&desc->lock);
59 } 59 }
@@ -70,7 +70,7 @@ unsigned long probe_irq_on(void)
70 raw_spin_lock_irq(&desc->lock); 70 raw_spin_lock_irq(&desc->lock);
71 if (!desc->action && irq_settings_can_probe(desc)) { 71 if (!desc->action && irq_settings_can_probe(desc)) {
72 desc->istate |= IRQS_AUTODETECT | IRQS_WAITING; 72 desc->istate |= IRQS_AUTODETECT | IRQS_WAITING;
73 if (irq_startup(desc, false)) 73 if (irq_startup(desc, IRQ_NORESEND, IRQ_START_FORCE))
74 desc->istate |= IRQS_PENDING; 74 desc->istate |= IRQS_PENDING;
75 } 75 }
76 raw_spin_unlock_irq(&desc->lock); 76 raw_spin_unlock_irq(&desc->lock);
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
index c94da688ee9b..2e30d925a40d 100644
--- a/kernel/irq/chip.c
+++ b/kernel/irq/chip.c
@@ -185,47 +185,162 @@ static void irq_state_set_masked(struct irq_desc *desc)
185 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED); 185 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
186} 186}
187 187
188int irq_startup(struct irq_desc *desc, bool resend) 188static void irq_state_clr_started(struct irq_desc *desc)
189{ 189{
190 int ret = 0; 190 irqd_clear(&desc->irq_data, IRQD_IRQ_STARTED);
191}
191 192
192 irq_state_clr_disabled(desc); 193static void irq_state_set_started(struct irq_desc *desc)
193 desc->depth = 0; 194{
195 irqd_set(&desc->irq_data, IRQD_IRQ_STARTED);
196}
197
198enum {
199 IRQ_STARTUP_NORMAL,
200 IRQ_STARTUP_MANAGED,
201 IRQ_STARTUP_ABORT,
202};
203
204#ifdef CONFIG_SMP
205static int
206__irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
207{
208 struct irq_data *d = irq_desc_get_irq_data(desc);
209
210 if (!irqd_affinity_is_managed(d))
211 return IRQ_STARTUP_NORMAL;
212
213 irqd_clr_managed_shutdown(d);
194 214
195 irq_domain_activate_irq(&desc->irq_data); 215 if (cpumask_any_and(aff, cpu_online_mask) > nr_cpu_ids) {
196 if (desc->irq_data.chip->irq_startup) { 216 /*
197 ret = desc->irq_data.chip->irq_startup(&desc->irq_data); 217 * Catch code which fiddles with enable_irq() on a managed
218 * and potentially shutdown IRQ. Chained interrupt
219 * installment or irq auto probing should not happen on
220 * managed irqs either. Emit a warning, break the affinity
221 * and start it up as a normal interrupt.
222 */
223 if (WARN_ON_ONCE(force))
224 return IRQ_STARTUP_NORMAL;
225 /*
226 * The interrupt was requested, but there is no online CPU
227 * in it's affinity mask. Put it into managed shutdown
228 * state and let the cpu hotplug mechanism start it up once
229 * a CPU in the mask becomes available.
230 */
231 irqd_set_managed_shutdown(d);
232 return IRQ_STARTUP_ABORT;
233 }
234 return IRQ_STARTUP_MANAGED;
235}
236#else
237static int
238__irq_startup_managed(struct irq_desc *desc, struct cpumask *aff, bool force)
239{
240 return IRQ_STARTUP_NORMAL;
241}
242#endif
243
244static int __irq_startup(struct irq_desc *desc)
245{
246 struct irq_data *d = irq_desc_get_irq_data(desc);
247 int ret = 0;
248
249 irq_domain_activate_irq(d);
250 if (d->chip->irq_startup) {
251 ret = d->chip->irq_startup(d);
252 irq_state_clr_disabled(desc);
198 irq_state_clr_masked(desc); 253 irq_state_clr_masked(desc);
199 } else { 254 } else {
200 irq_enable(desc); 255 irq_enable(desc);
201 } 256 }
257 irq_state_set_started(desc);
258 return ret;
259}
260
261int irq_startup(struct irq_desc *desc, bool resend, bool force)
262{
263 struct irq_data *d = irq_desc_get_irq_data(desc);
264 struct cpumask *aff = irq_data_get_affinity_mask(d);
265 int ret = 0;
266
267 desc->depth = 0;
268
269 if (irqd_is_started(d)) {
270 irq_enable(desc);
271 } else {
272 switch (__irq_startup_managed(desc, aff, force)) {
273 case IRQ_STARTUP_NORMAL:
274 ret = __irq_startup(desc);
275 irq_setup_affinity(desc);
276 break;
277 case IRQ_STARTUP_MANAGED:
278 ret = __irq_startup(desc);
279 irq_set_affinity_locked(d, aff, false);
280 break;
281 case IRQ_STARTUP_ABORT:
282 return 0;
283 }
284 }
202 if (resend) 285 if (resend)
203 check_irq_resend(desc); 286 check_irq_resend(desc);
287
204 return ret; 288 return ret;
205} 289}
206 290
291static void __irq_disable(struct irq_desc *desc, bool mask);
292
207void irq_shutdown(struct irq_desc *desc) 293void irq_shutdown(struct irq_desc *desc)
208{ 294{
209 irq_state_set_disabled(desc); 295 if (irqd_is_started(&desc->irq_data)) {
210 desc->depth = 1; 296 desc->depth = 1;
211 if (desc->irq_data.chip->irq_shutdown) 297 if (desc->irq_data.chip->irq_shutdown) {
212 desc->irq_data.chip->irq_shutdown(&desc->irq_data); 298 desc->irq_data.chip->irq_shutdown(&desc->irq_data);
213 else if (desc->irq_data.chip->irq_disable) 299 irq_state_set_disabled(desc);
214 desc->irq_data.chip->irq_disable(&desc->irq_data); 300 irq_state_set_masked(desc);
215 else 301 } else {
216 desc->irq_data.chip->irq_mask(&desc->irq_data); 302 __irq_disable(desc, true);
303 }
304 irq_state_clr_started(desc);
305 }
306 /*
307 * This must be called even if the interrupt was never started up,
308 * because the activation can happen before the interrupt is
309 * available for request/startup. It has it's own state tracking so
310 * it's safe to call it unconditionally.
311 */
217 irq_domain_deactivate_irq(&desc->irq_data); 312 irq_domain_deactivate_irq(&desc->irq_data);
218 irq_state_set_masked(desc);
219} 313}
220 314
221void irq_enable(struct irq_desc *desc) 315void irq_enable(struct irq_desc *desc)
222{ 316{
223 irq_state_clr_disabled(desc); 317 if (!irqd_irq_disabled(&desc->irq_data)) {
224 if (desc->irq_data.chip->irq_enable) 318 unmask_irq(desc);
225 desc->irq_data.chip->irq_enable(&desc->irq_data); 319 } else {
226 else 320 irq_state_clr_disabled(desc);
227 desc->irq_data.chip->irq_unmask(&desc->irq_data); 321 if (desc->irq_data.chip->irq_enable) {
228 irq_state_clr_masked(desc); 322 desc->irq_data.chip->irq_enable(&desc->irq_data);
323 irq_state_clr_masked(desc);
324 } else {
325 unmask_irq(desc);
326 }
327 }
328}
329
330static void __irq_disable(struct irq_desc *desc, bool mask)
331{
332 if (irqd_irq_disabled(&desc->irq_data)) {
333 if (mask)
334 mask_irq(desc);
335 } else {
336 irq_state_set_disabled(desc);
337 if (desc->irq_data.chip->irq_disable) {
338 desc->irq_data.chip->irq_disable(&desc->irq_data);
339 irq_state_set_masked(desc);
340 } else if (mask) {
341 mask_irq(desc);
342 }
343 }
229} 344}
230 345
231/** 346/**
@@ -250,13 +365,7 @@ void irq_enable(struct irq_desc *desc)
250 */ 365 */
251void irq_disable(struct irq_desc *desc) 366void irq_disable(struct irq_desc *desc)
252{ 367{
253 irq_state_set_disabled(desc); 368 __irq_disable(desc, irq_settings_disable_unlazy(desc));
254 if (desc->irq_data.chip->irq_disable) {
255 desc->irq_data.chip->irq_disable(&desc->irq_data);
256 irq_state_set_masked(desc);
257 } else if (irq_settings_disable_unlazy(desc)) {
258 mask_irq(desc);
259 }
260} 369}
261 370
262void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu) 371void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu)
@@ -279,18 +388,21 @@ void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu)
279 388
280static inline void mask_ack_irq(struct irq_desc *desc) 389static inline void mask_ack_irq(struct irq_desc *desc)
281{ 390{
282 if (desc->irq_data.chip->irq_mask_ack) 391 if (desc->irq_data.chip->irq_mask_ack) {
283 desc->irq_data.chip->irq_mask_ack(&desc->irq_data); 392 desc->irq_data.chip->irq_mask_ack(&desc->irq_data);
284 else { 393 irq_state_set_masked(desc);
285 desc->irq_data.chip->irq_mask(&desc->irq_data); 394 } else {
395 mask_irq(desc);
286 if (desc->irq_data.chip->irq_ack) 396 if (desc->irq_data.chip->irq_ack)
287 desc->irq_data.chip->irq_ack(&desc->irq_data); 397 desc->irq_data.chip->irq_ack(&desc->irq_data);
288 } 398 }
289 irq_state_set_masked(desc);
290} 399}
291 400
292void mask_irq(struct irq_desc *desc) 401void mask_irq(struct irq_desc *desc)
293{ 402{
403 if (irqd_irq_masked(&desc->irq_data))
404 return;
405
294 if (desc->irq_data.chip->irq_mask) { 406 if (desc->irq_data.chip->irq_mask) {
295 desc->irq_data.chip->irq_mask(&desc->irq_data); 407 desc->irq_data.chip->irq_mask(&desc->irq_data);
296 irq_state_set_masked(desc); 408 irq_state_set_masked(desc);
@@ -299,6 +411,9 @@ void mask_irq(struct irq_desc *desc)
299 411
300void unmask_irq(struct irq_desc *desc) 412void unmask_irq(struct irq_desc *desc)
301{ 413{
414 if (!irqd_irq_masked(&desc->irq_data))
415 return;
416
302 if (desc->irq_data.chip->irq_unmask) { 417 if (desc->irq_data.chip->irq_unmask) {
303 desc->irq_data.chip->irq_unmask(&desc->irq_data); 418 desc->irq_data.chip->irq_unmask(&desc->irq_data);
304 irq_state_clr_masked(desc); 419 irq_state_clr_masked(desc);
@@ -312,10 +427,7 @@ void unmask_threaded_irq(struct irq_desc *desc)
312 if (chip->flags & IRQCHIP_EOI_THREADED) 427 if (chip->flags & IRQCHIP_EOI_THREADED)
313 chip->irq_eoi(&desc->irq_data); 428 chip->irq_eoi(&desc->irq_data);
314 429
315 if (chip->irq_unmask) { 430 unmask_irq(desc);
316 chip->irq_unmask(&desc->irq_data);
317 irq_state_clr_masked(desc);
318 }
319} 431}
320 432
321/* 433/*
@@ -851,7 +963,7 @@ __irq_do_set_handler(struct irq_desc *desc, irq_flow_handler_t handle,
851 irq_settings_set_norequest(desc); 963 irq_settings_set_norequest(desc);
852 irq_settings_set_nothread(desc); 964 irq_settings_set_nothread(desc);
853 desc->action = &chained_action; 965 desc->action = &chained_action;
854 irq_startup(desc, true); 966 irq_startup(desc, IRQ_RESEND, IRQ_START_FORCE);
855 } 967 }
856} 968}
857 969
@@ -903,6 +1015,13 @@ void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set)
903 1015
904 if (!desc) 1016 if (!desc)
905 return; 1017 return;
1018
1019 /*
1020 * Warn when a driver sets the no autoenable flag on an already
1021 * active interrupt.
1022 */
1023 WARN_ON_ONCE(!desc->depth && (set & _IRQ_NOAUTOEN));
1024
906 irq_settings_clr_and_set(desc, clr, set); 1025 irq_settings_clr_and_set(desc, clr, set);
907 1026
908 irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU | 1027 irqd_clear(&desc->irq_data, IRQD_NO_BALANCING | IRQD_PER_CPU |
diff --git a/kernel/irq/cpuhotplug.c b/kernel/irq/cpuhotplug.c
index 011f8c4c63da..aee8f7ec40af 100644
--- a/kernel/irq/cpuhotplug.c
+++ b/kernel/irq/cpuhotplug.c
@@ -14,37 +14,99 @@
14 14
15#include "internals.h" 15#include "internals.h"
16 16
17/* For !GENERIC_IRQ_EFFECTIVE_AFF_MASK this looks at general affinity mask */
18static inline bool irq_needs_fixup(struct irq_data *d)
19{
20 const struct cpumask *m = irq_data_get_effective_affinity_mask(d);
21
22 return cpumask_test_cpu(smp_processor_id(), m);
23}
24
17static bool migrate_one_irq(struct irq_desc *desc) 25static bool migrate_one_irq(struct irq_desc *desc)
18{ 26{
19 struct irq_data *d = irq_desc_get_irq_data(desc); 27 struct irq_data *d = irq_desc_get_irq_data(desc);
20 const struct cpumask *affinity = d->common->affinity; 28 struct irq_chip *chip = irq_data_get_irq_chip(d);
21 struct irq_chip *c; 29 bool maskchip = !irq_can_move_pcntxt(d) && !irqd_irq_masked(d);
22 bool ret = false; 30 const struct cpumask *affinity;
31 bool brokeaff = false;
32 int err;
23 33
24 /* 34 /*
25 * If this is a per-CPU interrupt, or the affinity does not 35 * IRQ chip might be already torn down, but the irq descriptor is
26 * include this CPU, then we have nothing to do. 36 * still in the radix tree. Also if the chip has no affinity setter,
37 * nothing can be done here.
27 */ 38 */
28 if (irqd_is_per_cpu(d) || 39 if (!chip || !chip->irq_set_affinity) {
29 !cpumask_test_cpu(smp_processor_id(), affinity)) 40 pr_debug("IRQ %u: Unable to migrate away\n", d->irq);
30 return false; 41 return false;
42 }
43
44 /*
45 * No move required, if:
46 * - Interrupt is per cpu
47 * - Interrupt is not started
48 * - Affinity mask does not include this CPU.
49 *
50 * Note: Do not check desc->action as this might be a chained
51 * interrupt.
52 */
53 if (irqd_is_per_cpu(d) || !irqd_is_started(d) || !irq_needs_fixup(d)) {
54 /*
55 * If an irq move is pending, abort it if the dying CPU is
56 * the sole target.
57 */
58 irq_fixup_move_pending(desc, false);
59 return false;
60 }
61
62 /*
63 * Complete an eventually pending irq move cleanup. If this
64 * interrupt was moved in hard irq context, then the vectors need
65 * to be cleaned up. It can't wait until this interrupt actually
66 * happens and this CPU was involved.
67 */
68 irq_force_complete_move(desc);
69
70 /*
71 * If there is a setaffinity pending, then try to reuse the pending
72 * mask, so the last change of the affinity does not get lost. If
73 * there is no move pending or the pending mask does not contain
74 * any online CPU, use the current affinity mask.
75 */
76 if (irq_fixup_move_pending(desc, true))
77 affinity = irq_desc_get_pending_mask(desc);
78 else
79 affinity = irq_data_get_affinity_mask(d);
80
81 /* Mask the chip for interrupts which cannot move in process context */
82 if (maskchip && chip->irq_mask)
83 chip->irq_mask(d);
31 84
32 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) { 85 if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
86 /*
87 * If the interrupt is managed, then shut it down and leave
88 * the affinity untouched.
89 */
90 if (irqd_affinity_is_managed(d)) {
91 irqd_set_managed_shutdown(d);
92 irq_shutdown(desc);
93 return false;
94 }
33 affinity = cpu_online_mask; 95 affinity = cpu_online_mask;
34 ret = true; 96 brokeaff = true;
35 } 97 }
36 98
37 c = irq_data_get_irq_chip(d); 99 err = irq_do_set_affinity(d, affinity, true);
38 if (!c->irq_set_affinity) { 100 if (err) {
39 pr_debug("IRQ%u: unable to set affinity\n", d->irq); 101 pr_warn_ratelimited("IRQ%u: set affinity failed(%d).\n",
40 } else { 102 d->irq, err);
41 int r = irq_do_set_affinity(d, affinity, false); 103 brokeaff = false;
42 if (r)
43 pr_warn_ratelimited("IRQ%u: set affinity failed(%d).\n",
44 d->irq, r);
45 } 104 }
46 105
47 return ret; 106 if (maskchip && chip->irq_unmask)
107 chip->irq_unmask(d);
108
109 return brokeaff;
48} 110}
49 111
50/** 112/**
@@ -59,11 +121,8 @@ static bool migrate_one_irq(struct irq_desc *desc)
59 */ 121 */
60void irq_migrate_all_off_this_cpu(void) 122void irq_migrate_all_off_this_cpu(void)
61{ 123{
62 unsigned int irq;
63 struct irq_desc *desc; 124 struct irq_desc *desc;
64 unsigned long flags; 125 unsigned int irq;
65
66 local_irq_save(flags);
67 126
68 for_each_active_irq(irq) { 127 for_each_active_irq(irq) {
69 bool affinity_broken; 128 bool affinity_broken;
@@ -73,10 +132,53 @@ void irq_migrate_all_off_this_cpu(void)
73 affinity_broken = migrate_one_irq(desc); 132 affinity_broken = migrate_one_irq(desc);
74 raw_spin_unlock(&desc->lock); 133 raw_spin_unlock(&desc->lock);
75 134
76 if (affinity_broken) 135 if (affinity_broken) {
77 pr_warn_ratelimited("IRQ%u no longer affine to CPU%u\n", 136 pr_warn_ratelimited("IRQ %u: no longer affine to CPU%u\n",
78 irq, smp_processor_id()); 137 irq, smp_processor_id());
138 }
139 }
140}
141
142static void irq_restore_affinity_of_irq(struct irq_desc *desc, unsigned int cpu)
143{
144 struct irq_data *data = irq_desc_get_irq_data(desc);
145 const struct cpumask *affinity = irq_data_get_affinity_mask(data);
146
147 if (!irqd_affinity_is_managed(data) || !desc->action ||
148 !irq_data_get_irq_chip(data) || !cpumask_test_cpu(cpu, affinity))
149 return;
150
151 if (irqd_is_managed_and_shutdown(data)) {
152 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
153 return;
154 }
155
156 /*
157 * If the interrupt can only be directed to a single target
158 * CPU then it is already assigned to a CPU in the affinity
159 * mask. No point in trying to move it around.
160 */
161 if (!irqd_is_single_target(data))
162 irq_set_affinity_locked(data, affinity, false);
163}
164
165/**
166 * irq_affinity_online_cpu - Restore affinity for managed interrupts
167 * @cpu: Upcoming CPU for which interrupts should be restored
168 */
169int irq_affinity_online_cpu(unsigned int cpu)
170{
171 struct irq_desc *desc;
172 unsigned int irq;
173
174 irq_lock_sparse();
175 for_each_active_irq(irq) {
176 desc = irq_to_desc(irq);
177 raw_spin_lock_irq(&desc->lock);
178 irq_restore_affinity_of_irq(desc, cpu);
179 raw_spin_unlock_irq(&desc->lock);
79 } 180 }
181 irq_unlock_sparse();
80 182
81 local_irq_restore(flags); 183 return 0;
82} 184}
diff --git a/kernel/irq/debugfs.c b/kernel/irq/debugfs.c
new file mode 100644
index 000000000000..4d384edc0c64
--- /dev/null
+++ b/kernel/irq/debugfs.c
@@ -0,0 +1,213 @@
1/*
2 * Copyright 2017 Thomas Gleixner <tglx@linutronix.de>
3 *
4 * This file is licensed under the GPL V2.
5 */
6#include <linux/irqdomain.h>
7#include <linux/irq.h>
8
9#include "internals.h"
10
11static struct dentry *irq_dir;
12
13struct irq_bit_descr {
14 unsigned int mask;
15 char *name;
16};
17#define BIT_MASK_DESCR(m) { .mask = m, .name = #m }
18
19static void irq_debug_show_bits(struct seq_file *m, int ind, unsigned int state,
20 const struct irq_bit_descr *sd, int size)
21{
22 int i;
23
24 for (i = 0; i < size; i++, sd++) {
25 if (state & sd->mask)
26 seq_printf(m, "%*s%s\n", ind + 12, "", sd->name);
27 }
28}
29
30#ifdef CONFIG_SMP
31static void irq_debug_show_masks(struct seq_file *m, struct irq_desc *desc)
32{
33 struct irq_data *data = irq_desc_get_irq_data(desc);
34 struct cpumask *msk;
35
36 msk = irq_data_get_affinity_mask(data);
37 seq_printf(m, "affinity: %*pbl\n", cpumask_pr_args(msk));
38#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
39 msk = irq_data_get_effective_affinity_mask(data);
40 seq_printf(m, "effectiv: %*pbl\n", cpumask_pr_args(msk));
41#endif
42#ifdef CONFIG_GENERIC_PENDING_IRQ
43 msk = desc->pending_mask;
44 seq_printf(m, "pending: %*pbl\n", cpumask_pr_args(msk));
45#endif
46}
47#else
48static void irq_debug_show_masks(struct seq_file *m, struct irq_desc *desc) { }
49#endif
50
51static const struct irq_bit_descr irqchip_flags[] = {
52 BIT_MASK_DESCR(IRQCHIP_SET_TYPE_MASKED),
53 BIT_MASK_DESCR(IRQCHIP_EOI_IF_HANDLED),
54 BIT_MASK_DESCR(IRQCHIP_MASK_ON_SUSPEND),
55 BIT_MASK_DESCR(IRQCHIP_ONOFFLINE_ENABLED),
56 BIT_MASK_DESCR(IRQCHIP_SKIP_SET_WAKE),
57 BIT_MASK_DESCR(IRQCHIP_ONESHOT_SAFE),
58 BIT_MASK_DESCR(IRQCHIP_EOI_THREADED),
59};
60
61static void
62irq_debug_show_chip(struct seq_file *m, struct irq_data *data, int ind)
63{
64 struct irq_chip *chip = data->chip;
65
66 if (!chip) {
67 seq_printf(m, "chip: None\n");
68 return;
69 }
70 seq_printf(m, "%*schip: %s\n", ind, "", chip->name);
71 seq_printf(m, "%*sflags: 0x%lx\n", ind + 1, "", chip->flags);
72 irq_debug_show_bits(m, ind, chip->flags, irqchip_flags,
73 ARRAY_SIZE(irqchip_flags));
74}
75
76static void
77irq_debug_show_data(struct seq_file *m, struct irq_data *data, int ind)
78{
79 seq_printf(m, "%*sdomain: %s\n", ind, "",
80 data->domain ? data->domain->name : "");
81 seq_printf(m, "%*shwirq: 0x%lx\n", ind + 1, "", data->hwirq);
82 irq_debug_show_chip(m, data, ind + 1);
83#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
84 if (!data->parent_data)
85 return;
86 seq_printf(m, "%*sparent:\n", ind + 1, "");
87 irq_debug_show_data(m, data->parent_data, ind + 4);
88#endif
89}
90
91static const struct irq_bit_descr irqdata_states[] = {
92 BIT_MASK_DESCR(IRQ_TYPE_EDGE_RISING),
93 BIT_MASK_DESCR(IRQ_TYPE_EDGE_FALLING),
94 BIT_MASK_DESCR(IRQ_TYPE_LEVEL_HIGH),
95 BIT_MASK_DESCR(IRQ_TYPE_LEVEL_LOW),
96 BIT_MASK_DESCR(IRQD_LEVEL),
97
98 BIT_MASK_DESCR(IRQD_ACTIVATED),
99 BIT_MASK_DESCR(IRQD_IRQ_STARTED),
100 BIT_MASK_DESCR(IRQD_IRQ_DISABLED),
101 BIT_MASK_DESCR(IRQD_IRQ_MASKED),
102 BIT_MASK_DESCR(IRQD_IRQ_INPROGRESS),
103
104 BIT_MASK_DESCR(IRQD_PER_CPU),
105 BIT_MASK_DESCR(IRQD_NO_BALANCING),
106
107 BIT_MASK_DESCR(IRQD_SINGLE_TARGET),
108 BIT_MASK_DESCR(IRQD_MOVE_PCNTXT),
109 BIT_MASK_DESCR(IRQD_AFFINITY_SET),
110 BIT_MASK_DESCR(IRQD_SETAFFINITY_PENDING),
111 BIT_MASK_DESCR(IRQD_AFFINITY_MANAGED),
112 BIT_MASK_DESCR(IRQD_MANAGED_SHUTDOWN),
113
114 BIT_MASK_DESCR(IRQD_FORWARDED_TO_VCPU),
115
116 BIT_MASK_DESCR(IRQD_WAKEUP_STATE),
117 BIT_MASK_DESCR(IRQD_WAKEUP_ARMED),
118};
119
120static const struct irq_bit_descr irqdesc_states[] = {
121 BIT_MASK_DESCR(_IRQ_NOPROBE),
122 BIT_MASK_DESCR(_IRQ_NOREQUEST),
123 BIT_MASK_DESCR(_IRQ_NOTHREAD),
124 BIT_MASK_DESCR(_IRQ_NOAUTOEN),
125 BIT_MASK_DESCR(_IRQ_NESTED_THREAD),
126 BIT_MASK_DESCR(_IRQ_PER_CPU_DEVID),
127 BIT_MASK_DESCR(_IRQ_IS_POLLED),
128 BIT_MASK_DESCR(_IRQ_DISABLE_UNLAZY),
129};
130
131static const struct irq_bit_descr irqdesc_istates[] = {
132 BIT_MASK_DESCR(IRQS_AUTODETECT),
133 BIT_MASK_DESCR(IRQS_SPURIOUS_DISABLED),
134 BIT_MASK_DESCR(IRQS_POLL_INPROGRESS),
135 BIT_MASK_DESCR(IRQS_ONESHOT),
136 BIT_MASK_DESCR(IRQS_REPLAY),
137 BIT_MASK_DESCR(IRQS_WAITING),
138 BIT_MASK_DESCR(IRQS_PENDING),
139 BIT_MASK_DESCR(IRQS_SUSPENDED),
140};
141
142
143static int irq_debug_show(struct seq_file *m, void *p)
144{
145 struct irq_desc *desc = m->private;
146 struct irq_data *data;
147
148 raw_spin_lock_irq(&desc->lock);
149 data = irq_desc_get_irq_data(desc);
150 seq_printf(m, "handler: %pf\n", desc->handle_irq);
151 seq_printf(m, "status: 0x%08x\n", desc->status_use_accessors);
152 irq_debug_show_bits(m, 0, desc->status_use_accessors, irqdesc_states,
153 ARRAY_SIZE(irqdesc_states));
154 seq_printf(m, "istate: 0x%08x\n", desc->istate);
155 irq_debug_show_bits(m, 0, desc->istate, irqdesc_istates,
156 ARRAY_SIZE(irqdesc_istates));
157 seq_printf(m, "ddepth: %u\n", desc->depth);
158 seq_printf(m, "wdepth: %u\n", desc->wake_depth);
159 seq_printf(m, "dstate: 0x%08x\n", irqd_get(data));
160 irq_debug_show_bits(m, 0, irqd_get(data), irqdata_states,
161 ARRAY_SIZE(irqdata_states));
162 seq_printf(m, "node: %d\n", irq_data_get_node(data));
163 irq_debug_show_masks(m, desc);
164 irq_debug_show_data(m, data, 0);
165 raw_spin_unlock_irq(&desc->lock);
166 return 0;
167}
168
169static int irq_debug_open(struct inode *inode, struct file *file)
170{
171 return single_open(file, irq_debug_show, inode->i_private);
172}
173
174static const struct file_operations dfs_irq_ops = {
175 .open = irq_debug_open,
176 .read = seq_read,
177 .llseek = seq_lseek,
178 .release = single_release,
179};
180
181void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc)
182{
183 char name [10];
184
185 if (!irq_dir || !desc || desc->debugfs_file)
186 return;
187
188 sprintf(name, "%d", irq);
189 desc->debugfs_file = debugfs_create_file(name, 0444, irq_dir, desc,
190 &dfs_irq_ops);
191}
192
193static int __init irq_debugfs_init(void)
194{
195 struct dentry *root_dir;
196 int irq;
197
198 root_dir = debugfs_create_dir("irq", NULL);
199 if (!root_dir)
200 return -ENOMEM;
201
202 irq_domain_debugfs_init(root_dir);
203
204 irq_dir = debugfs_create_dir("irqs", root_dir);
205
206 irq_lock_sparse();
207 for_each_active_irq(irq)
208 irq_add_debugfs_entry(irq, irq_to_desc(irq));
209 irq_unlock_sparse();
210
211 return 0;
212}
213__initcall(irq_debugfs_init);
diff --git a/kernel/irq/devres.c b/kernel/irq/devres.c
index 1613bfd48365..194c506d9d20 100644
--- a/kernel/irq/devres.c
+++ b/kernel/irq/devres.c
@@ -4,6 +4,8 @@
4#include <linux/gfp.h> 4#include <linux/gfp.h>
5#include <linux/irq.h> 5#include <linux/irq.h>
6 6
7#include "internals.h"
8
7/* 9/*
8 * Device resource management aware IRQ request/free implementation. 10 * Device resource management aware IRQ request/free implementation.
9 */ 11 */
@@ -198,3 +200,87 @@ int __devm_irq_alloc_descs(struct device *dev, int irq, unsigned int from,
198 return base; 200 return base;
199} 201}
200EXPORT_SYMBOL_GPL(__devm_irq_alloc_descs); 202EXPORT_SYMBOL_GPL(__devm_irq_alloc_descs);
203
204#ifdef CONFIG_GENERIC_IRQ_CHIP
205/**
206 * devm_irq_alloc_generic_chip - Allocate and initialize a generic chip
207 * for a managed device
208 * @dev: Device to allocate the generic chip for
209 * @name: Name of the irq chip
210 * @num_ct: Number of irq_chip_type instances associated with this
211 * @irq_base: Interrupt base nr for this chip
212 * @reg_base: Register base address (virtual)
213 * @handler: Default flow handler associated with this chip
214 *
215 * Returns an initialized irq_chip_generic structure. The chip defaults
216 * to the primary (index 0) irq_chip_type and @handler
217 */
218struct irq_chip_generic *
219devm_irq_alloc_generic_chip(struct device *dev, const char *name, int num_ct,
220 unsigned int irq_base, void __iomem *reg_base,
221 irq_flow_handler_t handler)
222{
223 struct irq_chip_generic *gc;
224 unsigned long sz = sizeof(*gc) + num_ct * sizeof(struct irq_chip_type);
225
226 gc = devm_kzalloc(dev, sz, GFP_KERNEL);
227 if (gc)
228 irq_init_generic_chip(gc, name, num_ct,
229 irq_base, reg_base, handler);
230
231 return gc;
232}
233EXPORT_SYMBOL_GPL(devm_irq_alloc_generic_chip);
234
235struct irq_generic_chip_devres {
236 struct irq_chip_generic *gc;
237 u32 msk;
238 unsigned int clr;
239 unsigned int set;
240};
241
242static void devm_irq_remove_generic_chip(struct device *dev, void *res)
243{
244 struct irq_generic_chip_devres *this = res;
245
246 irq_remove_generic_chip(this->gc, this->msk, this->clr, this->set);
247}
248
249/**
250 * devm_irq_setup_generic_chip - Setup a range of interrupts with a generic
251 * chip for a managed device
252 *
253 * @dev: Device to setup the generic chip for
254 * @gc: Generic irq chip holding all data
255 * @msk: Bitmask holding the irqs to initialize relative to gc->irq_base
256 * @flags: Flags for initialization
257 * @clr: IRQ_* bits to clear
258 * @set: IRQ_* bits to set
259 *
260 * Set up max. 32 interrupts starting from gc->irq_base. Note, this
261 * initializes all interrupts to the primary irq_chip_type and its
262 * associated handler.
263 */
264int devm_irq_setup_generic_chip(struct device *dev, struct irq_chip_generic *gc,
265 u32 msk, enum irq_gc_flags flags,
266 unsigned int clr, unsigned int set)
267{
268 struct irq_generic_chip_devres *dr;
269
270 dr = devres_alloc(devm_irq_remove_generic_chip,
271 sizeof(*dr), GFP_KERNEL);
272 if (!dr)
273 return -ENOMEM;
274
275 irq_setup_generic_chip(gc, msk, flags, clr, set);
276
277 dr->gc = gc;
278 dr->msk = msk;
279 dr->clr = clr;
280 dr->set = set;
281 devres_add(dev, dr);
282
283 return 0;
284}
285EXPORT_SYMBOL_GPL(devm_irq_setup_generic_chip);
286#endif /* CONFIG_GENERIC_IRQ_CHIP */
diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c
index ee32870079c9..f7086b78ad6e 100644
--- a/kernel/irq/generic-chip.c
+++ b/kernel/irq/generic-chip.c
@@ -201,10 +201,9 @@ static void irq_writel_be(u32 val, void __iomem *addr)
201 iowrite32be(val, addr); 201 iowrite32be(val, addr);
202} 202}
203 203
204static void 204void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
205irq_init_generic_chip(struct irq_chip_generic *gc, const char *name, 205 int num_ct, unsigned int irq_base,
206 int num_ct, unsigned int irq_base, 206 void __iomem *reg_base, irq_flow_handler_t handler)
207 void __iomem *reg_base, irq_flow_handler_t handler)
208{ 207{
209 raw_spin_lock_init(&gc->lock); 208 raw_spin_lock_init(&gc->lock);
210 gc->num_ct = num_ct; 209 gc->num_ct = num_ct;
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
index d3f24905852c..eb4d3e8945b8 100644
--- a/kernel/irq/handle.c
+++ b/kernel/irq/handle.c
@@ -138,6 +138,8 @@ irqreturn_t __handle_irq_event_percpu(struct irq_desc *desc, unsigned int *flags
138 unsigned int irq = desc->irq_data.irq; 138 unsigned int irq = desc->irq_data.irq;
139 struct irqaction *action; 139 struct irqaction *action;
140 140
141 record_irq_time(desc);
142
141 for_each_action_of_desc(desc, action) { 143 for_each_action_of_desc(desc, action) {
142 irqreturn_t res; 144 irqreturn_t res;
143 145
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h
index bc226e783bd2..9da14d125df4 100644
--- a/kernel/irq/internals.h
+++ b/kernel/irq/internals.h
@@ -8,6 +8,7 @@
8#include <linux/irqdesc.h> 8#include <linux/irqdesc.h>
9#include <linux/kernel_stat.h> 9#include <linux/kernel_stat.h>
10#include <linux/pm_runtime.h> 10#include <linux/pm_runtime.h>
11#include <linux/sched/clock.h>
11 12
12#ifdef CONFIG_SPARSE_IRQ 13#ifdef CONFIG_SPARSE_IRQ
13# define IRQ_BITMAP_BITS (NR_IRQS + 8196) 14# define IRQ_BITMAP_BITS (NR_IRQS + 8196)
@@ -57,6 +58,7 @@ enum {
57 IRQS_WAITING = 0x00000080, 58 IRQS_WAITING = 0x00000080,
58 IRQS_PENDING = 0x00000200, 59 IRQS_PENDING = 0x00000200,
59 IRQS_SUSPENDED = 0x00000800, 60 IRQS_SUSPENDED = 0x00000800,
61 IRQS_TIMINGS = 0x00001000,
60}; 62};
61 63
62#include "debug.h" 64#include "debug.h"
@@ -66,7 +68,14 @@ extern int __irq_set_trigger(struct irq_desc *desc, unsigned long flags);
66extern void __disable_irq(struct irq_desc *desc); 68extern void __disable_irq(struct irq_desc *desc);
67extern void __enable_irq(struct irq_desc *desc); 69extern void __enable_irq(struct irq_desc *desc);
68 70
69extern int irq_startup(struct irq_desc *desc, bool resend); 71#define IRQ_RESEND true
72#define IRQ_NORESEND false
73
74#define IRQ_START_FORCE true
75#define IRQ_START_COND false
76
77extern int irq_startup(struct irq_desc *desc, bool resend, bool force);
78
70extern void irq_shutdown(struct irq_desc *desc); 79extern void irq_shutdown(struct irq_desc *desc);
71extern void irq_enable(struct irq_desc *desc); 80extern void irq_enable(struct irq_desc *desc);
72extern void irq_disable(struct irq_desc *desc); 81extern void irq_disable(struct irq_desc *desc);
@@ -109,13 +118,19 @@ static inline void unregister_handler_proc(unsigned int irq,
109 118
110extern bool irq_can_set_affinity_usr(unsigned int irq); 119extern bool irq_can_set_affinity_usr(unsigned int irq);
111 120
112extern int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask); 121extern int irq_select_affinity_usr(unsigned int irq);
113 122
114extern void irq_set_thread_affinity(struct irq_desc *desc); 123extern void irq_set_thread_affinity(struct irq_desc *desc);
115 124
116extern int irq_do_set_affinity(struct irq_data *data, 125extern int irq_do_set_affinity(struct irq_data *data,
117 const struct cpumask *dest, bool force); 126 const struct cpumask *dest, bool force);
118 127
128#ifdef CONFIG_SMP
129extern int irq_setup_affinity(struct irq_desc *desc);
130#else
131static inline int irq_setup_affinity(struct irq_desc *desc) { return 0; }
132#endif
133
119/* Inline functions for support of irq chips on slow busses */ 134/* Inline functions for support of irq chips on slow busses */
120static inline void chip_bus_lock(struct irq_desc *desc) 135static inline void chip_bus_lock(struct irq_desc *desc)
121{ 136{
@@ -169,6 +184,11 @@ irq_put_desc_unlock(struct irq_desc *desc, unsigned long flags)
169 184
170#define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors) 185#define __irqd_to_state(d) ACCESS_PRIVATE((d)->common, state_use_accessors)
171 186
187static inline unsigned int irqd_get(struct irq_data *d)
188{
189 return __irqd_to_state(d);
190}
191
172/* 192/*
173 * Manipulation functions for irq_data.state 193 * Manipulation functions for irq_data.state
174 */ 194 */
@@ -182,6 +202,16 @@ static inline void irqd_clr_move_pending(struct irq_data *d)
182 __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING; 202 __irqd_to_state(d) &= ~IRQD_SETAFFINITY_PENDING;
183} 203}
184 204
205static inline void irqd_set_managed_shutdown(struct irq_data *d)
206{
207 __irqd_to_state(d) |= IRQD_MANAGED_SHUTDOWN;
208}
209
210static inline void irqd_clr_managed_shutdown(struct irq_data *d)
211{
212 __irqd_to_state(d) &= ~IRQD_MANAGED_SHUTDOWN;
213}
214
185static inline void irqd_clear(struct irq_data *d, unsigned int mask) 215static inline void irqd_clear(struct irq_data *d, unsigned int mask)
186{ 216{
187 __irqd_to_state(d) &= ~mask; 217 __irqd_to_state(d) &= ~mask;
@@ -226,3 +256,194 @@ irq_pm_install_action(struct irq_desc *desc, struct irqaction *action) { }
226static inline void 256static inline void
227irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { } 257irq_pm_remove_action(struct irq_desc *desc, struct irqaction *action) { }
228#endif 258#endif
259
260#ifdef CONFIG_IRQ_TIMINGS
261
262#define IRQ_TIMINGS_SHIFT 5
263#define IRQ_TIMINGS_SIZE (1 << IRQ_TIMINGS_SHIFT)
264#define IRQ_TIMINGS_MASK (IRQ_TIMINGS_SIZE - 1)
265
266/**
267 * struct irq_timings - irq timings storing structure
268 * @values: a circular buffer of u64 encoded <timestamp,irq> values
269 * @count: the number of elements in the array
270 */
271struct irq_timings {
272 u64 values[IRQ_TIMINGS_SIZE];
273 int count;
274};
275
276DECLARE_PER_CPU(struct irq_timings, irq_timings);
277
278extern void irq_timings_free(int irq);
279extern int irq_timings_alloc(int irq);
280
281static inline void irq_remove_timings(struct irq_desc *desc)
282{
283 desc->istate &= ~IRQS_TIMINGS;
284
285 irq_timings_free(irq_desc_get_irq(desc));
286}
287
288static inline void irq_setup_timings(struct irq_desc *desc, struct irqaction *act)
289{
290 int irq = irq_desc_get_irq(desc);
291 int ret;
292
293 /*
294 * We don't need the measurement because the idle code already
295 * knows the next expiry event.
296 */
297 if (act->flags & __IRQF_TIMER)
298 return;
299
300 /*
301 * In case the timing allocation fails, we just want to warn,
302 * not fail, so letting the system boot anyway.
303 */
304 ret = irq_timings_alloc(irq);
305 if (ret) {
306 pr_warn("Failed to allocate irq timing stats for irq%d (%d)",
307 irq, ret);
308 return;
309 }
310
311 desc->istate |= IRQS_TIMINGS;
312}
313
314extern void irq_timings_enable(void);
315extern void irq_timings_disable(void);
316
317DECLARE_STATIC_KEY_FALSE(irq_timing_enabled);
318
319/*
320 * The interrupt number and the timestamp are encoded into a single
321 * u64 variable to optimize the size.
322 * 48 bit time stamp and 16 bit IRQ number is way sufficient.
323 * Who cares an IRQ after 78 hours of idle time?
324 */
325static inline u64 irq_timing_encode(u64 timestamp, int irq)
326{
327 return (timestamp << 16) | irq;
328}
329
330static inline int irq_timing_decode(u64 value, u64 *timestamp)
331{
332 *timestamp = value >> 16;
333 return value & U16_MAX;
334}
335
336/*
337 * The function record_irq_time is only called in one place in the
338 * interrupts handler. We want this function always inline so the code
339 * inside is embedded in the function and the static key branching
340 * code can act at the higher level. Without the explicit
341 * __always_inline we can end up with a function call and a small
342 * overhead in the hotpath for nothing.
343 */
344static __always_inline void record_irq_time(struct irq_desc *desc)
345{
346 if (!static_branch_likely(&irq_timing_enabled))
347 return;
348
349 if (desc->istate & IRQS_TIMINGS) {
350 struct irq_timings *timings = this_cpu_ptr(&irq_timings);
351
352 timings->values[timings->count & IRQ_TIMINGS_MASK] =
353 irq_timing_encode(local_clock(),
354 irq_desc_get_irq(desc));
355
356 timings->count++;
357 }
358}
359#else
360static inline void irq_remove_timings(struct irq_desc *desc) {}
361static inline void irq_setup_timings(struct irq_desc *desc,
362 struct irqaction *act) {};
363static inline void record_irq_time(struct irq_desc *desc) {}
364#endif /* CONFIG_IRQ_TIMINGS */
365
366
367#ifdef CONFIG_GENERIC_IRQ_CHIP
368void irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
369 int num_ct, unsigned int irq_base,
370 void __iomem *reg_base, irq_flow_handler_t handler);
371#else
372static inline void
373irq_init_generic_chip(struct irq_chip_generic *gc, const char *name,
374 int num_ct, unsigned int irq_base,
375 void __iomem *reg_base, irq_flow_handler_t handler) { }
376#endif /* CONFIG_GENERIC_IRQ_CHIP */
377
378#ifdef CONFIG_GENERIC_PENDING_IRQ
379static inline bool irq_can_move_pcntxt(struct irq_data *data)
380{
381 return irqd_can_move_in_process_context(data);
382}
383static inline bool irq_move_pending(struct irq_data *data)
384{
385 return irqd_is_setaffinity_pending(data);
386}
387static inline void
388irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
389{
390 cpumask_copy(desc->pending_mask, mask);
391}
392static inline void
393irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
394{
395 cpumask_copy(mask, desc->pending_mask);
396}
397static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
398{
399 return desc->pending_mask;
400}
401bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear);
402#else /* CONFIG_GENERIC_PENDING_IRQ */
403static inline bool irq_can_move_pcntxt(struct irq_data *data)
404{
405 return true;
406}
407static inline bool irq_move_pending(struct irq_data *data)
408{
409 return false;
410}
411static inline void
412irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
413{
414}
415static inline void
416irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
417{
418}
419static inline struct cpumask *irq_desc_get_pending_mask(struct irq_desc *desc)
420{
421 return NULL;
422}
423static inline bool irq_fixup_move_pending(struct irq_desc *desc, bool fclear)
424{
425 return false;
426}
427#endif /* !CONFIG_GENERIC_PENDING_IRQ */
428
429#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
430#include <linux/debugfs.h>
431
432void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *desc);
433static inline void irq_remove_debugfs_entry(struct irq_desc *desc)
434{
435 debugfs_remove(desc->debugfs_file);
436}
437# ifdef CONFIG_IRQ_DOMAIN
438void irq_domain_debugfs_init(struct dentry *root);
439# else
440static inline void irq_domain_debugfs_init(struct dentry *root);
441# endif
442#else /* CONFIG_GENERIC_IRQ_DEBUGFS */
443static inline void irq_add_debugfs_entry(unsigned int irq, struct irq_desc *d)
444{
445}
446static inline void irq_remove_debugfs_entry(struct irq_desc *d)
447{
448}
449#endif /* CONFIG_GENERIC_IRQ_DEBUGFS */
diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index 00bb0aeea1d0..948b50e78549 100644
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -54,14 +54,25 @@ static void __init init_irq_default_affinity(void)
54#endif 54#endif
55 55
56#ifdef CONFIG_SMP 56#ifdef CONFIG_SMP
57static int alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) 57static int alloc_masks(struct irq_desc *desc, int node)
58{ 58{
59 if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity, 59 if (!zalloc_cpumask_var_node(&desc->irq_common_data.affinity,
60 gfp, node)) 60 GFP_KERNEL, node))
61 return -ENOMEM; 61 return -ENOMEM;
62 62
63#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
64 if (!zalloc_cpumask_var_node(&desc->irq_common_data.effective_affinity,
65 GFP_KERNEL, node)) {
66 free_cpumask_var(desc->irq_common_data.affinity);
67 return -ENOMEM;
68 }
69#endif
70
63#ifdef CONFIG_GENERIC_PENDING_IRQ 71#ifdef CONFIG_GENERIC_PENDING_IRQ
64 if (!zalloc_cpumask_var_node(&desc->pending_mask, gfp, node)) { 72 if (!zalloc_cpumask_var_node(&desc->pending_mask, GFP_KERNEL, node)) {
73#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
74 free_cpumask_var(desc->irq_common_data.effective_affinity);
75#endif
65 free_cpumask_var(desc->irq_common_data.affinity); 76 free_cpumask_var(desc->irq_common_data.affinity);
66 return -ENOMEM; 77 return -ENOMEM;
67 } 78 }
@@ -86,7 +97,7 @@ static void desc_smp_init(struct irq_desc *desc, int node,
86 97
87#else 98#else
88static inline int 99static inline int
89alloc_masks(struct irq_desc *desc, gfp_t gfp, int node) { return 0; } 100alloc_masks(struct irq_desc *desc, int node) { return 0; }
90static inline void 101static inline void
91desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { } 102desc_smp_init(struct irq_desc *desc, int node, const struct cpumask *affinity) { }
92#endif 103#endif
@@ -105,6 +116,7 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node,
105 desc->irq_data.chip_data = NULL; 116 desc->irq_data.chip_data = NULL;
106 irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS); 117 irq_settings_clr_and_set(desc, ~0, _IRQ_DEFAULT_INIT_FLAGS);
107 irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED); 118 irqd_set(&desc->irq_data, IRQD_IRQ_DISABLED);
119 irqd_set(&desc->irq_data, IRQD_IRQ_MASKED);
108 desc->handle_irq = handle_bad_irq; 120 desc->handle_irq = handle_bad_irq;
109 desc->depth = 1; 121 desc->depth = 1;
110 desc->irq_count = 0; 122 desc->irq_count = 0;
@@ -324,6 +336,9 @@ static void free_masks(struct irq_desc *desc)
324 free_cpumask_var(desc->pending_mask); 336 free_cpumask_var(desc->pending_mask);
325#endif 337#endif
326 free_cpumask_var(desc->irq_common_data.affinity); 338 free_cpumask_var(desc->irq_common_data.affinity);
339#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
340 free_cpumask_var(desc->irq_common_data.effective_affinity);
341#endif
327} 342}
328#else 343#else
329static inline void free_masks(struct irq_desc *desc) { } 344static inline void free_masks(struct irq_desc *desc) { }
@@ -344,9 +359,8 @@ static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
344 struct module *owner) 359 struct module *owner)
345{ 360{
346 struct irq_desc *desc; 361 struct irq_desc *desc;
347 gfp_t gfp = GFP_KERNEL;
348 362
349 desc = kzalloc_node(sizeof(*desc), gfp, node); 363 desc = kzalloc_node(sizeof(*desc), GFP_KERNEL, node);
350 if (!desc) 364 if (!desc)
351 return NULL; 365 return NULL;
352 /* allocate based on nr_cpu_ids */ 366 /* allocate based on nr_cpu_ids */
@@ -354,7 +368,7 @@ static struct irq_desc *alloc_desc(int irq, int node, unsigned int flags,
354 if (!desc->kstat_irqs) 368 if (!desc->kstat_irqs)
355 goto err_desc; 369 goto err_desc;
356 370
357 if (alloc_masks(desc, gfp, node)) 371 if (alloc_masks(desc, node))
358 goto err_kstat; 372 goto err_kstat;
359 373
360 raw_spin_lock_init(&desc->lock); 374 raw_spin_lock_init(&desc->lock);
@@ -394,6 +408,7 @@ static void free_desc(unsigned int irq)
394{ 408{
395 struct irq_desc *desc = irq_to_desc(irq); 409 struct irq_desc *desc = irq_to_desc(irq);
396 410
411 irq_remove_debugfs_entry(desc);
397 unregister_irq_proc(irq, desc); 412 unregister_irq_proc(irq, desc);
398 413
399 /* 414 /*
@@ -480,7 +495,8 @@ int __init early_irq_init(void)
480 495
481 /* Let arch update nr_irqs and return the nr of preallocated irqs */ 496 /* Let arch update nr_irqs and return the nr of preallocated irqs */
482 initcnt = arch_probe_nr_irqs(); 497 initcnt = arch_probe_nr_irqs();
483 printk(KERN_INFO "NR_IRQS:%d nr_irqs:%d %d\n", NR_IRQS, nr_irqs, initcnt); 498 printk(KERN_INFO "NR_IRQS: %d, nr_irqs: %d, preallocated irqs: %d\n",
499 NR_IRQS, nr_irqs, initcnt);
484 500
485 if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS)) 501 if (WARN_ON(nr_irqs > IRQ_BITMAP_BITS))
486 nr_irqs = IRQ_BITMAP_BITS; 502 nr_irqs = IRQ_BITMAP_BITS;
@@ -516,14 +532,14 @@ int __init early_irq_init(void)
516 532
517 init_irq_default_affinity(); 533 init_irq_default_affinity();
518 534
519 printk(KERN_INFO "NR_IRQS:%d\n", NR_IRQS); 535 printk(KERN_INFO "NR_IRQS: %d\n", NR_IRQS);
520 536
521 desc = irq_desc; 537 desc = irq_desc;
522 count = ARRAY_SIZE(irq_desc); 538 count = ARRAY_SIZE(irq_desc);
523 539
524 for (i = 0; i < count; i++) { 540 for (i = 0; i < count; i++) {
525 desc[i].kstat_irqs = alloc_percpu(unsigned int); 541 desc[i].kstat_irqs = alloc_percpu(unsigned int);
526 alloc_masks(&desc[i], GFP_KERNEL, node); 542 alloc_masks(&desc[i], node);
527 raw_spin_lock_init(&desc[i].lock); 543 raw_spin_lock_init(&desc[i].lock);
528 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class); 544 lockdep_set_class(&desc[i].lock, &irq_desc_lock_class);
529 desc_set_defaults(i, &desc[i], node, NULL, NULL); 545 desc_set_defaults(i, &desc[i], node, NULL, NULL);
diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c
index 31805f237396..14fe862aa2e3 100644
--- a/kernel/irq/irqdomain.c
+++ b/kernel/irq/irqdomain.c
@@ -26,39 +26,69 @@ static struct irq_domain *irq_default_domain;
26static void irq_domain_check_hierarchy(struct irq_domain *domain); 26static void irq_domain_check_hierarchy(struct irq_domain *domain);
27 27
28struct irqchip_fwid { 28struct irqchip_fwid {
29 struct fwnode_handle fwnode; 29 struct fwnode_handle fwnode;
30 char *name; 30 unsigned int type;
31 char *name;
31 void *data; 32 void *data;
32}; 33};
33 34
35#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
36static void debugfs_add_domain_dir(struct irq_domain *d);
37static void debugfs_remove_domain_dir(struct irq_domain *d);
38#else
39static inline void debugfs_add_domain_dir(struct irq_domain *d) { }
40static inline void debugfs_remove_domain_dir(struct irq_domain *d) { }
41#endif
42
34/** 43/**
35 * irq_domain_alloc_fwnode - Allocate a fwnode_handle suitable for 44 * irq_domain_alloc_fwnode - Allocate a fwnode_handle suitable for
36 * identifying an irq domain 45 * identifying an irq domain
37 * @data: optional user-provided data 46 * @type: Type of irqchip_fwnode. See linux/irqdomain.h
47 * @name: Optional user provided domain name
48 * @id: Optional user provided id if name != NULL
49 * @data: Optional user-provided data
38 * 50 *
39 * Allocate a struct device_node, and return a poiner to the embedded 51 * Allocate a struct irqchip_fwid, and return a poiner to the embedded
40 * fwnode_handle (or NULL on failure). 52 * fwnode_handle (or NULL on failure).
53 *
54 * Note: The types IRQCHIP_FWNODE_NAMED and IRQCHIP_FWNODE_NAMED_ID are
55 * solely to transport name information to irqdomain creation code. The
56 * node is not stored. For other types the pointer is kept in the irq
57 * domain struct.
41 */ 58 */
42struct fwnode_handle *irq_domain_alloc_fwnode(void *data) 59struct fwnode_handle *__irq_domain_alloc_fwnode(unsigned int type, int id,
60 const char *name, void *data)
43{ 61{
44 struct irqchip_fwid *fwid; 62 struct irqchip_fwid *fwid;
45 char *name; 63 char *n;
46 64
47 fwid = kzalloc(sizeof(*fwid), GFP_KERNEL); 65 fwid = kzalloc(sizeof(*fwid), GFP_KERNEL);
48 name = kasprintf(GFP_KERNEL, "irqchip@%p", data);
49 66
50 if (!fwid || !name) { 67 switch (type) {
68 case IRQCHIP_FWNODE_NAMED:
69 n = kasprintf(GFP_KERNEL, "%s", name);
70 break;
71 case IRQCHIP_FWNODE_NAMED_ID:
72 n = kasprintf(GFP_KERNEL, "%s-%d", name, id);
73 break;
74 default:
75 n = kasprintf(GFP_KERNEL, "irqchip@%p", data);
76 break;
77 }
78
79 if (!fwid || !n) {
51 kfree(fwid); 80 kfree(fwid);
52 kfree(name); 81 kfree(n);
53 return NULL; 82 return NULL;
54 } 83 }
55 84
56 fwid->name = name; 85 fwid->type = type;
86 fwid->name = n;
57 fwid->data = data; 87 fwid->data = data;
58 fwid->fwnode.type = FWNODE_IRQCHIP; 88 fwid->fwnode.type = FWNODE_IRQCHIP;
59 return &fwid->fwnode; 89 return &fwid->fwnode;
60} 90}
61EXPORT_SYMBOL_GPL(irq_domain_alloc_fwnode); 91EXPORT_SYMBOL_GPL(__irq_domain_alloc_fwnode);
62 92
63/** 93/**
64 * irq_domain_free_fwnode - Free a non-OF-backed fwnode_handle 94 * irq_domain_free_fwnode - Free a non-OF-backed fwnode_handle
@@ -97,26 +127,82 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size,
97 void *host_data) 127 void *host_data)
98{ 128{
99 struct device_node *of_node = to_of_node(fwnode); 129 struct device_node *of_node = to_of_node(fwnode);
130 struct irqchip_fwid *fwid;
100 struct irq_domain *domain; 131 struct irq_domain *domain;
101 132
133 static atomic_t unknown_domains;
134
102 domain = kzalloc_node(sizeof(*domain) + (sizeof(unsigned int) * size), 135 domain = kzalloc_node(sizeof(*domain) + (sizeof(unsigned int) * size),
103 GFP_KERNEL, of_node_to_nid(of_node)); 136 GFP_KERNEL, of_node_to_nid(of_node));
104 if (WARN_ON(!domain)) 137 if (WARN_ON(!domain))
105 return NULL; 138 return NULL;
106 139
140 if (fwnode && is_fwnode_irqchip(fwnode)) {
141 fwid = container_of(fwnode, struct irqchip_fwid, fwnode);
142
143 switch (fwid->type) {
144 case IRQCHIP_FWNODE_NAMED:
145 case IRQCHIP_FWNODE_NAMED_ID:
146 domain->name = kstrdup(fwid->name, GFP_KERNEL);
147 if (!domain->name) {
148 kfree(domain);
149 return NULL;
150 }
151 domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
152 break;
153 default:
154 domain->fwnode = fwnode;
155 domain->name = fwid->name;
156 break;
157 }
158 } else if (of_node) {
159 char *name;
160
161 /*
162 * DT paths contain '/', which debugfs is legitimately
163 * unhappy about. Replace them with ':', which does
164 * the trick and is not as offensive as '\'...
165 */
166 name = kstrdup(of_node_full_name(of_node), GFP_KERNEL);
167 if (!name) {
168 kfree(domain);
169 return NULL;
170 }
171
172 strreplace(name, '/', ':');
173
174 domain->name = name;
175 domain->fwnode = fwnode;
176 domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
177 }
178
179 if (!domain->name) {
180 if (fwnode) {
181 pr_err("Invalid fwnode type (%d) for irqdomain\n",
182 fwnode->type);
183 }
184 domain->name = kasprintf(GFP_KERNEL, "unknown-%d",
185 atomic_inc_return(&unknown_domains));
186 if (!domain->name) {
187 kfree(domain);
188 return NULL;
189 }
190 domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
191 }
192
107 of_node_get(of_node); 193 of_node_get(of_node);
108 194
109 /* Fill structure */ 195 /* Fill structure */
110 INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL); 196 INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL);
111 domain->ops = ops; 197 domain->ops = ops;
112 domain->host_data = host_data; 198 domain->host_data = host_data;
113 domain->fwnode = fwnode;
114 domain->hwirq_max = hwirq_max; 199 domain->hwirq_max = hwirq_max;
115 domain->revmap_size = size; 200 domain->revmap_size = size;
116 domain->revmap_direct_max_irq = direct_max; 201 domain->revmap_direct_max_irq = direct_max;
117 irq_domain_check_hierarchy(domain); 202 irq_domain_check_hierarchy(domain);
118 203
119 mutex_lock(&irq_domain_mutex); 204 mutex_lock(&irq_domain_mutex);
205 debugfs_add_domain_dir(domain);
120 list_add(&domain->link, &irq_domain_list); 206 list_add(&domain->link, &irq_domain_list);
121 mutex_unlock(&irq_domain_mutex); 207 mutex_unlock(&irq_domain_mutex);
122 208
@@ -136,6 +222,7 @@ EXPORT_SYMBOL_GPL(__irq_domain_add);
136void irq_domain_remove(struct irq_domain *domain) 222void irq_domain_remove(struct irq_domain *domain)
137{ 223{
138 mutex_lock(&irq_domain_mutex); 224 mutex_lock(&irq_domain_mutex);
225 debugfs_remove_domain_dir(domain);
139 226
140 WARN_ON(!radix_tree_empty(&domain->revmap_tree)); 227 WARN_ON(!radix_tree_empty(&domain->revmap_tree));
141 228
@@ -152,10 +239,43 @@ void irq_domain_remove(struct irq_domain *domain)
152 pr_debug("Removed domain %s\n", domain->name); 239 pr_debug("Removed domain %s\n", domain->name);
153 240
154 of_node_put(irq_domain_get_of_node(domain)); 241 of_node_put(irq_domain_get_of_node(domain));
242 if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED)
243 kfree(domain->name);
155 kfree(domain); 244 kfree(domain);
156} 245}
157EXPORT_SYMBOL_GPL(irq_domain_remove); 246EXPORT_SYMBOL_GPL(irq_domain_remove);
158 247
248void irq_domain_update_bus_token(struct irq_domain *domain,
249 enum irq_domain_bus_token bus_token)
250{
251 char *name;
252
253 if (domain->bus_token == bus_token)
254 return;
255
256 mutex_lock(&irq_domain_mutex);
257
258 domain->bus_token = bus_token;
259
260 name = kasprintf(GFP_KERNEL, "%s-%d", domain->name, bus_token);
261 if (!name) {
262 mutex_unlock(&irq_domain_mutex);
263 return;
264 }
265
266 debugfs_remove_domain_dir(domain);
267
268 if (domain->flags & IRQ_DOMAIN_NAME_ALLOCATED)
269 kfree(domain->name);
270 else
271 domain->flags |= IRQ_DOMAIN_NAME_ALLOCATED;
272
273 domain->name = name;
274 debugfs_add_domain_dir(domain);
275
276 mutex_unlock(&irq_domain_mutex);
277}
278
159/** 279/**
160 * irq_domain_add_simple() - Register an irq_domain and optionally map a range of irqs 280 * irq_domain_add_simple() - Register an irq_domain and optionally map a range of irqs
161 * @of_node: pointer to interrupt controller's device tree node. 281 * @of_node: pointer to interrupt controller's device tree node.
@@ -344,6 +464,7 @@ void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq)
344 464
345 irq_data->domain = NULL; 465 irq_data->domain = NULL;
346 irq_data->hwirq = 0; 466 irq_data->hwirq = 0;
467 domain->mapcount--;
347 468
348 /* Clear reverse map for this hwirq */ 469 /* Clear reverse map for this hwirq */
349 if (hwirq < domain->revmap_size) { 470 if (hwirq < domain->revmap_size) {
@@ -395,6 +516,7 @@ int irq_domain_associate(struct irq_domain *domain, unsigned int virq,
395 domain->name = irq_data->chip->name; 516 domain->name = irq_data->chip->name;
396 } 517 }
397 518
519 domain->mapcount++;
398 if (hwirq < domain->revmap_size) { 520 if (hwirq < domain->revmap_size) {
399 domain->linear_revmap[hwirq] = virq; 521 domain->linear_revmap[hwirq] = virq;
400 } else { 522 } else {
@@ -746,13 +868,54 @@ unsigned int irq_find_mapping(struct irq_domain *domain,
746EXPORT_SYMBOL_GPL(irq_find_mapping); 868EXPORT_SYMBOL_GPL(irq_find_mapping);
747 869
748#ifdef CONFIG_IRQ_DOMAIN_DEBUG 870#ifdef CONFIG_IRQ_DOMAIN_DEBUG
871static void virq_debug_show_one(struct seq_file *m, struct irq_desc *desc)
872{
873 struct irq_domain *domain;
874 struct irq_data *data;
875
876 domain = desc->irq_data.domain;
877 data = &desc->irq_data;
878
879 while (domain) {
880 unsigned int irq = data->irq;
881 unsigned long hwirq = data->hwirq;
882 struct irq_chip *chip;
883 bool direct;
884
885 if (data == &desc->irq_data)
886 seq_printf(m, "%5d ", irq);
887 else
888 seq_printf(m, "%5d+ ", irq);
889 seq_printf(m, "0x%05lx ", hwirq);
890
891 chip = irq_data_get_irq_chip(data);
892 seq_printf(m, "%-15s ", (chip && chip->name) ? chip->name : "none");
893
894 seq_printf(m, data ? "0x%p " : " %p ",
895 irq_data_get_irq_chip_data(data));
896
897 seq_printf(m, " %c ", (desc->action && desc->action->handler) ? '*' : ' ');
898 direct = (irq == hwirq) && (irq < domain->revmap_direct_max_irq);
899 seq_printf(m, "%6s%-8s ",
900 (hwirq < domain->revmap_size) ? "LINEAR" : "RADIX",
901 direct ? "(DIRECT)" : "");
902 seq_printf(m, "%s\n", domain->name);
903#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
904 domain = domain->parent;
905 data = data->parent_data;
906#else
907 domain = NULL;
908#endif
909 }
910}
911
749static int virq_debug_show(struct seq_file *m, void *private) 912static int virq_debug_show(struct seq_file *m, void *private)
750{ 913{
751 unsigned long flags; 914 unsigned long flags;
752 struct irq_desc *desc; 915 struct irq_desc *desc;
753 struct irq_domain *domain; 916 struct irq_domain *domain;
754 struct radix_tree_iter iter; 917 struct radix_tree_iter iter;
755 void *data, **slot; 918 void **slot;
756 int i; 919 int i;
757 920
758 seq_printf(m, " %-16s %-6s %-10s %-10s %s\n", 921 seq_printf(m, " %-16s %-6s %-10s %-10s %s\n",
@@ -760,15 +923,26 @@ static int virq_debug_show(struct seq_file *m, void *private)
760 mutex_lock(&irq_domain_mutex); 923 mutex_lock(&irq_domain_mutex);
761 list_for_each_entry(domain, &irq_domain_list, link) { 924 list_for_each_entry(domain, &irq_domain_list, link) {
762 struct device_node *of_node; 925 struct device_node *of_node;
926 const char *name;
927
763 int count = 0; 928 int count = 0;
929
764 of_node = irq_domain_get_of_node(domain); 930 of_node = irq_domain_get_of_node(domain);
931 if (of_node)
932 name = of_node_full_name(of_node);
933 else if (is_fwnode_irqchip(domain->fwnode))
934 name = container_of(domain->fwnode, struct irqchip_fwid,
935 fwnode)->name;
936 else
937 name = "";
938
765 radix_tree_for_each_slot(slot, &domain->revmap_tree, &iter, 0) 939 radix_tree_for_each_slot(slot, &domain->revmap_tree, &iter, 0)
766 count++; 940 count++;
767 seq_printf(m, "%c%-16s %6u %10u %10u %s\n", 941 seq_printf(m, "%c%-16s %6u %10u %10u %s\n",
768 domain == irq_default_domain ? '*' : ' ', domain->name, 942 domain == irq_default_domain ? '*' : ' ', domain->name,
769 domain->revmap_size + count, domain->revmap_size, 943 domain->revmap_size + count, domain->revmap_size,
770 domain->revmap_direct_max_irq, 944 domain->revmap_direct_max_irq,
771 of_node ? of_node_full_name(of_node) : ""); 945 name);
772 } 946 }
773 mutex_unlock(&irq_domain_mutex); 947 mutex_unlock(&irq_domain_mutex);
774 948
@@ -782,30 +956,7 @@ static int virq_debug_show(struct seq_file *m, void *private)
782 continue; 956 continue;
783 957
784 raw_spin_lock_irqsave(&desc->lock, flags); 958 raw_spin_lock_irqsave(&desc->lock, flags);
785 domain = desc->irq_data.domain; 959 virq_debug_show_one(m, desc);
786
787 if (domain) {
788 struct irq_chip *chip;
789 int hwirq = desc->irq_data.hwirq;
790 bool direct;
791
792 seq_printf(m, "%5d ", i);
793 seq_printf(m, "0x%05x ", hwirq);
794
795 chip = irq_desc_get_chip(desc);
796 seq_printf(m, "%-15s ", (chip && chip->name) ? chip->name : "none");
797
798 data = irq_desc_get_chip_data(desc);
799 seq_printf(m, data ? "0x%p " : " %p ", data);
800
801 seq_printf(m, " %c ", (desc->action && desc->action->handler) ? '*' : ' ');
802 direct = (i == hwirq) && (i < domain->revmap_direct_max_irq);
803 seq_printf(m, "%6s%-8s ",
804 (hwirq < domain->revmap_size) ? "LINEAR" : "RADIX",
805 direct ? "(DIRECT)" : "");
806 seq_printf(m, "%s\n", desc->irq_data.domain->name);
807 }
808
809 raw_spin_unlock_irqrestore(&desc->lock, flags); 960 raw_spin_unlock_irqrestore(&desc->lock, flags);
810 } 961 }
811 962
@@ -973,6 +1124,7 @@ static void irq_domain_insert_irq(int virq)
973 struct irq_domain *domain = data->domain; 1124 struct irq_domain *domain = data->domain;
974 irq_hw_number_t hwirq = data->hwirq; 1125 irq_hw_number_t hwirq = data->hwirq;
975 1126
1127 domain->mapcount++;
976 if (hwirq < domain->revmap_size) { 1128 if (hwirq < domain->revmap_size) {
977 domain->linear_revmap[hwirq] = virq; 1129 domain->linear_revmap[hwirq] = virq;
978 } else { 1130 } else {
@@ -1002,6 +1154,7 @@ static void irq_domain_remove_irq(int virq)
1002 struct irq_domain *domain = data->domain; 1154 struct irq_domain *domain = data->domain;
1003 irq_hw_number_t hwirq = data->hwirq; 1155 irq_hw_number_t hwirq = data->hwirq;
1004 1156
1157 domain->mapcount--;
1005 if (hwirq < domain->revmap_size) { 1158 if (hwirq < domain->revmap_size) {
1006 domain->linear_revmap[hwirq] = 0; 1159 domain->linear_revmap[hwirq] = 0;
1007 } else { 1160 } else {
@@ -1189,43 +1342,18 @@ void irq_domain_free_irqs_top(struct irq_domain *domain, unsigned int virq,
1189 irq_domain_free_irqs_common(domain, virq, nr_irqs); 1342 irq_domain_free_irqs_common(domain, virq, nr_irqs);
1190} 1343}
1191 1344
1192static bool irq_domain_is_auto_recursive(struct irq_domain *domain) 1345static void irq_domain_free_irqs_hierarchy(struct irq_domain *domain,
1193{
1194 return domain->flags & IRQ_DOMAIN_FLAG_AUTO_RECURSIVE;
1195}
1196
1197static void irq_domain_free_irqs_recursive(struct irq_domain *domain,
1198 unsigned int irq_base, 1346 unsigned int irq_base,
1199 unsigned int nr_irqs) 1347 unsigned int nr_irqs)
1200{ 1348{
1201 domain->ops->free(domain, irq_base, nr_irqs); 1349 domain->ops->free(domain, irq_base, nr_irqs);
1202 if (irq_domain_is_auto_recursive(domain)) {
1203 BUG_ON(!domain->parent);
1204 irq_domain_free_irqs_recursive(domain->parent, irq_base,
1205 nr_irqs);
1206 }
1207} 1350}
1208 1351
1209int irq_domain_alloc_irqs_recursive(struct irq_domain *domain, 1352int irq_domain_alloc_irqs_hierarchy(struct irq_domain *domain,
1210 unsigned int irq_base, 1353 unsigned int irq_base,
1211 unsigned int nr_irqs, void *arg) 1354 unsigned int nr_irqs, void *arg)
1212{ 1355{
1213 int ret = 0; 1356 return domain->ops->alloc(domain, irq_base, nr_irqs, arg);
1214 struct irq_domain *parent = domain->parent;
1215 bool recursive = irq_domain_is_auto_recursive(domain);
1216
1217 BUG_ON(recursive && !parent);
1218 if (recursive)
1219 ret = irq_domain_alloc_irqs_recursive(parent, irq_base,
1220 nr_irqs, arg);
1221 if (ret < 0)
1222 return ret;
1223
1224 ret = domain->ops->alloc(domain, irq_base, nr_irqs, arg);
1225 if (ret < 0 && recursive)
1226 irq_domain_free_irqs_recursive(parent, irq_base, nr_irqs);
1227
1228 return ret;
1229} 1357}
1230 1358
1231/** 1359/**
@@ -1286,7 +1414,7 @@ int __irq_domain_alloc_irqs(struct irq_domain *domain, int irq_base,
1286 } 1414 }
1287 1415
1288 mutex_lock(&irq_domain_mutex); 1416 mutex_lock(&irq_domain_mutex);
1289 ret = irq_domain_alloc_irqs_recursive(domain, virq, nr_irqs, arg); 1417 ret = irq_domain_alloc_irqs_hierarchy(domain, virq, nr_irqs, arg);
1290 if (ret < 0) { 1418 if (ret < 0) {
1291 mutex_unlock(&irq_domain_mutex); 1419 mutex_unlock(&irq_domain_mutex);
1292 goto out_free_irq_data; 1420 goto out_free_irq_data;
@@ -1321,7 +1449,7 @@ void irq_domain_free_irqs(unsigned int virq, unsigned int nr_irqs)
1321 mutex_lock(&irq_domain_mutex); 1449 mutex_lock(&irq_domain_mutex);
1322 for (i = 0; i < nr_irqs; i++) 1450 for (i = 0; i < nr_irqs; i++)
1323 irq_domain_remove_irq(virq + i); 1451 irq_domain_remove_irq(virq + i);
1324 irq_domain_free_irqs_recursive(data->domain, virq, nr_irqs); 1452 irq_domain_free_irqs_hierarchy(data->domain, virq, nr_irqs);
1325 mutex_unlock(&irq_domain_mutex); 1453 mutex_unlock(&irq_domain_mutex);
1326 1454
1327 irq_domain_free_irq_data(virq, nr_irqs); 1455 irq_domain_free_irq_data(virq, nr_irqs);
@@ -1341,15 +1469,11 @@ int irq_domain_alloc_irqs_parent(struct irq_domain *domain,
1341 unsigned int irq_base, unsigned int nr_irqs, 1469 unsigned int irq_base, unsigned int nr_irqs,
1342 void *arg) 1470 void *arg)
1343{ 1471{
1344 /* irq_domain_alloc_irqs_recursive() has called parent's alloc() */ 1472 if (!domain->parent)
1345 if (irq_domain_is_auto_recursive(domain)) 1473 return -ENOSYS;
1346 return 0;
1347 1474
1348 domain = domain->parent; 1475 return irq_domain_alloc_irqs_hierarchy(domain->parent, irq_base,
1349 if (domain) 1476 nr_irqs, arg);
1350 return irq_domain_alloc_irqs_recursive(domain, irq_base,
1351 nr_irqs, arg);
1352 return -ENOSYS;
1353} 1477}
1354EXPORT_SYMBOL_GPL(irq_domain_alloc_irqs_parent); 1478EXPORT_SYMBOL_GPL(irq_domain_alloc_irqs_parent);
1355 1479
@@ -1364,10 +1488,10 @@ EXPORT_SYMBOL_GPL(irq_domain_alloc_irqs_parent);
1364void irq_domain_free_irqs_parent(struct irq_domain *domain, 1488void irq_domain_free_irqs_parent(struct irq_domain *domain,
1365 unsigned int irq_base, unsigned int nr_irqs) 1489 unsigned int irq_base, unsigned int nr_irqs)
1366{ 1490{
1367 /* irq_domain_free_irqs_recursive() will call parent's free */ 1491 if (!domain->parent)
1368 if (!irq_domain_is_auto_recursive(domain) && domain->parent) 1492 return;
1369 irq_domain_free_irqs_recursive(domain->parent, irq_base, 1493
1370 nr_irqs); 1494 irq_domain_free_irqs_hierarchy(domain->parent, irq_base, nr_irqs);
1371} 1495}
1372EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent); 1496EXPORT_SYMBOL_GPL(irq_domain_free_irqs_parent);
1373 1497
@@ -1487,3 +1611,78 @@ static void irq_domain_check_hierarchy(struct irq_domain *domain)
1487{ 1611{
1488} 1612}
1489#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */ 1613#endif /* CONFIG_IRQ_DOMAIN_HIERARCHY */
1614
1615#ifdef CONFIG_GENERIC_IRQ_DEBUGFS
1616static struct dentry *domain_dir;
1617
1618static void
1619irq_domain_debug_show_one(struct seq_file *m, struct irq_domain *d, int ind)
1620{
1621 seq_printf(m, "%*sname: %s\n", ind, "", d->name);
1622 seq_printf(m, "%*ssize: %u\n", ind + 1, "",
1623 d->revmap_size + d->revmap_direct_max_irq);
1624 seq_printf(m, "%*smapped: %u\n", ind + 1, "", d->mapcount);
1625 seq_printf(m, "%*sflags: 0x%08x\n", ind +1 , "", d->flags);
1626#ifdef CONFIG_IRQ_DOMAIN_HIERARCHY
1627 if (!d->parent)
1628 return;
1629 seq_printf(m, "%*sparent: %s\n", ind + 1, "", d->parent->name);
1630 irq_domain_debug_show_one(m, d->parent, ind + 4);
1631#endif
1632}
1633
1634static int irq_domain_debug_show(struct seq_file *m, void *p)
1635{
1636 struct irq_domain *d = m->private;
1637
1638 /* Default domain? Might be NULL */
1639 if (!d) {
1640 if (!irq_default_domain)
1641 return 0;
1642 d = irq_default_domain;
1643 }
1644 irq_domain_debug_show_one(m, d, 0);
1645 return 0;
1646}
1647
1648static int irq_domain_debug_open(struct inode *inode, struct file *file)
1649{
1650 return single_open(file, irq_domain_debug_show, inode->i_private);
1651}
1652
1653static const struct file_operations dfs_domain_ops = {
1654 .open = irq_domain_debug_open,
1655 .read = seq_read,
1656 .llseek = seq_lseek,
1657 .release = single_release,
1658};
1659
1660static void debugfs_add_domain_dir(struct irq_domain *d)
1661{
1662 if (!d->name || !domain_dir || d->debugfs_file)
1663 return;
1664 d->debugfs_file = debugfs_create_file(d->name, 0444, domain_dir, d,
1665 &dfs_domain_ops);
1666}
1667
1668static void debugfs_remove_domain_dir(struct irq_domain *d)
1669{
1670 if (d->debugfs_file)
1671 debugfs_remove(d->debugfs_file);
1672}
1673
1674void __init irq_domain_debugfs_init(struct dentry *root)
1675{
1676 struct irq_domain *d;
1677
1678 domain_dir = debugfs_create_dir("domains", root);
1679 if (!domain_dir)
1680 return;
1681
1682 debugfs_create_file("default", 0444, domain_dir, NULL, &dfs_domain_ops);
1683 mutex_lock(&irq_domain_mutex);
1684 list_for_each_entry(d, &irq_domain_list, link)
1685 debugfs_add_domain_dir(d);
1686 mutex_unlock(&irq_domain_mutex);
1687}
1688#endif
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 425170d4439b..5c11c1730ba5 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -168,34 +168,6 @@ void irq_set_thread_affinity(struct irq_desc *desc)
168 set_bit(IRQTF_AFFINITY, &action->thread_flags); 168 set_bit(IRQTF_AFFINITY, &action->thread_flags);
169} 169}
170 170
171#ifdef CONFIG_GENERIC_PENDING_IRQ
172static inline bool irq_can_move_pcntxt(struct irq_data *data)
173{
174 return irqd_can_move_in_process_context(data);
175}
176static inline bool irq_move_pending(struct irq_data *data)
177{
178 return irqd_is_setaffinity_pending(data);
179}
180static inline void
181irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask)
182{
183 cpumask_copy(desc->pending_mask, mask);
184}
185static inline void
186irq_get_pending(struct cpumask *mask, struct irq_desc *desc)
187{
188 cpumask_copy(mask, desc->pending_mask);
189}
190#else
191static inline bool irq_can_move_pcntxt(struct irq_data *data) { return true; }
192static inline bool irq_move_pending(struct irq_data *data) { return false; }
193static inline void
194irq_copy_pending(struct irq_desc *desc, const struct cpumask *mask) { }
195static inline void
196irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { }
197#endif
198
199int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, 171int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask,
200 bool force) 172 bool force)
201{ 173{
@@ -345,15 +317,18 @@ EXPORT_SYMBOL_GPL(irq_set_affinity_notifier);
345/* 317/*
346 * Generic version of the affinity autoselector. 318 * Generic version of the affinity autoselector.
347 */ 319 */
348static int setup_affinity(struct irq_desc *desc, struct cpumask *mask) 320int irq_setup_affinity(struct irq_desc *desc)
349{ 321{
350 struct cpumask *set = irq_default_affinity; 322 struct cpumask *set = irq_default_affinity;
351 int node = irq_desc_get_node(desc); 323 int ret, node = irq_desc_get_node(desc);
324 static DEFINE_RAW_SPINLOCK(mask_lock);
325 static struct cpumask mask;
352 326
353 /* Excludes PER_CPU and NO_BALANCE interrupts */ 327 /* Excludes PER_CPU and NO_BALANCE interrupts */
354 if (!__irq_can_set_affinity(desc)) 328 if (!__irq_can_set_affinity(desc))
355 return 0; 329 return 0;
356 330
331 raw_spin_lock(&mask_lock);
357 /* 332 /*
358 * Preserve the managed affinity setting and a userspace affinity 333 * Preserve the managed affinity setting and a userspace affinity
359 * setup, but make sure that one of the targets is online. 334 * setup, but make sure that one of the targets is online.
@@ -367,46 +342,40 @@ static int setup_affinity(struct irq_desc *desc, struct cpumask *mask)
367 irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET); 342 irqd_clear(&desc->irq_data, IRQD_AFFINITY_SET);
368 } 343 }
369 344
370 cpumask_and(mask, cpu_online_mask, set); 345 cpumask_and(&mask, cpu_online_mask, set);
371 if (node != NUMA_NO_NODE) { 346 if (node != NUMA_NO_NODE) {
372 const struct cpumask *nodemask = cpumask_of_node(node); 347 const struct cpumask *nodemask = cpumask_of_node(node);
373 348
374 /* make sure at least one of the cpus in nodemask is online */ 349 /* make sure at least one of the cpus in nodemask is online */
375 if (cpumask_intersects(mask, nodemask)) 350 if (cpumask_intersects(&mask, nodemask))
376 cpumask_and(mask, mask, nodemask); 351 cpumask_and(&mask, &mask, nodemask);
377 } 352 }
378 irq_do_set_affinity(&desc->irq_data, mask, false); 353 ret = irq_do_set_affinity(&desc->irq_data, &mask, false);
379 return 0; 354 raw_spin_unlock(&mask_lock);
355 return ret;
380} 356}
381#else 357#else
382/* Wrapper for ALPHA specific affinity selector magic */ 358/* Wrapper for ALPHA specific affinity selector magic */
383static inline int setup_affinity(struct irq_desc *d, struct cpumask *mask) 359int irq_setup_affinity(struct irq_desc *desc)
384{ 360{
385 return irq_select_affinity(irq_desc_get_irq(d)); 361 return irq_select_affinity(irq_desc_get_irq(desc));
386} 362}
387#endif 363#endif
388 364
389/* 365/*
390 * Called when affinity is set via /proc/irq 366 * Called when a bogus affinity is set via /proc/irq
391 */ 367 */
392int irq_select_affinity_usr(unsigned int irq, struct cpumask *mask) 368int irq_select_affinity_usr(unsigned int irq)
393{ 369{
394 struct irq_desc *desc = irq_to_desc(irq); 370 struct irq_desc *desc = irq_to_desc(irq);
395 unsigned long flags; 371 unsigned long flags;
396 int ret; 372 int ret;
397 373
398 raw_spin_lock_irqsave(&desc->lock, flags); 374 raw_spin_lock_irqsave(&desc->lock, flags);
399 ret = setup_affinity(desc, mask); 375 ret = irq_setup_affinity(desc);
400 raw_spin_unlock_irqrestore(&desc->lock, flags); 376 raw_spin_unlock_irqrestore(&desc->lock, flags);
401 return ret; 377 return ret;
402} 378}
403
404#else
405static inline int
406setup_affinity(struct irq_desc *desc, struct cpumask *mask)
407{
408 return 0;
409}
410#endif 379#endif
411 380
412/** 381/**
@@ -533,9 +502,15 @@ void __enable_irq(struct irq_desc *desc)
533 goto err_out; 502 goto err_out;
534 /* Prevent probing on this irq: */ 503 /* Prevent probing on this irq: */
535 irq_settings_set_noprobe(desc); 504 irq_settings_set_noprobe(desc);
536 irq_enable(desc); 505 /*
537 check_irq_resend(desc); 506 * Call irq_startup() not irq_enable() here because the
538 /* fall-through */ 507 * interrupt might be marked NOAUTOEN. So irq_startup()
508 * needs to be invoked when it gets enabled the first
509 * time. If it was already started up, then irq_startup()
510 * will invoke irq_enable() under the hood.
511 */
512 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
513 break;
539 } 514 }
540 default: 515 default:
541 desc->depth--; 516 desc->depth--;
@@ -1122,7 +1097,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1122 struct irqaction *old, **old_ptr; 1097 struct irqaction *old, **old_ptr;
1123 unsigned long flags, thread_mask = 0; 1098 unsigned long flags, thread_mask = 0;
1124 int ret, nested, shared = 0; 1099 int ret, nested, shared = 0;
1125 cpumask_var_t mask;
1126 1100
1127 if (!desc) 1101 if (!desc)
1128 return -EINVAL; 1102 return -EINVAL;
@@ -1181,11 +1155,6 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1181 } 1155 }
1182 } 1156 }
1183 1157
1184 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) {
1185 ret = -ENOMEM;
1186 goto out_thread;
1187 }
1188
1189 /* 1158 /*
1190 * Drivers are often written to work w/o knowledge about the 1159 * Drivers are often written to work w/o knowledge about the
1191 * underlying irq chip implementation, so a request for a 1160 * underlying irq chip implementation, so a request for a
@@ -1250,7 +1219,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1250 */ 1219 */
1251 if (thread_mask == ~0UL) { 1220 if (thread_mask == ~0UL) {
1252 ret = -EBUSY; 1221 ret = -EBUSY;
1253 goto out_mask; 1222 goto out_unlock;
1254 } 1223 }
1255 /* 1224 /*
1256 * The thread_mask for the action is or'ed to 1225 * The thread_mask for the action is or'ed to
@@ -1294,7 +1263,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1294 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n", 1263 pr_err("Threaded irq requested with handler=NULL and !ONESHOT for irq %d\n",
1295 irq); 1264 irq);
1296 ret = -EINVAL; 1265 ret = -EINVAL;
1297 goto out_mask; 1266 goto out_unlock;
1298 } 1267 }
1299 1268
1300 if (!shared) { 1269 if (!shared) {
@@ -1302,7 +1271,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1302 if (ret) { 1271 if (ret) {
1303 pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n", 1272 pr_err("Failed to request resources for %s (irq %d) on irqchip %s\n",
1304 new->name, irq, desc->irq_data.chip->name); 1273 new->name, irq, desc->irq_data.chip->name);
1305 goto out_mask; 1274 goto out_unlock;
1306 } 1275 }
1307 1276
1308 init_waitqueue_head(&desc->wait_for_threads); 1277 init_waitqueue_head(&desc->wait_for_threads);
@@ -1314,7 +1283,7 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1314 1283
1315 if (ret) { 1284 if (ret) {
1316 irq_release_resources(desc); 1285 irq_release_resources(desc);
1317 goto out_mask; 1286 goto out_unlock;
1318 } 1287 }
1319 } 1288 }
1320 1289
@@ -1330,20 +1299,25 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1330 if (new->flags & IRQF_ONESHOT) 1299 if (new->flags & IRQF_ONESHOT)
1331 desc->istate |= IRQS_ONESHOT; 1300 desc->istate |= IRQS_ONESHOT;
1332 1301
1333 if (irq_settings_can_autoenable(desc))
1334 irq_startup(desc, true);
1335 else
1336 /* Undo nested disables: */
1337 desc->depth = 1;
1338
1339 /* Exclude IRQ from balancing if requested */ 1302 /* Exclude IRQ from balancing if requested */
1340 if (new->flags & IRQF_NOBALANCING) { 1303 if (new->flags & IRQF_NOBALANCING) {
1341 irq_settings_set_no_balancing(desc); 1304 irq_settings_set_no_balancing(desc);
1342 irqd_set(&desc->irq_data, IRQD_NO_BALANCING); 1305 irqd_set(&desc->irq_data, IRQD_NO_BALANCING);
1343 } 1306 }
1344 1307
1345 /* Set default affinity mask once everything is setup */ 1308 if (irq_settings_can_autoenable(desc)) {
1346 setup_affinity(desc, mask); 1309 irq_startup(desc, IRQ_RESEND, IRQ_START_COND);
1310 } else {
1311 /*
1312 * Shared interrupts do not go well with disabling
1313 * auto enable. The sharing interrupt might request
1314 * it while it's still disabled and then wait for
1315 * interrupts forever.
1316 */
1317 WARN_ON_ONCE(new->flags & IRQF_SHARED);
1318 /* Undo nested disables: */
1319 desc->depth = 1;
1320 }
1347 1321
1348 } else if (new->flags & IRQF_TRIGGER_MASK) { 1322 } else if (new->flags & IRQF_TRIGGER_MASK) {
1349 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK; 1323 unsigned int nmsk = new->flags & IRQF_TRIGGER_MASK;
@@ -1374,6 +1348,8 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1374 1348
1375 raw_spin_unlock_irqrestore(&desc->lock, flags); 1349 raw_spin_unlock_irqrestore(&desc->lock, flags);
1376 1350
1351 irq_setup_timings(desc, new);
1352
1377 /* 1353 /*
1378 * Strictly no need to wake it up, but hung_task complains 1354 * Strictly no need to wake it up, but hung_task complains
1379 * when no hard interrupt wakes the thread up. 1355 * when no hard interrupt wakes the thread up.
@@ -1384,10 +1360,9 @@ __setup_irq(unsigned int irq, struct irq_desc *desc, struct irqaction *new)
1384 wake_up_process(new->secondary->thread); 1360 wake_up_process(new->secondary->thread);
1385 1361
1386 register_irq_proc(irq, desc); 1362 register_irq_proc(irq, desc);
1363 irq_add_debugfs_entry(irq, desc);
1387 new->dir = NULL; 1364 new->dir = NULL;
1388 register_handler_proc(irq, new); 1365 register_handler_proc(irq, new);
1389 free_cpumask_var(mask);
1390
1391 return 0; 1366 return 0;
1392 1367
1393mismatch: 1368mismatch:
@@ -1400,9 +1375,8 @@ mismatch:
1400 } 1375 }
1401 ret = -EBUSY; 1376 ret = -EBUSY;
1402 1377
1403out_mask: 1378out_unlock:
1404 raw_spin_unlock_irqrestore(&desc->lock, flags); 1379 raw_spin_unlock_irqrestore(&desc->lock, flags);
1405 free_cpumask_var(mask);
1406 1380
1407out_thread: 1381out_thread:
1408 if (new->thread) { 1382 if (new->thread) {
@@ -1502,6 +1476,7 @@ static struct irqaction *__free_irq(unsigned int irq, void *dev_id)
1502 irq_settings_clr_disable_unlazy(desc); 1476 irq_settings_clr_disable_unlazy(desc);
1503 irq_shutdown(desc); 1477 irq_shutdown(desc);
1504 irq_release_resources(desc); 1478 irq_release_resources(desc);
1479 irq_remove_timings(desc);
1505 } 1480 }
1506 1481
1507#ifdef CONFIG_SMP 1482#ifdef CONFIG_SMP
diff --git a/kernel/irq/migration.c b/kernel/irq/migration.c
index 37ddb7bda651..6ca054a3f91d 100644
--- a/kernel/irq/migration.c
+++ b/kernel/irq/migration.c
@@ -4,6 +4,36 @@
4 4
5#include "internals.h" 5#include "internals.h"
6 6
7/**
8 * irq_fixup_move_pending - Cleanup irq move pending from a dying CPU
9 * @desc: Interrupt descpriptor to clean up
10 * @force_clear: If set clear the move pending bit unconditionally.
11 * If not set, clear it only when the dying CPU is the
12 * last one in the pending mask.
13 *
14 * Returns true if the pending bit was set and the pending mask contains an
15 * online CPU other than the dying CPU.
16 */
17bool irq_fixup_move_pending(struct irq_desc *desc, bool force_clear)
18{
19 struct irq_data *data = irq_desc_get_irq_data(desc);
20
21 if (!irqd_is_setaffinity_pending(data))
22 return false;
23
24 /*
25 * The outgoing CPU might be the last online target in a pending
26 * interrupt move. If that's the case clear the pending move bit.
27 */
28 if (cpumask_any_and(desc->pending_mask, cpu_online_mask) >= nr_cpu_ids) {
29 irqd_clr_move_pending(data);
30 return false;
31 }
32 if (force_clear)
33 irqd_clr_move_pending(data);
34 return true;
35}
36
7void irq_move_masked_irq(struct irq_data *idata) 37void irq_move_masked_irq(struct irq_data *idata)
8{ 38{
9 struct irq_desc *desc = irq_data_to_desc(idata); 39 struct irq_desc *desc = irq_data_to_desc(idata);
diff --git a/kernel/irq/msi.c b/kernel/irq/msi.c
index ddc2f5427f75..48eadf416c24 100644
--- a/kernel/irq/msi.c
+++ b/kernel/irq/msi.c
@@ -265,13 +265,20 @@ struct irq_domain *msi_create_irq_domain(struct fwnode_handle *fwnode,
265 struct msi_domain_info *info, 265 struct msi_domain_info *info,
266 struct irq_domain *parent) 266 struct irq_domain *parent)
267{ 267{
268 struct irq_domain *domain;
269
268 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) 270 if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
269 msi_domain_update_dom_ops(info); 271 msi_domain_update_dom_ops(info);
270 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) 272 if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
271 msi_domain_update_chip_ops(info); 273 msi_domain_update_chip_ops(info);
272 274
273 return irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0, 275 domain = irq_domain_create_hierarchy(parent, IRQ_DOMAIN_FLAG_MSI, 0,
274 fwnode, &msi_domain_ops, info); 276 fwnode, &msi_domain_ops, info);
277
278 if (domain && !domain->name && info->chip)
279 domain->name = info->chip->name;
280
281 return domain;
275} 282}
276 283
277int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev, 284int msi_domain_prepare_irqs(struct irq_domain *domain, struct device *dev,
@@ -308,7 +315,7 @@ int msi_domain_populate_irqs(struct irq_domain *domain, struct device *dev,
308 315
309 ops->set_desc(arg, desc); 316 ops->set_desc(arg, desc);
310 /* Assumes the domain mutex is held! */ 317 /* Assumes the domain mutex is held! */
311 ret = irq_domain_alloc_irqs_recursive(domain, virq, 1, arg); 318 ret = irq_domain_alloc_irqs_hierarchy(domain, virq, 1, arg);
312 if (ret) 319 if (ret)
313 break; 320 break;
314 321
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index c53edad7b459..7f9642a1e267 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -37,19 +37,47 @@ static struct proc_dir_entry *root_irq_dir;
37 37
38#ifdef CONFIG_SMP 38#ifdef CONFIG_SMP
39 39
40static int show_irq_affinity(int type, struct seq_file *m, void *v) 40enum {
41 AFFINITY,
42 AFFINITY_LIST,
43 EFFECTIVE,
44 EFFECTIVE_LIST,
45};
46
47static int show_irq_affinity(int type, struct seq_file *m)
41{ 48{
42 struct irq_desc *desc = irq_to_desc((long)m->private); 49 struct irq_desc *desc = irq_to_desc((long)m->private);
43 const struct cpumask *mask = desc->irq_common_data.affinity; 50 const struct cpumask *mask;
44 51
52 switch (type) {
53 case AFFINITY:
54 case AFFINITY_LIST:
55 mask = desc->irq_common_data.affinity;
45#ifdef CONFIG_GENERIC_PENDING_IRQ 56#ifdef CONFIG_GENERIC_PENDING_IRQ
46 if (irqd_is_setaffinity_pending(&desc->irq_data)) 57 if (irqd_is_setaffinity_pending(&desc->irq_data))
47 mask = desc->pending_mask; 58 mask = desc->pending_mask;
48#endif 59#endif
49 if (type) 60 break;
61 case EFFECTIVE:
62 case EFFECTIVE_LIST:
63#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
64 mask = desc->irq_common_data.effective_affinity;
65 break;
66#else
67 return -EINVAL;
68#endif
69 };
70
71 switch (type) {
72 case AFFINITY_LIST:
73 case EFFECTIVE_LIST:
50 seq_printf(m, "%*pbl\n", cpumask_pr_args(mask)); 74 seq_printf(m, "%*pbl\n", cpumask_pr_args(mask));
51 else 75 break;
76 case AFFINITY:
77 case EFFECTIVE:
52 seq_printf(m, "%*pb\n", cpumask_pr_args(mask)); 78 seq_printf(m, "%*pb\n", cpumask_pr_args(mask));
79 break;
80 }
53 return 0; 81 return 0;
54} 82}
55 83
@@ -80,12 +108,12 @@ static int irq_affinity_hint_proc_show(struct seq_file *m, void *v)
80int no_irq_affinity; 108int no_irq_affinity;
81static int irq_affinity_proc_show(struct seq_file *m, void *v) 109static int irq_affinity_proc_show(struct seq_file *m, void *v)
82{ 110{
83 return show_irq_affinity(0, m, v); 111 return show_irq_affinity(AFFINITY, m);
84} 112}
85 113
86static int irq_affinity_list_proc_show(struct seq_file *m, void *v) 114static int irq_affinity_list_proc_show(struct seq_file *m, void *v)
87{ 115{
88 return show_irq_affinity(1, m, v); 116 return show_irq_affinity(AFFINITY_LIST, m);
89} 117}
90 118
91 119
@@ -120,9 +148,11 @@ static ssize_t write_irq_affinity(int type, struct file *file,
120 * one online CPU still has to be targeted. 148 * one online CPU still has to be targeted.
121 */ 149 */
122 if (!cpumask_intersects(new_value, cpu_online_mask)) { 150 if (!cpumask_intersects(new_value, cpu_online_mask)) {
123 /* Special case for empty set - allow the architecture 151 /*
124 code to set default SMP affinity. */ 152 * Special case for empty set - allow the architecture code
125 err = irq_select_affinity_usr(irq, new_value) ? -EINVAL : count; 153 * to set default SMP affinity.
154 */
155 err = irq_select_affinity_usr(irq) ? -EINVAL : count;
126 } else { 156 } else {
127 irq_set_affinity(irq, new_value); 157 irq_set_affinity(irq, new_value);
128 err = count; 158 err = count;
@@ -183,6 +213,44 @@ static const struct file_operations irq_affinity_list_proc_fops = {
183 .write = irq_affinity_list_proc_write, 213 .write = irq_affinity_list_proc_write,
184}; 214};
185 215
216#ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
217static int irq_effective_aff_proc_show(struct seq_file *m, void *v)
218{
219 return show_irq_affinity(EFFECTIVE, m);
220}
221
222static int irq_effective_aff_list_proc_show(struct seq_file *m, void *v)
223{
224 return show_irq_affinity(EFFECTIVE_LIST, m);
225}
226
227static int irq_effective_aff_proc_open(struct inode *inode, struct file *file)
228{
229 return single_open(file, irq_effective_aff_proc_show, PDE_DATA(inode));
230}
231
232static int irq_effective_aff_list_proc_open(struct inode *inode,
233 struct file *file)
234{
235 return single_open(file, irq_effective_aff_list_proc_show,
236 PDE_DATA(inode));
237}
238
239static const struct file_operations irq_effective_aff_proc_fops = {
240 .open = irq_effective_aff_proc_open,
241 .read = seq_read,
242 .llseek = seq_lseek,
243 .release = single_release,
244};
245
246static const struct file_operations irq_effective_aff_list_proc_fops = {
247 .open = irq_effective_aff_list_proc_open,
248 .read = seq_read,
249 .llseek = seq_lseek,
250 .release = single_release,
251};
252#endif
253
186static int default_affinity_show(struct seq_file *m, void *v) 254static int default_affinity_show(struct seq_file *m, void *v)
187{ 255{
188 seq_printf(m, "%*pb\n", cpumask_pr_args(irq_default_affinity)); 256 seq_printf(m, "%*pb\n", cpumask_pr_args(irq_default_affinity));
@@ -324,6 +392,7 @@ void register_handler_proc(unsigned int irq, struct irqaction *action)
324void register_irq_proc(unsigned int irq, struct irq_desc *desc) 392void register_irq_proc(unsigned int irq, struct irq_desc *desc)
325{ 393{
326 static DEFINE_MUTEX(register_lock); 394 static DEFINE_MUTEX(register_lock);
395 void __maybe_unused *irqp = (void *)(unsigned long) irq;
327 char name [MAX_NAMELEN]; 396 char name [MAX_NAMELEN];
328 397
329 if (!root_irq_dir || (desc->irq_data.chip == &no_irq_chip)) 398 if (!root_irq_dir || (desc->irq_data.chip == &no_irq_chip))
@@ -349,20 +418,25 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc)
349#ifdef CONFIG_SMP 418#ifdef CONFIG_SMP
350 /* create /proc/irq/<irq>/smp_affinity */ 419 /* create /proc/irq/<irq>/smp_affinity */
351 proc_create_data("smp_affinity", 0644, desc->dir, 420 proc_create_data("smp_affinity", 0644, desc->dir,
352 &irq_affinity_proc_fops, (void *)(long)irq); 421 &irq_affinity_proc_fops, irqp);
353 422
354 /* create /proc/irq/<irq>/affinity_hint */ 423 /* create /proc/irq/<irq>/affinity_hint */
355 proc_create_data("affinity_hint", 0444, desc->dir, 424 proc_create_data("affinity_hint", 0444, desc->dir,
356 &irq_affinity_hint_proc_fops, (void *)(long)irq); 425 &irq_affinity_hint_proc_fops, irqp);
357 426
358 /* create /proc/irq/<irq>/smp_affinity_list */ 427 /* create /proc/irq/<irq>/smp_affinity_list */
359 proc_create_data("smp_affinity_list", 0644, desc->dir, 428 proc_create_data("smp_affinity_list", 0644, desc->dir,
360 &irq_affinity_list_proc_fops, (void *)(long)irq); 429 &irq_affinity_list_proc_fops, irqp);
361 430
362 proc_create_data("node", 0444, desc->dir, 431 proc_create_data("node", 0444, desc->dir,
363 &irq_node_proc_fops, (void *)(long)irq); 432 &irq_node_proc_fops, irqp);
433# ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
434 proc_create_data("effective_affinity", 0444, desc->dir,
435 &irq_effective_aff_proc_fops, irqp);
436 proc_create_data("effective_affinity_list", 0444, desc->dir,
437 &irq_effective_aff_list_proc_fops, irqp);
438# endif
364#endif 439#endif
365
366 proc_create_data("spurious", 0444, desc->dir, 440 proc_create_data("spurious", 0444, desc->dir,
367 &irq_spurious_proc_fops, (void *)(long)irq); 441 &irq_spurious_proc_fops, (void *)(long)irq);
368 442
@@ -381,6 +455,10 @@ void unregister_irq_proc(unsigned int irq, struct irq_desc *desc)
381 remove_proc_entry("affinity_hint", desc->dir); 455 remove_proc_entry("affinity_hint", desc->dir);
382 remove_proc_entry("smp_affinity_list", desc->dir); 456 remove_proc_entry("smp_affinity_list", desc->dir);
383 remove_proc_entry("node", desc->dir); 457 remove_proc_entry("node", desc->dir);
458# ifdef CONFIG_GENERIC_IRQ_EFFECTIVE_AFF_MASK
459 remove_proc_entry("effective_affinity", desc->dir);
460 remove_proc_entry("effective_affinity_list", desc->dir);
461# endif
384#endif 462#endif
385 remove_proc_entry("spurious", desc->dir); 463 remove_proc_entry("spurious", desc->dir);
386 464
diff --git a/kernel/irq/timings.c b/kernel/irq/timings.c
new file mode 100644
index 000000000000..c8c1d073fbf1
--- /dev/null
+++ b/kernel/irq/timings.c
@@ -0,0 +1,369 @@
1/*
2 * linux/kernel/irq/timings.c
3 *
4 * Copyright (C) 2016, Linaro Ltd - Daniel Lezcano <daniel.lezcano@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 */
11#include <linux/kernel.h>
12#include <linux/percpu.h>
13#include <linux/slab.h>
14#include <linux/static_key.h>
15#include <linux/interrupt.h>
16#include <linux/idr.h>
17#include <linux/irq.h>
18#include <linux/math64.h>
19
20#include <trace/events/irq.h>
21
22#include "internals.h"
23
24DEFINE_STATIC_KEY_FALSE(irq_timing_enabled);
25
26DEFINE_PER_CPU(struct irq_timings, irq_timings);
27
28struct irqt_stat {
29 u64 next_evt;
30 u64 last_ts;
31 u64 variance;
32 u32 avg;
33 u32 nr_samples;
34 int anomalies;
35 int valid;
36};
37
38static DEFINE_IDR(irqt_stats);
39
40void irq_timings_enable(void)
41{
42 static_branch_enable(&irq_timing_enabled);
43}
44
45void irq_timings_disable(void)
46{
47 static_branch_disable(&irq_timing_enabled);
48}
49
50/**
51 * irqs_update - update the irq timing statistics with a new timestamp
52 *
53 * @irqs: an irqt_stat struct pointer
54 * @ts: the new timestamp
55 *
56 * The statistics are computed online, in other words, the code is
57 * designed to compute the statistics on a stream of values rather
58 * than doing multiple passes on the values to compute the average,
59 * then the variance. The integer division introduces a loss of
60 * precision but with an acceptable error margin regarding the results
61 * we would have with the double floating precision: we are dealing
62 * with nanosec, so big numbers, consequently the mantisse is
63 * negligeable, especially when converting the time in usec
64 * afterwards.
65 *
66 * The computation happens at idle time. When the CPU is not idle, the
67 * interrupts' timestamps are stored in the circular buffer, when the
68 * CPU goes idle and this routine is called, all the buffer's values
69 * are injected in the statistical model continuying to extend the
70 * statistics from the previous busy-idle cycle.
71 *
72 * The observations showed a device will trigger a burst of periodic
73 * interrupts followed by one or two peaks of longer time, for
74 * instance when a SD card device flushes its cache, then the periodic
75 * intervals occur again. A one second inactivity period resets the
76 * stats, that gives us the certitude the statistical values won't
77 * exceed 1x10^9, thus the computation won't overflow.
78 *
79 * Basically, the purpose of the algorithm is to watch the periodic
80 * interrupts and eliminate the peaks.
81 *
82 * An interrupt is considered periodically stable if the interval of
83 * its occurences follow the normal distribution, thus the values
84 * comply with:
85 *
86 * avg - 3 x stddev < value < avg + 3 x stddev
87 *
88 * Which can be simplified to:
89 *
90 * -3 x stddev < value - avg < 3 x stddev
91 *
92 * abs(value - avg) < 3 x stddev
93 *
94 * In order to save a costly square root computation, we use the
95 * variance. For the record, stddev = sqrt(variance). The equation
96 * above becomes:
97 *
98 * abs(value - avg) < 3 x sqrt(variance)
99 *
100 * And finally we square it:
101 *
102 * (value - avg) ^ 2 < (3 x sqrt(variance)) ^ 2
103 *
104 * (value - avg) x (value - avg) < 9 x variance
105 *
106 * Statistically speaking, any values out of this interval is
107 * considered as an anomaly and is discarded. However, a normal
108 * distribution appears when the number of samples is 30 (it is the
109 * rule of thumb in statistics, cf. "30 samples" on Internet). When
110 * there are three consecutive anomalies, the statistics are resetted.
111 *
112 */
113static void irqs_update(struct irqt_stat *irqs, u64 ts)
114{
115 u64 old_ts = irqs->last_ts;
116 u64 variance = 0;
117 u64 interval;
118 s64 diff;
119
120 /*
121 * The timestamps are absolute time values, we need to compute
122 * the timing interval between two interrupts.
123 */
124 irqs->last_ts = ts;
125
126 /*
127 * The interval type is u64 in order to deal with the same
128 * type in our computation, that prevent mindfuck issues with
129 * overflow, sign and division.
130 */
131 interval = ts - old_ts;
132
133 /*
134 * The interrupt triggered more than one second apart, that
135 * ends the sequence as predictible for our purpose. In this
136 * case, assume we have the beginning of a sequence and the
137 * timestamp is the first value. As it is impossible to
138 * predict anything at this point, return.
139 *
140 * Note the first timestamp of the sequence will always fall
141 * in this test because the old_ts is zero. That is what we
142 * want as we need another timestamp to compute an interval.
143 */
144 if (interval >= NSEC_PER_SEC) {
145 memset(irqs, 0, sizeof(*irqs));
146 irqs->last_ts = ts;
147 return;
148 }
149
150 /*
151 * Pre-compute the delta with the average as the result is
152 * used several times in this function.
153 */
154 diff = interval - irqs->avg;
155
156 /*
157 * Increment the number of samples.
158 */
159 irqs->nr_samples++;
160
161 /*
162 * Online variance divided by the number of elements if there
163 * is more than one sample. Normally the formula is division
164 * by nr_samples - 1 but we assume the number of element will be
165 * more than 32 and dividing by 32 instead of 31 is enough
166 * precise.
167 */
168 if (likely(irqs->nr_samples > 1))
169 variance = irqs->variance >> IRQ_TIMINGS_SHIFT;
170
171 /*
172 * The rule of thumb in statistics for the normal distribution
173 * is having at least 30 samples in order to have the model to
174 * apply. Values outside the interval are considered as an
175 * anomaly.
176 */
177 if ((irqs->nr_samples >= 30) && ((diff * diff) > (9 * variance))) {
178 /*
179 * After three consecutive anomalies, we reset the
180 * stats as it is no longer stable enough.
181 */
182 if (irqs->anomalies++ >= 3) {
183 memset(irqs, 0, sizeof(*irqs));
184 irqs->last_ts = ts;
185 return;
186 }
187 } else {
188 /*
189 * The anomalies must be consecutives, so at this
190 * point, we reset the anomalies counter.
191 */
192 irqs->anomalies = 0;
193 }
194
195 /*
196 * The interrupt is considered stable enough to try to predict
197 * the next event on it.
198 */
199 irqs->valid = 1;
200
201 /*
202 * Online average algorithm:
203 *
204 * new_average = average + ((value - average) / count)
205 *
206 * The variance computation depends on the new average
207 * to be computed here first.
208 *
209 */
210 irqs->avg = irqs->avg + (diff >> IRQ_TIMINGS_SHIFT);
211
212 /*
213 * Online variance algorithm:
214 *
215 * new_variance = variance + (value - average) x (value - new_average)
216 *
217 * Warning: irqs->avg is updated with the line above, hence
218 * 'interval - irqs->avg' is no longer equal to 'diff'
219 */
220 irqs->variance = irqs->variance + (diff * (interval - irqs->avg));
221
222 /*
223 * Update the next event
224 */
225 irqs->next_evt = ts + irqs->avg;
226}
227
228/**
229 * irq_timings_next_event - Return when the next event is supposed to arrive
230 *
231 * During the last busy cycle, the number of interrupts is incremented
232 * and stored in the irq_timings structure. This information is
233 * necessary to:
234 *
235 * - know if the index in the table wrapped up:
236 *
237 * If more than the array size interrupts happened during the
238 * last busy/idle cycle, the index wrapped up and we have to
239 * begin with the next element in the array which is the last one
240 * in the sequence, otherwise it is a the index 0.
241 *
242 * - have an indication of the interrupts activity on this CPU
243 * (eg. irq/sec)
244 *
245 * The values are 'consumed' after inserting in the statistical model,
246 * thus the count is reinitialized.
247 *
248 * The array of values **must** be browsed in the time direction, the
249 * timestamp must increase between an element and the next one.
250 *
251 * Returns a nanosec time based estimation of the earliest interrupt,
252 * U64_MAX otherwise.
253 */
254u64 irq_timings_next_event(u64 now)
255{
256 struct irq_timings *irqts = this_cpu_ptr(&irq_timings);
257 struct irqt_stat *irqs;
258 struct irqt_stat __percpu *s;
259 u64 ts, next_evt = U64_MAX;
260 int i, irq = 0;
261
262 /*
263 * This function must be called with the local irq disabled in
264 * order to prevent the timings circular buffer to be updated
265 * while we are reading it.
266 */
267 WARN_ON_ONCE(!irqs_disabled());
268
269 /*
270 * Number of elements in the circular buffer: If it happens it
271 * was flushed before, then the number of elements could be
272 * smaller than IRQ_TIMINGS_SIZE, so the count is used,
273 * otherwise the array size is used as we wrapped. The index
274 * begins from zero when we did not wrap. That could be done
275 * in a nicer way with the proper circular array structure
276 * type but with the cost of extra computation in the
277 * interrupt handler hot path. We choose efficiency.
278 *
279 * Inject measured irq/timestamp to the statistical model
280 * while decrementing the counter because we consume the data
281 * from our circular buffer.
282 */
283 for (i = irqts->count & IRQ_TIMINGS_MASK,
284 irqts->count = min(IRQ_TIMINGS_SIZE, irqts->count);
285 irqts->count > 0; irqts->count--, i = (i + 1) & IRQ_TIMINGS_MASK) {
286
287 irq = irq_timing_decode(irqts->values[i], &ts);
288
289 s = idr_find(&irqt_stats, irq);
290 if (s) {
291 irqs = this_cpu_ptr(s);
292 irqs_update(irqs, ts);
293 }
294 }
295
296 /*
297 * Look in the list of interrupts' statistics, the earliest
298 * next event.
299 */
300 idr_for_each_entry(&irqt_stats, s, i) {
301
302 irqs = this_cpu_ptr(s);
303
304 if (!irqs->valid)
305 continue;
306
307 if (irqs->next_evt <= now) {
308 irq = i;
309 next_evt = now;
310
311 /*
312 * This interrupt mustn't use in the future
313 * until new events occur and update the
314 * statistics.
315 */
316 irqs->valid = 0;
317 break;
318 }
319
320 if (irqs->next_evt < next_evt) {
321 irq = i;
322 next_evt = irqs->next_evt;
323 }
324 }
325
326 return next_evt;
327}
328
329void irq_timings_free(int irq)
330{
331 struct irqt_stat __percpu *s;
332
333 s = idr_find(&irqt_stats, irq);
334 if (s) {
335 free_percpu(s);
336 idr_remove(&irqt_stats, irq);
337 }
338}
339
340int irq_timings_alloc(int irq)
341{
342 struct irqt_stat __percpu *s;
343 int id;
344
345 /*
346 * Some platforms can have the same private interrupt per cpu,
347 * so this function may be be called several times with the
348 * same interrupt number. Just bail out in case the per cpu
349 * stat structure is already allocated.
350 */
351 s = idr_find(&irqt_stats, irq);
352 if (s)
353 return 0;
354
355 s = alloc_percpu(*s);
356 if (!s)
357 return -ENOMEM;
358
359 idr_preload(GFP_KERNEL);
360 id = idr_alloc(&irqt_stats, s, irq, irq + 1, GFP_NOWAIT);
361 idr_preload_end();
362
363 if (id < 0) {
364 free_percpu(s);
365 return id;
366 }
367
368 return 0;
369}