aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/Kconfig24
-rw-r--r--arch/x86/include/asm/dma-mapping.h2
-rw-r--r--arch/x86/include/asm/io_apic.h9
-rw-r--r--arch/x86/include/asm/iommu.h2
-rw-r--r--arch/x86/include/asm/irq_vectors.h11
-rw-r--r--arch/x86/include/asm/pci.h2
-rw-r--r--arch/x86/include/asm/pci_64.h1
-rw-r--r--arch/x86/include/asm/uaccess.h2
-rw-r--r--arch/x86/include/asm/uaccess_32.h8
-rw-r--r--arch/x86/include/asm/uaccess_64.h6
-rw-r--r--arch/x86/kernel/Makefile3
-rw-r--r--arch/x86/kernel/hpet.c7
-rw-r--r--arch/x86/kernel/io_apic.c761
-rw-r--r--arch/x86/kernel/irq.c3
-rw-r--r--arch/x86/kernel/irq_32.c2
-rw-r--r--arch/x86/kernel/irq_64.c2
-rw-r--r--arch/x86/kernel/irqinit_32.c3
-rw-r--r--arch/x86/kernel/irqinit_64.c3
-rw-r--r--arch/x86/kernel/pci-dma.c13
-rw-r--r--arch/x86/kernel/pci-swiotlb_64.c29
-rw-r--r--arch/x86/kernel/quirks.c2
-rw-r--r--arch/x86/kernel/setup.c2
-rw-r--r--arch/x86/lib/usercopy_32.c8
-rw-r--r--arch/x86/lib/usercopy_64.c4
-rw-r--r--arch/x86/mm/init_32.c3
-rw-r--r--arch/x86/oprofile/op_model_amd.c89
26 files changed, 675 insertions, 326 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 98a0ed52b5c3..0f44add3e0b7 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -247,6 +247,28 @@ config X86_HAS_BOOT_CPU_ID
247 def_bool y 247 def_bool y
248 depends on X86_VOYAGER 248 depends on X86_VOYAGER
249 249
250config SPARSE_IRQ
251 bool "Support sparse irq numbering"
252 depends on PCI_MSI || HT_IRQ
253 help
254 This enables support for sparse irqs. This is useful for distro
255 kernels that want to define a high CONFIG_NR_CPUS value but still
256 want to have low kernel memory footprint on smaller machines.
257
258 ( Sparse IRQs can also be beneficial on NUMA boxes, as they spread
259 out the irq_desc[] array in a more NUMA-friendly way. )
260
261 If you don't know what to do here, say N.
262
263config NUMA_MIGRATE_IRQ_DESC
264 bool "Move irq desc when changing irq smp_affinity"
265 depends on SPARSE_IRQ && NUMA
266 default n
267 help
268 This enables moving irq_desc to cpu/node that irq will use handled.
269
270 If you don't know what to do here, say N.
271
250config X86_FIND_SMP_CONFIG 272config X86_FIND_SMP_CONFIG
251 def_bool y 273 def_bool y
252 depends on X86_MPPARSE || X86_VOYAGER 274 depends on X86_MPPARSE || X86_VOYAGER
@@ -479,7 +501,7 @@ config HPET_TIMER
479 The HPET provides a stable time base on SMP 501 The HPET provides a stable time base on SMP
480 systems, unlike the TSC, but it is more expensive to access, 502 systems, unlike the TSC, but it is more expensive to access,
481 as it is off-chip. You can find the HPET spec at 503 as it is off-chip. You can find the HPET spec at
482 <http://www.intel.com/hardwaredesign/hpetspec.htm>. 504 <http://www.intel.com/hardwaredesign/hpetspec_1.pdf>.
483 505
484 You can safely choose Y here. However, HPET will only be 506 You can safely choose Y here. However, HPET will only be
485 activated if the platform and the BIOS support this feature. 507 activated if the platform and the BIOS support this feature.
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h
index dc22c0733282..4035357f5b9d 100644
--- a/arch/x86/include/asm/dma-mapping.h
+++ b/arch/x86/include/asm/dma-mapping.h
@@ -65,7 +65,7 @@ static inline struct dma_mapping_ops *get_dma_ops(struct device *dev)
65 return dma_ops; 65 return dma_ops;
66 else 66 else
67 return dev->archdata.dma_ops; 67 return dev->archdata.dma_ops;
68#endif /* _ASM_X86_DMA_MAPPING_H */ 68#endif
69} 69}
70 70
71/* Make sure we keep the same behaviour */ 71/* Make sure we keep the same behaviour */
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index e475e009ae5d..7a1f44ac1f17 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -198,17 +198,14 @@ extern void restore_IO_APIC_setup(void);
198extern void reinit_intr_remapped_IO_APIC(int); 198extern void reinit_intr_remapped_IO_APIC(int);
199#endif 199#endif
200 200
201extern int probe_nr_irqs(void); 201extern void probe_nr_irqs_gsi(void);
202 202
203#else /* !CONFIG_X86_IO_APIC */ 203#else /* !CONFIG_X86_IO_APIC */
204#define io_apic_assign_pci_irqs 0 204#define io_apic_assign_pci_irqs 0
205static const int timer_through_8259 = 0; 205static const int timer_through_8259 = 0;
206static inline void ioapic_init_mappings(void) { } 206static inline void ioapic_init_mappings(void) { }
207 207
208static inline int probe_nr_irqs(void) 208static inline void probe_nr_irqs_gsi(void) { }
209{
210 return NR_IRQS;
211}
212#endif 209#endif
213 210
214#endif /* _ASM_X86_IO_APIC_H */ 211#endif /* _ASM_X86_IO_APIC_H */
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h
index 295b13193f4d..a6ee9e6f530f 100644
--- a/arch/x86/include/asm/iommu.h
+++ b/arch/x86/include/asm/iommu.h
@@ -7,8 +7,6 @@ extern struct dma_mapping_ops nommu_dma_ops;
7extern int force_iommu, no_iommu; 7extern int force_iommu, no_iommu;
8extern int iommu_detected; 8extern int iommu_detected;
9 9
10extern unsigned long iommu_nr_pages(unsigned long addr, unsigned long len);
11
12/* 10 seconds */ 10/* 10 seconds */
13#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) 11#define DMAR_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
14 12
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 0005adb0f941..f7ff65032b9d 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -101,12 +101,23 @@
101#define LAST_VM86_IRQ 15 101#define LAST_VM86_IRQ 15
102#define invalid_vm86_irq(irq) ((irq) < 3 || (irq) > 15) 102#define invalid_vm86_irq(irq) ((irq) < 3 || (irq) > 15)
103 103
104#define NR_IRQS_LEGACY 16
105
104#if defined(CONFIG_X86_IO_APIC) && !defined(CONFIG_X86_VOYAGER) 106#if defined(CONFIG_X86_IO_APIC) && !defined(CONFIG_X86_VOYAGER)
107
108#ifndef CONFIG_SPARSE_IRQ
105# if NR_CPUS < MAX_IO_APICS 109# if NR_CPUS < MAX_IO_APICS
106# define NR_IRQS (NR_VECTORS + (32 * NR_CPUS)) 110# define NR_IRQS (NR_VECTORS + (32 * NR_CPUS))
107# else 111# else
108# define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS)) 112# define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS))
109# endif 113# endif
114#else
115# if (8 * NR_CPUS) > (32 * MAX_IO_APICS)
116# define NR_IRQS (NR_VECTORS + (8 * NR_CPUS))
117# else
118# define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS))
119# endif
120#endif
110 121
111#elif defined(CONFIG_X86_VOYAGER) 122#elif defined(CONFIG_X86_VOYAGER)
112 123
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index 647781298e7e..66834c41c049 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -84,6 +84,8 @@ static inline void pci_dma_burst_advice(struct pci_dev *pdev,
84static inline void early_quirks(void) { } 84static inline void early_quirks(void) { }
85#endif 85#endif
86 86
87extern void pci_iommu_alloc(void);
88
87#endif /* __KERNEL__ */ 89#endif /* __KERNEL__ */
88 90
89#ifdef CONFIG_X86_32 91#ifdef CONFIG_X86_32
diff --git a/arch/x86/include/asm/pci_64.h b/arch/x86/include/asm/pci_64.h
index d02d936840a3..4da207982777 100644
--- a/arch/x86/include/asm/pci_64.h
+++ b/arch/x86/include/asm/pci_64.h
@@ -23,7 +23,6 @@ extern int (*pci_config_write)(int seg, int bus, int dev, int fn,
23 int reg, int len, u32 value); 23 int reg, int len, u32 value);
24 24
25extern void dma32_reserve_bootmem(void); 25extern void dma32_reserve_bootmem(void);
26extern void pci_iommu_alloc(void);
27 26
28/* The PCI address space does equal the physical memory 27/* The PCI address space does equal the physical memory
29 * address space. The networking and block device layers use 28 * address space. The networking and block device layers use
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index 580c3ee6c58c..4340055b7559 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -157,6 +157,7 @@ extern int __get_user_bad(void);
157 int __ret_gu; \ 157 int __ret_gu; \
158 unsigned long __val_gu; \ 158 unsigned long __val_gu; \
159 __chk_user_ptr(ptr); \ 159 __chk_user_ptr(ptr); \
160 might_fault(); \
160 switch (sizeof(*(ptr))) { \ 161 switch (sizeof(*(ptr))) { \
161 case 1: \ 162 case 1: \
162 __get_user_x(1, __ret_gu, __val_gu, ptr); \ 163 __get_user_x(1, __ret_gu, __val_gu, ptr); \
@@ -241,6 +242,7 @@ extern void __put_user_8(void);
241 int __ret_pu; \ 242 int __ret_pu; \
242 __typeof__(*(ptr)) __pu_val; \ 243 __typeof__(*(ptr)) __pu_val; \
243 __chk_user_ptr(ptr); \ 244 __chk_user_ptr(ptr); \
245 might_fault(); \
244 __pu_val = x; \ 246 __pu_val = x; \
245 switch (sizeof(*(ptr))) { \ 247 switch (sizeof(*(ptr))) { \
246 case 1: \ 248 case 1: \
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index d095a3aeea1b..5e06259e90e5 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -82,8 +82,8 @@ __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n)
82static __always_inline unsigned long __must_check 82static __always_inline unsigned long __must_check
83__copy_to_user(void __user *to, const void *from, unsigned long n) 83__copy_to_user(void __user *to, const void *from, unsigned long n)
84{ 84{
85 might_sleep(); 85 might_fault();
86 return __copy_to_user_inatomic(to, from, n); 86 return __copy_to_user_inatomic(to, from, n);
87} 87}
88 88
89static __always_inline unsigned long 89static __always_inline unsigned long
@@ -137,7 +137,7 @@ __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n)
137static __always_inline unsigned long 137static __always_inline unsigned long
138__copy_from_user(void *to, const void __user *from, unsigned long n) 138__copy_from_user(void *to, const void __user *from, unsigned long n)
139{ 139{
140 might_sleep(); 140 might_fault();
141 if (__builtin_constant_p(n)) { 141 if (__builtin_constant_p(n)) {
142 unsigned long ret; 142 unsigned long ret;
143 143
@@ -159,7 +159,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n)
159static __always_inline unsigned long __copy_from_user_nocache(void *to, 159static __always_inline unsigned long __copy_from_user_nocache(void *to,
160 const void __user *from, unsigned long n) 160 const void __user *from, unsigned long n)
161{ 161{
162 might_sleep(); 162 might_fault();
163 if (__builtin_constant_p(n)) { 163 if (__builtin_constant_p(n)) {
164 unsigned long ret; 164 unsigned long ret;
165 165
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index f8cfd00db450..84210c479fca 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -29,6 +29,8 @@ static __always_inline __must_check
29int __copy_from_user(void *dst, const void __user *src, unsigned size) 29int __copy_from_user(void *dst, const void __user *src, unsigned size)
30{ 30{
31 int ret = 0; 31 int ret = 0;
32
33 might_fault();
32 if (!__builtin_constant_p(size)) 34 if (!__builtin_constant_p(size))
33 return copy_user_generic(dst, (__force void *)src, size); 35 return copy_user_generic(dst, (__force void *)src, size);
34 switch (size) { 36 switch (size) {
@@ -71,6 +73,8 @@ static __always_inline __must_check
71int __copy_to_user(void __user *dst, const void *src, unsigned size) 73int __copy_to_user(void __user *dst, const void *src, unsigned size)
72{ 74{
73 int ret = 0; 75 int ret = 0;
76
77 might_fault();
74 if (!__builtin_constant_p(size)) 78 if (!__builtin_constant_p(size))
75 return copy_user_generic((__force void *)dst, src, size); 79 return copy_user_generic((__force void *)dst, src, size);
76 switch (size) { 80 switch (size) {
@@ -113,6 +117,8 @@ static __always_inline __must_check
113int __copy_in_user(void __user *dst, const void __user *src, unsigned size) 117int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
114{ 118{
115 int ret = 0; 119 int ret = 0;
120
121 might_fault();
116 if (!__builtin_constant_p(size)) 122 if (!__builtin_constant_p(size))
117 return copy_user_generic((__force void *)dst, 123 return copy_user_generic((__force void *)dst,
118 (__force void *)src, size); 124 (__force void *)src, size);
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 88dd768eab6d..d364df03c1d6 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -109,6 +109,8 @@ obj-$(CONFIG_MICROCODE) += microcode.o
109 109
110obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o 110obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o
111 111
112obj-$(CONFIG_SWIOTLB) += pci-swiotlb_64.o # NB rename without _64
113
112### 114###
113# 64 bit specific files 115# 64 bit specific files
114ifeq ($(CONFIG_X86_64),y) 116ifeq ($(CONFIG_X86_64),y)
@@ -122,7 +124,6 @@ ifeq ($(CONFIG_X86_64),y)
122 obj-$(CONFIG_GART_IOMMU) += pci-gart_64.o aperture_64.o 124 obj-$(CONFIG_GART_IOMMU) += pci-gart_64.o aperture_64.o
123 obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o 125 obj-$(CONFIG_CALGARY_IOMMU) += pci-calgary_64.o tce_64.o
124 obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o 126 obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o
125 obj-$(CONFIG_SWIOTLB) += pci-swiotlb_64.o
126 127
127 obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o 128 obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o
128endif 129endif
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c
index 3f0a3edf0a57..845ea097383e 100644
--- a/arch/x86/kernel/hpet.c
+++ b/arch/x86/kernel/hpet.c
@@ -813,7 +813,7 @@ int __init hpet_enable(void)
813 813
814out_nohpet: 814out_nohpet:
815 hpet_clear_mapping(); 815 hpet_clear_mapping();
816 boot_hpet_disable = 1; 816 hpet_address = 0;
817 return 0; 817 return 0;
818} 818}
819 819
@@ -836,10 +836,11 @@ static __init int hpet_late_init(void)
836 836
837 hpet_address = force_hpet_address; 837 hpet_address = force_hpet_address;
838 hpet_enable(); 838 hpet_enable();
839 if (!hpet_virt_address)
840 return -ENODEV;
841 } 839 }
842 840
841 if (!hpet_virt_address)
842 return -ENODEV;
843
843 hpet_reserve_platform_timers(hpet_readl(HPET_ID)); 844 hpet_reserve_platform_timers(hpet_readl(HPET_ID));
844 845
845 for_each_online_cpu(cpu) { 846 for_each_online_cpu(cpu) {
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c
index 679e7bbbbcd6..f6ea94b74da1 100644
--- a/arch/x86/kernel/io_apic.c
+++ b/arch/x86/kernel/io_apic.c
@@ -108,93 +108,252 @@ static int __init parse_noapic(char *str)
108early_param("noapic", parse_noapic); 108early_param("noapic", parse_noapic);
109 109
110struct irq_pin_list; 110struct irq_pin_list;
111
112/*
113 * This is performance-critical, we want to do it O(1)
114 *
115 * the indexing order of this array favors 1:1 mappings
116 * between pins and IRQs.
117 */
118
119struct irq_pin_list {
120 int apic, pin;
121 struct irq_pin_list *next;
122};
123
124static struct irq_pin_list *get_one_free_irq_2_pin(int cpu)
125{
126 struct irq_pin_list *pin;
127 int node;
128
129 node = cpu_to_node(cpu);
130
131 pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
132 printk(KERN_DEBUG " alloc irq_2_pin on cpu %d node %d\n", cpu, node);
133
134 return pin;
135}
136
111struct irq_cfg { 137struct irq_cfg {
112 unsigned int irq;
113 struct irq_pin_list *irq_2_pin; 138 struct irq_pin_list *irq_2_pin;
114 cpumask_t domain; 139 cpumask_t domain;
115 cpumask_t old_domain; 140 cpumask_t old_domain;
116 unsigned move_cleanup_count; 141 unsigned move_cleanup_count;
117 u8 vector; 142 u8 vector;
118 u8 move_in_progress : 1; 143 u8 move_in_progress : 1;
144#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
145 u8 move_desc_pending : 1;
146#endif
119}; 147};
120 148
121/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ 149/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
150#ifdef CONFIG_SPARSE_IRQ
151static struct irq_cfg irq_cfgx[] = {
152#else
122static struct irq_cfg irq_cfgx[NR_IRQS] = { 153static struct irq_cfg irq_cfgx[NR_IRQS] = {
123 [0] = { .irq = 0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, }, 154#endif
124 [1] = { .irq = 1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, }, 155 [0] = { .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, },
125 [2] = { .irq = 2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, }, 156 [1] = { .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, },
126 [3] = { .irq = 3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, }, 157 [2] = { .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, },
127 [4] = { .irq = 4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, }, 158 [3] = { .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, },
128 [5] = { .irq = 5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, }, 159 [4] = { .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, },
129 [6] = { .irq = 6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, }, 160 [5] = { .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, },
130 [7] = { .irq = 7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, }, 161 [6] = { .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, },
131 [8] = { .irq = 8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, }, 162 [7] = { .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, },
132 [9] = { .irq = 9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, }, 163 [8] = { .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, },
133 [10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, }, 164 [9] = { .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, },
134 [11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, }, 165 [10] = { .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
135 [12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, }, 166 [11] = { .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
136 [13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, }, 167 [12] = { .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
137 [14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, }, 168 [13] = { .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
138 [15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, }, 169 [14] = { .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
170 [15] = { .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
139}; 171};
140 172
141#define for_each_irq_cfg(irq, cfg) \ 173void __init arch_early_irq_init(void)
142 for (irq = 0, cfg = irq_cfgx; irq < nr_irqs; irq++, cfg++) 174{
175 struct irq_cfg *cfg;
176 struct irq_desc *desc;
177 int count;
178 int i;
179
180 cfg = irq_cfgx;
181 count = ARRAY_SIZE(irq_cfgx);
182
183 for (i = 0; i < count; i++) {
184 desc = irq_to_desc(i);
185 desc->chip_data = &cfg[i];
186 }
187}
143 188
189#ifdef CONFIG_SPARSE_IRQ
144static struct irq_cfg *irq_cfg(unsigned int irq) 190static struct irq_cfg *irq_cfg(unsigned int irq)
145{ 191{
146 return irq < nr_irqs ? irq_cfgx + irq : NULL; 192 struct irq_cfg *cfg = NULL;
193 struct irq_desc *desc;
194
195 desc = irq_to_desc(irq);
196 if (desc)
197 cfg = desc->chip_data;
198
199 return cfg;
147} 200}
148 201
149static struct irq_cfg *irq_cfg_alloc(unsigned int irq) 202static struct irq_cfg *get_one_free_irq_cfg(int cpu)
150{ 203{
151 return irq_cfg(irq); 204 struct irq_cfg *cfg;
205 int node;
206
207 node = cpu_to_node(cpu);
208
209 cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
210 printk(KERN_DEBUG " alloc irq_cfg on cpu %d node %d\n", cpu, node);
211
212 return cfg;
152} 213}
153 214
154/* 215void arch_init_chip_data(struct irq_desc *desc, int cpu)
155 * Rough estimation of how many shared IRQs there are, can be changed 216{
156 * anytime. 217 struct irq_cfg *cfg;
157 */
158#define MAX_PLUS_SHARED_IRQS NR_IRQS
159#define PIN_MAP_SIZE (MAX_PLUS_SHARED_IRQS + NR_IRQS)
160 218
161/* 219 cfg = desc->chip_data;
162 * This is performance-critical, we want to do it O(1) 220 if (!cfg) {
163 * 221 desc->chip_data = get_one_free_irq_cfg(cpu);
164 * the indexing order of this array favors 1:1 mappings 222 if (!desc->chip_data) {
165 * between pins and IRQs. 223 printk(KERN_ERR "can not alloc irq_cfg\n");
166 */ 224 BUG_ON(1);
225 }
226 }
227}
167 228
168struct irq_pin_list { 229#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
169 int apic, pin; 230
170 struct irq_pin_list *next; 231static void
171}; 232init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
233{
234 struct irq_pin_list *old_entry, *head, *tail, *entry;
235
236 cfg->irq_2_pin = NULL;
237 old_entry = old_cfg->irq_2_pin;
238 if (!old_entry)
239 return;
172 240
173static struct irq_pin_list irq_2_pin_head[PIN_MAP_SIZE]; 241 entry = get_one_free_irq_2_pin(cpu);
174static struct irq_pin_list *irq_2_pin_ptr; 242 if (!entry)
243 return;
175 244
176static void __init irq_2_pin_init(void) 245 entry->apic = old_entry->apic;
246 entry->pin = old_entry->pin;
247 head = entry;
248 tail = entry;
249 old_entry = old_entry->next;
250 while (old_entry) {
251 entry = get_one_free_irq_2_pin(cpu);
252 if (!entry) {
253 entry = head;
254 while (entry) {
255 head = entry->next;
256 kfree(entry);
257 entry = head;
258 }
259 /* still use the old one */
260 return;
261 }
262 entry->apic = old_entry->apic;
263 entry->pin = old_entry->pin;
264 tail->next = entry;
265 tail = entry;
266 old_entry = old_entry->next;
267 }
268
269 tail->next = NULL;
270 cfg->irq_2_pin = head;
271}
272
273static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
177{ 274{
178 struct irq_pin_list *pin = irq_2_pin_head; 275 struct irq_pin_list *entry, *next;
179 int i;
180 276
181 for (i = 1; i < PIN_MAP_SIZE; i++) 277 if (old_cfg->irq_2_pin == cfg->irq_2_pin)
182 pin[i-1].next = &pin[i]; 278 return;
279
280 entry = old_cfg->irq_2_pin;
183 281
184 irq_2_pin_ptr = &pin[0]; 282 while (entry) {
283 next = entry->next;
284 kfree(entry);
285 entry = next;
286 }
287 old_cfg->irq_2_pin = NULL;
185} 288}
186 289
187static struct irq_pin_list *get_one_free_irq_2_pin(void) 290void arch_init_copy_chip_data(struct irq_desc *old_desc,
291 struct irq_desc *desc, int cpu)
188{ 292{
189 struct irq_pin_list *pin = irq_2_pin_ptr; 293 struct irq_cfg *cfg;
294 struct irq_cfg *old_cfg;
190 295
191 if (!pin) 296 cfg = get_one_free_irq_cfg(cpu);
192 panic("can not get more irq_2_pin\n");
193 297
194 irq_2_pin_ptr = pin->next; 298 if (!cfg)
195 pin->next = NULL; 299 return;
196 return pin; 300
301 desc->chip_data = cfg;
302
303 old_cfg = old_desc->chip_data;
304
305 memcpy(cfg, old_cfg, sizeof(struct irq_cfg));
306
307 init_copy_irq_2_pin(old_cfg, cfg, cpu);
308}
309
310static void free_irq_cfg(struct irq_cfg *old_cfg)
311{
312 kfree(old_cfg);
313}
314
315void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
316{
317 struct irq_cfg *old_cfg, *cfg;
318
319 old_cfg = old_desc->chip_data;
320 cfg = desc->chip_data;
321
322 if (old_cfg == cfg)
323 return;
324
325 if (old_cfg) {
326 free_irq_2_pin(old_cfg, cfg);
327 free_irq_cfg(old_cfg);
328 old_desc->chip_data = NULL;
329 }
330}
331
332static void set_extra_move_desc(struct irq_desc *desc, cpumask_t mask)
333{
334 struct irq_cfg *cfg = desc->chip_data;
335
336 if (!cfg->move_in_progress) {
337 /* it means that domain is not changed */
338 if (!cpus_intersects(desc->affinity, mask))
339 cfg->move_desc_pending = 1;
340 }
197} 341}
342#endif
343
344#else
345static struct irq_cfg *irq_cfg(unsigned int irq)
346{
347 return irq < nr_irqs ? irq_cfgx + irq : NULL;
348}
349
350#endif
351
352#ifndef CONFIG_NUMA_MIGRATE_IRQ_DESC
353static inline void set_extra_move_desc(struct irq_desc *desc, cpumask_t mask)
354{
355}
356#endif
198 357
199struct io_apic { 358struct io_apic {
200 unsigned int index; 359 unsigned int index;
@@ -237,11 +396,10 @@ static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned
237 writel(value, &io_apic->data); 396 writel(value, &io_apic->data);
238} 397}
239 398
240static bool io_apic_level_ack_pending(unsigned int irq) 399static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
241{ 400{
242 struct irq_pin_list *entry; 401 struct irq_pin_list *entry;
243 unsigned long flags; 402 unsigned long flags;
244 struct irq_cfg *cfg = irq_cfg(irq);
245 403
246 spin_lock_irqsave(&ioapic_lock, flags); 404 spin_lock_irqsave(&ioapic_lock, flags);
247 entry = cfg->irq_2_pin; 405 entry = cfg->irq_2_pin;
@@ -323,13 +481,12 @@ static void ioapic_mask_entry(int apic, int pin)
323} 481}
324 482
325#ifdef CONFIG_SMP 483#ifdef CONFIG_SMP
326static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector) 484static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
327{ 485{
328 int apic, pin; 486 int apic, pin;
329 struct irq_cfg *cfg;
330 struct irq_pin_list *entry; 487 struct irq_pin_list *entry;
488 u8 vector = cfg->vector;
331 489
332 cfg = irq_cfg(irq);
333 entry = cfg->irq_2_pin; 490 entry = cfg->irq_2_pin;
334 for (;;) { 491 for (;;) {
335 unsigned int reg; 492 unsigned int reg;
@@ -359,24 +516,27 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
359 } 516 }
360} 517}
361 518
362static int assign_irq_vector(int irq, cpumask_t mask); 519static int assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask);
363 520
364static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) 521static void set_ioapic_affinity_irq_desc(struct irq_desc *desc, cpumask_t mask)
365{ 522{
366 struct irq_cfg *cfg; 523 struct irq_cfg *cfg;
367 unsigned long flags; 524 unsigned long flags;
368 unsigned int dest; 525 unsigned int dest;
369 cpumask_t tmp; 526 cpumask_t tmp;
370 struct irq_desc *desc; 527 unsigned int irq;
371 528
372 cpus_and(tmp, mask, cpu_online_map); 529 cpus_and(tmp, mask, cpu_online_map);
373 if (cpus_empty(tmp)) 530 if (cpus_empty(tmp))
374 return; 531 return;
375 532
376 cfg = irq_cfg(irq); 533 irq = desc->irq;
377 if (assign_irq_vector(irq, mask)) 534 cfg = desc->chip_data;
535 if (assign_irq_vector(irq, cfg, mask))
378 return; 536 return;
379 537
538 set_extra_move_desc(desc, mask);
539
380 cpus_and(tmp, cfg->domain, mask); 540 cpus_and(tmp, cfg->domain, mask);
381 dest = cpu_mask_to_apicid(tmp); 541 dest = cpu_mask_to_apicid(tmp);
382 /* 542 /*
@@ -384,12 +544,20 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
384 */ 544 */
385 dest = SET_APIC_LOGICAL_ID(dest); 545 dest = SET_APIC_LOGICAL_ID(dest);
386 546
387 desc = irq_to_desc(irq);
388 spin_lock_irqsave(&ioapic_lock, flags); 547 spin_lock_irqsave(&ioapic_lock, flags);
389 __target_IO_APIC_irq(irq, dest, cfg->vector); 548 __target_IO_APIC_irq(irq, dest, cfg);
390 desc->affinity = mask; 549 desc->affinity = mask;
391 spin_unlock_irqrestore(&ioapic_lock, flags); 550 spin_unlock_irqrestore(&ioapic_lock, flags);
392} 551}
552
553static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
554{
555 struct irq_desc *desc;
556
557 desc = irq_to_desc(irq);
558
559 set_ioapic_affinity_irq_desc(desc, mask);
560}
393#endif /* CONFIG_SMP */ 561#endif /* CONFIG_SMP */
394 562
395/* 563/*
@@ -397,16 +565,18 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
397 * shared ISA-space IRQs, so we have to support them. We are super 565 * shared ISA-space IRQs, so we have to support them. We are super
398 * fast in the common case, and fast for shared ISA-space IRQs. 566 * fast in the common case, and fast for shared ISA-space IRQs.
399 */ 567 */
400static void add_pin_to_irq(unsigned int irq, int apic, int pin) 568static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
401{ 569{
402 struct irq_cfg *cfg;
403 struct irq_pin_list *entry; 570 struct irq_pin_list *entry;
404 571
405 /* first time to refer irq_cfg, so with new */
406 cfg = irq_cfg_alloc(irq);
407 entry = cfg->irq_2_pin; 572 entry = cfg->irq_2_pin;
408 if (!entry) { 573 if (!entry) {
409 entry = get_one_free_irq_2_pin(); 574 entry = get_one_free_irq_2_pin(cpu);
575 if (!entry) {
576 printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
577 apic, pin);
578 return;
579 }
410 cfg->irq_2_pin = entry; 580 cfg->irq_2_pin = entry;
411 entry->apic = apic; 581 entry->apic = apic;
412 entry->pin = pin; 582 entry->pin = pin;
@@ -421,7 +591,7 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin)
421 entry = entry->next; 591 entry = entry->next;
422 } 592 }
423 593
424 entry->next = get_one_free_irq_2_pin(); 594 entry->next = get_one_free_irq_2_pin(cpu);
425 entry = entry->next; 595 entry = entry->next;
426 entry->apic = apic; 596 entry->apic = apic;
427 entry->pin = pin; 597 entry->pin = pin;
@@ -430,11 +600,10 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin)
430/* 600/*
431 * Reroute an IRQ to a different pin. 601 * Reroute an IRQ to a different pin.
432 */ 602 */
433static void __init replace_pin_at_irq(unsigned int irq, 603static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu,
434 int oldapic, int oldpin, 604 int oldapic, int oldpin,
435 int newapic, int newpin) 605 int newapic, int newpin)
436{ 606{
437 struct irq_cfg *cfg = irq_cfg(irq);
438 struct irq_pin_list *entry = cfg->irq_2_pin; 607 struct irq_pin_list *entry = cfg->irq_2_pin;
439 int replaced = 0; 608 int replaced = 0;
440 609
@@ -451,18 +620,16 @@ static void __init replace_pin_at_irq(unsigned int irq,
451 620
452 /* why? call replace before add? */ 621 /* why? call replace before add? */
453 if (!replaced) 622 if (!replaced)
454 add_pin_to_irq(irq, newapic, newpin); 623 add_pin_to_irq_cpu(cfg, cpu, newapic, newpin);
455} 624}
456 625
457static inline void io_apic_modify_irq(unsigned int irq, 626static inline void io_apic_modify_irq(struct irq_cfg *cfg,
458 int mask_and, int mask_or, 627 int mask_and, int mask_or,
459 void (*final)(struct irq_pin_list *entry)) 628 void (*final)(struct irq_pin_list *entry))
460{ 629{
461 int pin; 630 int pin;
462 struct irq_cfg *cfg;
463 struct irq_pin_list *entry; 631 struct irq_pin_list *entry;
464 632
465 cfg = irq_cfg(irq);
466 for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) { 633 for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) {
467 unsigned int reg; 634 unsigned int reg;
468 pin = entry->pin; 635 pin = entry->pin;
@@ -475,9 +642,9 @@ static inline void io_apic_modify_irq(unsigned int irq,
475 } 642 }
476} 643}
477 644
478static void __unmask_IO_APIC_irq(unsigned int irq) 645static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
479{ 646{
480 io_apic_modify_irq(irq, ~IO_APIC_REDIR_MASKED, 0, NULL); 647 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL);
481} 648}
482 649
483#ifdef CONFIG_X86_64 650#ifdef CONFIG_X86_64
@@ -492,47 +659,64 @@ void io_apic_sync(struct irq_pin_list *entry)
492 readl(&io_apic->data); 659 readl(&io_apic->data);
493} 660}
494 661
495static void __mask_IO_APIC_irq(unsigned int irq) 662static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
496{ 663{
497 io_apic_modify_irq(irq, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync); 664 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
498} 665}
499#else /* CONFIG_X86_32 */ 666#else /* CONFIG_X86_32 */
500static void __mask_IO_APIC_irq(unsigned int irq) 667static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
501{ 668{
502 io_apic_modify_irq(irq, ~0, IO_APIC_REDIR_MASKED, NULL); 669 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, NULL);
503} 670}
504 671
505static void __mask_and_edge_IO_APIC_irq(unsigned int irq) 672static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
506{ 673{
507 io_apic_modify_irq(irq, ~IO_APIC_REDIR_LEVEL_TRIGGER, 674 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
508 IO_APIC_REDIR_MASKED, NULL); 675 IO_APIC_REDIR_MASKED, NULL);
509} 676}
510 677
511static void __unmask_and_level_IO_APIC_irq(unsigned int irq) 678static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
512{ 679{
513 io_apic_modify_irq(irq, ~IO_APIC_REDIR_MASKED, 680 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
514 IO_APIC_REDIR_LEVEL_TRIGGER, NULL); 681 IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
515} 682}
516#endif /* CONFIG_X86_32 */ 683#endif /* CONFIG_X86_32 */
517 684
518static void mask_IO_APIC_irq (unsigned int irq) 685static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
519{ 686{
687 struct irq_cfg *cfg = desc->chip_data;
520 unsigned long flags; 688 unsigned long flags;
521 689
690 BUG_ON(!cfg);
691
522 spin_lock_irqsave(&ioapic_lock, flags); 692 spin_lock_irqsave(&ioapic_lock, flags);
523 __mask_IO_APIC_irq(irq); 693 __mask_IO_APIC_irq(cfg);
524 spin_unlock_irqrestore(&ioapic_lock, flags); 694 spin_unlock_irqrestore(&ioapic_lock, flags);
525} 695}
526 696
527static void unmask_IO_APIC_irq (unsigned int irq) 697static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
528{ 698{
699 struct irq_cfg *cfg = desc->chip_data;
529 unsigned long flags; 700 unsigned long flags;
530 701
531 spin_lock_irqsave(&ioapic_lock, flags); 702 spin_lock_irqsave(&ioapic_lock, flags);
532 __unmask_IO_APIC_irq(irq); 703 __unmask_IO_APIC_irq(cfg);
533 spin_unlock_irqrestore(&ioapic_lock, flags); 704 spin_unlock_irqrestore(&ioapic_lock, flags);
534} 705}
535 706
707static void mask_IO_APIC_irq(unsigned int irq)
708{
709 struct irq_desc *desc = irq_to_desc(irq);
710
711 mask_IO_APIC_irq_desc(desc);
712}
713static void unmask_IO_APIC_irq(unsigned int irq)
714{
715 struct irq_desc *desc = irq_to_desc(irq);
716
717 unmask_IO_APIC_irq_desc(desc);
718}
719
536static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) 720static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
537{ 721{
538 struct IO_APIC_route_entry entry; 722 struct IO_APIC_route_entry entry;
@@ -809,7 +993,7 @@ EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
809 */ 993 */
810static int EISA_ELCR(unsigned int irq) 994static int EISA_ELCR(unsigned int irq)
811{ 995{
812 if (irq < 16) { 996 if (irq < NR_IRQS_LEGACY) {
813 unsigned int port = 0x4d0 + (irq >> 3); 997 unsigned int port = 0x4d0 + (irq >> 3);
814 return (inb(port) >> (irq & 7)) & 1; 998 return (inb(port) >> (irq & 7)) & 1;
815 } 999 }
@@ -1034,7 +1218,7 @@ void unlock_vector_lock(void)
1034 spin_unlock(&vector_lock); 1218 spin_unlock(&vector_lock);
1035} 1219}
1036 1220
1037static int __assign_irq_vector(int irq, cpumask_t mask) 1221static int __assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask)
1038{ 1222{
1039 /* 1223 /*
1040 * NOTE! The local APIC isn't very good at handling 1224 * NOTE! The local APIC isn't very good at handling
@@ -1050,16 +1234,13 @@ static int __assign_irq_vector(int irq, cpumask_t mask)
1050 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0; 1234 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
1051 unsigned int old_vector; 1235 unsigned int old_vector;
1052 int cpu; 1236 int cpu;
1053 struct irq_cfg *cfg;
1054 1237
1055 cfg = irq_cfg(irq); 1238 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
1239 return -EBUSY;
1056 1240
1057 /* Only try and allocate irqs on cpus that are present */ 1241 /* Only try and allocate irqs on cpus that are present */
1058 cpus_and(mask, mask, cpu_online_map); 1242 cpus_and(mask, mask, cpu_online_map);
1059 1243
1060 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
1061 return -EBUSY;
1062
1063 old_vector = cfg->vector; 1244 old_vector = cfg->vector;
1064 if (old_vector) { 1245 if (old_vector) {
1065 cpumask_t tmp; 1246 cpumask_t tmp;
@@ -1113,24 +1294,22 @@ next:
1113 return -ENOSPC; 1294 return -ENOSPC;
1114} 1295}
1115 1296
1116static int assign_irq_vector(int irq, cpumask_t mask) 1297static int assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask)
1117{ 1298{
1118 int err; 1299 int err;
1119 unsigned long flags; 1300 unsigned long flags;
1120 1301
1121 spin_lock_irqsave(&vector_lock, flags); 1302 spin_lock_irqsave(&vector_lock, flags);
1122 err = __assign_irq_vector(irq, mask); 1303 err = __assign_irq_vector(irq, cfg, mask);
1123 spin_unlock_irqrestore(&vector_lock, flags); 1304 spin_unlock_irqrestore(&vector_lock, flags);
1124 return err; 1305 return err;
1125} 1306}
1126 1307
1127static void __clear_irq_vector(int irq) 1308static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1128{ 1309{
1129 struct irq_cfg *cfg;
1130 cpumask_t mask; 1310 cpumask_t mask;
1131 int cpu, vector; 1311 int cpu, vector;
1132 1312
1133 cfg = irq_cfg(irq);
1134 BUG_ON(!cfg->vector); 1313 BUG_ON(!cfg->vector);
1135 1314
1136 vector = cfg->vector; 1315 vector = cfg->vector;
@@ -1162,9 +1341,13 @@ void __setup_vector_irq(int cpu)
1162 /* This function must be called with vector_lock held */ 1341 /* This function must be called with vector_lock held */
1163 int irq, vector; 1342 int irq, vector;
1164 struct irq_cfg *cfg; 1343 struct irq_cfg *cfg;
1344 struct irq_desc *desc;
1165 1345
1166 /* Mark the inuse vectors */ 1346 /* Mark the inuse vectors */
1167 for_each_irq_cfg(irq, cfg) { 1347 for_each_irq_desc(irq, desc) {
1348 if (!desc)
1349 continue;
1350 cfg = desc->chip_data;
1168 if (!cpu_isset(cpu, cfg->domain)) 1351 if (!cpu_isset(cpu, cfg->domain))
1169 continue; 1352 continue;
1170 vector = cfg->vector; 1353 vector = cfg->vector;
@@ -1215,11 +1398,8 @@ static inline int IO_APIC_irq_trigger(int irq)
1215} 1398}
1216#endif 1399#endif
1217 1400
1218static void ioapic_register_intr(int irq, unsigned long trigger) 1401static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger)
1219{ 1402{
1220 struct irq_desc *desc;
1221
1222 desc = irq_to_desc(irq);
1223 1403
1224 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || 1404 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1225 trigger == IOAPIC_LEVEL) 1405 trigger == IOAPIC_LEVEL)
@@ -1311,7 +1491,7 @@ static int setup_ioapic_entry(int apic, int irq,
1311 return 0; 1491 return 0;
1312} 1492}
1313 1493
1314static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, 1494static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_desc *desc,
1315 int trigger, int polarity) 1495 int trigger, int polarity)
1316{ 1496{
1317 struct irq_cfg *cfg; 1497 struct irq_cfg *cfg;
@@ -1321,10 +1501,10 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
1321 if (!IO_APIC_IRQ(irq)) 1501 if (!IO_APIC_IRQ(irq))
1322 return; 1502 return;
1323 1503
1324 cfg = irq_cfg(irq); 1504 cfg = desc->chip_data;
1325 1505
1326 mask = TARGET_CPUS; 1506 mask = TARGET_CPUS;
1327 if (assign_irq_vector(irq, mask)) 1507 if (assign_irq_vector(irq, cfg, mask))
1328 return; 1508 return;
1329 1509
1330 cpus_and(mask, cfg->domain, mask); 1510 cpus_and(mask, cfg->domain, mask);
@@ -1341,12 +1521,12 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
1341 cfg->vector)) { 1521 cfg->vector)) {
1342 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n", 1522 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1343 mp_ioapics[apic].mp_apicid, pin); 1523 mp_ioapics[apic].mp_apicid, pin);
1344 __clear_irq_vector(irq); 1524 __clear_irq_vector(irq, cfg);
1345 return; 1525 return;
1346 } 1526 }
1347 1527
1348 ioapic_register_intr(irq, trigger); 1528 ioapic_register_intr(irq, desc, trigger);
1349 if (irq < 16) 1529 if (irq < NR_IRQS_LEGACY)
1350 disable_8259A_irq(irq); 1530 disable_8259A_irq(irq);
1351 1531
1352 ioapic_write_entry(apic, pin, entry); 1532 ioapic_write_entry(apic, pin, entry);
@@ -1356,6 +1536,9 @@ static void __init setup_IO_APIC_irqs(void)
1356{ 1536{
1357 int apic, pin, idx, irq; 1537 int apic, pin, idx, irq;
1358 int notcon = 0; 1538 int notcon = 0;
1539 struct irq_desc *desc;
1540 struct irq_cfg *cfg;
1541 int cpu = boot_cpu_id;
1359 1542
1360 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); 1543 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1361 1544
@@ -1387,9 +1570,15 @@ static void __init setup_IO_APIC_irqs(void)
1387 if (multi_timer_check(apic, irq)) 1570 if (multi_timer_check(apic, irq))
1388 continue; 1571 continue;
1389#endif 1572#endif
1390 add_pin_to_irq(irq, apic, pin); 1573 desc = irq_to_desc_alloc_cpu(irq, cpu);
1574 if (!desc) {
1575 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
1576 continue;
1577 }
1578 cfg = desc->chip_data;
1579 add_pin_to_irq_cpu(cfg, cpu, apic, pin);
1391 1580
1392 setup_IO_APIC_irq(apic, pin, irq, 1581 setup_IO_APIC_irq(apic, pin, irq, desc,
1393 irq_trigger(idx), irq_polarity(idx)); 1582 irq_trigger(idx), irq_polarity(idx));
1394 } 1583 }
1395 } 1584 }
@@ -1448,6 +1637,7 @@ __apicdebuginit(void) print_IO_APIC(void)
1448 union IO_APIC_reg_03 reg_03; 1637 union IO_APIC_reg_03 reg_03;
1449 unsigned long flags; 1638 unsigned long flags;
1450 struct irq_cfg *cfg; 1639 struct irq_cfg *cfg;
1640 struct irq_desc *desc;
1451 unsigned int irq; 1641 unsigned int irq;
1452 1642
1453 if (apic_verbosity == APIC_QUIET) 1643 if (apic_verbosity == APIC_QUIET)
@@ -1537,8 +1727,13 @@ __apicdebuginit(void) print_IO_APIC(void)
1537 } 1727 }
1538 } 1728 }
1539 printk(KERN_DEBUG "IRQ to pin mappings:\n"); 1729 printk(KERN_DEBUG "IRQ to pin mappings:\n");
1540 for_each_irq_cfg(irq, cfg) { 1730 for_each_irq_desc(irq, desc) {
1541 struct irq_pin_list *entry = cfg->irq_2_pin; 1731 struct irq_pin_list *entry;
1732
1733 if (!desc)
1734 continue;
1735 cfg = desc->chip_data;
1736 entry = cfg->irq_2_pin;
1542 if (!entry) 1737 if (!entry)
1543 continue; 1738 continue;
1544 printk(KERN_DEBUG "IRQ%d ", irq); 1739 printk(KERN_DEBUG "IRQ%d ", irq);
@@ -2022,14 +2217,16 @@ static unsigned int startup_ioapic_irq(unsigned int irq)
2022{ 2217{
2023 int was_pending = 0; 2218 int was_pending = 0;
2024 unsigned long flags; 2219 unsigned long flags;
2220 struct irq_cfg *cfg;
2025 2221
2026 spin_lock_irqsave(&ioapic_lock, flags); 2222 spin_lock_irqsave(&ioapic_lock, flags);
2027 if (irq < 16) { 2223 if (irq < NR_IRQS_LEGACY) {
2028 disable_8259A_irq(irq); 2224 disable_8259A_irq(irq);
2029 if (i8259A_irq_pending(irq)) 2225 if (i8259A_irq_pending(irq))
2030 was_pending = 1; 2226 was_pending = 1;
2031 } 2227 }
2032 __unmask_IO_APIC_irq(irq); 2228 cfg = irq_cfg(irq);
2229 __unmask_IO_APIC_irq(cfg);
2033 spin_unlock_irqrestore(&ioapic_lock, flags); 2230 spin_unlock_irqrestore(&ioapic_lock, flags);
2034 2231
2035 return was_pending; 2232 return was_pending;
@@ -2092,35 +2289,37 @@ static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
2092 * as simple as edge triggered migration and we can do the irq migration 2289 * as simple as edge triggered migration and we can do the irq migration
2093 * with a simple atomic update to IO-APIC RTE. 2290 * with a simple atomic update to IO-APIC RTE.
2094 */ 2291 */
2095static void migrate_ioapic_irq(int irq, cpumask_t mask) 2292static void migrate_ioapic_irq_desc(struct irq_desc *desc, cpumask_t mask)
2096{ 2293{
2097 struct irq_cfg *cfg; 2294 struct irq_cfg *cfg;
2098 struct irq_desc *desc;
2099 cpumask_t tmp, cleanup_mask; 2295 cpumask_t tmp, cleanup_mask;
2100 struct irte irte; 2296 struct irte irte;
2101 int modify_ioapic_rte; 2297 int modify_ioapic_rte;
2102 unsigned int dest; 2298 unsigned int dest;
2103 unsigned long flags; 2299 unsigned long flags;
2300 unsigned int irq;
2104 2301
2105 cpus_and(tmp, mask, cpu_online_map); 2302 cpus_and(tmp, mask, cpu_online_map);
2106 if (cpus_empty(tmp)) 2303 if (cpus_empty(tmp))
2107 return; 2304 return;
2108 2305
2306 irq = desc->irq;
2109 if (get_irte(irq, &irte)) 2307 if (get_irte(irq, &irte))
2110 return; 2308 return;
2111 2309
2112 if (assign_irq_vector(irq, mask)) 2310 cfg = desc->chip_data;
2311 if (assign_irq_vector(irq, cfg, mask))
2113 return; 2312 return;
2114 2313
2115 cfg = irq_cfg(irq); 2314 set_extra_move_desc(desc, mask);
2315
2116 cpus_and(tmp, cfg->domain, mask); 2316 cpus_and(tmp, cfg->domain, mask);
2117 dest = cpu_mask_to_apicid(tmp); 2317 dest = cpu_mask_to_apicid(tmp);
2118 2318
2119 desc = irq_to_desc(irq);
2120 modify_ioapic_rte = desc->status & IRQ_LEVEL; 2319 modify_ioapic_rte = desc->status & IRQ_LEVEL;
2121 if (modify_ioapic_rte) { 2320 if (modify_ioapic_rte) {
2122 spin_lock_irqsave(&ioapic_lock, flags); 2321 spin_lock_irqsave(&ioapic_lock, flags);
2123 __target_IO_APIC_irq(irq, dest, cfg->vector); 2322 __target_IO_APIC_irq(irq, dest, cfg);
2124 spin_unlock_irqrestore(&ioapic_lock, flags); 2323 spin_unlock_irqrestore(&ioapic_lock, flags);
2125 } 2324 }
2126 2325
@@ -2142,14 +2341,14 @@ static void migrate_ioapic_irq(int irq, cpumask_t mask)
2142 desc->affinity = mask; 2341 desc->affinity = mask;
2143} 2342}
2144 2343
2145static int migrate_irq_remapped_level(int irq) 2344static int migrate_irq_remapped_level_desc(struct irq_desc *desc)
2146{ 2345{
2147 int ret = -1; 2346 int ret = -1;
2148 struct irq_desc *desc = irq_to_desc(irq); 2347 struct irq_cfg *cfg = desc->chip_data;
2149 2348
2150 mask_IO_APIC_irq(irq); 2349 mask_IO_APIC_irq_desc(desc);
2151 2350
2152 if (io_apic_level_ack_pending(irq)) { 2351 if (io_apic_level_ack_pending(cfg)) {
2153 /* 2352 /*
2154 * Interrupt in progress. Migrating irq now will change the 2353 * Interrupt in progress. Migrating irq now will change the
2155 * vector information in the IO-APIC RTE and that will confuse 2354 * vector information in the IO-APIC RTE and that will confuse
@@ -2161,14 +2360,15 @@ static int migrate_irq_remapped_level(int irq)
2161 } 2360 }
2162 2361
2163 /* everthing is clear. we have right of way */ 2362 /* everthing is clear. we have right of way */
2164 migrate_ioapic_irq(irq, desc->pending_mask); 2363 migrate_ioapic_irq_desc(desc, desc->pending_mask);
2165 2364
2166 ret = 0; 2365 ret = 0;
2167 desc->status &= ~IRQ_MOVE_PENDING; 2366 desc->status &= ~IRQ_MOVE_PENDING;
2168 cpus_clear(desc->pending_mask); 2367 cpus_clear(desc->pending_mask);
2169 2368
2170unmask: 2369unmask:
2171 unmask_IO_APIC_irq(irq); 2370 unmask_IO_APIC_irq_desc(desc);
2371
2172 return ret; 2372 return ret;
2173} 2373}
2174 2374
@@ -2178,6 +2378,9 @@ static void ir_irq_migration(struct work_struct *work)
2178 struct irq_desc *desc; 2378 struct irq_desc *desc;
2179 2379
2180 for_each_irq_desc(irq, desc) { 2380 for_each_irq_desc(irq, desc) {
2381 if (!desc)
2382 continue;
2383
2181 if (desc->status & IRQ_MOVE_PENDING) { 2384 if (desc->status & IRQ_MOVE_PENDING) {
2182 unsigned long flags; 2385 unsigned long flags;
2183 2386
@@ -2198,18 +2401,22 @@ static void ir_irq_migration(struct work_struct *work)
2198/* 2401/*
2199 * Migrates the IRQ destination in the process context. 2402 * Migrates the IRQ destination in the process context.
2200 */ 2403 */
2201static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) 2404static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, cpumask_t mask)
2202{ 2405{
2203 struct irq_desc *desc = irq_to_desc(irq);
2204
2205 if (desc->status & IRQ_LEVEL) { 2406 if (desc->status & IRQ_LEVEL) {
2206 desc->status |= IRQ_MOVE_PENDING; 2407 desc->status |= IRQ_MOVE_PENDING;
2207 desc->pending_mask = mask; 2408 desc->pending_mask = mask;
2208 migrate_irq_remapped_level(irq); 2409 migrate_irq_remapped_level_desc(desc);
2209 return; 2410 return;
2210 } 2411 }
2211 2412
2212 migrate_ioapic_irq(irq, mask); 2413 migrate_ioapic_irq_desc(desc, mask);
2414}
2415static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
2416{
2417 struct irq_desc *desc = irq_to_desc(irq);
2418
2419 set_ir_ioapic_affinity_irq_desc(desc, mask);
2213} 2420}
2214#endif 2421#endif
2215 2422
@@ -2228,6 +2435,9 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void)
2228 struct irq_cfg *cfg; 2435 struct irq_cfg *cfg;
2229 irq = __get_cpu_var(vector_irq)[vector]; 2436 irq = __get_cpu_var(vector_irq)[vector];
2230 2437
2438 if (irq == -1)
2439 continue;
2440
2231 desc = irq_to_desc(irq); 2441 desc = irq_to_desc(irq);
2232 if (!desc) 2442 if (!desc)
2233 continue; 2443 continue;
@@ -2249,19 +2459,40 @@ unlock:
2249 irq_exit(); 2459 irq_exit();
2250} 2460}
2251 2461
2252static void irq_complete_move(unsigned int irq) 2462static void irq_complete_move(struct irq_desc **descp)
2253{ 2463{
2254 struct irq_cfg *cfg = irq_cfg(irq); 2464 struct irq_desc *desc = *descp;
2465 struct irq_cfg *cfg = desc->chip_data;
2255 unsigned vector, me; 2466 unsigned vector, me;
2256 2467
2257 if (likely(!cfg->move_in_progress)) 2468 if (likely(!cfg->move_in_progress)) {
2469#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
2470 if (likely(!cfg->move_desc_pending))
2471 return;
2472
2473 /* domain has not changed, but affinity did */
2474 me = smp_processor_id();
2475 if (cpu_isset(me, desc->affinity)) {
2476 *descp = desc = move_irq_desc(desc, me);
2477 /* get the new one */
2478 cfg = desc->chip_data;
2479 cfg->move_desc_pending = 0;
2480 }
2481#endif
2258 return; 2482 return;
2483 }
2259 2484
2260 vector = ~get_irq_regs()->orig_ax; 2485 vector = ~get_irq_regs()->orig_ax;
2261 me = smp_processor_id(); 2486 me = smp_processor_id();
2262 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) { 2487 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
2263 cpumask_t cleanup_mask; 2488 cpumask_t cleanup_mask;
2264 2489
2490#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
2491 *descp = desc = move_irq_desc(desc, me);
2492 /* get the new one */
2493 cfg = desc->chip_data;
2494#endif
2495
2265 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map); 2496 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2266 cfg->move_cleanup_count = cpus_weight(cleanup_mask); 2497 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2267 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR); 2498 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
@@ -2269,8 +2500,9 @@ static void irq_complete_move(unsigned int irq)
2269 } 2500 }
2270} 2501}
2271#else 2502#else
2272static inline void irq_complete_move(unsigned int irq) {} 2503static inline void irq_complete_move(struct irq_desc **descp) {}
2273#endif 2504#endif
2505
2274#ifdef CONFIG_INTR_REMAP 2506#ifdef CONFIG_INTR_REMAP
2275static void ack_x2apic_level(unsigned int irq) 2507static void ack_x2apic_level(unsigned int irq)
2276{ 2508{
@@ -2281,11 +2513,14 @@ static void ack_x2apic_edge(unsigned int irq)
2281{ 2513{
2282 ack_x2APIC_irq(); 2514 ack_x2APIC_irq();
2283} 2515}
2516
2284#endif 2517#endif
2285 2518
2286static void ack_apic_edge(unsigned int irq) 2519static void ack_apic_edge(unsigned int irq)
2287{ 2520{
2288 irq_complete_move(irq); 2521 struct irq_desc *desc = irq_to_desc(irq);
2522
2523 irq_complete_move(&desc);
2289 move_native_irq(irq); 2524 move_native_irq(irq);
2290 ack_APIC_irq(); 2525 ack_APIC_irq();
2291} 2526}
@@ -2294,18 +2529,21 @@ atomic_t irq_mis_count;
2294 2529
2295static void ack_apic_level(unsigned int irq) 2530static void ack_apic_level(unsigned int irq)
2296{ 2531{
2532 struct irq_desc *desc = irq_to_desc(irq);
2533
2297#ifdef CONFIG_X86_32 2534#ifdef CONFIG_X86_32
2298 unsigned long v; 2535 unsigned long v;
2299 int i; 2536 int i;
2300#endif 2537#endif
2538 struct irq_cfg *cfg;
2301 int do_unmask_irq = 0; 2539 int do_unmask_irq = 0;
2302 2540
2303 irq_complete_move(irq); 2541 irq_complete_move(&desc);
2304#ifdef CONFIG_GENERIC_PENDING_IRQ 2542#ifdef CONFIG_GENERIC_PENDING_IRQ
2305 /* If we are moving the irq we need to mask it */ 2543 /* If we are moving the irq we need to mask it */
2306 if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) { 2544 if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
2307 do_unmask_irq = 1; 2545 do_unmask_irq = 1;
2308 mask_IO_APIC_irq(irq); 2546 mask_IO_APIC_irq_desc(desc);
2309 } 2547 }
2310#endif 2548#endif
2311 2549
@@ -2329,7 +2567,8 @@ static void ack_apic_level(unsigned int irq)
2329 * operation to prevent an edge-triggered interrupt escaping meanwhile. 2567 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2330 * The idea is from Manfred Spraul. --macro 2568 * The idea is from Manfred Spraul. --macro
2331 */ 2569 */
2332 i = irq_cfg(irq)->vector; 2570 cfg = desc->chip_data;
2571 i = cfg->vector;
2333 2572
2334 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1)); 2573 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2335#endif 2574#endif
@@ -2368,17 +2607,18 @@ static void ack_apic_level(unsigned int irq)
2368 * accurate and is causing problems then it is a hardware bug 2607 * accurate and is causing problems then it is a hardware bug
2369 * and you can go talk to the chipset vendor about it. 2608 * and you can go talk to the chipset vendor about it.
2370 */ 2609 */
2371 if (!io_apic_level_ack_pending(irq)) 2610 cfg = desc->chip_data;
2611 if (!io_apic_level_ack_pending(cfg))
2372 move_masked_irq(irq); 2612 move_masked_irq(irq);
2373 unmask_IO_APIC_irq(irq); 2613 unmask_IO_APIC_irq_desc(desc);
2374 } 2614 }
2375 2615
2376#ifdef CONFIG_X86_32 2616#ifdef CONFIG_X86_32
2377 if (!(v & (1 << (i & 0x1f)))) { 2617 if (!(v & (1 << (i & 0x1f)))) {
2378 atomic_inc(&irq_mis_count); 2618 atomic_inc(&irq_mis_count);
2379 spin_lock(&ioapic_lock); 2619 spin_lock(&ioapic_lock);
2380 __mask_and_edge_IO_APIC_irq(irq); 2620 __mask_and_edge_IO_APIC_irq(cfg);
2381 __unmask_and_level_IO_APIC_irq(irq); 2621 __unmask_and_level_IO_APIC_irq(cfg);
2382 spin_unlock(&ioapic_lock); 2622 spin_unlock(&ioapic_lock);
2383 } 2623 }
2384#endif 2624#endif
@@ -2429,20 +2669,22 @@ static inline void init_IO_APIC_traps(void)
2429 * Also, we've got to be careful not to trash gate 2669 * Also, we've got to be careful not to trash gate
2430 * 0x80, because int 0x80 is hm, kind of importantish. ;) 2670 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2431 */ 2671 */
2432 for_each_irq_cfg(irq, cfg) { 2672 for_each_irq_desc(irq, desc) {
2433 if (IO_APIC_IRQ(irq) && !cfg->vector) { 2673 if (!desc)
2674 continue;
2675
2676 cfg = desc->chip_data;
2677 if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) {
2434 /* 2678 /*
2435 * Hmm.. We don't have an entry for this, 2679 * Hmm.. We don't have an entry for this,
2436 * so default to an old-fashioned 8259 2680 * so default to an old-fashioned 8259
2437 * interrupt if we can.. 2681 * interrupt if we can..
2438 */ 2682 */
2439 if (irq < 16) 2683 if (irq < NR_IRQS_LEGACY)
2440 make_8259A_irq(irq); 2684 make_8259A_irq(irq);
2441 else { 2685 else
2442 desc = irq_to_desc(irq);
2443 /* Strange. Oh, well.. */ 2686 /* Strange. Oh, well.. */
2444 desc->chip = &no_irq_chip; 2687 desc->chip = &no_irq_chip;
2445 }
2446 } 2688 }
2447 } 2689 }
2448} 2690}
@@ -2467,7 +2709,7 @@ static void unmask_lapic_irq(unsigned int irq)
2467 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED); 2709 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2468} 2710}
2469 2711
2470static void ack_lapic_irq (unsigned int irq) 2712static void ack_lapic_irq(unsigned int irq)
2471{ 2713{
2472 ack_APIC_irq(); 2714 ack_APIC_irq();
2473} 2715}
@@ -2479,11 +2721,8 @@ static struct irq_chip lapic_chip __read_mostly = {
2479 .ack = ack_lapic_irq, 2721 .ack = ack_lapic_irq,
2480}; 2722};
2481 2723
2482static void lapic_register_intr(int irq) 2724static void lapic_register_intr(int irq, struct irq_desc *desc)
2483{ 2725{
2484 struct irq_desc *desc;
2485
2486 desc = irq_to_desc(irq);
2487 desc->status &= ~IRQ_LEVEL; 2726 desc->status &= ~IRQ_LEVEL;
2488 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq, 2727 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2489 "edge"); 2728 "edge");
@@ -2587,7 +2826,9 @@ int timer_through_8259 __initdata;
2587 */ 2826 */
2588static inline void __init check_timer(void) 2827static inline void __init check_timer(void)
2589{ 2828{
2590 struct irq_cfg *cfg = irq_cfg(0); 2829 struct irq_desc *desc = irq_to_desc(0);
2830 struct irq_cfg *cfg = desc->chip_data;
2831 int cpu = boot_cpu_id;
2591 int apic1, pin1, apic2, pin2; 2832 int apic1, pin1, apic2, pin2;
2592 unsigned long flags; 2833 unsigned long flags;
2593 unsigned int ver; 2834 unsigned int ver;
@@ -2602,7 +2843,7 @@ static inline void __init check_timer(void)
2602 * get/set the timer IRQ vector: 2843 * get/set the timer IRQ vector:
2603 */ 2844 */
2604 disable_8259A_irq(0); 2845 disable_8259A_irq(0);
2605 assign_irq_vector(0, TARGET_CPUS); 2846 assign_irq_vector(0, cfg, TARGET_CPUS);
2606 2847
2607 /* 2848 /*
2608 * As IRQ0 is to be enabled in the 8259A, the virtual 2849 * As IRQ0 is to be enabled in the 8259A, the virtual
@@ -2653,10 +2894,10 @@ static inline void __init check_timer(void)
2653 * Ok, does IRQ0 through the IOAPIC work? 2894 * Ok, does IRQ0 through the IOAPIC work?
2654 */ 2895 */
2655 if (no_pin1) { 2896 if (no_pin1) {
2656 add_pin_to_irq(0, apic1, pin1); 2897 add_pin_to_irq_cpu(cfg, cpu, apic1, pin1);
2657 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector); 2898 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2658 } 2899 }
2659 unmask_IO_APIC_irq(0); 2900 unmask_IO_APIC_irq_desc(desc);
2660 if (timer_irq_works()) { 2901 if (timer_irq_works()) {
2661 if (nmi_watchdog == NMI_IO_APIC) { 2902 if (nmi_watchdog == NMI_IO_APIC) {
2662 setup_nmi(); 2903 setup_nmi();
@@ -2682,9 +2923,9 @@ static inline void __init check_timer(void)
2682 /* 2923 /*
2683 * legacy devices should be connected to IO APIC #0 2924 * legacy devices should be connected to IO APIC #0
2684 */ 2925 */
2685 replace_pin_at_irq(0, apic1, pin1, apic2, pin2); 2926 replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2);
2686 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector); 2927 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2687 unmask_IO_APIC_irq(0); 2928 unmask_IO_APIC_irq_desc(desc);
2688 enable_8259A_irq(0); 2929 enable_8259A_irq(0);
2689 if (timer_irq_works()) { 2930 if (timer_irq_works()) {
2690 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n"); 2931 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
@@ -2716,7 +2957,7 @@ static inline void __init check_timer(void)
2716 apic_printk(APIC_QUIET, KERN_INFO 2957 apic_printk(APIC_QUIET, KERN_INFO
2717 "...trying to set up timer as Virtual Wire IRQ...\n"); 2958 "...trying to set up timer as Virtual Wire IRQ...\n");
2718 2959
2719 lapic_register_intr(0); 2960 lapic_register_intr(0, desc);
2720 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ 2961 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2721 enable_8259A_irq(0); 2962 enable_8259A_irq(0);
2722 2963
@@ -2901,22 +3142,26 @@ unsigned int create_irq_nr(unsigned int irq_want)
2901 unsigned int irq; 3142 unsigned int irq;
2902 unsigned int new; 3143 unsigned int new;
2903 unsigned long flags; 3144 unsigned long flags;
2904 struct irq_cfg *cfg_new; 3145 struct irq_cfg *cfg_new = NULL;
2905 3146 int cpu = boot_cpu_id;
2906 irq_want = nr_irqs - 1; 3147 struct irq_desc *desc_new = NULL;
2907 3148
2908 irq = 0; 3149 irq = 0;
2909 spin_lock_irqsave(&vector_lock, flags); 3150 spin_lock_irqsave(&vector_lock, flags);
2910 for (new = irq_want; new > 0; new--) { 3151 for (new = irq_want; new < NR_IRQS; new++) {
2911 if (platform_legacy_irq(new)) 3152 if (platform_legacy_irq(new))
2912 continue; 3153 continue;
2913 cfg_new = irq_cfg(new); 3154
2914 if (cfg_new && cfg_new->vector != 0) 3155 desc_new = irq_to_desc_alloc_cpu(new, cpu);
3156 if (!desc_new) {
3157 printk(KERN_INFO "can not get irq_desc for %d\n", new);
2915 continue; 3158 continue;
2916 /* check if need to create one */ 3159 }
2917 if (!cfg_new) 3160 cfg_new = desc_new->chip_data;
2918 cfg_new = irq_cfg_alloc(new); 3161
2919 if (__assign_irq_vector(new, TARGET_CPUS) == 0) 3162 if (cfg_new->vector != 0)
3163 continue;
3164 if (__assign_irq_vector(new, cfg_new, TARGET_CPUS) == 0)
2920 irq = new; 3165 irq = new;
2921 break; 3166 break;
2922 } 3167 }
@@ -2924,15 +3169,21 @@ unsigned int create_irq_nr(unsigned int irq_want)
2924 3169
2925 if (irq > 0) { 3170 if (irq > 0) {
2926 dynamic_irq_init(irq); 3171 dynamic_irq_init(irq);
3172 /* restore it, in case dynamic_irq_init clear it */
3173 if (desc_new)
3174 desc_new->chip_data = cfg_new;
2927 } 3175 }
2928 return irq; 3176 return irq;
2929} 3177}
2930 3178
3179static int nr_irqs_gsi = NR_IRQS_LEGACY;
2931int create_irq(void) 3180int create_irq(void)
2932{ 3181{
3182 unsigned int irq_want;
2933 int irq; 3183 int irq;
2934 3184
2935 irq = create_irq_nr(nr_irqs - 1); 3185 irq_want = nr_irqs_gsi;
3186 irq = create_irq_nr(irq_want);
2936 3187
2937 if (irq == 0) 3188 if (irq == 0)
2938 irq = -1; 3189 irq = -1;
@@ -2943,14 +3194,22 @@ int create_irq(void)
2943void destroy_irq(unsigned int irq) 3194void destroy_irq(unsigned int irq)
2944{ 3195{
2945 unsigned long flags; 3196 unsigned long flags;
3197 struct irq_cfg *cfg;
3198 struct irq_desc *desc;
2946 3199
3200 /* store it, in case dynamic_irq_cleanup clear it */
3201 desc = irq_to_desc(irq);
3202 cfg = desc->chip_data;
2947 dynamic_irq_cleanup(irq); 3203 dynamic_irq_cleanup(irq);
3204 /* connect back irq_cfg */
3205 if (desc)
3206 desc->chip_data = cfg;
2948 3207
2949#ifdef CONFIG_INTR_REMAP 3208#ifdef CONFIG_INTR_REMAP
2950 free_irte(irq); 3209 free_irte(irq);
2951#endif 3210#endif
2952 spin_lock_irqsave(&vector_lock, flags); 3211 spin_lock_irqsave(&vector_lock, flags);
2953 __clear_irq_vector(irq); 3212 __clear_irq_vector(irq, cfg);
2954 spin_unlock_irqrestore(&vector_lock, flags); 3213 spin_unlock_irqrestore(&vector_lock, flags);
2955} 3214}
2956 3215
@@ -2965,12 +3224,12 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
2965 unsigned dest; 3224 unsigned dest;
2966 cpumask_t tmp; 3225 cpumask_t tmp;
2967 3226
3227 cfg = irq_cfg(irq);
2968 tmp = TARGET_CPUS; 3228 tmp = TARGET_CPUS;
2969 err = assign_irq_vector(irq, tmp); 3229 err = assign_irq_vector(irq, cfg, tmp);
2970 if (err) 3230 if (err)
2971 return err; 3231 return err;
2972 3232
2973 cfg = irq_cfg(irq);
2974 cpus_and(tmp, cfg->domain, tmp); 3233 cpus_and(tmp, cfg->domain, tmp);
2975 dest = cpu_mask_to_apicid(tmp); 3234 dest = cpu_mask_to_apicid(tmp);
2976 3235
@@ -3028,35 +3287,35 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
3028#ifdef CONFIG_SMP 3287#ifdef CONFIG_SMP
3029static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask) 3288static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3030{ 3289{
3290 struct irq_desc *desc = irq_to_desc(irq);
3031 struct irq_cfg *cfg; 3291 struct irq_cfg *cfg;
3032 struct msi_msg msg; 3292 struct msi_msg msg;
3033 unsigned int dest; 3293 unsigned int dest;
3034 cpumask_t tmp; 3294 cpumask_t tmp;
3035 struct irq_desc *desc;
3036 3295
3037 cpus_and(tmp, mask, cpu_online_map); 3296 cpus_and(tmp, mask, cpu_online_map);
3038 if (cpus_empty(tmp)) 3297 if (cpus_empty(tmp))
3039 return; 3298 return;
3040 3299
3041 if (assign_irq_vector(irq, mask)) 3300 cfg = desc->chip_data;
3301 if (assign_irq_vector(irq, cfg, mask))
3042 return; 3302 return;
3043 3303
3044 cfg = irq_cfg(irq); 3304 set_extra_move_desc(desc, mask);
3305
3045 cpus_and(tmp, cfg->domain, mask); 3306 cpus_and(tmp, cfg->domain, mask);
3046 dest = cpu_mask_to_apicid(tmp); 3307 dest = cpu_mask_to_apicid(tmp);
3047 3308
3048 read_msi_msg(irq, &msg); 3309 read_msi_msg_desc(desc, &msg);
3049 3310
3050 msg.data &= ~MSI_DATA_VECTOR_MASK; 3311 msg.data &= ~MSI_DATA_VECTOR_MASK;
3051 msg.data |= MSI_DATA_VECTOR(cfg->vector); 3312 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3052 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK; 3313 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3053 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3314 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3054 3315
3055 write_msi_msg(irq, &msg); 3316 write_msi_msg_desc(desc, &msg);
3056 desc = irq_to_desc(irq);
3057 desc->affinity = mask; 3317 desc->affinity = mask;
3058} 3318}
3059
3060#ifdef CONFIG_INTR_REMAP 3319#ifdef CONFIG_INTR_REMAP
3061/* 3320/*
3062 * Migrate the MSI irq to another cpumask. This migration is 3321 * Migrate the MSI irq to another cpumask. This migration is
@@ -3064,11 +3323,11 @@ static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3064 */ 3323 */
3065static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask) 3324static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3066{ 3325{
3326 struct irq_desc *desc = irq_to_desc(irq);
3067 struct irq_cfg *cfg; 3327 struct irq_cfg *cfg;
3068 unsigned int dest; 3328 unsigned int dest;
3069 cpumask_t tmp, cleanup_mask; 3329 cpumask_t tmp, cleanup_mask;
3070 struct irte irte; 3330 struct irte irte;
3071 struct irq_desc *desc;
3072 3331
3073 cpus_and(tmp, mask, cpu_online_map); 3332 cpus_and(tmp, mask, cpu_online_map);
3074 if (cpus_empty(tmp)) 3333 if (cpus_empty(tmp))
@@ -3077,10 +3336,12 @@ static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3077 if (get_irte(irq, &irte)) 3336 if (get_irte(irq, &irte))
3078 return; 3337 return;
3079 3338
3080 if (assign_irq_vector(irq, mask)) 3339 cfg = desc->chip_data;
3340 if (assign_irq_vector(irq, cfg, mask))
3081 return; 3341 return;
3082 3342
3083 cfg = irq_cfg(irq); 3343 set_extra_move_desc(desc, mask);
3344
3084 cpus_and(tmp, cfg->domain, mask); 3345 cpus_and(tmp, cfg->domain, mask);
3085 dest = cpu_mask_to_apicid(tmp); 3346 dest = cpu_mask_to_apicid(tmp);
3086 3347
@@ -3104,9 +3365,9 @@ static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3104 cfg->move_in_progress = 0; 3365 cfg->move_in_progress = 0;
3105 } 3366 }
3106 3367
3107 desc = irq_to_desc(irq);
3108 desc->affinity = mask; 3368 desc->affinity = mask;
3109} 3369}
3370
3110#endif 3371#endif
3111#endif /* CONFIG_SMP */ 3372#endif /* CONFIG_SMP */
3112 3373
@@ -3165,7 +3426,7 @@ static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
3165} 3426}
3166#endif 3427#endif
3167 3428
3168static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq) 3429static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3169{ 3430{
3170 int ret; 3431 int ret;
3171 struct msi_msg msg; 3432 struct msi_msg msg;
@@ -3174,7 +3435,7 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
3174 if (ret < 0) 3435 if (ret < 0)
3175 return ret; 3436 return ret;
3176 3437
3177 set_irq_msi(irq, desc); 3438 set_irq_msi(irq, msidesc);
3178 write_msi_msg(irq, &msg); 3439 write_msi_msg(irq, &msg);
3179 3440
3180#ifdef CONFIG_INTR_REMAP 3441#ifdef CONFIG_INTR_REMAP
@@ -3194,26 +3455,13 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
3194 return 0; 3455 return 0;
3195} 3456}
3196 3457
3197static unsigned int build_irq_for_pci_dev(struct pci_dev *dev) 3458int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc)
3198{
3199 unsigned int irq;
3200
3201 irq = dev->bus->number;
3202 irq <<= 8;
3203 irq |= dev->devfn;
3204 irq <<= 12;
3205
3206 return irq;
3207}
3208
3209int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
3210{ 3459{
3211 unsigned int irq; 3460 unsigned int irq;
3212 int ret; 3461 int ret;
3213 unsigned int irq_want; 3462 unsigned int irq_want;
3214 3463
3215 irq_want = build_irq_for_pci_dev(dev) + 0x100; 3464 irq_want = nr_irqs_gsi;
3216
3217 irq = create_irq_nr(irq_want); 3465 irq = create_irq_nr(irq_want);
3218 if (irq == 0) 3466 if (irq == 0)
3219 return -1; 3467 return -1;
@@ -3227,7 +3475,7 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
3227 goto error; 3475 goto error;
3228no_ir: 3476no_ir:
3229#endif 3477#endif
3230 ret = setup_msi_irq(dev, desc, irq); 3478 ret = setup_msi_irq(dev, msidesc, irq);
3231 if (ret < 0) { 3479 if (ret < 0) {
3232 destroy_irq(irq); 3480 destroy_irq(irq);
3233 return ret; 3481 return ret;
@@ -3245,7 +3493,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3245{ 3493{
3246 unsigned int irq; 3494 unsigned int irq;
3247 int ret, sub_handle; 3495 int ret, sub_handle;
3248 struct msi_desc *desc; 3496 struct msi_desc *msidesc;
3249 unsigned int irq_want; 3497 unsigned int irq_want;
3250 3498
3251#ifdef CONFIG_INTR_REMAP 3499#ifdef CONFIG_INTR_REMAP
@@ -3253,10 +3501,11 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3253 int index = 0; 3501 int index = 0;
3254#endif 3502#endif
3255 3503
3256 irq_want = build_irq_for_pci_dev(dev) + 0x100; 3504 irq_want = nr_irqs_gsi;
3257 sub_handle = 0; 3505 sub_handle = 0;
3258 list_for_each_entry(desc, &dev->msi_list, list) { 3506 list_for_each_entry(msidesc, &dev->msi_list, list) {
3259 irq = create_irq_nr(irq_want--); 3507 irq = create_irq_nr(irq_want);
3508 irq_want++;
3260 if (irq == 0) 3509 if (irq == 0)
3261 return -1; 3510 return -1;
3262#ifdef CONFIG_INTR_REMAP 3511#ifdef CONFIG_INTR_REMAP
@@ -3288,7 +3537,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3288 } 3537 }
3289no_ir: 3538no_ir:
3290#endif 3539#endif
3291 ret = setup_msi_irq(dev, desc, irq); 3540 ret = setup_msi_irq(dev, msidesc, irq);
3292 if (ret < 0) 3541 if (ret < 0)
3293 goto error; 3542 goto error;
3294 sub_handle++; 3543 sub_handle++;
@@ -3309,20 +3558,22 @@ void arch_teardown_msi_irq(unsigned int irq)
3309#ifdef CONFIG_SMP 3558#ifdef CONFIG_SMP
3310static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask) 3559static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
3311{ 3560{
3561 struct irq_desc *desc = irq_to_desc(irq);
3312 struct irq_cfg *cfg; 3562 struct irq_cfg *cfg;
3313 struct msi_msg msg; 3563 struct msi_msg msg;
3314 unsigned int dest; 3564 unsigned int dest;
3315 cpumask_t tmp; 3565 cpumask_t tmp;
3316 struct irq_desc *desc;
3317 3566
3318 cpus_and(tmp, mask, cpu_online_map); 3567 cpus_and(tmp, mask, cpu_online_map);
3319 if (cpus_empty(tmp)) 3568 if (cpus_empty(tmp))
3320 return; 3569 return;
3321 3570
3322 if (assign_irq_vector(irq, mask)) 3571 cfg = desc->chip_data;
3572 if (assign_irq_vector(irq, cfg, mask))
3323 return; 3573 return;
3324 3574
3325 cfg = irq_cfg(irq); 3575 set_extra_move_desc(desc, mask);
3576
3326 cpus_and(tmp, cfg->domain, mask); 3577 cpus_and(tmp, cfg->domain, mask);
3327 dest = cpu_mask_to_apicid(tmp); 3578 dest = cpu_mask_to_apicid(tmp);
3328 3579
@@ -3334,9 +3585,9 @@ static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
3334 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3585 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3335 3586
3336 dmar_msi_write(irq, &msg); 3587 dmar_msi_write(irq, &msg);
3337 desc = irq_to_desc(irq);
3338 desc->affinity = mask; 3588 desc->affinity = mask;
3339} 3589}
3590
3340#endif /* CONFIG_SMP */ 3591#endif /* CONFIG_SMP */
3341 3592
3342struct irq_chip dmar_msi_type = { 3593struct irq_chip dmar_msi_type = {
@@ -3370,8 +3621,8 @@ int arch_setup_dmar_msi(unsigned int irq)
3370#ifdef CONFIG_SMP 3621#ifdef CONFIG_SMP
3371static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask) 3622static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
3372{ 3623{
3624 struct irq_desc *desc = irq_to_desc(irq);
3373 struct irq_cfg *cfg; 3625 struct irq_cfg *cfg;
3374 struct irq_desc *desc;
3375 struct msi_msg msg; 3626 struct msi_msg msg;
3376 unsigned int dest; 3627 unsigned int dest;
3377 cpumask_t tmp; 3628 cpumask_t tmp;
@@ -3380,10 +3631,12 @@ static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
3380 if (cpus_empty(tmp)) 3631 if (cpus_empty(tmp))
3381 return; 3632 return;
3382 3633
3383 if (assign_irq_vector(irq, mask)) 3634 cfg = desc->chip_data;
3635 if (assign_irq_vector(irq, cfg, mask))
3384 return; 3636 return;
3385 3637
3386 cfg = irq_cfg(irq); 3638 set_extra_move_desc(desc, mask);
3639
3387 cpus_and(tmp, cfg->domain, mask); 3640 cpus_and(tmp, cfg->domain, mask);
3388 dest = cpu_mask_to_apicid(tmp); 3641 dest = cpu_mask_to_apicid(tmp);
3389 3642
@@ -3395,9 +3648,9 @@ static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
3395 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3648 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3396 3649
3397 hpet_msi_write(irq, &msg); 3650 hpet_msi_write(irq, &msg);
3398 desc = irq_to_desc(irq);
3399 desc->affinity = mask; 3651 desc->affinity = mask;
3400} 3652}
3653
3401#endif /* CONFIG_SMP */ 3654#endif /* CONFIG_SMP */
3402 3655
3403struct irq_chip hpet_msi_type = { 3656struct irq_chip hpet_msi_type = {
@@ -3452,26 +3705,28 @@ static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3452 3705
3453static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask) 3706static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
3454{ 3707{
3708 struct irq_desc *desc = irq_to_desc(irq);
3455 struct irq_cfg *cfg; 3709 struct irq_cfg *cfg;
3456 unsigned int dest; 3710 unsigned int dest;
3457 cpumask_t tmp; 3711 cpumask_t tmp;
3458 struct irq_desc *desc;
3459 3712
3460 cpus_and(tmp, mask, cpu_online_map); 3713 cpus_and(tmp, mask, cpu_online_map);
3461 if (cpus_empty(tmp)) 3714 if (cpus_empty(tmp))
3462 return; 3715 return;
3463 3716
3464 if (assign_irq_vector(irq, mask)) 3717 cfg = desc->chip_data;
3718 if (assign_irq_vector(irq, cfg, mask))
3465 return; 3719 return;
3466 3720
3467 cfg = irq_cfg(irq); 3721 set_extra_move_desc(desc, mask);
3722
3468 cpus_and(tmp, cfg->domain, mask); 3723 cpus_and(tmp, cfg->domain, mask);
3469 dest = cpu_mask_to_apicid(tmp); 3724 dest = cpu_mask_to_apicid(tmp);
3470 3725
3471 target_ht_irq(irq, dest, cfg->vector); 3726 target_ht_irq(irq, dest, cfg->vector);
3472 desc = irq_to_desc(irq);
3473 desc->affinity = mask; 3727 desc->affinity = mask;
3474} 3728}
3729
3475#endif 3730#endif
3476 3731
3477static struct irq_chip ht_irq_chip = { 3732static struct irq_chip ht_irq_chip = {
@@ -3491,13 +3746,13 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3491 int err; 3746 int err;
3492 cpumask_t tmp; 3747 cpumask_t tmp;
3493 3748
3749 cfg = irq_cfg(irq);
3494 tmp = TARGET_CPUS; 3750 tmp = TARGET_CPUS;
3495 err = assign_irq_vector(irq, tmp); 3751 err = assign_irq_vector(irq, cfg, tmp);
3496 if (!err) { 3752 if (!err) {
3497 struct ht_irq_msg msg; 3753 struct ht_irq_msg msg;
3498 unsigned dest; 3754 unsigned dest;
3499 3755
3500 cfg = irq_cfg(irq);
3501 cpus_and(tmp, cfg->domain, tmp); 3756 cpus_and(tmp, cfg->domain, tmp);
3502 dest = cpu_mask_to_apicid(tmp); 3757 dest = cpu_mask_to_apicid(tmp);
3503 3758
@@ -3543,7 +3798,9 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3543 unsigned long flags; 3798 unsigned long flags;
3544 int err; 3799 int err;
3545 3800
3546 err = assign_irq_vector(irq, *eligible_cpu); 3801 cfg = irq_cfg(irq);
3802
3803 err = assign_irq_vector(irq, cfg, *eligible_cpu);
3547 if (err != 0) 3804 if (err != 0)
3548 return err; 3805 return err;
3549 3806
@@ -3552,8 +3809,6 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3552 irq_name); 3809 irq_name);
3553 spin_unlock_irqrestore(&vector_lock, flags); 3810 spin_unlock_irqrestore(&vector_lock, flags);
3554 3811
3555 cfg = irq_cfg(irq);
3556
3557 mmr_value = 0; 3812 mmr_value = 0;
3558 entry = (struct uv_IO_APIC_route_entry *)&mmr_value; 3813 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3559 BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long)); 3814 BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
@@ -3605,9 +3860,16 @@ int __init io_apic_get_redir_entries (int ioapic)
3605 return reg_01.bits.entries; 3860 return reg_01.bits.entries;
3606} 3861}
3607 3862
3608int __init probe_nr_irqs(void) 3863void __init probe_nr_irqs_gsi(void)
3609{ 3864{
3610 return NR_IRQS; 3865 int idx;
3866 int nr = 0;
3867
3868 for (idx = 0; idx < nr_ioapics; idx++)
3869 nr += io_apic_get_redir_entries(idx) + 1;
3870
3871 if (nr > nr_irqs_gsi)
3872 nr_irqs_gsi = nr;
3611} 3873}
3612 3874
3613/* -------------------------------------------------------------------------- 3875/* --------------------------------------------------------------------------
@@ -3706,19 +3968,31 @@ int __init io_apic_get_version(int ioapic)
3706 3968
3707int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity) 3969int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
3708{ 3970{
3971 struct irq_desc *desc;
3972 struct irq_cfg *cfg;
3973 int cpu = boot_cpu_id;
3974
3709 if (!IO_APIC_IRQ(irq)) { 3975 if (!IO_APIC_IRQ(irq)) {
3710 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n", 3976 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3711 ioapic); 3977 ioapic);
3712 return -EINVAL; 3978 return -EINVAL;
3713 } 3979 }
3714 3980
3981 desc = irq_to_desc_alloc_cpu(irq, cpu);
3982 if (!desc) {
3983 printk(KERN_INFO "can not get irq_desc %d\n", irq);
3984 return 0;
3985 }
3986
3715 /* 3987 /*
3716 * IRQs < 16 are already in the irq_2_pin[] map 3988 * IRQs < 16 are already in the irq_2_pin[] map
3717 */ 3989 */
3718 if (irq >= 16) 3990 if (irq >= NR_IRQS_LEGACY) {
3719 add_pin_to_irq(irq, ioapic, pin); 3991 cfg = desc->chip_data;
3992 add_pin_to_irq_cpu(cfg, cpu, ioapic, pin);
3993 }
3720 3994
3721 setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity); 3995 setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity);
3722 3996
3723 return 0; 3997 return 0;
3724} 3998}
@@ -3772,9 +4046,10 @@ void __init setup_ioapic_dest(void)
3772 * when you have too many devices, because at that time only boot 4046 * when you have too many devices, because at that time only boot
3773 * cpu is online. 4047 * cpu is online.
3774 */ 4048 */
3775 cfg = irq_cfg(irq); 4049 desc = irq_to_desc(irq);
4050 cfg = desc->chip_data;
3776 if (!cfg->vector) { 4051 if (!cfg->vector) {
3777 setup_IO_APIC_irq(ioapic, pin, irq, 4052 setup_IO_APIC_irq(ioapic, pin, irq, desc,
3778 irq_trigger(irq_entry), 4053 irq_trigger(irq_entry),
3779 irq_polarity(irq_entry)); 4054 irq_polarity(irq_entry));
3780 continue; 4055 continue;
@@ -3784,7 +4059,6 @@ void __init setup_ioapic_dest(void)
3784 /* 4059 /*
3785 * Honour affinities which have been set in early boot 4060 * Honour affinities which have been set in early boot
3786 */ 4061 */
3787 desc = irq_to_desc(irq);
3788 if (desc->status & 4062 if (desc->status &
3789 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET)) 4063 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
3790 mask = desc->affinity; 4064 mask = desc->affinity;
@@ -3793,10 +4067,10 @@ void __init setup_ioapic_dest(void)
3793 4067
3794#ifdef CONFIG_INTR_REMAP 4068#ifdef CONFIG_INTR_REMAP
3795 if (intr_remapping_enabled) 4069 if (intr_remapping_enabled)
3796 set_ir_ioapic_affinity_irq(irq, mask); 4070 set_ir_ioapic_affinity_irq_desc(desc, mask);
3797 else 4071 else
3798#endif 4072#endif
3799 set_ioapic_affinity_irq(irq, mask); 4073 set_ioapic_affinity_irq_desc(desc, mask);
3800 } 4074 }
3801 4075
3802 } 4076 }
@@ -3845,7 +4119,6 @@ void __init ioapic_init_mappings(void)
3845 struct resource *ioapic_res; 4119 struct resource *ioapic_res;
3846 int i; 4120 int i;
3847 4121
3848 irq_2_pin_init();
3849 ioapic_res = ioapic_setup_resources(); 4122 ioapic_res = ioapic_setup_resources();
3850 for (i = 0; i < nr_ioapics; i++) { 4123 for (i = 0; i < nr_ioapics; i++) {
3851 if (smp_found_config) { 4124 if (smp_found_config) {
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index d1d4dc52f649..3f1d9d18df67 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -118,6 +118,9 @@ int show_interrupts(struct seq_file *p, void *v)
118 } 118 }
119 119
120 desc = irq_to_desc(i); 120 desc = irq_to_desc(i);
121 if (!desc)
122 return 0;
123
121 spin_lock_irqsave(&desc->lock, flags); 124 spin_lock_irqsave(&desc->lock, flags);
122#ifndef CONFIG_SMP 125#ifndef CONFIG_SMP
123 any_count = kstat_irqs(i); 126 any_count = kstat_irqs(i);
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index a51382672de0..119fc9c8ff7f 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -242,6 +242,8 @@ void fixup_irqs(cpumask_t map)
242 for_each_irq_desc(irq, desc) { 242 for_each_irq_desc(irq, desc) {
243 cpumask_t mask; 243 cpumask_t mask;
244 244
245 if (!desc)
246 continue;
245 if (irq == 2) 247 if (irq == 2)
246 continue; 248 continue;
247 249
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 1df869e5bd0b..a174a217eb1a 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -91,6 +91,8 @@ void fixup_irqs(cpumask_t map)
91 int break_affinity = 0; 91 int break_affinity = 0;
92 int set_affinity = 1; 92 int set_affinity = 1;
93 93
94 if (!desc)
95 continue;
94 if (irq == 2) 96 if (irq == 2)
95 continue; 97 continue;
96 98
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
index 607db63044a5..203384ed2b5d 100644
--- a/arch/x86/kernel/irqinit_32.c
+++ b/arch/x86/kernel/irqinit_32.c
@@ -68,8 +68,7 @@ void __init init_ISA_irqs (void)
68 /* 68 /*
69 * 16 old-style INTA-cycle interrupts: 69 * 16 old-style INTA-cycle interrupts:
70 */ 70 */
71 for (i = 0; i < 16; i++) { 71 for (i = 0; i < NR_IRQS_LEGACY; i++) {
72 /* first time call this irq_desc */
73 struct irq_desc *desc = irq_to_desc(i); 72 struct irq_desc *desc = irq_to_desc(i);
74 73
75 desc->status = IRQ_DISABLED; 74 desc->status = IRQ_DISABLED;
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c
index 8670b3ce626e..6190e6ef546c 100644
--- a/arch/x86/kernel/irqinit_64.c
+++ b/arch/x86/kernel/irqinit_64.c
@@ -76,8 +76,7 @@ void __init init_ISA_irqs(void)
76 init_bsp_APIC(); 76 init_bsp_APIC();
77 init_8259A(0); 77 init_8259A(0);
78 78
79 for (i = 0; i < 16; i++) { 79 for (i = 0; i < NR_IRQS_LEGACY; i++) {
80 /* first time call this irq_desc */
81 struct irq_desc *desc = irq_to_desc(i); 80 struct irq_desc *desc = irq_to_desc(i);
82 81
83 desc->status = IRQ_DISABLED; 82 desc->status = IRQ_DISABLED;
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index 7a3dfceb90e4..19a1044a0cd9 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -101,11 +101,15 @@ static void __init dma32_free_bootmem(void)
101 dma32_bootmem_ptr = NULL; 101 dma32_bootmem_ptr = NULL;
102 dma32_bootmem_size = 0; 102 dma32_bootmem_size = 0;
103} 103}
104#endif
104 105
105void __init pci_iommu_alloc(void) 106void __init pci_iommu_alloc(void)
106{ 107{
108#ifdef CONFIG_X86_64
107 /* free the range so iommu could get some range less than 4G */ 109 /* free the range so iommu could get some range less than 4G */
108 dma32_free_bootmem(); 110 dma32_free_bootmem();
111#endif
112
109 /* 113 /*
110 * The order of these functions is important for 114 * The order of these functions is important for
111 * fall-back/fail-over reasons 115 * fall-back/fail-over reasons
@@ -121,15 +125,6 @@ void __init pci_iommu_alloc(void)
121 pci_swiotlb_init(); 125 pci_swiotlb_init();
122} 126}
123 127
124unsigned long iommu_nr_pages(unsigned long addr, unsigned long len)
125{
126 unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);
127
128 return size >> PAGE_SHIFT;
129}
130EXPORT_SYMBOL(iommu_nr_pages);
131#endif
132
133void *dma_generic_alloc_coherent(struct device *dev, size_t size, 128void *dma_generic_alloc_coherent(struct device *dev, size_t size,
134 dma_addr_t *dma_addr, gfp_t flag) 129 dma_addr_t *dma_addr, gfp_t flag)
135{ 130{
diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb_64.c
index 3c539d111abb..242c3440687f 100644
--- a/arch/x86/kernel/pci-swiotlb_64.c
+++ b/arch/x86/kernel/pci-swiotlb_64.c
@@ -3,6 +3,8 @@
3#include <linux/pci.h> 3#include <linux/pci.h>
4#include <linux/cache.h> 4#include <linux/cache.h>
5#include <linux/module.h> 5#include <linux/module.h>
6#include <linux/swiotlb.h>
7#include <linux/bootmem.h>
6#include <linux/dma-mapping.h> 8#include <linux/dma-mapping.h>
7 9
8#include <asm/iommu.h> 10#include <asm/iommu.h>
@@ -11,6 +13,31 @@
11 13
12int swiotlb __read_mostly; 14int swiotlb __read_mostly;
13 15
16void *swiotlb_alloc_boot(size_t size, unsigned long nslabs)
17{
18 return alloc_bootmem_low_pages(size);
19}
20
21void *swiotlb_alloc(unsigned order, unsigned long nslabs)
22{
23 return (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN, order);
24}
25
26dma_addr_t swiotlb_phys_to_bus(phys_addr_t paddr)
27{
28 return paddr;
29}
30
31phys_addr_t swiotlb_bus_to_phys(dma_addr_t baddr)
32{
33 return baddr;
34}
35
36int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size)
37{
38 return 0;
39}
40
14static dma_addr_t 41static dma_addr_t
15swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size, 42swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size,
16 int direction) 43 int direction)
@@ -50,8 +77,10 @@ struct dma_mapping_ops swiotlb_dma_ops = {
50void __init pci_swiotlb_init(void) 77void __init pci_swiotlb_init(void)
51{ 78{
52 /* don't initialize swiotlb if iommu=off (no_iommu=1) */ 79 /* don't initialize swiotlb if iommu=off (no_iommu=1) */
80#ifdef CONFIG_X86_64
53 if (!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN) 81 if (!iommu_detected && !no_iommu && max_pfn > MAX_DMA32_PFN)
54 swiotlb = 1; 82 swiotlb = 1;
83#endif
55 if (swiotlb_force) 84 if (swiotlb_force)
56 swiotlb = 1; 85 swiotlb = 1;
57 if (swiotlb) { 86 if (swiotlb) {
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c
index 67465ed89310..309949e9e1c1 100644
--- a/arch/x86/kernel/quirks.c
+++ b/arch/x86/kernel/quirks.c
@@ -168,6 +168,8 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH7_31,
168 ich_force_enable_hpet); 168 ich_force_enable_hpet);
169DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH8_1, 169DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH8_1,
170 ich_force_enable_hpet); 170 ich_force_enable_hpet);
171DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH8_4,
172 ich_force_enable_hpet);
171DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH9_7, 173DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH9_7,
172 ich_force_enable_hpet); 174 ich_force_enable_hpet);
173 175
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 08e02e8453c9..ae0d8042cf69 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -953,7 +953,7 @@ void __init setup_arch(char **cmdline_p)
953 ioapic_init_mappings(); 953 ioapic_init_mappings();
954 954
955 /* need to wait for io_apic is mapped */ 955 /* need to wait for io_apic is mapped */
956 nr_irqs = probe_nr_irqs(); 956 probe_nr_irqs_gsi();
957 957
958 kvm_guest_init(); 958 kvm_guest_init();
959 959
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 9e68075544f6..4a20b2f9a381 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -39,7 +39,7 @@ static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned lon
39#define __do_strncpy_from_user(dst, src, count, res) \ 39#define __do_strncpy_from_user(dst, src, count, res) \
40do { \ 40do { \
41 int __d0, __d1, __d2; \ 41 int __d0, __d1, __d2; \
42 might_sleep(); \ 42 might_fault(); \
43 __asm__ __volatile__( \ 43 __asm__ __volatile__( \
44 " testl %1,%1\n" \ 44 " testl %1,%1\n" \
45 " jz 2f\n" \ 45 " jz 2f\n" \
@@ -126,7 +126,7 @@ EXPORT_SYMBOL(strncpy_from_user);
126#define __do_clear_user(addr,size) \ 126#define __do_clear_user(addr,size) \
127do { \ 127do { \
128 int __d0; \ 128 int __d0; \
129 might_sleep(); \ 129 might_fault(); \
130 __asm__ __volatile__( \ 130 __asm__ __volatile__( \
131 "0: rep; stosl\n" \ 131 "0: rep; stosl\n" \
132 " movl %2,%0\n" \ 132 " movl %2,%0\n" \
@@ -155,7 +155,7 @@ do { \
155unsigned long 155unsigned long
156clear_user(void __user *to, unsigned long n) 156clear_user(void __user *to, unsigned long n)
157{ 157{
158 might_sleep(); 158 might_fault();
159 if (access_ok(VERIFY_WRITE, to, n)) 159 if (access_ok(VERIFY_WRITE, to, n))
160 __do_clear_user(to, n); 160 __do_clear_user(to, n);
161 return n; 161 return n;
@@ -197,7 +197,7 @@ long strnlen_user(const char __user *s, long n)
197 unsigned long mask = -__addr_ok(s); 197 unsigned long mask = -__addr_ok(s);
198 unsigned long res, tmp; 198 unsigned long res, tmp;
199 199
200 might_sleep(); 200 might_fault();
201 201
202 __asm__ __volatile__( 202 __asm__ __volatile__(
203 " testl %0, %0\n" 203 " testl %0, %0\n"
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index f4df6e7c718b..64d6c84e6353 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -15,7 +15,7 @@
15#define __do_strncpy_from_user(dst,src,count,res) \ 15#define __do_strncpy_from_user(dst,src,count,res) \
16do { \ 16do { \
17 long __d0, __d1, __d2; \ 17 long __d0, __d1, __d2; \
18 might_sleep(); \ 18 might_fault(); \
19 __asm__ __volatile__( \ 19 __asm__ __volatile__( \
20 " testq %1,%1\n" \ 20 " testq %1,%1\n" \
21 " jz 2f\n" \ 21 " jz 2f\n" \
@@ -64,7 +64,7 @@ EXPORT_SYMBOL(strncpy_from_user);
64unsigned long __clear_user(void __user *addr, unsigned long size) 64unsigned long __clear_user(void __user *addr, unsigned long size)
65{ 65{
66 long __d0; 66 long __d0;
67 might_sleep(); 67 might_fault();
68 /* no memory constraint because it doesn't change any memory gcc knows 68 /* no memory constraint because it doesn't change any memory gcc knows
69 about */ 69 about */
70 asm volatile( 70 asm volatile(
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 800e1d94c1b5..8655b5bb0963 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -21,6 +21,7 @@
21#include <linux/init.h> 21#include <linux/init.h>
22#include <linux/highmem.h> 22#include <linux/highmem.h>
23#include <linux/pagemap.h> 23#include <linux/pagemap.h>
24#include <linux/pci.h>
24#include <linux/pfn.h> 25#include <linux/pfn.h>
25#include <linux/poison.h> 26#include <linux/poison.h>
26#include <linux/bootmem.h> 27#include <linux/bootmem.h>
@@ -967,6 +968,8 @@ void __init mem_init(void)
967 int codesize, reservedpages, datasize, initsize; 968 int codesize, reservedpages, datasize, initsize;
968 int tmp; 969 int tmp;
969 970
971 pci_iommu_alloc();
972
970#ifdef CONFIG_FLATMEM 973#ifdef CONFIG_FLATMEM
971 BUG_ON(!mem_map); 974 BUG_ON(!mem_map);
972#endif 975#endif
diff --git a/arch/x86/oprofile/op_model_amd.c b/arch/x86/oprofile/op_model_amd.c
index 509513760a6e..98658f25f542 100644
--- a/arch/x86/oprofile/op_model_amd.c
+++ b/arch/x86/oprofile/op_model_amd.c
@@ -65,11 +65,13 @@ static unsigned long reset_value[NUM_COUNTERS];
65#define IBS_FETCH_BEGIN 3 65#define IBS_FETCH_BEGIN 3
66#define IBS_OP_BEGIN 4 66#define IBS_OP_BEGIN 4
67 67
68/* The function interface needs to be fixed, something like add 68/*
69 data. Should then be added to linux/oprofile.h. */ 69 * The function interface needs to be fixed, something like add
70 * data. Should then be added to linux/oprofile.h.
71 */
70extern void 72extern void
71oprofile_add_ibs_sample(struct pt_regs *const regs, 73oprofile_add_ibs_sample(struct pt_regs * const regs,
72 unsigned int *const ibs_sample, int ibs_code); 74 unsigned int * const ibs_sample, int ibs_code);
73 75
74struct ibs_fetch_sample { 76struct ibs_fetch_sample {
75 /* MSRC001_1031 IBS Fetch Linear Address Register */ 77 /* MSRC001_1031 IBS Fetch Linear Address Register */
@@ -104,11 +106,6 @@ struct ibs_op_sample {
104 unsigned int ibs_dc_phys_high; 106 unsigned int ibs_dc_phys_high;
105}; 107};
106 108
107/*
108 * unitialize the APIC for the IBS interrupts if needed on AMD Family10h+
109*/
110static void clear_ibs_nmi(void);
111
112static int ibs_allowed; /* AMD Family10h and later */ 109static int ibs_allowed; /* AMD Family10h and later */
113 110
114struct op_ibs_config { 111struct op_ibs_config {
@@ -223,7 +220,7 @@ op_amd_handle_ibs(struct pt_regs * const regs,
223 (unsigned int *)&ibs_fetch, 220 (unsigned int *)&ibs_fetch,
224 IBS_FETCH_BEGIN); 221 IBS_FETCH_BEGIN);
225 222
226 /*reenable the IRQ */ 223 /* reenable the IRQ */
227 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high); 224 rdmsr(MSR_AMD64_IBSFETCHCTL, low, high);
228 high &= ~IBS_FETCH_HIGH_VALID_BIT; 225 high &= ~IBS_FETCH_HIGH_VALID_BIT;
229 high |= IBS_FETCH_HIGH_ENABLE; 226 high |= IBS_FETCH_HIGH_ENABLE;
@@ -331,8 +328,10 @@ static void op_amd_stop(struct op_msrs const * const msrs)
331 unsigned int low, high; 328 unsigned int low, high;
332 int i; 329 int i;
333 330
334 /* Subtle: stop on all counters to avoid race with 331 /*
335 * setting our pm callback */ 332 * Subtle: stop on all counters to avoid race with setting our
333 * pm callback
334 */
336 for (i = 0 ; i < NUM_COUNTERS ; ++i) { 335 for (i = 0 ; i < NUM_COUNTERS ; ++i) {
337 if (!reset_value[i]) 336 if (!reset_value[i])
338 continue; 337 continue;
@@ -343,13 +342,15 @@ static void op_amd_stop(struct op_msrs const * const msrs)
343 342
344#ifdef CONFIG_OPROFILE_IBS 343#ifdef CONFIG_OPROFILE_IBS
345 if (ibs_allowed && ibs_config.fetch_enabled) { 344 if (ibs_allowed && ibs_config.fetch_enabled) {
346 low = 0; /* clear max count and enable */ 345 /* clear max count and enable */
346 low = 0;
347 high = 0; 347 high = 0;
348 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high); 348 wrmsr(MSR_AMD64_IBSFETCHCTL, low, high);
349 } 349 }
350 350
351 if (ibs_allowed && ibs_config.op_enabled) { 351 if (ibs_allowed && ibs_config.op_enabled) {
352 low = 0; /* clear max count and enable */ 352 /* clear max count and enable */
353 low = 0;
353 high = 0; 354 high = 0;
354 wrmsr(MSR_AMD64_IBSOPCTL, low, high); 355 wrmsr(MSR_AMD64_IBSOPCTL, low, high);
355 } 356 }
@@ -370,18 +371,7 @@ static void op_amd_shutdown(struct op_msrs const * const msrs)
370 } 371 }
371} 372}
372 373
373#ifndef CONFIG_OPROFILE_IBS 374#ifdef CONFIG_OPROFILE_IBS
374
375/* no IBS support */
376
377static int op_amd_init(struct oprofile_operations *ops)
378{
379 return 0;
380}
381
382static void op_amd_exit(void) {}
383
384#else
385 375
386static u8 ibs_eilvt_off; 376static u8 ibs_eilvt_off;
387 377
@@ -395,7 +385,7 @@ static inline void apic_clear_ibs_nmi_per_cpu(void *arg)
395 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1); 385 setup_APIC_eilvt_ibs(0, APIC_EILVT_MSG_FIX, 1);
396} 386}
397 387
398static int pfm_amd64_setup_eilvt(void) 388static int init_ibs_nmi(void)
399{ 389{
400#define IBSCTL_LVTOFFSETVAL (1 << 8) 390#define IBSCTL_LVTOFFSETVAL (1 << 8)
401#define IBSCTL 0x1cc 391#define IBSCTL 0x1cc
@@ -443,18 +433,22 @@ static int pfm_amd64_setup_eilvt(void)
443 return 0; 433 return 0;
444} 434}
445 435
446/* 436/* uninitialize the APIC for the IBS interrupts if needed */
447 * initialize the APIC for the IBS interrupts 437static void clear_ibs_nmi(void)
448 * if available (AMD Family10h rev B0 and later) 438{
449 */ 439 if (ibs_allowed)
450static void setup_ibs(void) 440 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1);
441}
442
443/* initialize the APIC for the IBS interrupts if available */
444static void ibs_init(void)
451{ 445{
452 ibs_allowed = boot_cpu_has(X86_FEATURE_IBS); 446 ibs_allowed = boot_cpu_has(X86_FEATURE_IBS);
453 447
454 if (!ibs_allowed) 448 if (!ibs_allowed)
455 return; 449 return;
456 450
457 if (pfm_amd64_setup_eilvt()) { 451 if (init_ibs_nmi()) {
458 ibs_allowed = 0; 452 ibs_allowed = 0;
459 return; 453 return;
460 } 454 }
@@ -462,14 +456,12 @@ static void setup_ibs(void)
462 printk(KERN_INFO "oprofile: AMD IBS detected\n"); 456 printk(KERN_INFO "oprofile: AMD IBS detected\n");
463} 457}
464 458
465 459static void ibs_exit(void)
466/*
467 * unitialize the APIC for the IBS interrupts if needed on AMD Family10h
468 * rev B0 and later */
469static void clear_ibs_nmi(void)
470{ 460{
471 if (ibs_allowed) 461 if (!ibs_allowed)
472 on_each_cpu(apic_clear_ibs_nmi_per_cpu, NULL, 1); 462 return;
463
464 clear_ibs_nmi();
473} 465}
474 466
475static int (*create_arch_files)(struct super_block *sb, struct dentry *root); 467static int (*create_arch_files)(struct super_block *sb, struct dentry *root);
@@ -519,7 +511,7 @@ static int setup_ibs_files(struct super_block *sb, struct dentry *root)
519 511
520static int op_amd_init(struct oprofile_operations *ops) 512static int op_amd_init(struct oprofile_operations *ops)
521{ 513{
522 setup_ibs(); 514 ibs_init();
523 create_arch_files = ops->create_files; 515 create_arch_files = ops->create_files;
524 ops->create_files = setup_ibs_files; 516 ops->create_files = setup_ibs_files;
525 return 0; 517 return 0;
@@ -527,10 +519,21 @@ static int op_amd_init(struct oprofile_operations *ops)
527 519
528static void op_amd_exit(void) 520static void op_amd_exit(void)
529{ 521{
530 clear_ibs_nmi(); 522 ibs_exit();
531} 523}
532 524
533#endif 525#else
526
527/* no IBS support */
528
529static int op_amd_init(struct oprofile_operations *ops)
530{
531 return 0;
532}
533
534static void op_amd_exit(void) {}
535
536#endif /* CONFIG_OPROFILE_IBS */
534 537
535struct op_x86_model_spec const op_amd_spec = { 538struct op_x86_model_spec const op_amd_spec = {
536 .init = op_amd_init, 539 .init = op_amd_init,
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-21 03:27:25 -0500