diff options
33 files changed, 1221 insertions, 330 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 66c14961a9b..0f44add3e0b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
@@ -247,6 +247,28 @@ config X86_HAS_BOOT_CPU_ID | |||
247 | def_bool y | 247 | def_bool y |
248 | depends on X86_VOYAGER | 248 | depends on X86_VOYAGER |
249 | 249 | ||
250 | config SPARSE_IRQ | ||
251 | bool "Support sparse irq numbering" | ||
252 | depends on PCI_MSI || HT_IRQ | ||
253 | help | ||
254 | This enables support for sparse irqs. This is useful for distro | ||
255 | kernels that want to define a high CONFIG_NR_CPUS value but still | ||
256 | want to have low kernel memory footprint on smaller machines. | ||
257 | |||
258 | ( Sparse IRQs can also be beneficial on NUMA boxes, as they spread | ||
259 | out the irq_desc[] array in a more NUMA-friendly way. ) | ||
260 | |||
261 | If you don't know what to do here, say N. | ||
262 | |||
263 | config NUMA_MIGRATE_IRQ_DESC | ||
264 | bool "Move irq desc when changing irq smp_affinity" | ||
265 | depends on SPARSE_IRQ && NUMA | ||
266 | default n | ||
267 | help | ||
268 | This enables moving irq_desc to cpu/node that irq will use handled. | ||
269 | |||
270 | If you don't know what to do here, say N. | ||
271 | |||
250 | config X86_FIND_SMP_CONFIG | 272 | config X86_FIND_SMP_CONFIG |
251 | def_bool y | 273 | def_bool y |
252 | depends on X86_MPPARSE || X86_VOYAGER | 274 | depends on X86_MPPARSE || X86_VOYAGER |
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h index e475e009ae5..7a1f44ac1f1 100644 --- a/arch/x86/include/asm/io_apic.h +++ b/arch/x86/include/asm/io_apic.h | |||
@@ -198,17 +198,14 @@ extern void restore_IO_APIC_setup(void); | |||
198 | extern void reinit_intr_remapped_IO_APIC(int); | 198 | extern void reinit_intr_remapped_IO_APIC(int); |
199 | #endif | 199 | #endif |
200 | 200 | ||
201 | extern int probe_nr_irqs(void); | 201 | extern void probe_nr_irqs_gsi(void); |
202 | 202 | ||
203 | #else /* !CONFIG_X86_IO_APIC */ | 203 | #else /* !CONFIG_X86_IO_APIC */ |
204 | #define io_apic_assign_pci_irqs 0 | 204 | #define io_apic_assign_pci_irqs 0 |
205 | static const int timer_through_8259 = 0; | 205 | static const int timer_through_8259 = 0; |
206 | static inline void ioapic_init_mappings(void) { } | 206 | static inline void ioapic_init_mappings(void) { } |
207 | 207 | ||
208 | static inline int probe_nr_irqs(void) | 208 | static inline void probe_nr_irqs_gsi(void) { } |
209 | { | ||
210 | return NR_IRQS; | ||
211 | } | ||
212 | #endif | 209 | #endif |
213 | 210 | ||
214 | #endif /* _ASM_X86_IO_APIC_H */ | 211 | #endif /* _ASM_X86_IO_APIC_H */ |
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 0005adb0f94..f7ff65032b9 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h | |||
@@ -101,12 +101,23 @@ | |||
101 | #define LAST_VM86_IRQ 15 | 101 | #define LAST_VM86_IRQ 15 |
102 | #define invalid_vm86_irq(irq) ((irq) < 3 || (irq) > 15) | 102 | #define invalid_vm86_irq(irq) ((irq) < 3 || (irq) > 15) |
103 | 103 | ||
104 | #define NR_IRQS_LEGACY 16 | ||
105 | |||
104 | #if defined(CONFIG_X86_IO_APIC) && !defined(CONFIG_X86_VOYAGER) | 106 | #if defined(CONFIG_X86_IO_APIC) && !defined(CONFIG_X86_VOYAGER) |
107 | |||
108 | #ifndef CONFIG_SPARSE_IRQ | ||
105 | # if NR_CPUS < MAX_IO_APICS | 109 | # if NR_CPUS < MAX_IO_APICS |
106 | # define NR_IRQS (NR_VECTORS + (32 * NR_CPUS)) | 110 | # define NR_IRQS (NR_VECTORS + (32 * NR_CPUS)) |
107 | # else | 111 | # else |
108 | # define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS)) | 112 | # define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS)) |
109 | # endif | 113 | # endif |
114 | #else | ||
115 | # if (8 * NR_CPUS) > (32 * MAX_IO_APICS) | ||
116 | # define NR_IRQS (NR_VECTORS + (8 * NR_CPUS)) | ||
117 | # else | ||
118 | # define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS)) | ||
119 | # endif | ||
120 | #endif | ||
110 | 121 | ||
111 | #elif defined(CONFIG_X86_VOYAGER) | 122 | #elif defined(CONFIG_X86_VOYAGER) |
112 | 123 | ||
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c index 679e7bbbbcd..f6ea94b74da 100644 --- a/arch/x86/kernel/io_apic.c +++ b/arch/x86/kernel/io_apic.c | |||
@@ -108,93 +108,252 @@ static int __init parse_noapic(char *str) | |||
108 | early_param("noapic", parse_noapic); | 108 | early_param("noapic", parse_noapic); |
109 | 109 | ||
110 | struct irq_pin_list; | 110 | struct irq_pin_list; |
111 | |||
112 | /* | ||
113 | * This is performance-critical, we want to do it O(1) | ||
114 | * | ||
115 | * the indexing order of this array favors 1:1 mappings | ||
116 | * between pins and IRQs. | ||
117 | */ | ||
118 | |||
119 | struct irq_pin_list { | ||
120 | int apic, pin; | ||
121 | struct irq_pin_list *next; | ||
122 | }; | ||
123 | |||
124 | static struct irq_pin_list *get_one_free_irq_2_pin(int cpu) | ||
125 | { | ||
126 | struct irq_pin_list *pin; | ||
127 | int node; | ||
128 | |||
129 | node = cpu_to_node(cpu); | ||
130 | |||
131 | pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node); | ||
132 | printk(KERN_DEBUG " alloc irq_2_pin on cpu %d node %d\n", cpu, node); | ||
133 | |||
134 | return pin; | ||
135 | } | ||
136 | |||
111 | struct irq_cfg { | 137 | struct irq_cfg { |
112 | unsigned int irq; | ||
113 | struct irq_pin_list *irq_2_pin; | 138 | struct irq_pin_list *irq_2_pin; |
114 | cpumask_t domain; | 139 | cpumask_t domain; |
115 | cpumask_t old_domain; | 140 | cpumask_t old_domain; |
116 | unsigned move_cleanup_count; | 141 | unsigned move_cleanup_count; |
117 | u8 vector; | 142 | u8 vector; |
118 | u8 move_in_progress : 1; | 143 | u8 move_in_progress : 1; |
144 | #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC | ||
145 | u8 move_desc_pending : 1; | ||
146 | #endif | ||
119 | }; | 147 | }; |
120 | 148 | ||
121 | /* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ | 149 | /* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ |
150 | #ifdef CONFIG_SPARSE_IRQ | ||
151 | static struct irq_cfg irq_cfgx[] = { | ||
152 | #else | ||
122 | static struct irq_cfg irq_cfgx[NR_IRQS] = { | 153 | static struct irq_cfg irq_cfgx[NR_IRQS] = { |
123 | [0] = { .irq = 0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, }, | 154 | #endif |
124 | [1] = { .irq = 1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, }, | 155 | [0] = { .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, }, |
125 | [2] = { .irq = 2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, }, | 156 | [1] = { .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, }, |
126 | [3] = { .irq = 3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, }, | 157 | [2] = { .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, }, |
127 | [4] = { .irq = 4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, }, | 158 | [3] = { .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, }, |
128 | [5] = { .irq = 5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, }, | 159 | [4] = { .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, }, |
129 | [6] = { .irq = 6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, }, | 160 | [5] = { .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, }, |
130 | [7] = { .irq = 7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, }, | 161 | [6] = { .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, }, |
131 | [8] = { .irq = 8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, }, | 162 | [7] = { .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, }, |
132 | [9] = { .irq = 9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, }, | 163 | [8] = { .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, }, |
133 | [10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, }, | 164 | [9] = { .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, }, |
134 | [11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, }, | 165 | [10] = { .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, }, |
135 | [12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, }, | 166 | [11] = { .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, }, |
136 | [13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, }, | 167 | [12] = { .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, }, |
137 | [14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, }, | 168 | [13] = { .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, }, |
138 | [15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, }, | 169 | [14] = { .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, }, |
170 | [15] = { .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, }, | ||
139 | }; | 171 | }; |
140 | 172 | ||
141 | #define for_each_irq_cfg(irq, cfg) \ | 173 | void __init arch_early_irq_init(void) |
142 | for (irq = 0, cfg = irq_cfgx; irq < nr_irqs; irq++, cfg++) | 174 | { |
175 | struct irq_cfg *cfg; | ||
176 | struct irq_desc *desc; | ||
177 | int count; | ||
178 | int i; | ||
179 | |||
180 | cfg = irq_cfgx; | ||
181 | count = ARRAY_SIZE(irq_cfgx); | ||
182 | |||
183 | for (i = 0; i < count; i++) { | ||
184 | desc = irq_to_desc(i); | ||
185 | desc->chip_data = &cfg[i]; | ||
186 | } | ||
187 | } | ||
143 | 188 | ||
189 | #ifdef CONFIG_SPARSE_IRQ | ||
144 | static struct irq_cfg *irq_cfg(unsigned int irq) | 190 | static struct irq_cfg *irq_cfg(unsigned int irq) |
145 | { | 191 | { |
146 | return irq < nr_irqs ? irq_cfgx + irq : NULL; | 192 | struct irq_cfg *cfg = NULL; |
193 | struct irq_desc *desc; | ||
194 | |||
195 | desc = irq_to_desc(irq); | ||
196 | if (desc) | ||
197 | cfg = desc->chip_data; | ||
198 | |||
199 | return cfg; | ||
147 | } | 200 | } |
148 | 201 | ||
149 | static struct irq_cfg *irq_cfg_alloc(unsigned int irq) | 202 | static struct irq_cfg *get_one_free_irq_cfg(int cpu) |
150 | { | 203 | { |
151 | return irq_cfg(irq); | 204 | struct irq_cfg *cfg; |
205 | int node; | ||
206 | |||
207 | node = cpu_to_node(cpu); | ||
208 | |||
209 | cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node); | ||
210 | printk(KERN_DEBUG " alloc irq_cfg on cpu %d node %d\n", cpu, node); | ||
211 | |||
212 | return cfg; | ||
152 | } | 213 | } |
153 | 214 | ||
154 | /* | 215 | void arch_init_chip_data(struct irq_desc *desc, int cpu) |
155 | * Rough estimation of how many shared IRQs there are, can be changed | 216 | { |
156 | * anytime. | 217 | struct irq_cfg *cfg; |
157 | */ | ||
158 | #define MAX_PLUS_SHARED_IRQS NR_IRQS | ||
159 | #define PIN_MAP_SIZE (MAX_PLUS_SHARED_IRQS + NR_IRQS) | ||
160 | 218 | ||
161 | /* | 219 | cfg = desc->chip_data; |
162 | * This is performance-critical, we want to do it O(1) | 220 | if (!cfg) { |
163 | * | 221 | desc->chip_data = get_one_free_irq_cfg(cpu); |
164 | * the indexing order of this array favors 1:1 mappings | 222 | if (!desc->chip_data) { |
165 | * between pins and IRQs. | 223 | printk(KERN_ERR "can not alloc irq_cfg\n"); |
166 | */ | 224 | BUG_ON(1); |
225 | } | ||
226 | } | ||
227 | } | ||
167 | 228 | ||
168 | struct irq_pin_list { | 229 | #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC |
169 | int apic, pin; | 230 | |
170 | struct irq_pin_list *next; | 231 | static void |
171 | }; | 232 | init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu) |
233 | { | ||
234 | struct irq_pin_list *old_entry, *head, *tail, *entry; | ||
235 | |||
236 | cfg->irq_2_pin = NULL; | ||
237 | old_entry = old_cfg->irq_2_pin; | ||
238 | if (!old_entry) | ||
239 | return; | ||
172 | 240 | ||
173 | static struct irq_pin_list irq_2_pin_head[PIN_MAP_SIZE]; | 241 | entry = get_one_free_irq_2_pin(cpu); |
174 | static struct irq_pin_list *irq_2_pin_ptr; | 242 | if (!entry) |
243 | return; | ||
175 | 244 | ||
176 | static void __init irq_2_pin_init(void) | 245 | entry->apic = old_entry->apic; |
246 | entry->pin = old_entry->pin; | ||
247 | head = entry; | ||
248 | tail = entry; | ||
249 | old_entry = old_entry->next; | ||
250 | while (old_entry) { | ||
251 | entry = get_one_free_irq_2_pin(cpu); | ||
252 | if (!entry) { | ||
253 | entry = head; | ||
254 | while (entry) { | ||
255 | head = entry->next; | ||
256 | kfree(entry); | ||
257 | entry = head; | ||
258 | } | ||
259 | /* still use the old one */ | ||
260 | return; | ||
261 | } | ||
262 | entry->apic = old_entry->apic; | ||
263 | entry->pin = old_entry->pin; | ||
264 | tail->next = entry; | ||
265 | tail = entry; | ||
266 | old_entry = old_entry->next; | ||
267 | } | ||
268 | |||
269 | tail->next = NULL; | ||
270 | cfg->irq_2_pin = head; | ||
271 | } | ||
272 | |||
273 | static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg) | ||
177 | { | 274 | { |
178 | struct irq_pin_list *pin = irq_2_pin_head; | 275 | struct irq_pin_list *entry, *next; |
179 | int i; | ||
180 | 276 | ||
181 | for (i = 1; i < PIN_MAP_SIZE; i++) | 277 | if (old_cfg->irq_2_pin == cfg->irq_2_pin) |
182 | pin[i-1].next = &pin[i]; | 278 | return; |
279 | |||
280 | entry = old_cfg->irq_2_pin; | ||
183 | 281 | ||
184 | irq_2_pin_ptr = &pin[0]; | 282 | while (entry) { |
283 | next = entry->next; | ||
284 | kfree(entry); | ||
285 | entry = next; | ||
286 | } | ||
287 | old_cfg->irq_2_pin = NULL; | ||
185 | } | 288 | } |
186 | 289 | ||
187 | static struct irq_pin_list *get_one_free_irq_2_pin(void) | 290 | void arch_init_copy_chip_data(struct irq_desc *old_desc, |
291 | struct irq_desc *desc, int cpu) | ||
188 | { | 292 | { |
189 | struct irq_pin_list *pin = irq_2_pin_ptr; | 293 | struct irq_cfg *cfg; |
294 | struct irq_cfg *old_cfg; | ||
190 | 295 | ||
191 | if (!pin) | 296 | cfg = get_one_free_irq_cfg(cpu); |
192 | panic("can not get more irq_2_pin\n"); | ||
193 | 297 | ||
194 | irq_2_pin_ptr = pin->next; | 298 | if (!cfg) |
195 | pin->next = NULL; | 299 | return; |
196 | return pin; | 300 | |
301 | desc->chip_data = cfg; | ||
302 | |||
303 | old_cfg = old_desc->chip_data; | ||
304 | |||
305 | memcpy(cfg, old_cfg, sizeof(struct irq_cfg)); | ||
306 | |||
307 | init_copy_irq_2_pin(old_cfg, cfg, cpu); | ||
308 | } | ||
309 | |||
310 | static void free_irq_cfg(struct irq_cfg *old_cfg) | ||
311 | { | ||
312 | kfree(old_cfg); | ||
313 | } | ||
314 | |||
315 | void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc) | ||
316 | { | ||
317 | struct irq_cfg *old_cfg, *cfg; | ||
318 | |||
319 | old_cfg = old_desc->chip_data; | ||
320 | cfg = desc->chip_data; | ||
321 | |||
322 | if (old_cfg == cfg) | ||
323 | return; | ||
324 | |||
325 | if (old_cfg) { | ||
326 | free_irq_2_pin(old_cfg, cfg); | ||
327 | free_irq_cfg(old_cfg); | ||
328 | old_desc->chip_data = NULL; | ||
329 | } | ||
330 | } | ||
331 | |||
332 | static void set_extra_move_desc(struct irq_desc *desc, cpumask_t mask) | ||
333 | { | ||
334 | struct irq_cfg *cfg = desc->chip_data; | ||
335 | |||
336 | if (!cfg->move_in_progress) { | ||
337 | /* it means that domain is not changed */ | ||
338 | if (!cpus_intersects(desc->affinity, mask)) | ||
339 | cfg->move_desc_pending = 1; | ||
340 | } | ||
197 | } | 341 | } |
342 | #endif | ||
343 | |||
344 | #else | ||
345 | static struct irq_cfg *irq_cfg(unsigned int irq) | ||
346 | { | ||
347 | return irq < nr_irqs ? irq_cfgx + irq : NULL; | ||
348 | } | ||
349 | |||
350 | #endif | ||
351 | |||
352 | #ifndef CONFIG_NUMA_MIGRATE_IRQ_DESC | ||
353 | static inline void set_extra_move_desc(struct irq_desc *desc, cpumask_t mask) | ||
354 | { | ||
355 | } | ||
356 | #endif | ||
198 | 357 | ||
199 | struct io_apic { | 358 | struct io_apic { |
200 | unsigned int index; | 359 | unsigned int index; |
@@ -237,11 +396,10 @@ static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned | |||
237 | writel(value, &io_apic->data); | 396 | writel(value, &io_apic->data); |
238 | } | 397 | } |
239 | 398 | ||
240 | static bool io_apic_level_ack_pending(unsigned int irq) | 399 | static bool io_apic_level_ack_pending(struct irq_cfg *cfg) |
241 | { | 400 | { |
242 | struct irq_pin_list *entry; | 401 | struct irq_pin_list *entry; |
243 | unsigned long flags; | 402 | unsigned long flags; |
244 | struct irq_cfg *cfg = irq_cfg(irq); | ||
245 | 403 | ||
246 | spin_lock_irqsave(&ioapic_lock, flags); | 404 | spin_lock_irqsave(&ioapic_lock, flags); |
247 | entry = cfg->irq_2_pin; | 405 | entry = cfg->irq_2_pin; |
@@ -323,13 +481,12 @@ static void ioapic_mask_entry(int apic, int pin) | |||
323 | } | 481 | } |
324 | 482 | ||
325 | #ifdef CONFIG_SMP | 483 | #ifdef CONFIG_SMP |
326 | static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector) | 484 | static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg) |
327 | { | 485 | { |
328 | int apic, pin; | 486 | int apic, pin; |
329 | struct irq_cfg *cfg; | ||
330 | struct irq_pin_list *entry; | 487 | struct irq_pin_list *entry; |
488 | u8 vector = cfg->vector; | ||
331 | 489 | ||
332 | cfg = irq_cfg(irq); | ||
333 | entry = cfg->irq_2_pin; | 490 | entry = cfg->irq_2_pin; |
334 | for (;;) { | 491 | for (;;) { |
335 | unsigned int reg; | 492 | unsigned int reg; |
@@ -359,24 +516,27 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector) | |||
359 | } | 516 | } |
360 | } | 517 | } |
361 | 518 | ||
362 | static int assign_irq_vector(int irq, cpumask_t mask); | 519 | static int assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask); |
363 | 520 | ||
364 | static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) | 521 | static void set_ioapic_affinity_irq_desc(struct irq_desc *desc, cpumask_t mask) |
365 | { | 522 | { |
366 | struct irq_cfg *cfg; | 523 | struct irq_cfg *cfg; |
367 | unsigned long flags; | 524 | unsigned long flags; |
368 | unsigned int dest; | 525 | unsigned int dest; |
369 | cpumask_t tmp; | 526 | cpumask_t tmp; |
370 | struct irq_desc *desc; | 527 | unsigned int irq; |
371 | 528 | ||
372 | cpus_and(tmp, mask, cpu_online_map); | 529 | cpus_and(tmp, mask, cpu_online_map); |
373 | if (cpus_empty(tmp)) | 530 | if (cpus_empty(tmp)) |
374 | return; | 531 | return; |
375 | 532 | ||
376 | cfg = irq_cfg(irq); | 533 | irq = desc->irq; |
377 | if (assign_irq_vector(irq, mask)) | 534 | cfg = desc->chip_data; |
535 | if (assign_irq_vector(irq, cfg, mask)) | ||
378 | return; | 536 | return; |
379 | 537 | ||
538 | set_extra_move_desc(desc, mask); | ||
539 | |||
380 | cpus_and(tmp, cfg->domain, mask); | 540 | cpus_and(tmp, cfg->domain, mask); |
381 | dest = cpu_mask_to_apicid(tmp); | 541 | dest = cpu_mask_to_apicid(tmp); |
382 | /* | 542 | /* |
@@ -384,12 +544,20 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) | |||
384 | */ | 544 | */ |
385 | dest = SET_APIC_LOGICAL_ID(dest); | 545 | dest = SET_APIC_LOGICAL_ID(dest); |
386 | 546 | ||
387 | desc = irq_to_desc(irq); | ||
388 | spin_lock_irqsave(&ioapic_lock, flags); | 547 | spin_lock_irqsave(&ioapic_lock, flags); |
389 | __target_IO_APIC_irq(irq, dest, cfg->vector); | 548 | __target_IO_APIC_irq(irq, dest, cfg); |
390 | desc->affinity = mask; | 549 | desc->affinity = mask; |
391 | spin_unlock_irqrestore(&ioapic_lock, flags); | 550 | spin_unlock_irqrestore(&ioapic_lock, flags); |
392 | } | 551 | } |
552 | |||
553 | static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) | ||
554 | { | ||
555 | struct irq_desc *desc; | ||
556 | |||
557 | desc = irq_to_desc(irq); | ||
558 | |||
559 | set_ioapic_affinity_irq_desc(desc, mask); | ||
560 | } | ||
393 | #endif /* CONFIG_SMP */ | 561 | #endif /* CONFIG_SMP */ |
394 | 562 | ||
395 | /* | 563 | /* |
@@ -397,16 +565,18 @@ static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) | |||
397 | * shared ISA-space IRQs, so we have to support them. We are super | 565 | * shared ISA-space IRQs, so we have to support them. We are super |
398 | * fast in the common case, and fast for shared ISA-space IRQs. | 566 | * fast in the common case, and fast for shared ISA-space IRQs. |
399 | */ | 567 | */ |
400 | static void add_pin_to_irq(unsigned int irq, int apic, int pin) | 568 | static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin) |
401 | { | 569 | { |
402 | struct irq_cfg *cfg; | ||
403 | struct irq_pin_list *entry; | 570 | struct irq_pin_list *entry; |
404 | 571 | ||
405 | /* first time to refer irq_cfg, so with new */ | ||
406 | cfg = irq_cfg_alloc(irq); | ||
407 | entry = cfg->irq_2_pin; | 572 | entry = cfg->irq_2_pin; |
408 | if (!entry) { | 573 | if (!entry) { |
409 | entry = get_one_free_irq_2_pin(); | 574 | entry = get_one_free_irq_2_pin(cpu); |
575 | if (!entry) { | ||
576 | printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n", | ||
577 | apic, pin); | ||
578 | return; | ||
579 | } | ||
410 | cfg->irq_2_pin = entry; | 580 | cfg->irq_2_pin = entry; |
411 | entry->apic = apic; | 581 | entry->apic = apic; |
412 | entry->pin = pin; | 582 | entry->pin = pin; |
@@ -421,7 +591,7 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin) | |||
421 | entry = entry->next; | 591 | entry = entry->next; |
422 | } | 592 | } |
423 | 593 | ||
424 | entry->next = get_one_free_irq_2_pin(); | 594 | entry->next = get_one_free_irq_2_pin(cpu); |
425 | entry = entry->next; | 595 | entry = entry->next; |
426 | entry->apic = apic; | 596 | entry->apic = apic; |
427 | entry->pin = pin; | 597 | entry->pin = pin; |
@@ -430,11 +600,10 @@ static void add_pin_to_irq(unsigned int irq, int apic, int pin) | |||
430 | /* | 600 | /* |
431 | * Reroute an IRQ to a different pin. | 601 | * Reroute an IRQ to a different pin. |
432 | */ | 602 | */ |
433 | static void __init replace_pin_at_irq(unsigned int irq, | 603 | static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu, |
434 | int oldapic, int oldpin, | 604 | int oldapic, int oldpin, |
435 | int newapic, int newpin) | 605 | int newapic, int newpin) |
436 | { | 606 | { |
437 | struct irq_cfg *cfg = irq_cfg(irq); | ||
438 | struct irq_pin_list *entry = cfg->irq_2_pin; | 607 | struct irq_pin_list *entry = cfg->irq_2_pin; |
439 | int replaced = 0; | 608 | int replaced = 0; |
440 | 609 | ||
@@ -451,18 +620,16 @@ static void __init replace_pin_at_irq(unsigned int irq, | |||
451 | 620 | ||
452 | /* why? call replace before add? */ | 621 | /* why? call replace before add? */ |
453 | if (!replaced) | 622 | if (!replaced) |
454 | add_pin_to_irq(irq, newapic, newpin); | 623 | add_pin_to_irq_cpu(cfg, cpu, newapic, newpin); |
455 | } | 624 | } |
456 | 625 | ||
457 | static inline void io_apic_modify_irq(unsigned int irq, | 626 | static inline void io_apic_modify_irq(struct irq_cfg *cfg, |
458 | int mask_and, int mask_or, | 627 | int mask_and, int mask_or, |
459 | void (*final)(struct irq_pin_list *entry)) | 628 | void (*final)(struct irq_pin_list *entry)) |
460 | { | 629 | { |
461 | int pin; | 630 | int pin; |
462 | struct irq_cfg *cfg; | ||
463 | struct irq_pin_list *entry; | 631 | struct irq_pin_list *entry; |
464 | 632 | ||
465 | cfg = irq_cfg(irq); | ||
466 | for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) { | 633 | for (entry = cfg->irq_2_pin; entry != NULL; entry = entry->next) { |
467 | unsigned int reg; | 634 | unsigned int reg; |
468 | pin = entry->pin; | 635 | pin = entry->pin; |
@@ -475,9 +642,9 @@ static inline void io_apic_modify_irq(unsigned int irq, | |||
475 | } | 642 | } |
476 | } | 643 | } |
477 | 644 | ||
478 | static void __unmask_IO_APIC_irq(unsigned int irq) | 645 | static void __unmask_IO_APIC_irq(struct irq_cfg *cfg) |
479 | { | 646 | { |
480 | io_apic_modify_irq(irq, ~IO_APIC_REDIR_MASKED, 0, NULL); | 647 | io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, 0, NULL); |
481 | } | 648 | } |
482 | 649 | ||
483 | #ifdef CONFIG_X86_64 | 650 | #ifdef CONFIG_X86_64 |
@@ -492,47 +659,64 @@ void io_apic_sync(struct irq_pin_list *entry) | |||
492 | readl(&io_apic->data); | 659 | readl(&io_apic->data); |
493 | } | 660 | } |
494 | 661 | ||
495 | static void __mask_IO_APIC_irq(unsigned int irq) | 662 | static void __mask_IO_APIC_irq(struct irq_cfg *cfg) |
496 | { | 663 | { |
497 | io_apic_modify_irq(irq, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync); | 664 | io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync); |
498 | } | 665 | } |
499 | #else /* CONFIG_X86_32 */ | 666 | #else /* CONFIG_X86_32 */ |
500 | static void __mask_IO_APIC_irq(unsigned int irq) | 667 | static void __mask_IO_APIC_irq(struct irq_cfg *cfg) |
501 | { | 668 | { |
502 | io_apic_modify_irq(irq, ~0, IO_APIC_REDIR_MASKED, NULL); | 669 | io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, NULL); |
503 | } | 670 | } |
504 | 671 | ||
505 | static void __mask_and_edge_IO_APIC_irq(unsigned int irq) | 672 | static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg) |
506 | { | 673 | { |
507 | io_apic_modify_irq(irq, ~IO_APIC_REDIR_LEVEL_TRIGGER, | 674 | io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER, |
508 | IO_APIC_REDIR_MASKED, NULL); | 675 | IO_APIC_REDIR_MASKED, NULL); |
509 | } | 676 | } |
510 | 677 | ||
511 | static void __unmask_and_level_IO_APIC_irq(unsigned int irq) | 678 | static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg) |
512 | { | 679 | { |
513 | io_apic_modify_irq(irq, ~IO_APIC_REDIR_MASKED, | 680 | io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED, |
514 | IO_APIC_REDIR_LEVEL_TRIGGER, NULL); | 681 | IO_APIC_REDIR_LEVEL_TRIGGER, NULL); |
515 | } | 682 | } |
516 | #endif /* CONFIG_X86_32 */ | 683 | #endif /* CONFIG_X86_32 */ |
517 | 684 | ||
518 | static void mask_IO_APIC_irq (unsigned int irq) | 685 | static void mask_IO_APIC_irq_desc(struct irq_desc *desc) |
519 | { | 686 | { |
687 | struct irq_cfg *cfg = desc->chip_data; | ||
520 | unsigned long flags; | 688 | unsigned long flags; |
521 | 689 | ||
690 | BUG_ON(!cfg); | ||
691 | |||
522 | spin_lock_irqsave(&ioapic_lock, flags); | 692 | spin_lock_irqsave(&ioapic_lock, flags); |
523 | __mask_IO_APIC_irq(irq); | 693 | __mask_IO_APIC_irq(cfg); |
524 | spin_unlock_irqrestore(&ioapic_lock, flags); | 694 | spin_unlock_irqrestore(&ioapic_lock, flags); |
525 | } | 695 | } |
526 | 696 | ||
527 | static void unmask_IO_APIC_irq (unsigned int irq) | 697 | static void unmask_IO_APIC_irq_desc(struct irq_desc *desc) |
528 | { | 698 | { |
699 | struct irq_cfg *cfg = desc->chip_data; | ||
529 | unsigned long flags; | 700 | unsigned long flags; |
530 | 701 | ||
531 | spin_lock_irqsave(&ioapic_lock, flags); | 702 | spin_lock_irqsave(&ioapic_lock, flags); |
532 | __unmask_IO_APIC_irq(irq); | 703 | __unmask_IO_APIC_irq(cfg); |
533 | spin_unlock_irqrestore(&ioapic_lock, flags); | 704 | spin_unlock_irqrestore(&ioapic_lock, flags); |
534 | } | 705 | } |
535 | 706 | ||
707 | static void mask_IO_APIC_irq(unsigned int irq) | ||
708 | { | ||
709 | struct irq_desc *desc = irq_to_desc(irq); | ||
710 | |||
711 | mask_IO_APIC_irq_desc(desc); | ||
712 | } | ||
713 | static void unmask_IO_APIC_irq(unsigned int irq) | ||
714 | { | ||
715 | struct irq_desc *desc = irq_to_desc(irq); | ||
716 | |||
717 | unmask_IO_APIC_irq_desc(desc); | ||
718 | } | ||
719 | |||
536 | static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) | 720 | static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin) |
537 | { | 721 | { |
538 | struct IO_APIC_route_entry entry; | 722 | struct IO_APIC_route_entry entry; |
@@ -809,7 +993,7 @@ EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector); | |||
809 | */ | 993 | */ |
810 | static int EISA_ELCR(unsigned int irq) | 994 | static int EISA_ELCR(unsigned int irq) |
811 | { | 995 | { |
812 | if (irq < 16) { | 996 | if (irq < NR_IRQS_LEGACY) { |
813 | unsigned int port = 0x4d0 + (irq >> 3); | 997 | unsigned int port = 0x4d0 + (irq >> 3); |
814 | return (inb(port) >> (irq & 7)) & 1; | 998 | return (inb(port) >> (irq & 7)) & 1; |
815 | } | 999 | } |
@@ -1034,7 +1218,7 @@ void unlock_vector_lock(void) | |||
1034 | spin_unlock(&vector_lock); | 1218 | spin_unlock(&vector_lock); |
1035 | } | 1219 | } |
1036 | 1220 | ||
1037 | static int __assign_irq_vector(int irq, cpumask_t mask) | 1221 | static int __assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask) |
1038 | { | 1222 | { |
1039 | /* | 1223 | /* |
1040 | * NOTE! The local APIC isn't very good at handling | 1224 | * NOTE! The local APIC isn't very good at handling |
@@ -1050,16 +1234,13 @@ static int __assign_irq_vector(int irq, cpumask_t mask) | |||
1050 | static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0; | 1234 | static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0; |
1051 | unsigned int old_vector; | 1235 | unsigned int old_vector; |
1052 | int cpu; | 1236 | int cpu; |
1053 | struct irq_cfg *cfg; | ||
1054 | 1237 | ||
1055 | cfg = irq_cfg(irq); | 1238 | if ((cfg->move_in_progress) || cfg->move_cleanup_count) |
1239 | return -EBUSY; | ||
1056 | 1240 | ||
1057 | /* Only try and allocate irqs on cpus that are present */ | 1241 | /* Only try and allocate irqs on cpus that are present */ |
1058 | cpus_and(mask, mask, cpu_online_map); | 1242 | cpus_and(mask, mask, cpu_online_map); |
1059 | 1243 | ||
1060 | if ((cfg->move_in_progress) || cfg->move_cleanup_count) | ||
1061 | return -EBUSY; | ||
1062 | |||
1063 | old_vector = cfg->vector; | 1244 | old_vector = cfg->vector; |
1064 | if (old_vector) { | 1245 | if (old_vector) { |
1065 | cpumask_t tmp; | 1246 | cpumask_t tmp; |
@@ -1113,24 +1294,22 @@ next: | |||
1113 | return -ENOSPC; | 1294 | return -ENOSPC; |
1114 | } | 1295 | } |
1115 | 1296 | ||
1116 | static int assign_irq_vector(int irq, cpumask_t mask) | 1297 | static int assign_irq_vector(int irq, struct irq_cfg *cfg, cpumask_t mask) |
1117 | { | 1298 | { |
1118 | int err; | 1299 | int err; |
1119 | unsigned long flags; | 1300 | unsigned long flags; |
1120 | 1301 | ||
1121 | spin_lock_irqsave(&vector_lock, flags); | 1302 | spin_lock_irqsave(&vector_lock, flags); |
1122 | err = __assign_irq_vector(irq, mask); | 1303 | err = __assign_irq_vector(irq, cfg, mask); |
1123 | spin_unlock_irqrestore(&vector_lock, flags); | 1304 | spin_unlock_irqrestore(&vector_lock, flags); |
1124 | return err; | 1305 | return err; |
1125 | } | 1306 | } |
1126 | 1307 | ||
1127 | static void __clear_irq_vector(int irq) | 1308 | static void __clear_irq_vector(int irq, struct irq_cfg *cfg) |
1128 | { | 1309 | { |
1129 | struct irq_cfg *cfg; | ||
1130 | cpumask_t mask; | 1310 | cpumask_t mask; |
1131 | int cpu, vector; | 1311 | int cpu, vector; |
1132 | 1312 | ||
1133 | cfg = irq_cfg(irq); | ||
1134 | BUG_ON(!cfg->vector); | 1313 | BUG_ON(!cfg->vector); |
1135 | 1314 | ||
1136 | vector = cfg->vector; | 1315 | vector = cfg->vector; |
@@ -1162,9 +1341,13 @@ void __setup_vector_irq(int cpu) | |||
1162 | /* This function must be called with vector_lock held */ | 1341 | /* This function must be called with vector_lock held */ |
1163 | int irq, vector; | 1342 | int irq, vector; |
1164 | struct irq_cfg *cfg; | 1343 | struct irq_cfg *cfg; |
1344 | struct irq_desc *desc; | ||
1165 | 1345 | ||
1166 | /* Mark the inuse vectors */ | 1346 | /* Mark the inuse vectors */ |
1167 | for_each_irq_cfg(irq, cfg) { | 1347 | for_each_irq_desc(irq, desc) { |
1348 | if (!desc) | ||
1349 | continue; | ||
1350 | cfg = desc->chip_data; | ||
1168 | if (!cpu_isset(cpu, cfg->domain)) | 1351 | if (!cpu_isset(cpu, cfg->domain)) |
1169 | continue; | 1352 | continue; |
1170 | vector = cfg->vector; | 1353 | vector = cfg->vector; |
@@ -1215,11 +1398,8 @@ static inline int IO_APIC_irq_trigger(int irq) | |||
1215 | } | 1398 | } |
1216 | #endif | 1399 | #endif |
1217 | 1400 | ||
1218 | static void ioapic_register_intr(int irq, unsigned long trigger) | 1401 | static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long trigger) |
1219 | { | 1402 | { |
1220 | struct irq_desc *desc; | ||
1221 | |||
1222 | desc = irq_to_desc(irq); | ||
1223 | 1403 | ||
1224 | if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || | 1404 | if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || |
1225 | trigger == IOAPIC_LEVEL) | 1405 | trigger == IOAPIC_LEVEL) |
@@ -1311,7 +1491,7 @@ static int setup_ioapic_entry(int apic, int irq, | |||
1311 | return 0; | 1491 | return 0; |
1312 | } | 1492 | } |
1313 | 1493 | ||
1314 | static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, | 1494 | static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, struct irq_desc *desc, |
1315 | int trigger, int polarity) | 1495 | int trigger, int polarity) |
1316 | { | 1496 | { |
1317 | struct irq_cfg *cfg; | 1497 | struct irq_cfg *cfg; |
@@ -1321,10 +1501,10 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, | |||
1321 | if (!IO_APIC_IRQ(irq)) | 1501 | if (!IO_APIC_IRQ(irq)) |
1322 | return; | 1502 | return; |
1323 | 1503 | ||
1324 | cfg = irq_cfg(irq); | 1504 | cfg = desc->chip_data; |
1325 | 1505 | ||
1326 | mask = TARGET_CPUS; | 1506 | mask = TARGET_CPUS; |
1327 | if (assign_irq_vector(irq, mask)) | 1507 | if (assign_irq_vector(irq, cfg, mask)) |
1328 | return; | 1508 | return; |
1329 | 1509 | ||
1330 | cpus_and(mask, cfg->domain, mask); | 1510 | cpus_and(mask, cfg->domain, mask); |
@@ -1341,12 +1521,12 @@ static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq, | |||
1341 | cfg->vector)) { | 1521 | cfg->vector)) { |
1342 | printk("Failed to setup ioapic entry for ioapic %d, pin %d\n", | 1522 | printk("Failed to setup ioapic entry for ioapic %d, pin %d\n", |
1343 | mp_ioapics[apic].mp_apicid, pin); | 1523 | mp_ioapics[apic].mp_apicid, pin); |
1344 | __clear_irq_vector(irq); | 1524 | __clear_irq_vector(irq, cfg); |
1345 | return; | 1525 | return; |
1346 | } | 1526 | } |
1347 | 1527 | ||
1348 | ioapic_register_intr(irq, trigger); | 1528 | ioapic_register_intr(irq, desc, trigger); |
1349 | if (irq < 16) | 1529 | if (irq < NR_IRQS_LEGACY) |
1350 | disable_8259A_irq(irq); | 1530 | disable_8259A_irq(irq); |
1351 | 1531 | ||
1352 | ioapic_write_entry(apic, pin, entry); | 1532 | ioapic_write_entry(apic, pin, entry); |
@@ -1356,6 +1536,9 @@ static void __init setup_IO_APIC_irqs(void) | |||
1356 | { | 1536 | { |
1357 | int apic, pin, idx, irq; | 1537 | int apic, pin, idx, irq; |
1358 | int notcon = 0; | 1538 | int notcon = 0; |
1539 | struct irq_desc *desc; | ||
1540 | struct irq_cfg *cfg; | ||
1541 | int cpu = boot_cpu_id; | ||
1359 | 1542 | ||
1360 | apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); | 1543 | apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); |
1361 | 1544 | ||
@@ -1387,9 +1570,15 @@ static void __init setup_IO_APIC_irqs(void) | |||
1387 | if (multi_timer_check(apic, irq)) | 1570 | if (multi_timer_check(apic, irq)) |
1388 | continue; | 1571 | continue; |
1389 | #endif | 1572 | #endif |
1390 | add_pin_to_irq(irq, apic, pin); | 1573 | desc = irq_to_desc_alloc_cpu(irq, cpu); |
1574 | if (!desc) { | ||
1575 | printk(KERN_INFO "can not get irq_desc for %d\n", irq); | ||
1576 | continue; | ||
1577 | } | ||
1578 | cfg = desc->chip_data; | ||
1579 | add_pin_to_irq_cpu(cfg, cpu, apic, pin); | ||
1391 | 1580 | ||
1392 | setup_IO_APIC_irq(apic, pin, irq, | 1581 | setup_IO_APIC_irq(apic, pin, irq, desc, |
1393 | irq_trigger(idx), irq_polarity(idx)); | 1582 | irq_trigger(idx), irq_polarity(idx)); |
1394 | } | 1583 | } |
1395 | } | 1584 | } |
@@ -1448,6 +1637,7 @@ __apicdebuginit(void) print_IO_APIC(void) | |||
1448 | union IO_APIC_reg_03 reg_03; | 1637 | union IO_APIC_reg_03 reg_03; |
1449 | unsigned long flags; | 1638 | unsigned long flags; |
1450 | struct irq_cfg *cfg; | 1639 | struct irq_cfg *cfg; |
1640 | struct irq_desc *desc; | ||
1451 | unsigned int irq; | 1641 | unsigned int irq; |
1452 | 1642 | ||
1453 | if (apic_verbosity == APIC_QUIET) | 1643 | if (apic_verbosity == APIC_QUIET) |
@@ -1537,8 +1727,13 @@ __apicdebuginit(void) print_IO_APIC(void) | |||
1537 | } | 1727 | } |
1538 | } | 1728 | } |
1539 | printk(KERN_DEBUG "IRQ to pin mappings:\n"); | 1729 | printk(KERN_DEBUG "IRQ to pin mappings:\n"); |
1540 | for_each_irq_cfg(irq, cfg) { | 1730 | for_each_irq_desc(irq, desc) { |
1541 | struct irq_pin_list *entry = cfg->irq_2_pin; | 1731 | struct irq_pin_list *entry; |
1732 | |||
1733 | if (!desc) | ||
1734 | continue; | ||
1735 | cfg = desc->chip_data; | ||
1736 | entry = cfg->irq_2_pin; | ||
1542 | if (!entry) | 1737 | if (!entry) |
1543 | continue; | 1738 | continue; |
1544 | printk(KERN_DEBUG "IRQ%d ", irq); | 1739 | printk(KERN_DEBUG "IRQ%d ", irq); |
@@ -2022,14 +2217,16 @@ static unsigned int startup_ioapic_irq(unsigned int irq) | |||
2022 | { | 2217 | { |
2023 | int was_pending = 0; | 2218 | int was_pending = 0; |
2024 | unsigned long flags; | 2219 | unsigned long flags; |
2220 | struct irq_cfg *cfg; | ||
2025 | 2221 | ||
2026 | spin_lock_irqsave(&ioapic_lock, flags); | 2222 | spin_lock_irqsave(&ioapic_lock, flags); |
2027 | if (irq < 16) { | 2223 | if (irq < NR_IRQS_LEGACY) { |
2028 | disable_8259A_irq(irq); | 2224 | disable_8259A_irq(irq); |
2029 | if (i8259A_irq_pending(irq)) | 2225 | if (i8259A_irq_pending(irq)) |
2030 | was_pending = 1; | 2226 | was_pending = 1; |
2031 | } | 2227 | } |
2032 | __unmask_IO_APIC_irq(irq); | 2228 | cfg = irq_cfg(irq); |
2229 | __unmask_IO_APIC_irq(cfg); | ||
2033 | spin_unlock_irqrestore(&ioapic_lock, flags); | 2230 | spin_unlock_irqrestore(&ioapic_lock, flags); |
2034 | 2231 | ||
2035 | return was_pending; | 2232 | return was_pending; |
@@ -2092,35 +2289,37 @@ static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration); | |||
2092 | * as simple as edge triggered migration and we can do the irq migration | 2289 | * as simple as edge triggered migration and we can do the irq migration |
2093 | * with a simple atomic update to IO-APIC RTE. | 2290 | * with a simple atomic update to IO-APIC RTE. |
2094 | */ | 2291 | */ |
2095 | static void migrate_ioapic_irq(int irq, cpumask_t mask) | 2292 | static void migrate_ioapic_irq_desc(struct irq_desc *desc, cpumask_t mask) |
2096 | { | 2293 | { |
2097 | struct irq_cfg *cfg; | 2294 | struct irq_cfg *cfg; |
2098 | struct irq_desc *desc; | ||
2099 | cpumask_t tmp, cleanup_mask; | 2295 | cpumask_t tmp, cleanup_mask; |
2100 | struct irte irte; | 2296 | struct irte irte; |
2101 | int modify_ioapic_rte; | 2297 | int modify_ioapic_rte; |
2102 | unsigned int dest; | 2298 | unsigned int dest; |
2103 | unsigned long flags; | 2299 | unsigned long flags; |
2300 | unsigned int irq; | ||
2104 | 2301 | ||
2105 | cpus_and(tmp, mask, cpu_online_map); | 2302 | cpus_and(tmp, mask, cpu_online_map); |
2106 | if (cpus_empty(tmp)) | 2303 | if (cpus_empty(tmp)) |
2107 | return; | 2304 | return; |
2108 | 2305 | ||
2306 | irq = desc->irq; | ||
2109 | if (get_irte(irq, &irte)) | 2307 | if (get_irte(irq, &irte)) |
2110 | return; | 2308 | return; |
2111 | 2309 | ||
2112 | if (assign_irq_vector(irq, mask)) | 2310 | cfg = desc->chip_data; |
2311 | if (assign_irq_vector(irq, cfg, mask)) | ||
2113 | return; | 2312 | return; |
2114 | 2313 | ||
2115 | cfg = irq_cfg(irq); | 2314 | set_extra_move_desc(desc, mask); |
2315 | |||
2116 | cpus_and(tmp, cfg->domain, mask); | 2316 | cpus_and(tmp, cfg->domain, mask); |
2117 | dest = cpu_mask_to_apicid(tmp); | 2317 | dest = cpu_mask_to_apicid(tmp); |
2118 | 2318 | ||
2119 | desc = irq_to_desc(irq); | ||
2120 | modify_ioapic_rte = desc->status & IRQ_LEVEL; | 2319 | modify_ioapic_rte = desc->status & IRQ_LEVEL; |
2121 | if (modify_ioapic_rte) { | 2320 | if (modify_ioapic_rte) { |
2122 | spin_lock_irqsave(&ioapic_lock, flags); | 2321 | spin_lock_irqsave(&ioapic_lock, flags); |
2123 | __target_IO_APIC_irq(irq, dest, cfg->vector); | 2322 | __target_IO_APIC_irq(irq, dest, cfg); |
2124 | spin_unlock_irqrestore(&ioapic_lock, flags); | 2323 | spin_unlock_irqrestore(&ioapic_lock, flags); |
2125 | } | 2324 | } |
2126 | 2325 | ||
@@ -2142,14 +2341,14 @@ static void migrate_ioapic_irq(int irq, cpumask_t mask) | |||
2142 | desc->affinity = mask; | 2341 | desc->affinity = mask; |
2143 | } | 2342 | } |
2144 | 2343 | ||
2145 | static int migrate_irq_remapped_level(int irq) | 2344 | static int migrate_irq_remapped_level_desc(struct irq_desc *desc) |
2146 | { | 2345 | { |
2147 | int ret = -1; | 2346 | int ret = -1; |
2148 | struct irq_desc *desc = irq_to_desc(irq); | 2347 | struct irq_cfg *cfg = desc->chip_data; |
2149 | 2348 | ||
2150 | mask_IO_APIC_irq(irq); | 2349 | mask_IO_APIC_irq_desc(desc); |
2151 | 2350 | ||
2152 | if (io_apic_level_ack_pending(irq)) { | 2351 | if (io_apic_level_ack_pending(cfg)) { |
2153 | /* | 2352 | /* |
2154 | * Interrupt in progress. Migrating irq now will change the | 2353 | * Interrupt in progress. Migrating irq now will change the |
2155 | * vector information in the IO-APIC RTE and that will confuse | 2354 | * vector information in the IO-APIC RTE and that will confuse |
@@ -2161,14 +2360,15 @@ static int migrate_irq_remapped_level(int irq) | |||
2161 | } | 2360 | } |
2162 | 2361 | ||
2163 | /* everthing is clear. we have right of way */ | 2362 | /* everthing is clear. we have right of way */ |
2164 | migrate_ioapic_irq(irq, desc->pending_mask); | 2363 | migrate_ioapic_irq_desc(desc, desc->pending_mask); |
2165 | 2364 | ||
2166 | ret = 0; | 2365 | ret = 0; |
2167 | desc->status &= ~IRQ_MOVE_PENDING; | 2366 | desc->status &= ~IRQ_MOVE_PENDING; |
2168 | cpus_clear(desc->pending_mask); | 2367 | cpus_clear(desc->pending_mask); |
2169 | 2368 | ||
2170 | unmask: | 2369 | unmask: |
2171 | unmask_IO_APIC_irq(irq); | 2370 | unmask_IO_APIC_irq_desc(desc); |
2371 | |||
2172 | return ret; | 2372 | return ret; |
2173 | } | 2373 | } |
2174 | 2374 | ||
@@ -2178,6 +2378,9 @@ static void ir_irq_migration(struct work_struct *work) | |||
2178 | struct irq_desc *desc; | 2378 | struct irq_desc *desc; |
2179 | 2379 | ||
2180 | for_each_irq_desc(irq, desc) { | 2380 | for_each_irq_desc(irq, desc) { |
2381 | if (!desc) | ||
2382 | continue; | ||
2383 | |||
2181 | if (desc->status & IRQ_MOVE_PENDING) { | 2384 | if (desc->status & IRQ_MOVE_PENDING) { |
2182 | unsigned long flags; | 2385 | unsigned long flags; |
2183 | 2386 | ||
@@ -2198,18 +2401,22 @@ static void ir_irq_migration(struct work_struct *work) | |||
2198 | /* | 2401 | /* |
2199 | * Migrates the IRQ destination in the process context. | 2402 | * Migrates the IRQ destination in the process context. |
2200 | */ | 2403 | */ |
2201 | static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) | 2404 | static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, cpumask_t mask) |
2202 | { | 2405 | { |
2203 | struct irq_desc *desc = irq_to_desc(irq); | ||
2204 | |||
2205 | if (desc->status & IRQ_LEVEL) { | 2406 | if (desc->status & IRQ_LEVEL) { |
2206 | desc->status |= IRQ_MOVE_PENDING; | 2407 | desc->status |= IRQ_MOVE_PENDING; |
2207 | desc->pending_mask = mask; | 2408 | desc->pending_mask = mask; |
2208 | migrate_irq_remapped_level(irq); | 2409 | migrate_irq_remapped_level_desc(desc); |
2209 | return; | 2410 | return; |
2210 | } | 2411 | } |
2211 | 2412 | ||
2212 | migrate_ioapic_irq(irq, mask); | 2413 | migrate_ioapic_irq_desc(desc, mask); |
2414 | } | ||
2415 | static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask) | ||
2416 | { | ||
2417 | struct irq_desc *desc = irq_to_desc(irq); | ||
2418 | |||
2419 | set_ir_ioapic_affinity_irq_desc(desc, mask); | ||
2213 | } | 2420 | } |
2214 | #endif | 2421 | #endif |
2215 | 2422 | ||
@@ -2228,6 +2435,9 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void) | |||
2228 | struct irq_cfg *cfg; | 2435 | struct irq_cfg *cfg; |
2229 | irq = __get_cpu_var(vector_irq)[vector]; | 2436 | irq = __get_cpu_var(vector_irq)[vector]; |
2230 | 2437 | ||
2438 | if (irq == -1) | ||
2439 | continue; | ||
2440 | |||
2231 | desc = irq_to_desc(irq); | 2441 | desc = irq_to_desc(irq); |
2232 | if (!desc) | 2442 | if (!desc) |
2233 | continue; | 2443 | continue; |
@@ -2249,19 +2459,40 @@ unlock: | |||
2249 | irq_exit(); | 2459 | irq_exit(); |
2250 | } | 2460 | } |
2251 | 2461 | ||
2252 | static void irq_complete_move(unsigned int irq) | 2462 | static void irq_complete_move(struct irq_desc **descp) |
2253 | { | 2463 | { |
2254 | struct irq_cfg *cfg = irq_cfg(irq); | 2464 | struct irq_desc *desc = *descp; |
2465 | struct irq_cfg *cfg = desc->chip_data; | ||
2255 | unsigned vector, me; | 2466 | unsigned vector, me; |
2256 | 2467 | ||
2257 | if (likely(!cfg->move_in_progress)) | 2468 | if (likely(!cfg->move_in_progress)) { |
2469 | #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC | ||
2470 | if (likely(!cfg->move_desc_pending)) | ||
2471 | return; | ||
2472 | |||
2473 | /* domain has not changed, but affinity did */ | ||
2474 | me = smp_processor_id(); | ||
2475 | if (cpu_isset(me, desc->affinity)) { | ||
2476 | *descp = desc = move_irq_desc(desc, me); | ||
2477 | /* get the new one */ | ||
2478 | cfg = desc->chip_data; | ||
2479 | cfg->move_desc_pending = 0; | ||
2480 | } | ||
2481 | #endif | ||
2258 | return; | 2482 | return; |
2483 | } | ||
2259 | 2484 | ||
2260 | vector = ~get_irq_regs()->orig_ax; | 2485 | vector = ~get_irq_regs()->orig_ax; |
2261 | me = smp_processor_id(); | 2486 | me = smp_processor_id(); |
2262 | if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) { | 2487 | if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) { |
2263 | cpumask_t cleanup_mask; | 2488 | cpumask_t cleanup_mask; |
2264 | 2489 | ||
2490 | #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC | ||
2491 | *descp = desc = move_irq_desc(desc, me); | ||
2492 | /* get the new one */ | ||
2493 | cfg = desc->chip_data; | ||
2494 | #endif | ||
2495 | |||
2265 | cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map); | 2496 | cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map); |
2266 | cfg->move_cleanup_count = cpus_weight(cleanup_mask); | 2497 | cfg->move_cleanup_count = cpus_weight(cleanup_mask); |
2267 | send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR); | 2498 | send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR); |
@@ -2269,8 +2500,9 @@ static void irq_complete_move(unsigned int irq) | |||
2269 | } | 2500 | } |
2270 | } | 2501 | } |
2271 | #else | 2502 | #else |
2272 | static inline void irq_complete_move(unsigned int irq) {} | 2503 | static inline void irq_complete_move(struct irq_desc **descp) {} |
2273 | #endif | 2504 | #endif |
2505 | |||
2274 | #ifdef CONFIG_INTR_REMAP | 2506 | #ifdef CONFIG_INTR_REMAP |
2275 | static void ack_x2apic_level(unsigned int irq) | 2507 | static void ack_x2apic_level(unsigned int irq) |
2276 | { | 2508 | { |
@@ -2281,11 +2513,14 @@ static void ack_x2apic_edge(unsigned int irq) | |||
2281 | { | 2513 | { |
2282 | ack_x2APIC_irq(); | 2514 | ack_x2APIC_irq(); |
2283 | } | 2515 | } |
2516 | |||
2284 | #endif | 2517 | #endif |
2285 | 2518 | ||
2286 | static void ack_apic_edge(unsigned int irq) | 2519 | static void ack_apic_edge(unsigned int irq) |
2287 | { | 2520 | { |
2288 | irq_complete_move(irq); | 2521 | struct irq_desc *desc = irq_to_desc(irq); |
2522 | |||
2523 | irq_complete_move(&desc); | ||
2289 | move_native_irq(irq); | 2524 | move_native_irq(irq); |
2290 | ack_APIC_irq(); | 2525 | ack_APIC_irq(); |
2291 | } | 2526 | } |
@@ -2294,18 +2529,21 @@ atomic_t irq_mis_count; | |||
2294 | 2529 | ||
2295 | static void ack_apic_level(unsigned int irq) | 2530 | static void ack_apic_level(unsigned int irq) |
2296 | { | 2531 | { |
2532 | struct irq_desc *desc = irq_to_desc(irq); | ||
2533 | |||
2297 | #ifdef CONFIG_X86_32 | 2534 | #ifdef CONFIG_X86_32 |
2298 | unsigned long v; | 2535 | unsigned long v; |
2299 | int i; | 2536 | int i; |
2300 | #endif | 2537 | #endif |
2538 | struct irq_cfg *cfg; | ||
2301 | int do_unmask_irq = 0; | 2539 | int do_unmask_irq = 0; |
2302 | 2540 | ||
2303 | irq_complete_move(irq); | 2541 | irq_complete_move(&desc); |
2304 | #ifdef CONFIG_GENERIC_PENDING_IRQ | 2542 | #ifdef CONFIG_GENERIC_PENDING_IRQ |
2305 | /* If we are moving the irq we need to mask it */ | 2543 | /* If we are moving the irq we need to mask it */ |
2306 | if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) { | 2544 | if (unlikely(desc->status & IRQ_MOVE_PENDING)) { |
2307 | do_unmask_irq = 1; | 2545 | do_unmask_irq = 1; |
2308 | mask_IO_APIC_irq(irq); | 2546 | mask_IO_APIC_irq_desc(desc); |
2309 | } | 2547 | } |
2310 | #endif | 2548 | #endif |
2311 | 2549 | ||
@@ -2329,7 +2567,8 @@ static void ack_apic_level(unsigned int irq) | |||
2329 | * operation to prevent an edge-triggered interrupt escaping meanwhile. | 2567 | * operation to prevent an edge-triggered interrupt escaping meanwhile. |
2330 | * The idea is from Manfred Spraul. --macro | 2568 | * The idea is from Manfred Spraul. --macro |
2331 | */ | 2569 | */ |
2332 | i = irq_cfg(irq)->vector; | 2570 | cfg = desc->chip_data; |
2571 | i = cfg->vector; | ||
2333 | 2572 | ||
2334 | v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1)); | 2573 | v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1)); |
2335 | #endif | 2574 | #endif |
@@ -2368,17 +2607,18 @@ static void ack_apic_level(unsigned int irq) | |||
2368 | * accurate and is causing problems then it is a hardware bug | 2607 | * accurate and is causing problems then it is a hardware bug |
2369 | * and you can go talk to the chipset vendor about it. | 2608 | * and you can go talk to the chipset vendor about it. |
2370 | */ | 2609 | */ |
2371 | if (!io_apic_level_ack_pending(irq)) | 2610 | cfg = desc->chip_data; |
2611 | if (!io_apic_level_ack_pending(cfg)) | ||
2372 | move_masked_irq(irq); | 2612 | move_masked_irq(irq); |
2373 | unmask_IO_APIC_irq(irq); | 2613 | unmask_IO_APIC_irq_desc(desc); |
2374 | } | 2614 | } |
2375 | 2615 | ||
2376 | #ifdef CONFIG_X86_32 | 2616 | #ifdef CONFIG_X86_32 |
2377 | if (!(v & (1 << (i & 0x1f)))) { | 2617 | if (!(v & (1 << (i & 0x1f)))) { |
2378 | atomic_inc(&irq_mis_count); | 2618 | atomic_inc(&irq_mis_count); |
2379 | spin_lock(&ioapic_lock); | 2619 | spin_lock(&ioapic_lock); |
2380 | __mask_and_edge_IO_APIC_irq(irq); | 2620 | __mask_and_edge_IO_APIC_irq(cfg); |
2381 | __unmask_and_level_IO_APIC_irq(irq); | 2621 | __unmask_and_level_IO_APIC_irq(cfg); |
2382 | spin_unlock(&ioapic_lock); | 2622 | spin_unlock(&ioapic_lock); |
2383 | } | 2623 | } |
2384 | #endif | 2624 | #endif |
@@ -2429,20 +2669,22 @@ static inline void init_IO_APIC_traps(void) | |||
2429 | * Also, we've got to be careful not to trash gate | 2669 | * Also, we've got to be careful not to trash gate |
2430 | * 0x80, because int 0x80 is hm, kind of importantish. ;) | 2670 | * 0x80, because int 0x80 is hm, kind of importantish. ;) |
2431 | */ | 2671 | */ |
2432 | for_each_irq_cfg(irq, cfg) { | 2672 | for_each_irq_desc(irq, desc) { |
2433 | if (IO_APIC_IRQ(irq) && !cfg->vector) { | 2673 | if (!desc) |
2674 | continue; | ||
2675 | |||
2676 | cfg = desc->chip_data; | ||
2677 | if (IO_APIC_IRQ(irq) && cfg && !cfg->vector) { | ||
2434 | /* | 2678 | /* |
2435 | * Hmm.. We don't have an entry for this, | 2679 | * Hmm.. We don't have an entry for this, |
2436 | * so default to an old-fashioned 8259 | 2680 | * so default to an old-fashioned 8259 |
2437 | * interrupt if we can.. | 2681 | * interrupt if we can.. |
2438 | */ | 2682 | */ |
2439 | if (irq < 16) | 2683 | if (irq < NR_IRQS_LEGACY) |
2440 | make_8259A_irq(irq); | 2684 | make_8259A_irq(irq); |
2441 | else { | 2685 | else |
2442 | desc = irq_to_desc(irq); | ||
2443 | /* Strange. Oh, well.. */ | 2686 | /* Strange. Oh, well.. */ |
2444 | desc->chip = &no_irq_chip; | 2687 | desc->chip = &no_irq_chip; |
2445 | } | ||
2446 | } | 2688 | } |
2447 | } | 2689 | } |
2448 | } | 2690 | } |
@@ -2467,7 +2709,7 @@ static void unmask_lapic_irq(unsigned int irq) | |||
2467 | apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED); | 2709 | apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED); |
2468 | } | 2710 | } |
2469 | 2711 | ||
2470 | static void ack_lapic_irq (unsigned int irq) | 2712 | static void ack_lapic_irq(unsigned int irq) |
2471 | { | 2713 | { |
2472 | ack_APIC_irq(); | 2714 | ack_APIC_irq(); |
2473 | } | 2715 | } |
@@ -2479,11 +2721,8 @@ static struct irq_chip lapic_chip __read_mostly = { | |||
2479 | .ack = ack_lapic_irq, | 2721 | .ack = ack_lapic_irq, |
2480 | }; | 2722 | }; |
2481 | 2723 | ||
2482 | static void lapic_register_intr(int irq) | 2724 | static void lapic_register_intr(int irq, struct irq_desc *desc) |
2483 | { | 2725 | { |
2484 | struct irq_desc *desc; | ||
2485 | |||
2486 | desc = irq_to_desc(irq); | ||
2487 | desc->status &= ~IRQ_LEVEL; | 2726 | desc->status &= ~IRQ_LEVEL; |
2488 | set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq, | 2727 | set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq, |
2489 | "edge"); | 2728 | "edge"); |
@@ -2587,7 +2826,9 @@ int timer_through_8259 __initdata; | |||
2587 | */ | 2826 | */ |
2588 | static inline void __init check_timer(void) | 2827 | static inline void __init check_timer(void) |
2589 | { | 2828 | { |
2590 | struct irq_cfg *cfg = irq_cfg(0); | 2829 | struct irq_desc *desc = irq_to_desc(0); |
2830 | struct irq_cfg *cfg = desc->chip_data; | ||
2831 | int cpu = boot_cpu_id; | ||
2591 | int apic1, pin1, apic2, pin2; | 2832 | int apic1, pin1, apic2, pin2; |
2592 | unsigned long flags; | 2833 | unsigned long flags; |
2593 | unsigned int ver; | 2834 | unsigned int ver; |
@@ -2602,7 +2843,7 @@ static inline void __init check_timer(void) | |||
2602 | * get/set the timer IRQ vector: | 2843 | * get/set the timer IRQ vector: |
2603 | */ | 2844 | */ |
2604 | disable_8259A_irq(0); | 2845 | disable_8259A_irq(0); |
2605 | assign_irq_vector(0, TARGET_CPUS); | 2846 | assign_irq_vector(0, cfg, TARGET_CPUS); |
2606 | 2847 | ||
2607 | /* | 2848 | /* |
2608 | * As IRQ0 is to be enabled in the 8259A, the virtual | 2849 | * As IRQ0 is to be enabled in the 8259A, the virtual |
@@ -2653,10 +2894,10 @@ static inline void __init check_timer(void) | |||
2653 | * Ok, does IRQ0 through the IOAPIC work? | 2894 | * Ok, does IRQ0 through the IOAPIC work? |
2654 | */ | 2895 | */ |
2655 | if (no_pin1) { | 2896 | if (no_pin1) { |
2656 | add_pin_to_irq(0, apic1, pin1); | 2897 | add_pin_to_irq_cpu(cfg, cpu, apic1, pin1); |
2657 | setup_timer_IRQ0_pin(apic1, pin1, cfg->vector); | 2898 | setup_timer_IRQ0_pin(apic1, pin1, cfg->vector); |
2658 | } | 2899 | } |
2659 | unmask_IO_APIC_irq(0); | 2900 | unmask_IO_APIC_irq_desc(desc); |
2660 | if (timer_irq_works()) { | 2901 | if (timer_irq_works()) { |
2661 | if (nmi_watchdog == NMI_IO_APIC) { | 2902 | if (nmi_watchdog == NMI_IO_APIC) { |
2662 | setup_nmi(); | 2903 | setup_nmi(); |
@@ -2682,9 +2923,9 @@ static inline void __init check_timer(void) | |||
2682 | /* | 2923 | /* |
2683 | * legacy devices should be connected to IO APIC #0 | 2924 | * legacy devices should be connected to IO APIC #0 |
2684 | */ | 2925 | */ |
2685 | replace_pin_at_irq(0, apic1, pin1, apic2, pin2); | 2926 | replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2); |
2686 | setup_timer_IRQ0_pin(apic2, pin2, cfg->vector); | 2927 | setup_timer_IRQ0_pin(apic2, pin2, cfg->vector); |
2687 | unmask_IO_APIC_irq(0); | 2928 | unmask_IO_APIC_irq_desc(desc); |
2688 | enable_8259A_irq(0); | 2929 | enable_8259A_irq(0); |
2689 | if (timer_irq_works()) { | 2930 | if (timer_irq_works()) { |
2690 | apic_printk(APIC_QUIET, KERN_INFO "....... works.\n"); | 2931 | apic_printk(APIC_QUIET, KERN_INFO "....... works.\n"); |
@@ -2716,7 +2957,7 @@ static inline void __init check_timer(void) | |||
2716 | apic_printk(APIC_QUIET, KERN_INFO | 2957 | apic_printk(APIC_QUIET, KERN_INFO |
2717 | "...trying to set up timer as Virtual Wire IRQ...\n"); | 2958 | "...trying to set up timer as Virtual Wire IRQ...\n"); |
2718 | 2959 | ||
2719 | lapic_register_intr(0); | 2960 | lapic_register_intr(0, desc); |
2720 | apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ | 2961 | apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ |
2721 | enable_8259A_irq(0); | 2962 | enable_8259A_irq(0); |
2722 | 2963 | ||
@@ -2901,22 +3142,26 @@ unsigned int create_irq_nr(unsigned int irq_want) | |||
2901 | unsigned int irq; | 3142 | unsigned int irq; |
2902 | unsigned int new; | 3143 | unsigned int new; |
2903 | unsigned long flags; | 3144 | unsigned long flags; |
2904 | struct irq_cfg *cfg_new; | 3145 | struct irq_cfg *cfg_new = NULL; |
2905 | 3146 | int cpu = boot_cpu_id; | |
2906 | irq_want = nr_irqs - 1; | 3147 | struct irq_desc *desc_new = NULL; |
2907 | 3148 | ||
2908 | irq = 0; | 3149 | irq = 0; |
2909 | spin_lock_irqsave(&vector_lock, flags); | 3150 | spin_lock_irqsave(&vector_lock, flags); |
2910 | for (new = irq_want; new > 0; new--) { | 3151 | for (new = irq_want; new < NR_IRQS; new++) { |
2911 | if (platform_legacy_irq(new)) | 3152 | if (platform_legacy_irq(new)) |
2912 | continue; | 3153 | continue; |
2913 | cfg_new = irq_cfg(new); | 3154 | |
2914 | if (cfg_new && cfg_new->vector != 0) | 3155 | desc_new = irq_to_desc_alloc_cpu(new, cpu); |
3156 | if (!desc_new) { | ||
3157 | printk(KERN_INFO "can not get irq_desc for %d\n", new); | ||
2915 | continue; | 3158 | continue; |
2916 | /* check if need to create one */ | 3159 | } |
2917 | if (!cfg_new) | 3160 | cfg_new = desc_new->chip_data; |
2918 | cfg_new = irq_cfg_alloc(new); | 3161 | |
2919 | if (__assign_irq_vector(new, TARGET_CPUS) == 0) | 3162 | if (cfg_new->vector != 0) |
3163 | continue; | ||
3164 | if (__assign_irq_vector(new, cfg_new, TARGET_CPUS) == 0) | ||
2920 | irq = new; | 3165 | irq = new; |
2921 | break; | 3166 | break; |
2922 | } | 3167 | } |
@@ -2924,15 +3169,21 @@ unsigned int create_irq_nr(unsigned int irq_want) | |||
2924 | 3169 | ||
2925 | if (irq > 0) { | 3170 | if (irq > 0) { |
2926 | dynamic_irq_init(irq); | 3171 | dynamic_irq_init(irq); |
3172 | /* restore it, in case dynamic_irq_init clear it */ | ||
3173 | if (desc_new) | ||
3174 | desc_new->chip_data = cfg_new; | ||
2927 | } | 3175 | } |
2928 | return irq; | 3176 | return irq; |
2929 | } | 3177 | } |
2930 | 3178 | ||
3179 | static int nr_irqs_gsi = NR_IRQS_LEGACY; | ||
2931 | int create_irq(void) | 3180 | int create_irq(void) |
2932 | { | 3181 | { |
3182 | unsigned int irq_want; | ||
2933 | int irq; | 3183 | int irq; |
2934 | 3184 | ||
2935 | irq = create_irq_nr(nr_irqs - 1); | 3185 | irq_want = nr_irqs_gsi; |
3186 | irq = create_irq_nr(irq_want); | ||
2936 | 3187 | ||
2937 | if (irq == 0) | 3188 | if (irq == 0) |
2938 | irq = -1; | 3189 | irq = -1; |
@@ -2943,14 +3194,22 @@ int create_irq(void) | |||
2943 | void destroy_irq(unsigned int irq) | 3194 | void destroy_irq(unsigned int irq) |
2944 | { | 3195 | { |
2945 | unsigned long flags; | 3196 | unsigned long flags; |
3197 | struct irq_cfg *cfg; | ||
3198 | struct irq_desc *desc; | ||
2946 | 3199 | ||
3200 | /* store it, in case dynamic_irq_cleanup clear it */ | ||
3201 | desc = irq_to_desc(irq); | ||
3202 | cfg = desc->chip_data; | ||
2947 | dynamic_irq_cleanup(irq); | 3203 | dynamic_irq_cleanup(irq); |
3204 | /* connect back irq_cfg */ | ||
3205 | if (desc) | ||
3206 | desc->chip_data = cfg; | ||
2948 | 3207 | ||
2949 | #ifdef CONFIG_INTR_REMAP | 3208 | #ifdef CONFIG_INTR_REMAP |
2950 | free_irte(irq); | 3209 | free_irte(irq); |
2951 | #endif | 3210 | #endif |
2952 | spin_lock_irqsave(&vector_lock, flags); | 3211 | spin_lock_irqsave(&vector_lock, flags); |
2953 | __clear_irq_vector(irq); | 3212 | __clear_irq_vector(irq, cfg); |
2954 | spin_unlock_irqrestore(&vector_lock, flags); | 3213 | spin_unlock_irqrestore(&vector_lock, flags); |
2955 | } | 3214 | } |
2956 | 3215 | ||
@@ -2965,12 +3224,12 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms | |||
2965 | unsigned dest; | 3224 | unsigned dest; |
2966 | cpumask_t tmp; | 3225 | cpumask_t tmp; |
2967 | 3226 | ||
3227 | cfg = irq_cfg(irq); | ||
2968 | tmp = TARGET_CPUS; | 3228 | tmp = TARGET_CPUS; |
2969 | err = assign_irq_vector(irq, tmp); | 3229 | err = assign_irq_vector(irq, cfg, tmp); |
2970 | if (err) | 3230 | if (err) |
2971 | return err; | 3231 | return err; |
2972 | 3232 | ||
2973 | cfg = irq_cfg(irq); | ||
2974 | cpus_and(tmp, cfg->domain, tmp); | 3233 | cpus_and(tmp, cfg->domain, tmp); |
2975 | dest = cpu_mask_to_apicid(tmp); | 3234 | dest = cpu_mask_to_apicid(tmp); |
2976 | 3235 | ||
@@ -3028,35 +3287,35 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms | |||
3028 | #ifdef CONFIG_SMP | 3287 | #ifdef CONFIG_SMP |
3029 | static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask) | 3288 | static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask) |
3030 | { | 3289 | { |
3290 | struct irq_desc *desc = irq_to_desc(irq); | ||
3031 | struct irq_cfg *cfg; | 3291 | struct irq_cfg *cfg; |
3032 | struct msi_msg msg; | 3292 | struct msi_msg msg; |
3033 | unsigned int dest; | 3293 | unsigned int dest; |
3034 | cpumask_t tmp; | 3294 | cpumask_t tmp; |
3035 | struct irq_desc *desc; | ||
3036 | 3295 | ||
3037 | cpus_and(tmp, mask, cpu_online_map); | 3296 | cpus_and(tmp, mask, cpu_online_map); |
3038 | if (cpus_empty(tmp)) | 3297 | if (cpus_empty(tmp)) |
3039 | return; | 3298 | return; |
3040 | 3299 | ||
3041 | if (assign_irq_vector(irq, mask)) | 3300 | cfg = desc->chip_data; |
3301 | if (assign_irq_vector(irq, cfg, mask)) | ||
3042 | return; | 3302 | return; |
3043 | 3303 | ||
3044 | cfg = irq_cfg(irq); | 3304 | set_extra_move_desc(desc, mask); |
3305 | |||
3045 | cpus_and(tmp, cfg->domain, mask); | 3306 | cpus_and(tmp, cfg->domain, mask); |
3046 | dest = cpu_mask_to_apicid(tmp); | 3307 | dest = cpu_mask_to_apicid(tmp); |
3047 | 3308 | ||
3048 | read_msi_msg(irq, &msg); | 3309 | read_msi_msg_desc(desc, &msg); |
3049 | 3310 | ||
3050 | msg.data &= ~MSI_DATA_VECTOR_MASK; | 3311 | msg.data &= ~MSI_DATA_VECTOR_MASK; |
3051 | msg.data |= MSI_DATA_VECTOR(cfg->vector); | 3312 | msg.data |= MSI_DATA_VECTOR(cfg->vector); |
3052 | msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK; | 3313 | msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK; |
3053 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); | 3314 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); |
3054 | 3315 | ||
3055 | write_msi_msg(irq, &msg); | 3316 | write_msi_msg_desc(desc, &msg); |
3056 | desc = irq_to_desc(irq); | ||
3057 | desc->affinity = mask; | 3317 | desc->affinity = mask; |
3058 | } | 3318 | } |
3059 | |||
3060 | #ifdef CONFIG_INTR_REMAP | 3319 | #ifdef CONFIG_INTR_REMAP |
3061 | /* | 3320 | /* |
3062 | * Migrate the MSI irq to another cpumask. This migration is | 3321 | * Migrate the MSI irq to another cpumask. This migration is |
@@ -3064,11 +3323,11 @@ static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask) | |||
3064 | */ | 3323 | */ |
3065 | static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask) | 3324 | static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask) |
3066 | { | 3325 | { |
3326 | struct irq_desc *desc = irq_to_desc(irq); | ||
3067 | struct irq_cfg *cfg; | 3327 | struct irq_cfg *cfg; |
3068 | unsigned int dest; | 3328 | unsigned int dest; |
3069 | cpumask_t tmp, cleanup_mask; | 3329 | cpumask_t tmp, cleanup_mask; |
3070 | struct irte irte; | 3330 | struct irte irte; |
3071 | struct irq_desc *desc; | ||
3072 | 3331 | ||
3073 | cpus_and(tmp, mask, cpu_online_map); | 3332 | cpus_and(tmp, mask, cpu_online_map); |
3074 | if (cpus_empty(tmp)) | 3333 | if (cpus_empty(tmp)) |
@@ -3077,10 +3336,12 @@ static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask) | |||
3077 | if (get_irte(irq, &irte)) | 3336 | if (get_irte(irq, &irte)) |
3078 | return; | 3337 | return; |
3079 | 3338 | ||
3080 | if (assign_irq_vector(irq, mask)) | 3339 | cfg = desc->chip_data; |
3340 | if (assign_irq_vector(irq, cfg, mask)) | ||
3081 | return; | 3341 | return; |
3082 | 3342 | ||
3083 | cfg = irq_cfg(irq); | 3343 | set_extra_move_desc(desc, mask); |
3344 | |||
3084 | cpus_and(tmp, cfg->domain, mask); | 3345 | cpus_and(tmp, cfg->domain, mask); |
3085 | dest = cpu_mask_to_apicid(tmp); | 3346 | dest = cpu_mask_to_apicid(tmp); |
3086 | 3347 | ||
@@ -3104,9 +3365,9 @@ static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask) | |||
3104 | cfg->move_in_progress = 0; | 3365 | cfg->move_in_progress = 0; |
3105 | } | 3366 | } |
3106 | 3367 | ||
3107 | desc = irq_to_desc(irq); | ||
3108 | desc->affinity = mask; | 3368 | desc->affinity = mask; |
3109 | } | 3369 | } |
3370 | |||
3110 | #endif | 3371 | #endif |
3111 | #endif /* CONFIG_SMP */ | 3372 | #endif /* CONFIG_SMP */ |
3112 | 3373 | ||
@@ -3165,7 +3426,7 @@ static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec) | |||
3165 | } | 3426 | } |
3166 | #endif | 3427 | #endif |
3167 | 3428 | ||
3168 | static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq) | 3429 | static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq) |
3169 | { | 3430 | { |
3170 | int ret; | 3431 | int ret; |
3171 | struct msi_msg msg; | 3432 | struct msi_msg msg; |
@@ -3174,7 +3435,7 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq) | |||
3174 | if (ret < 0) | 3435 | if (ret < 0) |
3175 | return ret; | 3436 | return ret; |
3176 | 3437 | ||
3177 | set_irq_msi(irq, desc); | 3438 | set_irq_msi(irq, msidesc); |
3178 | write_msi_msg(irq, &msg); | 3439 | write_msi_msg(irq, &msg); |
3179 | 3440 | ||
3180 | #ifdef CONFIG_INTR_REMAP | 3441 | #ifdef CONFIG_INTR_REMAP |
@@ -3194,26 +3455,13 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq) | |||
3194 | return 0; | 3455 | return 0; |
3195 | } | 3456 | } |
3196 | 3457 | ||
3197 | static unsigned int build_irq_for_pci_dev(struct pci_dev *dev) | 3458 | int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc) |
3198 | { | ||
3199 | unsigned int irq; | ||
3200 | |||
3201 | irq = dev->bus->number; | ||
3202 | irq <<= 8; | ||
3203 | irq |= dev->devfn; | ||
3204 | irq <<= 12; | ||
3205 | |||
3206 | return irq; | ||
3207 | } | ||
3208 | |||
3209 | int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) | ||
3210 | { | 3459 | { |
3211 | unsigned int irq; | 3460 | unsigned int irq; |
3212 | int ret; | 3461 | int ret; |
3213 | unsigned int irq_want; | 3462 | unsigned int irq_want; |
3214 | 3463 | ||
3215 | irq_want = build_irq_for_pci_dev(dev) + 0x100; | 3464 | irq_want = nr_irqs_gsi; |
3216 | |||
3217 | irq = create_irq_nr(irq_want); | 3465 | irq = create_irq_nr(irq_want); |
3218 | if (irq == 0) | 3466 | if (irq == 0) |
3219 | return -1; | 3467 | return -1; |
@@ -3227,7 +3475,7 @@ int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc) | |||
3227 | goto error; | 3475 | goto error; |
3228 | no_ir: | 3476 | no_ir: |
3229 | #endif | 3477 | #endif |
3230 | ret = setup_msi_irq(dev, desc, irq); | 3478 | ret = setup_msi_irq(dev, msidesc, irq); |
3231 | if (ret < 0) { | 3479 | if (ret < 0) { |
3232 | destroy_irq(irq); | 3480 | destroy_irq(irq); |
3233 | return ret; | 3481 | return ret; |
@@ -3245,7 +3493,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3245 | { | 3493 | { |
3246 | unsigned int irq; | 3494 | unsigned int irq; |
3247 | int ret, sub_handle; | 3495 | int ret, sub_handle; |
3248 | struct msi_desc *desc; | 3496 | struct msi_desc *msidesc; |
3249 | unsigned int irq_want; | 3497 | unsigned int irq_want; |
3250 | 3498 | ||
3251 | #ifdef CONFIG_INTR_REMAP | 3499 | #ifdef CONFIG_INTR_REMAP |
@@ -3253,10 +3501,11 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3253 | int index = 0; | 3501 | int index = 0; |
3254 | #endif | 3502 | #endif |
3255 | 3503 | ||
3256 | irq_want = build_irq_for_pci_dev(dev) + 0x100; | 3504 | irq_want = nr_irqs_gsi; |
3257 | sub_handle = 0; | 3505 | sub_handle = 0; |
3258 | list_for_each_entry(desc, &dev->msi_list, list) { | 3506 | list_for_each_entry(msidesc, &dev->msi_list, list) { |
3259 | irq = create_irq_nr(irq_want--); | 3507 | irq = create_irq_nr(irq_want); |
3508 | irq_want++; | ||
3260 | if (irq == 0) | 3509 | if (irq == 0) |
3261 | return -1; | 3510 | return -1; |
3262 | #ifdef CONFIG_INTR_REMAP | 3511 | #ifdef CONFIG_INTR_REMAP |
@@ -3288,7 +3537,7 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3288 | } | 3537 | } |
3289 | no_ir: | 3538 | no_ir: |
3290 | #endif | 3539 | #endif |
3291 | ret = setup_msi_irq(dev, desc, irq); | 3540 | ret = setup_msi_irq(dev, msidesc, irq); |
3292 | if (ret < 0) | 3541 | if (ret < 0) |
3293 | goto error; | 3542 | goto error; |
3294 | sub_handle++; | 3543 | sub_handle++; |
@@ -3309,20 +3558,22 @@ void arch_teardown_msi_irq(unsigned int irq) | |||
3309 | #ifdef CONFIG_SMP | 3558 | #ifdef CONFIG_SMP |
3310 | static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask) | 3559 | static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask) |
3311 | { | 3560 | { |
3561 | struct irq_desc *desc = irq_to_desc(irq); | ||
3312 | struct irq_cfg *cfg; | 3562 | struct irq_cfg *cfg; |
3313 | struct msi_msg msg; | 3563 | struct msi_msg msg; |
3314 | unsigned int dest; | 3564 | unsigned int dest; |
3315 | cpumask_t tmp; | 3565 | cpumask_t tmp; |
3316 | struct irq_desc *desc; | ||
3317 | 3566 | ||
3318 | cpus_and(tmp, mask, cpu_online_map); | 3567 | cpus_and(tmp, mask, cpu_online_map); |
3319 | if (cpus_empty(tmp)) | 3568 | if (cpus_empty(tmp)) |
3320 | return; | 3569 | return; |
3321 | 3570 | ||
3322 | if (assign_irq_vector(irq, mask)) | 3571 | cfg = desc->chip_data; |
3572 | if (assign_irq_vector(irq, cfg, mask)) | ||
3323 | return; | 3573 | return; |
3324 | 3574 | ||
3325 | cfg = irq_cfg(irq); | 3575 | set_extra_move_desc(desc, mask); |
3576 | |||
3326 | cpus_and(tmp, cfg->domain, mask); | 3577 | cpus_and(tmp, cfg->domain, mask); |
3327 | dest = cpu_mask_to_apicid(tmp); | 3578 | dest = cpu_mask_to_apicid(tmp); |
3328 | 3579 | ||
@@ -3334,9 +3585,9 @@ static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask) | |||
3334 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); | 3585 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); |
3335 | 3586 | ||
3336 | dmar_msi_write(irq, &msg); | 3587 | dmar_msi_write(irq, &msg); |
3337 | desc = irq_to_desc(irq); | ||
3338 | desc->affinity = mask; | 3588 | desc->affinity = mask; |
3339 | } | 3589 | } |
3590 | |||
3340 | #endif /* CONFIG_SMP */ | 3591 | #endif /* CONFIG_SMP */ |
3341 | 3592 | ||
3342 | struct irq_chip dmar_msi_type = { | 3593 | struct irq_chip dmar_msi_type = { |
@@ -3370,8 +3621,8 @@ int arch_setup_dmar_msi(unsigned int irq) | |||
3370 | #ifdef CONFIG_SMP | 3621 | #ifdef CONFIG_SMP |
3371 | static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask) | 3622 | static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask) |
3372 | { | 3623 | { |
3624 | struct irq_desc *desc = irq_to_desc(irq); | ||
3373 | struct irq_cfg *cfg; | 3625 | struct irq_cfg *cfg; |
3374 | struct irq_desc *desc; | ||
3375 | struct msi_msg msg; | 3626 | struct msi_msg msg; |
3376 | unsigned int dest; | 3627 | unsigned int dest; |
3377 | cpumask_t tmp; | 3628 | cpumask_t tmp; |
@@ -3380,10 +3631,12 @@ static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask) | |||
3380 | if (cpus_empty(tmp)) | 3631 | if (cpus_empty(tmp)) |
3381 | return; | 3632 | return; |
3382 | 3633 | ||
3383 | if (assign_irq_vector(irq, mask)) | 3634 | cfg = desc->chip_data; |
3635 | if (assign_irq_vector(irq, cfg, mask)) | ||
3384 | return; | 3636 | return; |
3385 | 3637 | ||
3386 | cfg = irq_cfg(irq); | 3638 | set_extra_move_desc(desc, mask); |
3639 | |||
3387 | cpus_and(tmp, cfg->domain, mask); | 3640 | cpus_and(tmp, cfg->domain, mask); |
3388 | dest = cpu_mask_to_apicid(tmp); | 3641 | dest = cpu_mask_to_apicid(tmp); |
3389 | 3642 | ||
@@ -3395,9 +3648,9 @@ static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask) | |||
3395 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); | 3648 | msg.address_lo |= MSI_ADDR_DEST_ID(dest); |
3396 | 3649 | ||
3397 | hpet_msi_write(irq, &msg); | 3650 | hpet_msi_write(irq, &msg); |
3398 | desc = irq_to_desc(irq); | ||
3399 | desc->affinity = mask; | 3651 | desc->affinity = mask; |
3400 | } | 3652 | } |
3653 | |||
3401 | #endif /* CONFIG_SMP */ | 3654 | #endif /* CONFIG_SMP */ |
3402 | 3655 | ||
3403 | struct irq_chip hpet_msi_type = { | 3656 | struct irq_chip hpet_msi_type = { |
@@ -3452,26 +3705,28 @@ static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector) | |||
3452 | 3705 | ||
3453 | static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask) | 3706 | static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask) |
3454 | { | 3707 | { |
3708 | struct irq_desc *desc = irq_to_desc(irq); | ||
3455 | struct irq_cfg *cfg; | 3709 | struct irq_cfg *cfg; |
3456 | unsigned int dest; | 3710 | unsigned int dest; |
3457 | cpumask_t tmp; | 3711 | cpumask_t tmp; |
3458 | struct irq_desc *desc; | ||
3459 | 3712 | ||
3460 | cpus_and(tmp, mask, cpu_online_map); | 3713 | cpus_and(tmp, mask, cpu_online_map); |
3461 | if (cpus_empty(tmp)) | 3714 | if (cpus_empty(tmp)) |
3462 | return; | 3715 | return; |
3463 | 3716 | ||
3464 | if (assign_irq_vector(irq, mask)) | 3717 | cfg = desc->chip_data; |
3718 | if (assign_irq_vector(irq, cfg, mask)) | ||
3465 | return; | 3719 | return; |
3466 | 3720 | ||
3467 | cfg = irq_cfg(irq); | 3721 | set_extra_move_desc(desc, mask); |
3722 | |||
3468 | cpus_and(tmp, cfg->domain, mask); | 3723 | cpus_and(tmp, cfg->domain, mask); |
3469 | dest = cpu_mask_to_apicid(tmp); | 3724 | dest = cpu_mask_to_apicid(tmp); |
3470 | 3725 | ||
3471 | target_ht_irq(irq, dest, cfg->vector); | 3726 | target_ht_irq(irq, dest, cfg->vector); |
3472 | desc = irq_to_desc(irq); | ||
3473 | desc->affinity = mask; | 3727 | desc->affinity = mask; |
3474 | } | 3728 | } |
3729 | |||
3475 | #endif | 3730 | #endif |
3476 | 3731 | ||
3477 | static struct irq_chip ht_irq_chip = { | 3732 | static struct irq_chip ht_irq_chip = { |
@@ -3491,13 +3746,13 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev) | |||
3491 | int err; | 3746 | int err; |
3492 | cpumask_t tmp; | 3747 | cpumask_t tmp; |
3493 | 3748 | ||
3749 | cfg = irq_cfg(irq); | ||
3494 | tmp = TARGET_CPUS; | 3750 | tmp = TARGET_CPUS; |
3495 | err = assign_irq_vector(irq, tmp); | 3751 | err = assign_irq_vector(irq, cfg, tmp); |
3496 | if (!err) { | 3752 | if (!err) { |
3497 | struct ht_irq_msg msg; | 3753 | struct ht_irq_msg msg; |
3498 | unsigned dest; | 3754 | unsigned dest; |
3499 | 3755 | ||
3500 | cfg = irq_cfg(irq); | ||
3501 | cpus_and(tmp, cfg->domain, tmp); | 3756 | cpus_and(tmp, cfg->domain, tmp); |
3502 | dest = cpu_mask_to_apicid(tmp); | 3757 | dest = cpu_mask_to_apicid(tmp); |
3503 | 3758 | ||
@@ -3543,7 +3798,9 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade, | |||
3543 | unsigned long flags; | 3798 | unsigned long flags; |
3544 | int err; | 3799 | int err; |
3545 | 3800 | ||
3546 | err = assign_irq_vector(irq, *eligible_cpu); | 3801 | cfg = irq_cfg(irq); |
3802 | |||
3803 | err = assign_irq_vector(irq, cfg, *eligible_cpu); | ||
3547 | if (err != 0) | 3804 | if (err != 0) |
3548 | return err; | 3805 | return err; |
3549 | 3806 | ||
@@ -3552,8 +3809,6 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade, | |||
3552 | irq_name); | 3809 | irq_name); |
3553 | spin_unlock_irqrestore(&vector_lock, flags); | 3810 | spin_unlock_irqrestore(&vector_lock, flags); |
3554 | 3811 | ||
3555 | cfg = irq_cfg(irq); | ||
3556 | |||
3557 | mmr_value = 0; | 3812 | mmr_value = 0; |
3558 | entry = (struct uv_IO_APIC_route_entry *)&mmr_value; | 3813 | entry = (struct uv_IO_APIC_route_entry *)&mmr_value; |
3559 | BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long)); | 3814 | BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long)); |
@@ -3605,9 +3860,16 @@ int __init io_apic_get_redir_entries (int ioapic) | |||
3605 | return reg_01.bits.entries; | 3860 | return reg_01.bits.entries; |
3606 | } | 3861 | } |
3607 | 3862 | ||
3608 | int __init probe_nr_irqs(void) | 3863 | void __init probe_nr_irqs_gsi(void) |
3609 | { | 3864 | { |
3610 | return NR_IRQS; | 3865 | int idx; |
3866 | int nr = 0; | ||
3867 | |||
3868 | for (idx = 0; idx < nr_ioapics; idx++) | ||
3869 | nr += io_apic_get_redir_entries(idx) + 1; | ||
3870 | |||
3871 | if (nr > nr_irqs_gsi) | ||
3872 | nr_irqs_gsi = nr; | ||
3611 | } | 3873 | } |
3612 | 3874 | ||
3613 | /* -------------------------------------------------------------------------- | 3875 | /* -------------------------------------------------------------------------- |
@@ -3706,19 +3968,31 @@ int __init io_apic_get_version(int ioapic) | |||
3706 | 3968 | ||
3707 | int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity) | 3969 | int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity) |
3708 | { | 3970 | { |
3971 | struct irq_desc *desc; | ||
3972 | struct irq_cfg *cfg; | ||
3973 | int cpu = boot_cpu_id; | ||
3974 | |||
3709 | if (!IO_APIC_IRQ(irq)) { | 3975 | if (!IO_APIC_IRQ(irq)) { |
3710 | apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n", | 3976 | apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n", |
3711 | ioapic); | 3977 | ioapic); |
3712 | return -EINVAL; | 3978 | return -EINVAL; |
3713 | } | 3979 | } |
3714 | 3980 | ||
3981 | desc = irq_to_desc_alloc_cpu(irq, cpu); | ||
3982 | if (!desc) { | ||
3983 | printk(KERN_INFO "can not get irq_desc %d\n", irq); | ||
3984 | return 0; | ||
3985 | } | ||
3986 | |||
3715 | /* | 3987 | /* |
3716 | * IRQs < 16 are already in the irq_2_pin[] map | 3988 | * IRQs < 16 are already in the irq_2_pin[] map |
3717 | */ | 3989 | */ |
3718 | if (irq >= 16) | 3990 | if (irq >= NR_IRQS_LEGACY) { |
3719 | add_pin_to_irq(irq, ioapic, pin); | 3991 | cfg = desc->chip_data; |
3992 | add_pin_to_irq_cpu(cfg, cpu, ioapic, pin); | ||
3993 | } | ||
3720 | 3994 | ||
3721 | setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity); | 3995 | setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity); |
3722 | 3996 | ||
3723 | return 0; | 3997 | return 0; |
3724 | } | 3998 | } |
@@ -3772,9 +4046,10 @@ void __init setup_ioapic_dest(void) | |||
3772 | * when you have too many devices, because at that time only boot | 4046 | * when you have too many devices, because at that time only boot |
3773 | * cpu is online. | 4047 | * cpu is online. |
3774 | */ | 4048 | */ |
3775 | cfg = irq_cfg(irq); | 4049 | desc = irq_to_desc(irq); |
4050 | cfg = desc->chip_data; | ||
3776 | if (!cfg->vector) { | 4051 | if (!cfg->vector) { |
3777 | setup_IO_APIC_irq(ioapic, pin, irq, | 4052 | setup_IO_APIC_irq(ioapic, pin, irq, desc, |
3778 | irq_trigger(irq_entry), | 4053 | irq_trigger(irq_entry), |
3779 | irq_polarity(irq_entry)); | 4054 | irq_polarity(irq_entry)); |
3780 | continue; | 4055 | continue; |
@@ -3784,7 +4059,6 @@ void __init setup_ioapic_dest(void) | |||
3784 | /* | 4059 | /* |
3785 | * Honour affinities which have been set in early boot | 4060 | * Honour affinities which have been set in early boot |
3786 | */ | 4061 | */ |
3787 | desc = irq_to_desc(irq); | ||
3788 | if (desc->status & | 4062 | if (desc->status & |
3789 | (IRQ_NO_BALANCING | IRQ_AFFINITY_SET)) | 4063 | (IRQ_NO_BALANCING | IRQ_AFFINITY_SET)) |
3790 | mask = desc->affinity; | 4064 | mask = desc->affinity; |
@@ -3793,10 +4067,10 @@ void __init setup_ioapic_dest(void) | |||
3793 | 4067 | ||
3794 | #ifdef CONFIG_INTR_REMAP | 4068 | #ifdef CONFIG_INTR_REMAP |
3795 | if (intr_remapping_enabled) | 4069 | if (intr_remapping_enabled) |
3796 | set_ir_ioapic_affinity_irq(irq, mask); | 4070 | set_ir_ioapic_affinity_irq_desc(desc, mask); |
3797 | else | 4071 | else |
3798 | #endif | 4072 | #endif |
3799 | set_ioapic_affinity_irq(irq, mask); | 4073 | set_ioapic_affinity_irq_desc(desc, mask); |
3800 | } | 4074 | } |
3801 | 4075 | ||
3802 | } | 4076 | } |
@@ -3845,7 +4119,6 @@ void __init ioapic_init_mappings(void) | |||
3845 | struct resource *ioapic_res; | 4119 | struct resource *ioapic_res; |
3846 | int i; | 4120 | int i; |
3847 | 4121 | ||
3848 | irq_2_pin_init(); | ||
3849 | ioapic_res = ioapic_setup_resources(); | 4122 | ioapic_res = ioapic_setup_resources(); |
3850 | for (i = 0; i < nr_ioapics; i++) { | 4123 | for (i = 0; i < nr_ioapics; i++) { |
3851 | if (smp_found_config) { | 4124 | if (smp_found_config) { |
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index d1d4dc52f64..3f1d9d18df6 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c | |||
@@ -118,6 +118,9 @@ int show_interrupts(struct seq_file *p, void *v) | |||
118 | } | 118 | } |
119 | 119 | ||
120 | desc = irq_to_desc(i); | 120 | desc = irq_to_desc(i); |
121 | if (!desc) | ||
122 | return 0; | ||
123 | |||
121 | spin_lock_irqsave(&desc->lock, flags); | 124 | spin_lock_irqsave(&desc->lock, flags); |
122 | #ifndef CONFIG_SMP | 125 | #ifndef CONFIG_SMP |
123 | any_count = kstat_irqs(i); | 126 | any_count = kstat_irqs(i); |
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index a51382672de..119fc9c8ff7 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c | |||
@@ -242,6 +242,8 @@ void fixup_irqs(cpumask_t map) | |||
242 | for_each_irq_desc(irq, desc) { | 242 | for_each_irq_desc(irq, desc) { |
243 | cpumask_t mask; | 243 | cpumask_t mask; |
244 | 244 | ||
245 | if (!desc) | ||
246 | continue; | ||
245 | if (irq == 2) | 247 | if (irq == 2) |
246 | continue; | 248 | continue; |
247 | 249 | ||
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 1df869e5bd0..a174a217eb1 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c | |||
@@ -91,6 +91,8 @@ void fixup_irqs(cpumask_t map) | |||
91 | int break_affinity = 0; | 91 | int break_affinity = 0; |
92 | int set_affinity = 1; | 92 | int set_affinity = 1; |
93 | 93 | ||
94 | if (!desc) | ||
95 | continue; | ||
94 | if (irq == 2) | 96 | if (irq == 2) |
95 | continue; | 97 | continue; |
96 | 98 | ||
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c index 607db63044a..203384ed2b5 100644 --- a/arch/x86/kernel/irqinit_32.c +++ b/arch/x86/kernel/irqinit_32.c | |||
@@ -68,8 +68,7 @@ void __init init_ISA_irqs (void) | |||
68 | /* | 68 | /* |
69 | * 16 old-style INTA-cycle interrupts: | 69 | * 16 old-style INTA-cycle interrupts: |
70 | */ | 70 | */ |
71 | for (i = 0; i < 16; i++) { | 71 | for (i = 0; i < NR_IRQS_LEGACY; i++) { |
72 | /* first time call this irq_desc */ | ||
73 | struct irq_desc *desc = irq_to_desc(i); | 72 | struct irq_desc *desc = irq_to_desc(i); |
74 | 73 | ||
75 | desc->status = IRQ_DISABLED; | 74 | desc->status = IRQ_DISABLED; |
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c index 8670b3ce626..6190e6ef546 100644 --- a/arch/x86/kernel/irqinit_64.c +++ b/arch/x86/kernel/irqinit_64.c | |||
@@ -76,8 +76,7 @@ void __init init_ISA_irqs(void) | |||
76 | init_bsp_APIC(); | 76 | init_bsp_APIC(); |
77 | init_8259A(0); | 77 | init_8259A(0); |
78 | 78 | ||
79 | for (i = 0; i < 16; i++) { | 79 | for (i = 0; i < NR_IRQS_LEGACY; i++) { |
80 | /* first time call this irq_desc */ | ||
81 | struct irq_desc *desc = irq_to_desc(i); | 80 | struct irq_desc *desc = irq_to_desc(i); |
82 | 81 | ||
83 | desc->status = IRQ_DISABLED; | 82 | desc->status = IRQ_DISABLED; |
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 08e02e8453c..ae0d8042cf6 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c | |||
@@ -953,7 +953,7 @@ void __init setup_arch(char **cmdline_p) | |||
953 | ioapic_init_mappings(); | 953 | ioapic_init_mappings(); |
954 | 954 | ||
955 | /* need to wait for io_apic is mapped */ | 955 | /* need to wait for io_apic is mapped */ |
956 | nr_irqs = probe_nr_irqs(); | 956 | probe_nr_irqs_gsi(); |
957 | 957 | ||
958 | kvm_guest_init(); | 958 | kvm_guest_init(); |
959 | 959 | ||
diff --git a/drivers/char/random.c b/drivers/char/random.c index 675076f5fca..d26891bfcd4 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c | |||
@@ -558,23 +558,9 @@ struct timer_rand_state { | |||
558 | unsigned dont_count_entropy:1; | 558 | unsigned dont_count_entropy:1; |
559 | }; | 559 | }; |
560 | 560 | ||
561 | static struct timer_rand_state *irq_timer_state[NR_IRQS]; | 561 | #ifndef CONFIG_SPARSE_IRQ |
562 | 562 | struct timer_rand_state *irq_timer_state[NR_IRQS]; | |
563 | static struct timer_rand_state *get_timer_rand_state(unsigned int irq) | 563 | #endif |
564 | { | ||
565 | if (irq >= nr_irqs) | ||
566 | return NULL; | ||
567 | |||
568 | return irq_timer_state[irq]; | ||
569 | } | ||
570 | |||
571 | static void set_timer_rand_state(unsigned int irq, struct timer_rand_state *state) | ||
572 | { | ||
573 | if (irq >= nr_irqs) | ||
574 | return; | ||
575 | |||
576 | irq_timer_state[irq] = state; | ||
577 | } | ||
578 | 564 | ||
579 | static struct timer_rand_state input_timer_state; | 565 | static struct timer_rand_state input_timer_state; |
580 | 566 | ||
@@ -933,8 +919,10 @@ void rand_initialize_irq(int irq) | |||
933 | { | 919 | { |
934 | struct timer_rand_state *state; | 920 | struct timer_rand_state *state; |
935 | 921 | ||
922 | #ifndef CONFIG_SPARSE_IRQ | ||
936 | if (irq >= nr_irqs) | 923 | if (irq >= nr_irqs) |
937 | return; | 924 | return; |
925 | #endif | ||
938 | 926 | ||
939 | state = get_timer_rand_state(irq); | 927 | state = get_timer_rand_state(irq); |
940 | 928 | ||
diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c index 2de5a3238c9..f78371b2252 100644 --- a/drivers/pci/intr_remapping.c +++ b/drivers/pci/intr_remapping.c | |||
@@ -5,6 +5,7 @@ | |||
5 | #include <linux/pci.h> | 5 | #include <linux/pci.h> |
6 | #include <linux/irq.h> | 6 | #include <linux/irq.h> |
7 | #include <asm/io_apic.h> | 7 | #include <asm/io_apic.h> |
8 | #include <asm/smp.h> | ||
8 | #include <linux/intel-iommu.h> | 9 | #include <linux/intel-iommu.h> |
9 | #include "intr_remapping.h" | 10 | #include "intr_remapping.h" |
10 | 11 | ||
@@ -19,17 +20,75 @@ struct irq_2_iommu { | |||
19 | u8 irte_mask; | 20 | u8 irte_mask; |
20 | }; | 21 | }; |
21 | 22 | ||
22 | static struct irq_2_iommu irq_2_iommuX[NR_IRQS]; | 23 | #ifdef CONFIG_SPARSE_IRQ |
24 | static struct irq_2_iommu *get_one_free_irq_2_iommu(int cpu) | ||
25 | { | ||
26 | struct irq_2_iommu *iommu; | ||
27 | int node; | ||
28 | |||
29 | node = cpu_to_node(cpu); | ||
30 | |||
31 | iommu = kzalloc_node(sizeof(*iommu), GFP_ATOMIC, node); | ||
32 | printk(KERN_DEBUG "alloc irq_2_iommu on cpu %d node %d\n", cpu, node); | ||
33 | |||
34 | return iommu; | ||
35 | } | ||
23 | 36 | ||
24 | static struct irq_2_iommu *irq_2_iommu(unsigned int irq) | 37 | static struct irq_2_iommu *irq_2_iommu(unsigned int irq) |
25 | { | 38 | { |
26 | return (irq < nr_irqs) ? irq_2_iommuX + irq : NULL; | 39 | struct irq_desc *desc; |
40 | |||
41 | desc = irq_to_desc(irq); | ||
42 | |||
43 | if (WARN_ON_ONCE(!desc)) | ||
44 | return NULL; | ||
45 | |||
46 | return desc->irq_2_iommu; | ||
47 | } | ||
48 | |||
49 | static struct irq_2_iommu *irq_2_iommu_alloc_cpu(unsigned int irq, int cpu) | ||
50 | { | ||
51 | struct irq_desc *desc; | ||
52 | struct irq_2_iommu *irq_iommu; | ||
53 | |||
54 | /* | ||
55 | * alloc irq desc if not allocated already. | ||
56 | */ | ||
57 | desc = irq_to_desc_alloc_cpu(irq, cpu); | ||
58 | if (!desc) { | ||
59 | printk(KERN_INFO "can not get irq_desc for %d\n", irq); | ||
60 | return NULL; | ||
61 | } | ||
62 | |||
63 | irq_iommu = desc->irq_2_iommu; | ||
64 | |||
65 | if (!irq_iommu) | ||
66 | desc->irq_2_iommu = get_one_free_irq_2_iommu(cpu); | ||
67 | |||
68 | return desc->irq_2_iommu; | ||
27 | } | 69 | } |
28 | 70 | ||
29 | static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq) | 71 | static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq) |
30 | { | 72 | { |
73 | return irq_2_iommu_alloc_cpu(irq, boot_cpu_id); | ||
74 | } | ||
75 | |||
76 | #else /* !CONFIG_SPARSE_IRQ */ | ||
77 | |||
78 | static struct irq_2_iommu irq_2_iommuX[NR_IRQS]; | ||
79 | |||
80 | static struct irq_2_iommu *irq_2_iommu(unsigned int irq) | ||
81 | { | ||
82 | if (irq < nr_irqs) | ||
83 | return &irq_2_iommuX[irq]; | ||
84 | |||
85 | return NULL; | ||
86 | } | ||
87 | static struct irq_2_iommu *irq_2_iommu_alloc(unsigned int irq) | ||
88 | { | ||
31 | return irq_2_iommu(irq); | 89 | return irq_2_iommu(irq); |
32 | } | 90 | } |
91 | #endif | ||
33 | 92 | ||
34 | static DEFINE_SPINLOCK(irq_2_ir_lock); | 93 | static DEFINE_SPINLOCK(irq_2_ir_lock); |
35 | 94 | ||
@@ -86,9 +145,11 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) | |||
86 | if (!count) | 145 | if (!count) |
87 | return -1; | 146 | return -1; |
88 | 147 | ||
148 | #ifndef CONFIG_SPARSE_IRQ | ||
89 | /* protect irq_2_iommu_alloc later */ | 149 | /* protect irq_2_iommu_alloc later */ |
90 | if (irq >= nr_irqs) | 150 | if (irq >= nr_irqs) |
91 | return -1; | 151 | return -1; |
152 | #endif | ||
92 | 153 | ||
93 | /* | 154 | /* |
94 | * start the IRTE search from index 0. | 155 | * start the IRTE search from index 0. |
@@ -130,6 +191,12 @@ int alloc_irte(struct intel_iommu *iommu, int irq, u16 count) | |||
130 | table->base[i].present = 1; | 191 | table->base[i].present = 1; |
131 | 192 | ||
132 | irq_iommu = irq_2_iommu_alloc(irq); | 193 | irq_iommu = irq_2_iommu_alloc(irq); |
194 | if (!irq_iommu) { | ||
195 | spin_unlock(&irq_2_ir_lock); | ||
196 | printk(KERN_ERR "can't allocate irq_2_iommu\n"); | ||
197 | return -1; | ||
198 | } | ||
199 | |||
133 | irq_iommu->iommu = iommu; | 200 | irq_iommu->iommu = iommu; |
134 | irq_iommu->irte_index = index; | 201 | irq_iommu->irte_index = index; |
135 | irq_iommu->sub_handle = 0; | 202 | irq_iommu->sub_handle = 0; |
@@ -177,6 +244,12 @@ int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle) | |||
177 | 244 | ||
178 | irq_iommu = irq_2_iommu_alloc(irq); | 245 | irq_iommu = irq_2_iommu_alloc(irq); |
179 | 246 | ||
247 | if (!irq_iommu) { | ||
248 | spin_unlock(&irq_2_ir_lock); | ||
249 | printk(KERN_ERR "can't allocate irq_2_iommu\n"); | ||
250 | return -1; | ||
251 | } | ||
252 | |||
180 | irq_iommu->iommu = iommu; | 253 | irq_iommu->iommu = iommu; |
181 | irq_iommu->irte_index = index; | 254 | irq_iommu->irte_index = index; |
182 | irq_iommu->sub_handle = subhandle; | 255 | irq_iommu->sub_handle = subhandle; |
diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 74801f7df9c..11a51f8ed3b 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c | |||
@@ -103,11 +103,11 @@ static void msix_set_enable(struct pci_dev *dev, int enable) | |||
103 | } | 103 | } |
104 | } | 104 | } |
105 | 105 | ||
106 | static void msix_flush_writes(unsigned int irq) | 106 | static void msix_flush_writes(struct irq_desc *desc) |
107 | { | 107 | { |
108 | struct msi_desc *entry; | 108 | struct msi_desc *entry; |
109 | 109 | ||
110 | entry = get_irq_msi(irq); | 110 | entry = get_irq_desc_msi(desc); |
111 | BUG_ON(!entry || !entry->dev); | 111 | BUG_ON(!entry || !entry->dev); |
112 | switch (entry->msi_attrib.type) { | 112 | switch (entry->msi_attrib.type) { |
113 | case PCI_CAP_ID_MSI: | 113 | case PCI_CAP_ID_MSI: |
@@ -135,11 +135,11 @@ static void msix_flush_writes(unsigned int irq) | |||
135 | * Returns 1 if it succeeded in masking the interrupt and 0 if the device | 135 | * Returns 1 if it succeeded in masking the interrupt and 0 if the device |
136 | * doesn't support MSI masking. | 136 | * doesn't support MSI masking. |
137 | */ | 137 | */ |
138 | static int msi_set_mask_bits(unsigned int irq, u32 mask, u32 flag) | 138 | static int msi_set_mask_bits(struct irq_desc *desc, u32 mask, u32 flag) |
139 | { | 139 | { |
140 | struct msi_desc *entry; | 140 | struct msi_desc *entry; |
141 | 141 | ||
142 | entry = get_irq_msi(irq); | 142 | entry = get_irq_desc_msi(desc); |
143 | BUG_ON(!entry || !entry->dev); | 143 | BUG_ON(!entry || !entry->dev); |
144 | switch (entry->msi_attrib.type) { | 144 | switch (entry->msi_attrib.type) { |
145 | case PCI_CAP_ID_MSI: | 145 | case PCI_CAP_ID_MSI: |
@@ -172,9 +172,9 @@ static int msi_set_mask_bits(unsigned int irq, u32 mask, u32 flag) | |||
172 | return 1; | 172 | return 1; |
173 | } | 173 | } |
174 | 174 | ||
175 | void read_msi_msg(unsigned int irq, struct msi_msg *msg) | 175 | void read_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg) |
176 | { | 176 | { |
177 | struct msi_desc *entry = get_irq_msi(irq); | 177 | struct msi_desc *entry = get_irq_desc_msi(desc); |
178 | switch(entry->msi_attrib.type) { | 178 | switch(entry->msi_attrib.type) { |
179 | case PCI_CAP_ID_MSI: | 179 | case PCI_CAP_ID_MSI: |
180 | { | 180 | { |
@@ -211,9 +211,16 @@ void read_msi_msg(unsigned int irq, struct msi_msg *msg) | |||
211 | } | 211 | } |
212 | } | 212 | } |
213 | 213 | ||
214 | void write_msi_msg(unsigned int irq, struct msi_msg *msg) | 214 | void read_msi_msg(unsigned int irq, struct msi_msg *msg) |
215 | { | 215 | { |
216 | struct msi_desc *entry = get_irq_msi(irq); | 216 | struct irq_desc *desc = irq_to_desc(irq); |
217 | |||
218 | read_msi_msg_desc(desc, msg); | ||
219 | } | ||
220 | |||
221 | void write_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg) | ||
222 | { | ||
223 | struct msi_desc *entry = get_irq_desc_msi(desc); | ||
217 | switch (entry->msi_attrib.type) { | 224 | switch (entry->msi_attrib.type) { |
218 | case PCI_CAP_ID_MSI: | 225 | case PCI_CAP_ID_MSI: |
219 | { | 226 | { |
@@ -252,21 +259,31 @@ void write_msi_msg(unsigned int irq, struct msi_msg *msg) | |||
252 | entry->msg = *msg; | 259 | entry->msg = *msg; |
253 | } | 260 | } |
254 | 261 | ||
262 | void write_msi_msg(unsigned int irq, struct msi_msg *msg) | ||
263 | { | ||
264 | struct irq_desc *desc = irq_to_desc(irq); | ||
265 | |||
266 | write_msi_msg_desc(desc, msg); | ||
267 | } | ||
268 | |||
255 | void mask_msi_irq(unsigned int irq) | 269 | void mask_msi_irq(unsigned int irq) |
256 | { | 270 | { |
257 | msi_set_mask_bits(irq, 1, 1); | 271 | struct irq_desc *desc = irq_to_desc(irq); |
258 | msix_flush_writes(irq); | 272 | |
273 | msi_set_mask_bits(desc, 1, 1); | ||
274 | msix_flush_writes(desc); | ||
259 | } | 275 | } |
260 | 276 | ||
261 | void unmask_msi_irq(unsigned int irq) | 277 | void unmask_msi_irq(unsigned int irq) |
262 | { | 278 | { |
263 | msi_set_mask_bits(irq, 1, 0); | 279 | struct irq_desc *desc = irq_to_desc(irq); |
264 | msix_flush_writes(irq); | 280 | |
281 | msi_set_mask_bits(desc, 1, 0); | ||
282 | msix_flush_writes(desc); | ||
265 | } | 283 | } |
266 | 284 | ||
267 | static int msi_free_irqs(struct pci_dev* dev); | 285 | static int msi_free_irqs(struct pci_dev* dev); |
268 | 286 | ||
269 | |||
270 | static struct msi_desc* alloc_msi_entry(void) | 287 | static struct msi_desc* alloc_msi_entry(void) |
271 | { | 288 | { |
272 | struct msi_desc *entry; | 289 | struct msi_desc *entry; |
@@ -303,9 +320,11 @@ static void __pci_restore_msi_state(struct pci_dev *dev) | |||
303 | pci_intx_for_msi(dev, 0); | 320 | pci_intx_for_msi(dev, 0); |
304 | msi_set_enable(dev, 0); | 321 | msi_set_enable(dev, 0); |
305 | write_msi_msg(dev->irq, &entry->msg); | 322 | write_msi_msg(dev->irq, &entry->msg); |
306 | if (entry->msi_attrib.maskbit) | 323 | if (entry->msi_attrib.maskbit) { |
307 | msi_set_mask_bits(dev->irq, entry->msi_attrib.maskbits_mask, | 324 | struct irq_desc *desc = irq_to_desc(dev->irq); |
325 | msi_set_mask_bits(desc, entry->msi_attrib.maskbits_mask, | ||
308 | entry->msi_attrib.masked); | 326 | entry->msi_attrib.masked); |
327 | } | ||
309 | 328 | ||
310 | pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); | 329 | pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &control); |
311 | control &= ~PCI_MSI_FLAGS_QSIZE; | 330 | control &= ~PCI_MSI_FLAGS_QSIZE; |
@@ -327,8 +346,9 @@ static void __pci_restore_msix_state(struct pci_dev *dev) | |||
327 | msix_set_enable(dev, 0); | 346 | msix_set_enable(dev, 0); |
328 | 347 | ||
329 | list_for_each_entry(entry, &dev->msi_list, list) { | 348 | list_for_each_entry(entry, &dev->msi_list, list) { |
349 | struct irq_desc *desc = irq_to_desc(entry->irq); | ||
330 | write_msi_msg(entry->irq, &entry->msg); | 350 | write_msi_msg(entry->irq, &entry->msg); |
331 | msi_set_mask_bits(entry->irq, 1, entry->msi_attrib.masked); | 351 | msi_set_mask_bits(desc, 1, entry->msi_attrib.masked); |
332 | } | 352 | } |
333 | 353 | ||
334 | BUG_ON(list_empty(&dev->msi_list)); | 354 | BUG_ON(list_empty(&dev->msi_list)); |
@@ -596,7 +616,8 @@ void pci_msi_shutdown(struct pci_dev* dev) | |||
596 | /* Return the the pci reset with msi irqs unmasked */ | 616 | /* Return the the pci reset with msi irqs unmasked */ |
597 | if (entry->msi_attrib.maskbit) { | 617 | if (entry->msi_attrib.maskbit) { |
598 | u32 mask = entry->msi_attrib.maskbits_mask; | 618 | u32 mask = entry->msi_attrib.maskbits_mask; |
599 | msi_set_mask_bits(dev->irq, mask, ~mask); | 619 | struct irq_desc *desc = irq_to_desc(dev->irq); |
620 | msi_set_mask_bits(desc, mask, ~mask); | ||
600 | } | 621 | } |
601 | if (!entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) | 622 | if (!entry->dev || entry->msi_attrib.type != PCI_CAP_ID_MSI) |
602 | return; | 623 | return; |
diff --git a/drivers/xen/events.c b/drivers/xen/events.c index 1e3b934a4cf..46625cd3874 100644 --- a/drivers/xen/events.c +++ b/drivers/xen/events.c | |||
@@ -141,8 +141,12 @@ static void init_evtchn_cpu_bindings(void) | |||
141 | int i; | 141 | int i; |
142 | 142 | ||
143 | /* By default all event channels notify CPU#0. */ | 143 | /* By default all event channels notify CPU#0. */ |
144 | for_each_irq_desc(i, desc) | 144 | for_each_irq_desc(i, desc) { |
145 | if (!desc) | ||
146 | continue; | ||
147 | |||
145 | desc->affinity = cpumask_of_cpu(0); | 148 | desc->affinity = cpumask_of_cpu(0); |
149 | } | ||
146 | #endif | 150 | #endif |
147 | 151 | ||
148 | memset(cpu_evtchn, 0, sizeof(cpu_evtchn)); | 152 | memset(cpu_evtchn, 0, sizeof(cpu_evtchn)); |
@@ -229,15 +233,20 @@ static void unmask_evtchn(int port) | |||
229 | static int find_unbound_irq(void) | 233 | static int find_unbound_irq(void) |
230 | { | 234 | { |
231 | int irq; | 235 | int irq; |
236 | struct irq_desc *desc; | ||
232 | 237 | ||
233 | /* Only allocate from dynirq range */ | 238 | /* Only allocate from dynirq range */ |
234 | for_each_irq_nr(irq) | 239 | for (irq = 0; irq < nr_irqs; irq++) |
235 | if (irq_bindcount[irq] == 0) | 240 | if (irq_bindcount[irq] == 0) |
236 | break; | 241 | break; |
237 | 242 | ||
238 | if (irq == nr_irqs) | 243 | if (irq == nr_irqs) |
239 | panic("No available IRQ to bind to: increase nr_irqs!\n"); | 244 | panic("No available IRQ to bind to: increase nr_irqs!\n"); |
240 | 245 | ||
246 | desc = irq_to_desc_alloc_cpu(irq, 0); | ||
247 | if (WARN_ON(desc == NULL)) | ||
248 | return -1; | ||
249 | |||
241 | return irq; | 250 | return irq; |
242 | } | 251 | } |
243 | 252 | ||
@@ -792,7 +801,7 @@ void xen_irq_resume(void) | |||
792 | mask_evtchn(evtchn); | 801 | mask_evtchn(evtchn); |
793 | 802 | ||
794 | /* No IRQ <-> event-channel mappings. */ | 803 | /* No IRQ <-> event-channel mappings. */ |
795 | for_each_irq_nr(irq) | 804 | for (irq = 0; irq < nr_irqs; irq++) |
796 | irq_info[irq].evtchn = 0; /* zap event-channel binding */ | 805 | irq_info[irq].evtchn = 0; /* zap event-channel binding */ |
797 | 806 | ||
798 | for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++) | 807 | for (evtchn = 0; evtchn < NR_EVENT_CHANNELS; evtchn++) |
@@ -824,7 +833,7 @@ void __init xen_init_IRQ(void) | |||
824 | mask_evtchn(i); | 833 | mask_evtchn(i); |
825 | 834 | ||
826 | /* Dynamic IRQ space is currently unbound. Zero the refcnts. */ | 835 | /* Dynamic IRQ space is currently unbound. Zero the refcnts. */ |
827 | for_each_irq_nr(i) | 836 | for (i = 0; i < nr_irqs; i++) |
828 | irq_bindcount[i] = 0; | 837 | irq_bindcount[i] = 0; |
829 | 838 | ||
830 | irq_ctx_init(smp_processor_id()); | 839 | irq_ctx_init(smp_processor_id()); |
diff --git a/fs/proc/stat.c b/fs/proc/stat.c index 81904f07679..3bb1cf1e742 100644 --- a/fs/proc/stat.c +++ b/fs/proc/stat.c | |||
@@ -44,10 +44,13 @@ static int show_stat(struct seq_file *p, void *v) | |||
44 | softirq = cputime64_add(softirq, kstat_cpu(i).cpustat.softirq); | 44 | softirq = cputime64_add(softirq, kstat_cpu(i).cpustat.softirq); |
45 | steal = cputime64_add(steal, kstat_cpu(i).cpustat.steal); | 45 | steal = cputime64_add(steal, kstat_cpu(i).cpustat.steal); |
46 | guest = cputime64_add(guest, kstat_cpu(i).cpustat.guest); | 46 | guest = cputime64_add(guest, kstat_cpu(i).cpustat.guest); |
47 | 47 | for_each_irq_nr(j) { | |
48 | for_each_irq_nr(j) | 48 | #ifdef CONFIG_SPARSE_IRQ |
49 | if (!irq_to_desc(j)) | ||
50 | continue; | ||
51 | #endif | ||
49 | sum += kstat_irqs_cpu(j, i); | 52 | sum += kstat_irqs_cpu(j, i); |
50 | 53 | } | |
51 | sum += arch_irq_stat_cpu(i); | 54 | sum += arch_irq_stat_cpu(i); |
52 | } | 55 | } |
53 | sum += arch_irq_stat(); | 56 | sum += arch_irq_stat(); |
@@ -92,7 +95,12 @@ static int show_stat(struct seq_file *p, void *v) | |||
92 | /* sum again ? it could be updated? */ | 95 | /* sum again ? it could be updated? */ |
93 | for_each_irq_nr(j) { | 96 | for_each_irq_nr(j) { |
94 | per_irq_sum = 0; | 97 | per_irq_sum = 0; |
95 | 98 | #ifdef CONFIG_SPARSE_IRQ | |
99 | if (!irq_to_desc(j)) { | ||
100 | seq_printf(p, " %u", per_irq_sum); | ||
101 | continue; | ||
102 | } | ||
103 | #endif | ||
96 | for_each_possible_cpu(i) | 104 | for_each_possible_cpu(i) |
97 | per_irq_sum += kstat_irqs_cpu(j, i); | 105 | per_irq_sum += kstat_irqs_cpu(j, i); |
98 | 106 | ||
diff --git a/include/linux/Kbuild b/include/linux/Kbuild index e531783e5d7..95ac82340c3 100644 --- a/include/linux/Kbuild +++ b/include/linux/Kbuild | |||
@@ -313,6 +313,7 @@ unifdef-y += ptrace.h | |||
313 | unifdef-y += qnx4_fs.h | 313 | unifdef-y += qnx4_fs.h |
314 | unifdef-y += quota.h | 314 | unifdef-y += quota.h |
315 | unifdef-y += random.h | 315 | unifdef-y += random.h |
316 | unifdef-y += irqnr.h | ||
316 | unifdef-y += reboot.h | 317 | unifdef-y += reboot.h |
317 | unifdef-y += reiserfs_fs.h | 318 | unifdef-y += reiserfs_fs.h |
318 | unifdef-y += reiserfs_xattr.h | 319 | unifdef-y += reiserfs_xattr.h |
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index d6210a97a8c..be3c484b524 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h | |||
@@ -14,6 +14,8 @@ | |||
14 | #include <linux/irqflags.h> | 14 | #include <linux/irqflags.h> |
15 | #include <linux/smp.h> | 15 | #include <linux/smp.h> |
16 | #include <linux/percpu.h> | 16 | #include <linux/percpu.h> |
17 | #include <linux/irqnr.h> | ||
18 | |||
17 | #include <asm/atomic.h> | 19 | #include <asm/atomic.h> |
18 | #include <asm/ptrace.h> | 20 | #include <asm/ptrace.h> |
19 | #include <asm/system.h> | 21 | #include <asm/system.h> |
diff --git a/include/linux/irq.h b/include/linux/irq.h index 3dddfa703eb..98564dc6447 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h | |||
@@ -129,9 +129,14 @@ struct irq_chip { | |||
129 | const char *typename; | 129 | const char *typename; |
130 | }; | 130 | }; |
131 | 131 | ||
132 | struct timer_rand_state; | ||
133 | struct irq_2_iommu; | ||
132 | /** | 134 | /** |
133 | * struct irq_desc - interrupt descriptor | 135 | * struct irq_desc - interrupt descriptor |
134 | * @irq: interrupt number for this descriptor | 136 | * @irq: interrupt number for this descriptor |
137 | * @timer_rand_state: pointer to timer rand state struct | ||
138 | * @kstat_irqs: irq stats per cpu | ||
139 | * @irq_2_iommu: iommu with this irq | ||
135 | * @handle_irq: highlevel irq-events handler [if NULL, __do_IRQ()] | 140 | * @handle_irq: highlevel irq-events handler [if NULL, __do_IRQ()] |
136 | * @chip: low level interrupt hardware access | 141 | * @chip: low level interrupt hardware access |
137 | * @msi_desc: MSI descriptor | 142 | * @msi_desc: MSI descriptor |
@@ -143,8 +148,8 @@ struct irq_chip { | |||
143 | * @depth: disable-depth, for nested irq_disable() calls | 148 | * @depth: disable-depth, for nested irq_disable() calls |
144 | * @wake_depth: enable depth, for multiple set_irq_wake() callers | 149 | * @wake_depth: enable depth, for multiple set_irq_wake() callers |
145 | * @irq_count: stats field to detect stalled irqs | 150 | * @irq_count: stats field to detect stalled irqs |
146 | * @irqs_unhandled: stats field for spurious unhandled interrupts | ||
147 | * @last_unhandled: aging timer for unhandled count | 151 | * @last_unhandled: aging timer for unhandled count |
152 | * @irqs_unhandled: stats field for spurious unhandled interrupts | ||
148 | * @lock: locking for SMP | 153 | * @lock: locking for SMP |
149 | * @affinity: IRQ affinity on SMP | 154 | * @affinity: IRQ affinity on SMP |
150 | * @cpu: cpu index useful for balancing | 155 | * @cpu: cpu index useful for balancing |
@@ -154,6 +159,13 @@ struct irq_chip { | |||
154 | */ | 159 | */ |
155 | struct irq_desc { | 160 | struct irq_desc { |
156 | unsigned int irq; | 161 | unsigned int irq; |
162 | #ifdef CONFIG_SPARSE_IRQ | ||
163 | struct timer_rand_state *timer_rand_state; | ||
164 | unsigned int *kstat_irqs; | ||
165 | # ifdef CONFIG_INTR_REMAP | ||
166 | struct irq_2_iommu *irq_2_iommu; | ||
167 | # endif | ||
168 | #endif | ||
157 | irq_flow_handler_t handle_irq; | 169 | irq_flow_handler_t handle_irq; |
158 | struct irq_chip *chip; | 170 | struct irq_chip *chip; |
159 | struct msi_desc *msi_desc; | 171 | struct msi_desc *msi_desc; |
@@ -165,8 +177,8 @@ struct irq_desc { | |||
165 | unsigned int depth; /* nested irq disables */ | 177 | unsigned int depth; /* nested irq disables */ |
166 | unsigned int wake_depth; /* nested wake enables */ | 178 | unsigned int wake_depth; /* nested wake enables */ |
167 | unsigned int irq_count; /* For detecting broken IRQs */ | 179 | unsigned int irq_count; /* For detecting broken IRQs */ |
168 | unsigned int irqs_unhandled; | ||
169 | unsigned long last_unhandled; /* Aging timer for unhandled count */ | 180 | unsigned long last_unhandled; /* Aging timer for unhandled count */ |
181 | unsigned int irqs_unhandled; | ||
170 | spinlock_t lock; | 182 | spinlock_t lock; |
171 | #ifdef CONFIG_SMP | 183 | #ifdef CONFIG_SMP |
172 | cpumask_t affinity; | 184 | cpumask_t affinity; |
@@ -181,12 +193,51 @@ struct irq_desc { | |||
181 | const char *name; | 193 | const char *name; |
182 | } ____cacheline_internodealigned_in_smp; | 194 | } ____cacheline_internodealigned_in_smp; |
183 | 195 | ||
196 | extern void early_irq_init(void); | ||
197 | extern void arch_early_irq_init(void); | ||
198 | extern void arch_init_chip_data(struct irq_desc *desc, int cpu); | ||
199 | extern void arch_init_copy_chip_data(struct irq_desc *old_desc, | ||
200 | struct irq_desc *desc, int cpu); | ||
201 | extern void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc); | ||
184 | 202 | ||
203 | #ifndef CONFIG_SPARSE_IRQ | ||
185 | extern struct irq_desc irq_desc[NR_IRQS]; | 204 | extern struct irq_desc irq_desc[NR_IRQS]; |
186 | 205 | ||
187 | static inline struct irq_desc *irq_to_desc(unsigned int irq) | 206 | static inline struct irq_desc *irq_to_desc(unsigned int irq) |
188 | { | 207 | { |
189 | return (irq < nr_irqs) ? irq_desc + irq : NULL; | 208 | return (irq < NR_IRQS) ? irq_desc + irq : NULL; |
209 | } | ||
210 | static inline struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) | ||
211 | { | ||
212 | return irq_to_desc(irq); | ||
213 | } | ||
214 | |||
215 | #else | ||
216 | |||
217 | extern struct irq_desc *irq_to_desc(unsigned int irq); | ||
218 | extern struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu); | ||
219 | extern struct irq_desc *move_irq_desc(struct irq_desc *old_desc, int cpu); | ||
220 | |||
221 | # define for_each_irq_desc(irq, desc) \ | ||
222 | for (irq = 0, desc = irq_to_desc(irq); irq < nr_irqs; irq++, desc = irq_to_desc(irq)) | ||
223 | # define for_each_irq_desc_reverse(irq, desc) \ | ||
224 | for (irq = nr_irqs - 1, desc = irq_to_desc(irq); irq >= 0; irq--, desc = irq_to_desc(irq)) | ||
225 | |||
226 | #define kstat_irqs_this_cpu(DESC) \ | ||
227 | ((DESC)->kstat_irqs[smp_processor_id()]) | ||
228 | #define kstat_incr_irqs_this_cpu(irqno, DESC) \ | ||
229 | ((DESC)->kstat_irqs[smp_processor_id()]++) | ||
230 | |||
231 | #endif | ||
232 | |||
233 | static inline struct irq_desc * | ||
234 | irq_remap_to_desc(unsigned int irq, struct irq_desc *desc) | ||
235 | { | ||
236 | #ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC | ||
237 | return irq_to_desc(irq); | ||
238 | #else | ||
239 | return desc; | ||
240 | #endif | ||
190 | } | 241 | } |
191 | 242 | ||
192 | /* | 243 | /* |
@@ -380,6 +431,11 @@ extern int set_irq_msi(unsigned int irq, struct msi_desc *entry); | |||
380 | #define get_irq_data(irq) (irq_to_desc(irq)->handler_data) | 431 | #define get_irq_data(irq) (irq_to_desc(irq)->handler_data) |
381 | #define get_irq_msi(irq) (irq_to_desc(irq)->msi_desc) | 432 | #define get_irq_msi(irq) (irq_to_desc(irq)->msi_desc) |
382 | 433 | ||
434 | #define get_irq_desc_chip(desc) ((desc)->chip) | ||
435 | #define get_irq_desc_chip_data(desc) ((desc)->chip_data) | ||
436 | #define get_irq_desc_data(desc) ((desc)->handler_data) | ||
437 | #define get_irq_desc_msi(desc) ((desc)->msi_desc) | ||
438 | |||
383 | #endif /* CONFIG_GENERIC_HARDIRQS */ | 439 | #endif /* CONFIG_GENERIC_HARDIRQS */ |
384 | 440 | ||
385 | #endif /* !CONFIG_S390 */ | 441 | #endif /* !CONFIG_S390 */ |
diff --git a/include/linux/irqnr.h b/include/linux/irqnr.h index 452c280c811..95d2b74641f 100644 --- a/include/linux/irqnr.h +++ b/include/linux/irqnr.h | |||
@@ -1,24 +1,38 @@ | |||
1 | #ifndef _LINUX_IRQNR_H | 1 | #ifndef _LINUX_IRQNR_H |
2 | #define _LINUX_IRQNR_H | 2 | #define _LINUX_IRQNR_H |
3 | 3 | ||
4 | /* | ||
5 | * Generic irq_desc iterators: | ||
6 | */ | ||
7 | #ifdef __KERNEL__ | ||
8 | |||
4 | #ifndef CONFIG_GENERIC_HARDIRQS | 9 | #ifndef CONFIG_GENERIC_HARDIRQS |
5 | #include <asm/irq.h> | 10 | #include <asm/irq.h> |
6 | # define nr_irqs NR_IRQS | 11 | # define nr_irqs NR_IRQS |
7 | 12 | ||
8 | # define for_each_irq_desc(irq, desc) \ | 13 | # define for_each_irq_desc(irq, desc) \ |
9 | for (irq = 0; irq < nr_irqs; irq++) | 14 | for (irq = 0; irq < nr_irqs; irq++) |
15 | |||
16 | # define for_each_irq_desc_reverse(irq, desc) \ | ||
17 | for (irq = nr_irqs - 1; irq >= 0; irq--) | ||
10 | #else | 18 | #else |
19 | |||
11 | extern int nr_irqs; | 20 | extern int nr_irqs; |
12 | 21 | ||
22 | #ifndef CONFIG_SPARSE_IRQ | ||
23 | |||
24 | struct irq_desc; | ||
13 | # define for_each_irq_desc(irq, desc) \ | 25 | # define for_each_irq_desc(irq, desc) \ |
14 | for (irq = 0, desc = irq_desc; irq < nr_irqs; irq++, desc++) | 26 | for (irq = 0, desc = irq_desc; irq < nr_irqs; irq++, desc++) |
15 | 27 | # define for_each_irq_desc_reverse(irq, desc) \ | |
16 | # define for_each_irq_desc_reverse(irq, desc) \ | 28 | for (irq = nr_irqs - 1, desc = irq_desc + (nr_irqs - 1); \ |
17 | for (irq = nr_irqs - 1, desc = irq_desc + (nr_irqs - 1); \ | 29 | irq >= 0; irq--, desc--) |
18 | irq >= 0; irq--, desc--) | 30 | #endif |
19 | #endif | 31 | #endif |
20 | 32 | ||
21 | #define for_each_irq_nr(irq) \ | 33 | #define for_each_irq_nr(irq) \ |
22 | for (irq = 0; irq < nr_irqs; irq++) | 34 | for (irq = 0; irq < nr_irqs; irq++) |
35 | |||
36 | #endif /* __KERNEL__ */ | ||
23 | 37 | ||
24 | #endif | 38 | #endif |
diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 4a145caeee0..4ee4b3d2316 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h | |||
@@ -28,7 +28,9 @@ struct cpu_usage_stat { | |||
28 | 28 | ||
29 | struct kernel_stat { | 29 | struct kernel_stat { |
30 | struct cpu_usage_stat cpustat; | 30 | struct cpu_usage_stat cpustat; |
31 | unsigned int irqs[NR_IRQS]; | 31 | #ifndef CONFIG_SPARSE_IRQ |
32 | unsigned int irqs[NR_IRQS]; | ||
33 | #endif | ||
32 | }; | 34 | }; |
33 | 35 | ||
34 | DECLARE_PER_CPU(struct kernel_stat, kstat); | 36 | DECLARE_PER_CPU(struct kernel_stat, kstat); |
@@ -39,6 +41,10 @@ DECLARE_PER_CPU(struct kernel_stat, kstat); | |||
39 | 41 | ||
40 | extern unsigned long long nr_context_switches(void); | 42 | extern unsigned long long nr_context_switches(void); |
41 | 43 | ||
44 | #ifndef CONFIG_SPARSE_IRQ | ||
45 | #define kstat_irqs_this_cpu(irq) \ | ||
46 | (kstat_this_cpu.irqs[irq]) | ||
47 | |||
42 | struct irq_desc; | 48 | struct irq_desc; |
43 | 49 | ||
44 | static inline void kstat_incr_irqs_this_cpu(unsigned int irq, | 50 | static inline void kstat_incr_irqs_this_cpu(unsigned int irq, |
@@ -46,11 +52,17 @@ static inline void kstat_incr_irqs_this_cpu(unsigned int irq, | |||
46 | { | 52 | { |
47 | kstat_this_cpu.irqs[irq]++; | 53 | kstat_this_cpu.irqs[irq]++; |
48 | } | 54 | } |
55 | #endif | ||
56 | |||
49 | 57 | ||
58 | #ifndef CONFIG_SPARSE_IRQ | ||
50 | static inline unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) | 59 | static inline unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) |
51 | { | 60 | { |
52 | return kstat_cpu(cpu).irqs[irq]; | 61 | return kstat_cpu(cpu).irqs[irq]; |
53 | } | 62 | } |
63 | #else | ||
64 | extern unsigned int kstat_irqs_cpu(unsigned int irq, int cpu); | ||
65 | #endif | ||
54 | 66 | ||
55 | /* | 67 | /* |
56 | * Number of interrupts per specific IRQ source, since bootup | 68 | * Number of interrupts per specific IRQ source, since bootup |
diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 37a0361f468..23bf02fb124 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h | |||
@@ -390,7 +390,7 @@ do { \ | |||
390 | 390 | ||
391 | #endif /* CONFIG_LOCK_STAT */ | 391 | #endif /* CONFIG_LOCK_STAT */ |
392 | 392 | ||
393 | #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_GENERIC_HARDIRQS) | 393 | #ifdef CONFIG_GENERIC_HARDIRQS |
394 | extern void early_init_irq_lock_class(void); | 394 | extern void early_init_irq_lock_class(void); |
395 | #else | 395 | #else |
396 | static inline void early_init_irq_lock_class(void) | 396 | static inline void early_init_irq_lock_class(void) |
diff --git a/include/linux/msi.h b/include/linux/msi.h index 8f293922720..d2b8a1e8ca1 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h | |||
@@ -10,8 +10,11 @@ struct msi_msg { | |||
10 | }; | 10 | }; |
11 | 11 | ||
12 | /* Helper functions */ | 12 | /* Helper functions */ |
13 | struct irq_desc; | ||
13 | extern void mask_msi_irq(unsigned int irq); | 14 | extern void mask_msi_irq(unsigned int irq); |
14 | extern void unmask_msi_irq(unsigned int irq); | 15 | extern void unmask_msi_irq(unsigned int irq); |
16 | extern void read_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg); | ||
17 | extern void write_msi_msg_desc(struct irq_desc *desc, struct msi_msg *msg); | ||
15 | extern void read_msi_msg(unsigned int irq, struct msi_msg *msg); | 18 | extern void read_msi_msg(unsigned int irq, struct msi_msg *msg); |
16 | extern void write_msi_msg(unsigned int irq, struct msi_msg *msg); | 19 | extern void write_msi_msg(unsigned int irq, struct msi_msg *msg); |
17 | 20 | ||
diff --git a/include/linux/random.h b/include/linux/random.h index 36f125c0c60..adbf3bd3c6b 100644 --- a/include/linux/random.h +++ b/include/linux/random.h | |||
@@ -8,6 +8,7 @@ | |||
8 | #define _LINUX_RANDOM_H | 8 | #define _LINUX_RANDOM_H |
9 | 9 | ||
10 | #include <linux/ioctl.h> | 10 | #include <linux/ioctl.h> |
11 | #include <linux/irqnr.h> | ||
11 | 12 | ||
12 | /* ioctl()'s for the random number generator */ | 13 | /* ioctl()'s for the random number generator */ |
13 | 14 | ||
@@ -44,6 +45,56 @@ struct rand_pool_info { | |||
44 | 45 | ||
45 | extern void rand_initialize_irq(int irq); | 46 | extern void rand_initialize_irq(int irq); |
46 | 47 | ||
48 | struct timer_rand_state; | ||
49 | #ifndef CONFIG_SPARSE_IRQ | ||
50 | |||
51 | extern struct timer_rand_state *irq_timer_state[]; | ||
52 | |||
53 | static inline struct timer_rand_state *get_timer_rand_state(unsigned int irq) | ||
54 | { | ||
55 | if (irq >= nr_irqs) | ||
56 | return NULL; | ||
57 | |||
58 | return irq_timer_state[irq]; | ||
59 | } | ||
60 | |||
61 | static inline void set_timer_rand_state(unsigned int irq, struct timer_rand_state *state) | ||
62 | { | ||
63 | if (irq >= nr_irqs) | ||
64 | return; | ||
65 | |||
66 | irq_timer_state[irq] = state; | ||
67 | } | ||
68 | |||
69 | #else | ||
70 | |||
71 | #include <linux/irq.h> | ||
72 | static inline struct timer_rand_state *get_timer_rand_state(unsigned int irq) | ||
73 | { | ||
74 | struct irq_desc *desc; | ||
75 | |||
76 | desc = irq_to_desc(irq); | ||
77 | |||
78 | if (!desc) | ||
79 | return NULL; | ||
80 | |||
81 | return desc->timer_rand_state; | ||
82 | } | ||
83 | |||
84 | static inline void set_timer_rand_state(unsigned int irq, struct timer_rand_state *state) | ||
85 | { | ||
86 | struct irq_desc *desc; | ||
87 | |||
88 | desc = irq_to_desc(irq); | ||
89 | |||
90 | if (!desc) | ||
91 | return; | ||
92 | |||
93 | desc->timer_rand_state = state; | ||
94 | } | ||
95 | #endif | ||
96 | |||
97 | |||
47 | extern void add_input_randomness(unsigned int type, unsigned int code, | 98 | extern void add_input_randomness(unsigned int type, unsigned int code, |
48 | unsigned int value); | 99 | unsigned int value); |
49 | extern void add_interrupt_randomness(int irq); | 100 | extern void add_interrupt_randomness(int irq); |
diff --git a/init/main.c b/init/main.c index 17e9757bfde..2a7ce0f8e45 100644 --- a/init/main.c +++ b/init/main.c | |||
@@ -540,6 +540,15 @@ void __init __weak thread_info_cache_init(void) | |||
540 | { | 540 | { |
541 | } | 541 | } |
542 | 542 | ||
543 | void __init __weak arch_early_irq_init(void) | ||
544 | { | ||
545 | } | ||
546 | |||
547 | void __init __weak early_irq_init(void) | ||
548 | { | ||
549 | arch_early_irq_init(); | ||
550 | } | ||
551 | |||
543 | asmlinkage void __init start_kernel(void) | 552 | asmlinkage void __init start_kernel(void) |
544 | { | 553 | { |
545 | char * command_line; | 554 | char * command_line; |
@@ -604,6 +613,8 @@ asmlinkage void __init start_kernel(void) | |||
604 | sort_main_extable(); | 613 | sort_main_extable(); |
605 | trap_init(); | 614 | trap_init(); |
606 | rcu_init(); | 615 | rcu_init(); |
616 | /* init some links before init_ISA_irqs() */ | ||
617 | early_irq_init(); | ||
607 | init_IRQ(); | 618 | init_IRQ(); |
608 | pidhash_init(); | 619 | pidhash_init(); |
609 | init_timers(); | 620 | init_timers(); |
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile index 681c52dbfe2..4dd5b1edac9 100644 --- a/kernel/irq/Makefile +++ b/kernel/irq/Makefile | |||
@@ -3,3 +3,4 @@ obj-y := handle.o manage.o spurious.o resend.o chip.o devres.o | |||
3 | obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o | 3 | obj-$(CONFIG_GENERIC_IRQ_PROBE) += autoprobe.o |
4 | obj-$(CONFIG_PROC_FS) += proc.o | 4 | obj-$(CONFIG_PROC_FS) += proc.o |
5 | obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o | 5 | obj-$(CONFIG_GENERIC_PENDING_IRQ) += migration.o |
6 | obj-$(CONFIG_NUMA_MIGRATE_IRQ_DESC) += numa_migrate.o | ||
diff --git a/kernel/irq/autoprobe.c b/kernel/irq/autoprobe.c index cc0f7321b8c..650ce4102a6 100644 --- a/kernel/irq/autoprobe.c +++ b/kernel/irq/autoprobe.c | |||
@@ -40,6 +40,9 @@ unsigned long probe_irq_on(void) | |||
40 | * flush such a longstanding irq before considering it as spurious. | 40 | * flush such a longstanding irq before considering it as spurious. |
41 | */ | 41 | */ |
42 | for_each_irq_desc_reverse(i, desc) { | 42 | for_each_irq_desc_reverse(i, desc) { |
43 | if (!desc) | ||
44 | continue; | ||
45 | |||
43 | spin_lock_irq(&desc->lock); | 46 | spin_lock_irq(&desc->lock); |
44 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { | 47 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { |
45 | /* | 48 | /* |
@@ -68,6 +71,9 @@ unsigned long probe_irq_on(void) | |||
68 | * happened in the previous stage, it may have masked itself) | 71 | * happened in the previous stage, it may have masked itself) |
69 | */ | 72 | */ |
70 | for_each_irq_desc_reverse(i, desc) { | 73 | for_each_irq_desc_reverse(i, desc) { |
74 | if (!desc) | ||
75 | continue; | ||
76 | |||
71 | spin_lock_irq(&desc->lock); | 77 | spin_lock_irq(&desc->lock); |
72 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { | 78 | if (!desc->action && !(desc->status & IRQ_NOPROBE)) { |
73 | desc->status |= IRQ_AUTODETECT | IRQ_WAITING; | 79 | desc->status |= IRQ_AUTODETECT | IRQ_WAITING; |
@@ -86,6 +92,9 @@ unsigned long probe_irq_on(void) | |||
86 | * Now filter out any obviously spurious interrupts | 92 | * Now filter out any obviously spurious interrupts |
87 | */ | 93 | */ |
88 | for_each_irq_desc(i, desc) { | 94 | for_each_irq_desc(i, desc) { |
95 | if (!desc) | ||
96 | continue; | ||
97 | |||
89 | spin_lock_irq(&desc->lock); | 98 | spin_lock_irq(&desc->lock); |
90 | status = desc->status; | 99 | status = desc->status; |
91 | 100 | ||
@@ -124,6 +133,9 @@ unsigned int probe_irq_mask(unsigned long val) | |||
124 | int i; | 133 | int i; |
125 | 134 | ||
126 | for_each_irq_desc(i, desc) { | 135 | for_each_irq_desc(i, desc) { |
136 | if (!desc) | ||
137 | continue; | ||
138 | |||
127 | spin_lock_irq(&desc->lock); | 139 | spin_lock_irq(&desc->lock); |
128 | status = desc->status; | 140 | status = desc->status; |
129 | 141 | ||
@@ -166,6 +178,9 @@ int probe_irq_off(unsigned long val) | |||
166 | unsigned int status; | 178 | unsigned int status; |
167 | 179 | ||
168 | for_each_irq_desc(i, desc) { | 180 | for_each_irq_desc(i, desc) { |
181 | if (!desc) | ||
182 | continue; | ||
183 | |||
169 | spin_lock_irq(&desc->lock); | 184 | spin_lock_irq(&desc->lock); |
170 | status = desc->status; | 185 | status = desc->status; |
171 | 186 | ||
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 10b5092e9bf..6eb3c7952b6 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c | |||
@@ -24,9 +24,10 @@ | |||
24 | */ | 24 | */ |
25 | void dynamic_irq_init(unsigned int irq) | 25 | void dynamic_irq_init(unsigned int irq) |
26 | { | 26 | { |
27 | struct irq_desc *desc = irq_to_desc(irq); | 27 | struct irq_desc *desc; |
28 | unsigned long flags; | 28 | unsigned long flags; |
29 | 29 | ||
30 | desc = irq_to_desc(irq); | ||
30 | if (!desc) { | 31 | if (!desc) { |
31 | WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq); | 32 | WARN(1, KERN_ERR "Trying to initialize invalid IRQ%d\n", irq); |
32 | return; | 33 | return; |
@@ -124,6 +125,7 @@ int set_irq_type(unsigned int irq, unsigned int type) | |||
124 | return -ENODEV; | 125 | return -ENODEV; |
125 | } | 126 | } |
126 | 127 | ||
128 | type &= IRQ_TYPE_SENSE_MASK; | ||
127 | if (type == IRQ_TYPE_NONE) | 129 | if (type == IRQ_TYPE_NONE) |
128 | return 0; | 130 | return 0; |
129 | 131 | ||
@@ -352,6 +354,7 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc) | |||
352 | 354 | ||
353 | spin_lock(&desc->lock); | 355 | spin_lock(&desc->lock); |
354 | mask_ack_irq(desc, irq); | 356 | mask_ack_irq(desc, irq); |
357 | desc = irq_remap_to_desc(irq, desc); | ||
355 | 358 | ||
356 | if (unlikely(desc->status & IRQ_INPROGRESS)) | 359 | if (unlikely(desc->status & IRQ_INPROGRESS)) |
357 | goto out_unlock; | 360 | goto out_unlock; |
@@ -429,6 +432,7 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) | |||
429 | desc->status &= ~IRQ_INPROGRESS; | 432 | desc->status &= ~IRQ_INPROGRESS; |
430 | out: | 433 | out: |
431 | desc->chip->eoi(irq); | 434 | desc->chip->eoi(irq); |
435 | desc = irq_remap_to_desc(irq, desc); | ||
432 | 436 | ||
433 | spin_unlock(&desc->lock); | 437 | spin_unlock(&desc->lock); |
434 | } | 438 | } |
@@ -465,12 +469,14 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc) | |||
465 | !desc->action)) { | 469 | !desc->action)) { |
466 | desc->status |= (IRQ_PENDING | IRQ_MASKED); | 470 | desc->status |= (IRQ_PENDING | IRQ_MASKED); |
467 | mask_ack_irq(desc, irq); | 471 | mask_ack_irq(desc, irq); |
472 | desc = irq_remap_to_desc(irq, desc); | ||
468 | goto out_unlock; | 473 | goto out_unlock; |
469 | } | 474 | } |
470 | kstat_incr_irqs_this_cpu(irq, desc); | 475 | kstat_incr_irqs_this_cpu(irq, desc); |
471 | 476 | ||
472 | /* Start handling the irq */ | 477 | /* Start handling the irq */ |
473 | desc->chip->ack(irq); | 478 | desc->chip->ack(irq); |
479 | desc = irq_remap_to_desc(irq, desc); | ||
474 | 480 | ||
475 | /* Mark the IRQ currently in progress.*/ | 481 | /* Mark the IRQ currently in progress.*/ |
476 | desc->status |= IRQ_INPROGRESS; | 482 | desc->status |= IRQ_INPROGRESS; |
@@ -531,8 +537,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) | |||
531 | if (!noirqdebug) | 537 | if (!noirqdebug) |
532 | note_interrupt(irq, desc, action_ret); | 538 | note_interrupt(irq, desc, action_ret); |
533 | 539 | ||
534 | if (desc->chip->eoi) | 540 | if (desc->chip->eoi) { |
535 | desc->chip->eoi(irq); | 541 | desc->chip->eoi(irq); |
542 | desc = irq_remap_to_desc(irq, desc); | ||
543 | } | ||
536 | } | 544 | } |
537 | 545 | ||
538 | void | 546 | void |
@@ -567,8 +575,10 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, | |||
567 | 575 | ||
568 | /* Uninstall? */ | 576 | /* Uninstall? */ |
569 | if (handle == handle_bad_irq) { | 577 | if (handle == handle_bad_irq) { |
570 | if (desc->chip != &no_irq_chip) | 578 | if (desc->chip != &no_irq_chip) { |
571 | mask_ack_irq(desc, irq); | 579 | mask_ack_irq(desc, irq); |
580 | desc = irq_remap_to_desc(irq, desc); | ||
581 | } | ||
572 | desc->status |= IRQ_DISABLED; | 582 | desc->status |= IRQ_DISABLED; |
573 | desc->depth = 1; | 583 | desc->depth = 1; |
574 | } | 584 | } |
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index c815b42d0f5..6492400cb50 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c | |||
@@ -15,9 +15,16 @@ | |||
15 | #include <linux/random.h> | 15 | #include <linux/random.h> |
16 | #include <linux/interrupt.h> | 16 | #include <linux/interrupt.h> |
17 | #include <linux/kernel_stat.h> | 17 | #include <linux/kernel_stat.h> |
18 | #include <linux/rculist.h> | ||
19 | #include <linux/hash.h> | ||
18 | 20 | ||
19 | #include "internals.h" | 21 | #include "internals.h" |
20 | 22 | ||
23 | /* | ||
24 | * lockdep: we want to handle all irq_desc locks as a single lock-class: | ||
25 | */ | ||
26 | struct lock_class_key irq_desc_lock_class; | ||
27 | |||
21 | /** | 28 | /** |
22 | * handle_bad_irq - handle spurious and unhandled irqs | 29 | * handle_bad_irq - handle spurious and unhandled irqs |
23 | * @irq: the interrupt number | 30 | * @irq: the interrupt number |
@@ -49,6 +56,155 @@ void handle_bad_irq(unsigned int irq, struct irq_desc *desc) | |||
49 | int nr_irqs = NR_IRQS; | 56 | int nr_irqs = NR_IRQS; |
50 | EXPORT_SYMBOL_GPL(nr_irqs); | 57 | EXPORT_SYMBOL_GPL(nr_irqs); |
51 | 58 | ||
59 | void __init __attribute__((weak)) arch_early_irq_init(void) | ||
60 | { | ||
61 | } | ||
62 | |||
63 | #ifdef CONFIG_SPARSE_IRQ | ||
64 | static struct irq_desc irq_desc_init = { | ||
65 | .irq = -1, | ||
66 | .status = IRQ_DISABLED, | ||
67 | .chip = &no_irq_chip, | ||
68 | .handle_irq = handle_bad_irq, | ||
69 | .depth = 1, | ||
70 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), | ||
71 | #ifdef CONFIG_SMP | ||
72 | .affinity = CPU_MASK_ALL | ||
73 | #endif | ||
74 | }; | ||
75 | |||
76 | void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr) | ||
77 | { | ||
78 | unsigned long bytes; | ||
79 | char *ptr; | ||
80 | int node; | ||
81 | |||
82 | /* Compute how many bytes we need per irq and allocate them */ | ||
83 | bytes = nr * sizeof(unsigned int); | ||
84 | |||
85 | node = cpu_to_node(cpu); | ||
86 | ptr = kzalloc_node(bytes, GFP_ATOMIC, node); | ||
87 | printk(KERN_DEBUG " alloc kstat_irqs on cpu %d node %d\n", cpu, node); | ||
88 | |||
89 | if (ptr) | ||
90 | desc->kstat_irqs = (unsigned int *)ptr; | ||
91 | } | ||
92 | |||
93 | void __attribute__((weak)) arch_init_chip_data(struct irq_desc *desc, int cpu) | ||
94 | { | ||
95 | } | ||
96 | |||
97 | static void init_one_irq_desc(int irq, struct irq_desc *desc, int cpu) | ||
98 | { | ||
99 | memcpy(desc, &irq_desc_init, sizeof(struct irq_desc)); | ||
100 | desc->irq = irq; | ||
101 | #ifdef CONFIG_SMP | ||
102 | desc->cpu = cpu; | ||
103 | #endif | ||
104 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); | ||
105 | init_kstat_irqs(desc, cpu, nr_cpu_ids); | ||
106 | if (!desc->kstat_irqs) { | ||
107 | printk(KERN_ERR "can not alloc kstat_irqs\n"); | ||
108 | BUG_ON(1); | ||
109 | } | ||
110 | arch_init_chip_data(desc, cpu); | ||
111 | } | ||
112 | |||
113 | /* | ||
114 | * Protect the sparse_irqs: | ||
115 | */ | ||
116 | DEFINE_SPINLOCK(sparse_irq_lock); | ||
117 | |||
118 | struct irq_desc *irq_desc_ptrs[NR_IRQS] __read_mostly; | ||
119 | |||
120 | static struct irq_desc irq_desc_legacy[NR_IRQS_LEGACY] __cacheline_aligned_in_smp = { | ||
121 | [0 ... NR_IRQS_LEGACY-1] = { | ||
122 | .irq = -1, | ||
123 | .status = IRQ_DISABLED, | ||
124 | .chip = &no_irq_chip, | ||
125 | .handle_irq = handle_bad_irq, | ||
126 | .depth = 1, | ||
127 | .lock = __SPIN_LOCK_UNLOCKED(irq_desc_init.lock), | ||
128 | #ifdef CONFIG_SMP | ||
129 | .affinity = CPU_MASK_ALL | ||
130 | #endif | ||
131 | } | ||
132 | }; | ||
133 | |||
134 | /* FIXME: use bootmem alloc ...*/ | ||
135 | static unsigned int kstat_irqs_legacy[NR_IRQS_LEGACY][NR_CPUS]; | ||
136 | |||
137 | void __init early_irq_init(void) | ||
138 | { | ||
139 | struct irq_desc *desc; | ||
140 | int legacy_count; | ||
141 | int i; | ||
142 | |||
143 | desc = irq_desc_legacy; | ||
144 | legacy_count = ARRAY_SIZE(irq_desc_legacy); | ||
145 | |||
146 | for (i = 0; i < legacy_count; i++) { | ||
147 | desc[i].irq = i; | ||
148 | desc[i].kstat_irqs = kstat_irqs_legacy[i]; | ||
149 | |||
150 | irq_desc_ptrs[i] = desc + i; | ||
151 | } | ||
152 | |||
153 | for (i = legacy_count; i < NR_IRQS; i++) | ||
154 | irq_desc_ptrs[i] = NULL; | ||
155 | |||
156 | arch_early_irq_init(); | ||
157 | } | ||
158 | |||
159 | struct irq_desc *irq_to_desc(unsigned int irq) | ||
160 | { | ||
161 | return (irq < NR_IRQS) ? irq_desc_ptrs[irq] : NULL; | ||
162 | } | ||
163 | |||
164 | struct irq_desc *irq_to_desc_alloc_cpu(unsigned int irq, int cpu) | ||
165 | { | ||
166 | struct irq_desc *desc; | ||
167 | unsigned long flags; | ||
168 | int node; | ||
169 | |||
170 | if (irq >= NR_IRQS) { | ||
171 | printk(KERN_WARNING "irq >= NR_IRQS in irq_to_desc_alloc: %d %d\n", | ||
172 | irq, NR_IRQS); | ||
173 | WARN_ON(1); | ||
174 | return NULL; | ||
175 | } | ||
176 | |||
177 | desc = irq_desc_ptrs[irq]; | ||
178 | if (desc) | ||
179 | return desc; | ||
180 | |||
181 | spin_lock_irqsave(&sparse_irq_lock, flags); | ||
182 | |||
183 | /* We have to check it to avoid races with another CPU */ | ||
184 | desc = irq_desc_ptrs[irq]; | ||
185 | if (desc) | ||
186 | goto out_unlock; | ||
187 | |||
188 | node = cpu_to_node(cpu); | ||
189 | desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node); | ||
190 | printk(KERN_DEBUG " alloc irq_desc for %d on cpu %d node %d\n", | ||
191 | irq, cpu, node); | ||
192 | if (!desc) { | ||
193 | printk(KERN_ERR "can not alloc irq_desc\n"); | ||
194 | BUG_ON(1); | ||
195 | } | ||
196 | init_one_irq_desc(irq, desc, cpu); | ||
197 | |||
198 | irq_desc_ptrs[irq] = desc; | ||
199 | |||
200 | out_unlock: | ||
201 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | ||
202 | |||
203 | return desc; | ||
204 | } | ||
205 | |||
206 | #else | ||
207 | |||
52 | struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { | 208 | struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { |
53 | [0 ... NR_IRQS-1] = { | 209 | [0 ... NR_IRQS-1] = { |
54 | .status = IRQ_DISABLED, | 210 | .status = IRQ_DISABLED, |
@@ -62,6 +218,8 @@ struct irq_desc irq_desc[NR_IRQS] __cacheline_aligned_in_smp = { | |||
62 | } | 218 | } |
63 | }; | 219 | }; |
64 | 220 | ||
221 | #endif | ||
222 | |||
65 | /* | 223 | /* |
66 | * What should we do if we get a hw irq event on an illegal vector? | 224 | * What should we do if we get a hw irq event on an illegal vector? |
67 | * Each architecture has to answer this themself. | 225 | * Each architecture has to answer this themself. |
@@ -179,8 +337,11 @@ unsigned int __do_IRQ(unsigned int irq) | |||
179 | /* | 337 | /* |
180 | * No locking required for CPU-local interrupts: | 338 | * No locking required for CPU-local interrupts: |
181 | */ | 339 | */ |
182 | if (desc->chip->ack) | 340 | if (desc->chip->ack) { |
183 | desc->chip->ack(irq); | 341 | desc->chip->ack(irq); |
342 | /* get new one */ | ||
343 | desc = irq_remap_to_desc(irq, desc); | ||
344 | } | ||
184 | if (likely(!(desc->status & IRQ_DISABLED))) { | 345 | if (likely(!(desc->status & IRQ_DISABLED))) { |
185 | action_ret = handle_IRQ_event(irq, desc->action); | 346 | action_ret = handle_IRQ_event(irq, desc->action); |
186 | if (!noirqdebug) | 347 | if (!noirqdebug) |
@@ -191,8 +352,10 @@ unsigned int __do_IRQ(unsigned int irq) | |||
191 | } | 352 | } |
192 | 353 | ||
193 | spin_lock(&desc->lock); | 354 | spin_lock(&desc->lock); |
194 | if (desc->chip->ack) | 355 | if (desc->chip->ack) { |
195 | desc->chip->ack(irq); | 356 | desc->chip->ack(irq); |
357 | desc = irq_remap_to_desc(irq, desc); | ||
358 | } | ||
196 | /* | 359 | /* |
197 | * REPLAY is when Linux resends an IRQ that was dropped earlier | 360 | * REPLAY is when Linux resends an IRQ that was dropped earlier |
198 | * WAITING is used by probe to mark irqs that are being tested | 361 | * WAITING is used by probe to mark irqs that are being tested |
@@ -259,19 +422,25 @@ out: | |||
259 | } | 422 | } |
260 | #endif | 423 | #endif |
261 | 424 | ||
262 | |||
263 | #ifdef CONFIG_TRACE_IRQFLAGS | ||
264 | /* | ||
265 | * lockdep: we want to handle all irq_desc locks as a single lock-class: | ||
266 | */ | ||
267 | static struct lock_class_key irq_desc_lock_class; | ||
268 | |||
269 | void early_init_irq_lock_class(void) | 425 | void early_init_irq_lock_class(void) |
270 | { | 426 | { |
271 | struct irq_desc *desc; | 427 | struct irq_desc *desc; |
272 | int i; | 428 | int i; |
273 | 429 | ||
274 | for_each_irq_desc(i, desc) | 430 | for_each_irq_desc(i, desc) { |
431 | if (!desc) | ||
432 | continue; | ||
433 | |||
275 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); | 434 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); |
435 | } | ||
436 | } | ||
437 | |||
438 | #ifdef CONFIG_SPARSE_IRQ | ||
439 | unsigned int kstat_irqs_cpu(unsigned int irq, int cpu) | ||
440 | { | ||
441 | struct irq_desc *desc = irq_to_desc(irq); | ||
442 | return desc->kstat_irqs[cpu]; | ||
276 | } | 443 | } |
277 | #endif | 444 | #endif |
445 | EXPORT_SYMBOL(kstat_irqs_cpu); | ||
446 | |||
diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index 64c1c7253da..e6d0a43cc12 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h | |||
@@ -13,6 +13,11 @@ extern void compat_irq_chip_set_default_handler(struct irq_desc *desc); | |||
13 | extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, | 13 | extern int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, |
14 | unsigned long flags); | 14 | unsigned long flags); |
15 | 15 | ||
16 | extern struct lock_class_key irq_desc_lock_class; | ||
17 | extern void init_kstat_irqs(struct irq_desc *desc, int cpu, int nr); | ||
18 | extern spinlock_t sparse_irq_lock; | ||
19 | extern struct irq_desc *irq_desc_ptrs[NR_IRQS]; | ||
20 | |||
16 | #ifdef CONFIG_PROC_FS | 21 | #ifdef CONFIG_PROC_FS |
17 | extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); | 22 | extern void register_irq_proc(unsigned int irq, struct irq_desc *desc); |
18 | extern void register_handler_proc(unsigned int irq, struct irqaction *action); | 23 | extern void register_handler_proc(unsigned int irq, struct irqaction *action); |
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index e9d1c8205a3..540f6c49f3f 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c | |||
@@ -370,16 +370,18 @@ int __irq_set_trigger(struct irq_desc *desc, unsigned int irq, | |||
370 | return 0; | 370 | return 0; |
371 | } | 371 | } |
372 | 372 | ||
373 | ret = chip->set_type(irq, flags & IRQF_TRIGGER_MASK); | 373 | /* caller masked out all except trigger mode flags */ |
374 | ret = chip->set_type(irq, flags); | ||
374 | 375 | ||
375 | if (ret) | 376 | if (ret) |
376 | pr_err("setting trigger mode %d for irq %u failed (%pF)\n", | 377 | pr_err("setting trigger mode %d for irq %u failed (%pF)\n", |
377 | (int)(flags & IRQF_TRIGGER_MASK), | 378 | (int)flags, irq, chip->set_type); |
378 | irq, chip->set_type); | ||
379 | else { | 379 | else { |
380 | if (flags & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH)) | ||
381 | flags |= IRQ_LEVEL; | ||
380 | /* note that IRQF_TRIGGER_MASK == IRQ_TYPE_SENSE_MASK */ | 382 | /* note that IRQF_TRIGGER_MASK == IRQ_TYPE_SENSE_MASK */ |
381 | desc->status &= ~IRQ_TYPE_SENSE_MASK; | 383 | desc->status &= ~(IRQ_LEVEL | IRQ_TYPE_SENSE_MASK); |
382 | desc->status |= flags & IRQ_TYPE_SENSE_MASK; | 384 | desc->status |= flags; |
383 | } | 385 | } |
384 | 386 | ||
385 | return ret; | 387 | return ret; |
@@ -459,7 +461,8 @@ __setup_irq(unsigned int irq, struct irq_desc * desc, struct irqaction *new) | |||
459 | 461 | ||
460 | /* Setup the type (level, edge polarity) if configured: */ | 462 | /* Setup the type (level, edge polarity) if configured: */ |
461 | if (new->flags & IRQF_TRIGGER_MASK) { | 463 | if (new->flags & IRQF_TRIGGER_MASK) { |
462 | ret = __irq_set_trigger(desc, irq, new->flags); | 464 | ret = __irq_set_trigger(desc, irq, |
465 | new->flags & IRQF_TRIGGER_MASK); | ||
463 | 466 | ||
464 | if (ret) { | 467 | if (ret) { |
465 | spin_unlock_irqrestore(&desc->lock, flags); | 468 | spin_unlock_irqrestore(&desc->lock, flags); |
diff --git a/kernel/irq/numa_migrate.c b/kernel/irq/numa_migrate.c new file mode 100644 index 00000000000..089c3746358 --- /dev/null +++ b/kernel/irq/numa_migrate.c | |||
@@ -0,0 +1,122 @@ | |||
1 | /* | ||
2 | * NUMA irq-desc migration code | ||
3 | * | ||
4 | * Migrate IRQ data structures (irq_desc, chip_data, etc.) over to | ||
5 | * the new "home node" of the IRQ. | ||
6 | */ | ||
7 | |||
8 | #include <linux/irq.h> | ||
9 | #include <linux/module.h> | ||
10 | #include <linux/random.h> | ||
11 | #include <linux/interrupt.h> | ||
12 | #include <linux/kernel_stat.h> | ||
13 | |||
14 | #include "internals.h" | ||
15 | |||
16 | static void init_copy_kstat_irqs(struct irq_desc *old_desc, | ||
17 | struct irq_desc *desc, | ||
18 | int cpu, int nr) | ||
19 | { | ||
20 | unsigned long bytes; | ||
21 | |||
22 | init_kstat_irqs(desc, cpu, nr); | ||
23 | |||
24 | if (desc->kstat_irqs != old_desc->kstat_irqs) { | ||
25 | /* Compute how many bytes we need per irq and allocate them */ | ||
26 | bytes = nr * sizeof(unsigned int); | ||
27 | |||
28 | memcpy(desc->kstat_irqs, old_desc->kstat_irqs, bytes); | ||
29 | } | ||
30 | } | ||
31 | |||
32 | static void free_kstat_irqs(struct irq_desc *old_desc, struct irq_desc *desc) | ||
33 | { | ||
34 | if (old_desc->kstat_irqs == desc->kstat_irqs) | ||
35 | return; | ||
36 | |||
37 | kfree(old_desc->kstat_irqs); | ||
38 | old_desc->kstat_irqs = NULL; | ||
39 | } | ||
40 | |||
41 | static void init_copy_one_irq_desc(int irq, struct irq_desc *old_desc, | ||
42 | struct irq_desc *desc, int cpu) | ||
43 | { | ||
44 | memcpy(desc, old_desc, sizeof(struct irq_desc)); | ||
45 | desc->cpu = cpu; | ||
46 | lockdep_set_class(&desc->lock, &irq_desc_lock_class); | ||
47 | init_copy_kstat_irqs(old_desc, desc, cpu, nr_cpu_ids); | ||
48 | arch_init_copy_chip_data(old_desc, desc, cpu); | ||
49 | } | ||
50 | |||
51 | static void free_one_irq_desc(struct irq_desc *old_desc, struct irq_desc *desc) | ||
52 | { | ||
53 | free_kstat_irqs(old_desc, desc); | ||
54 | arch_free_chip_data(old_desc, desc); | ||
55 | } | ||
56 | |||
57 | static struct irq_desc *__real_move_irq_desc(struct irq_desc *old_desc, | ||
58 | int cpu) | ||
59 | { | ||
60 | struct irq_desc *desc; | ||
61 | unsigned int irq; | ||
62 | unsigned long flags; | ||
63 | int node; | ||
64 | |||
65 | irq = old_desc->irq; | ||
66 | |||
67 | spin_lock_irqsave(&sparse_irq_lock, flags); | ||
68 | |||
69 | /* We have to check it to avoid races with another CPU */ | ||
70 | desc = irq_desc_ptrs[irq]; | ||
71 | |||
72 | if (desc && old_desc != desc) | ||
73 | goto out_unlock; | ||
74 | |||
75 | node = cpu_to_node(cpu); | ||
76 | desc = kzalloc_node(sizeof(*desc), GFP_ATOMIC, node); | ||
77 | printk(KERN_DEBUG " move irq_desc for %d to cpu %d node %d\n", | ||
78 | irq, cpu, node); | ||
79 | if (!desc) { | ||
80 | printk(KERN_ERR "can not get new irq_desc for moving\n"); | ||
81 | /* still use old one */ | ||
82 | desc = old_desc; | ||
83 | goto out_unlock; | ||
84 | } | ||
85 | init_copy_one_irq_desc(irq, old_desc, desc, cpu); | ||
86 | |||
87 | irq_desc_ptrs[irq] = desc; | ||
88 | |||
89 | /* free the old one */ | ||
90 | free_one_irq_desc(old_desc, desc); | ||
91 | kfree(old_desc); | ||
92 | |||
93 | out_unlock: | ||
94 | spin_unlock_irqrestore(&sparse_irq_lock, flags); | ||
95 | |||
96 | return desc; | ||
97 | } | ||
98 | |||
99 | struct irq_desc *move_irq_desc(struct irq_desc *desc, int cpu) | ||
100 | { | ||
101 | int old_cpu; | ||
102 | int node, old_node; | ||
103 | |||
104 | /* those all static, do move them */ | ||
105 | if (desc->irq < NR_IRQS_LEGACY) | ||
106 | return desc; | ||
107 | |||
108 | old_cpu = desc->cpu; | ||
109 | printk(KERN_DEBUG | ||
110 | "try to move irq_desc from cpu %d to %d\n", old_cpu, cpu); | ||
111 | if (old_cpu != cpu) { | ||
112 | node = cpu_to_node(cpu); | ||
113 | old_node = cpu_to_node(old_cpu); | ||
114 | if (old_node != node) | ||
115 | desc = __real_move_irq_desc(desc, cpu); | ||
116 | else | ||
117 | desc->cpu = cpu; | ||
118 | } | ||
119 | |||
120 | return desc; | ||
121 | } | ||
122 | |||
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c index d257e7d6a8a..f6b3440f05b 100644 --- a/kernel/irq/proc.c +++ b/kernel/irq/proc.c | |||
@@ -243,7 +243,11 @@ void init_irq_proc(void) | |||
243 | /* | 243 | /* |
244 | * Create entries for all existing IRQs. | 244 | * Create entries for all existing IRQs. |
245 | */ | 245 | */ |
246 | for_each_irq_desc(irq, desc) | 246 | for_each_irq_desc(irq, desc) { |
247 | if (!desc) | ||
248 | continue; | ||
249 | |||
247 | register_irq_proc(irq, desc); | 250 | register_irq_proc(irq, desc); |
251 | } | ||
248 | } | 252 | } |
249 | 253 | ||
diff --git a/kernel/irq/spurious.c b/kernel/irq/spurious.c index dd364c11e56..3738107531f 100644 --- a/kernel/irq/spurious.c +++ b/kernel/irq/spurious.c | |||
@@ -91,6 +91,9 @@ static int misrouted_irq(int irq) | |||
91 | int i, ok = 0; | 91 | int i, ok = 0; |
92 | 92 | ||
93 | for_each_irq_desc(i, desc) { | 93 | for_each_irq_desc(i, desc) { |
94 | if (!desc) | ||
95 | continue; | ||
96 | |||
94 | if (!i) | 97 | if (!i) |
95 | continue; | 98 | continue; |
96 | 99 | ||
@@ -112,6 +115,8 @@ static void poll_spurious_irqs(unsigned long dummy) | |||
112 | for_each_irq_desc(i, desc) { | 115 | for_each_irq_desc(i, desc) { |
113 | unsigned int status; | 116 | unsigned int status; |
114 | 117 | ||
118 | if (!desc) | ||
119 | continue; | ||
115 | if (!i) | 120 | if (!i) |
116 | continue; | 121 | continue; |
117 | 122 | ||