aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/apic/io_apic.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel/apic/io_apic.c')
-rw-r--r--arch/x86/kernel/apic/io_apic.c802
1 files changed, 445 insertions, 357 deletions
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index dc69f28489f5..eb2789c3f721 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -36,6 +36,7 @@
36#include <linux/freezer.h> 36#include <linux/freezer.h>
37#include <linux/kthread.h> 37#include <linux/kthread.h>
38#include <linux/jiffies.h> /* time_after() */ 38#include <linux/jiffies.h> /* time_after() */
39#include <linux/slab.h>
39#ifdef CONFIG_ACPI 40#ifdef CONFIG_ACPI
40#include <acpi/acpi_bus.h> 41#include <acpi/acpi_bus.h>
41#endif 42#endif
@@ -60,8 +61,6 @@
60#include <asm/irq_remapping.h> 61#include <asm/irq_remapping.h>
61#include <asm/hpet.h> 62#include <asm/hpet.h>
62#include <asm/hw_irq.h> 63#include <asm/hw_irq.h>
63#include <asm/uv/uv_hub.h>
64#include <asm/uv/uv_irq.h>
65 64
66#include <asm/apic.h> 65#include <asm/apic.h>
67 66
@@ -75,8 +74,8 @@
75 */ 74 */
76int sis_apic_bug = -1; 75int sis_apic_bug = -1;
77 76
78static DEFINE_SPINLOCK(ioapic_lock); 77static DEFINE_RAW_SPINLOCK(ioapic_lock);
79static DEFINE_SPINLOCK(vector_lock); 78static DEFINE_RAW_SPINLOCK(vector_lock);
80 79
81/* 80/*
82 * # of IRQ routing registers 81 * # of IRQ routing registers
@@ -96,8 +95,6 @@ struct mpc_intsrc mp_irqs[MAX_IRQ_SOURCES];
96/* # of MP IRQ source entries */ 95/* # of MP IRQ source entries */
97int mp_irq_entries; 96int mp_irq_entries;
98 97
99/* Number of legacy interrupts */
100static int nr_legacy_irqs __read_mostly = NR_IRQS_LEGACY;
101/* GSI interrupts */ 98/* GSI interrupts */
102static int nr_irqs_gsi = NR_IRQS_LEGACY; 99static int nr_irqs_gsi = NR_IRQS_LEGACY;
103 100
@@ -140,49 +137,12 @@ static struct irq_pin_list *get_one_free_irq_2_pin(int node)
140 return pin; 137 return pin;
141} 138}
142 139
143/*
144 * This is performance-critical, we want to do it O(1)
145 *
146 * Most irqs are mapped 1:1 with pins.
147 */
148struct irq_cfg {
149 struct irq_pin_list *irq_2_pin;
150 cpumask_var_t domain;
151 cpumask_var_t old_domain;
152 unsigned move_cleanup_count;
153 u8 vector;
154 u8 move_in_progress : 1;
155};
156
157/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ 140/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
158#ifdef CONFIG_SPARSE_IRQ 141#ifdef CONFIG_SPARSE_IRQ
159static struct irq_cfg irq_cfgx[] = { 142static struct irq_cfg irq_cfgx[NR_IRQS_LEGACY];
160#else 143#else
161static struct irq_cfg irq_cfgx[NR_IRQS] = { 144static struct irq_cfg irq_cfgx[NR_IRQS];
162#endif 145#endif
163 [0] = { .vector = IRQ0_VECTOR, },
164 [1] = { .vector = IRQ1_VECTOR, },
165 [2] = { .vector = IRQ2_VECTOR, },
166 [3] = { .vector = IRQ3_VECTOR, },
167 [4] = { .vector = IRQ4_VECTOR, },
168 [5] = { .vector = IRQ5_VECTOR, },
169 [6] = { .vector = IRQ6_VECTOR, },
170 [7] = { .vector = IRQ7_VECTOR, },
171 [8] = { .vector = IRQ8_VECTOR, },
172 [9] = { .vector = IRQ9_VECTOR, },
173 [10] = { .vector = IRQ10_VECTOR, },
174 [11] = { .vector = IRQ11_VECTOR, },
175 [12] = { .vector = IRQ12_VECTOR, },
176 [13] = { .vector = IRQ13_VECTOR, },
177 [14] = { .vector = IRQ14_VECTOR, },
178 [15] = { .vector = IRQ15_VECTOR, },
179};
180
181void __init io_apic_disable_legacy(void)
182{
183 nr_legacy_irqs = 0;
184 nr_irqs_gsi = 0;
185}
186 146
187int __init arch_early_irq_init(void) 147int __init arch_early_irq_init(void)
188{ 148{
@@ -192,6 +152,11 @@ int __init arch_early_irq_init(void)
192 int node; 152 int node;
193 int i; 153 int i;
194 154
155 if (!legacy_pic->nr_legacy_irqs) {
156 nr_irqs_gsi = 0;
157 io_apic_irqs = ~0UL;
158 }
159
195 cfg = irq_cfgx; 160 cfg = irq_cfgx;
196 count = ARRAY_SIZE(irq_cfgx); 161 count = ARRAY_SIZE(irq_cfgx);
197 node= cpu_to_node(boot_cpu_id); 162 node= cpu_to_node(boot_cpu_id);
@@ -201,15 +166,21 @@ int __init arch_early_irq_init(void)
201 desc->chip_data = &cfg[i]; 166 desc->chip_data = &cfg[i];
202 zalloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node); 167 zalloc_cpumask_var_node(&cfg[i].domain, GFP_NOWAIT, node);
203 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node); 168 zalloc_cpumask_var_node(&cfg[i].old_domain, GFP_NOWAIT, node);
204 if (i < nr_legacy_irqs) 169 /*
205 cpumask_setall(cfg[i].domain); 170 * For legacy IRQ's, start with assigning irq0 to irq15 to
171 * IRQ0_VECTOR to IRQ15_VECTOR on cpu 0.
172 */
173 if (i < legacy_pic->nr_legacy_irqs) {
174 cfg[i].vector = IRQ0_VECTOR + i;
175 cpumask_set_cpu(0, cfg[i].domain);
176 }
206 } 177 }
207 178
208 return 0; 179 return 0;
209} 180}
210 181
211#ifdef CONFIG_SPARSE_IRQ 182#ifdef CONFIG_SPARSE_IRQ
212static struct irq_cfg *irq_cfg(unsigned int irq) 183struct irq_cfg *irq_cfg(unsigned int irq)
213{ 184{
214 struct irq_cfg *cfg = NULL; 185 struct irq_cfg *cfg = NULL;
215 struct irq_desc *desc; 186 struct irq_desc *desc;
@@ -361,7 +332,7 @@ void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
361/* end for move_irq_desc */ 332/* end for move_irq_desc */
362 333
363#else 334#else
364static struct irq_cfg *irq_cfg(unsigned int irq) 335struct irq_cfg *irq_cfg(unsigned int irq)
365{ 336{
366 return irq < nr_irqs ? irq_cfgx + irq : NULL; 337 return irq < nr_irqs ? irq_cfgx + irq : NULL;
367} 338}
@@ -422,7 +393,7 @@ static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
422 struct irq_pin_list *entry; 393 struct irq_pin_list *entry;
423 unsigned long flags; 394 unsigned long flags;
424 395
425 spin_lock_irqsave(&ioapic_lock, flags); 396 raw_spin_lock_irqsave(&ioapic_lock, flags);
426 for_each_irq_pin(entry, cfg->irq_2_pin) { 397 for_each_irq_pin(entry, cfg->irq_2_pin) {
427 unsigned int reg; 398 unsigned int reg;
428 int pin; 399 int pin;
@@ -431,11 +402,11 @@ static bool io_apic_level_ack_pending(struct irq_cfg *cfg)
431 reg = io_apic_read(entry->apic, 0x10 + pin*2); 402 reg = io_apic_read(entry->apic, 0x10 + pin*2);
432 /* Is the remote IRR bit set? */ 403 /* Is the remote IRR bit set? */
433 if (reg & IO_APIC_REDIR_REMOTE_IRR) { 404 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
434 spin_unlock_irqrestore(&ioapic_lock, flags); 405 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
435 return true; 406 return true;
436 } 407 }
437 } 408 }
438 spin_unlock_irqrestore(&ioapic_lock, flags); 409 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
439 410
440 return false; 411 return false;
441} 412}
@@ -449,10 +420,10 @@ static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
449{ 420{
450 union entry_union eu; 421 union entry_union eu;
451 unsigned long flags; 422 unsigned long flags;
452 spin_lock_irqsave(&ioapic_lock, flags); 423 raw_spin_lock_irqsave(&ioapic_lock, flags);
453 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin); 424 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
454 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin); 425 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
455 spin_unlock_irqrestore(&ioapic_lock, flags); 426 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
456 return eu.entry; 427 return eu.entry;
457} 428}
458 429
@@ -475,9 +446,9 @@ __ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
475void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e) 446void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
476{ 447{
477 unsigned long flags; 448 unsigned long flags;
478 spin_lock_irqsave(&ioapic_lock, flags); 449 raw_spin_lock_irqsave(&ioapic_lock, flags);
479 __ioapic_write_entry(apic, pin, e); 450 __ioapic_write_entry(apic, pin, e);
480 spin_unlock_irqrestore(&ioapic_lock, flags); 451 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
481} 452}
482 453
483/* 454/*
@@ -490,10 +461,10 @@ static void ioapic_mask_entry(int apic, int pin)
490 unsigned long flags; 461 unsigned long flags;
491 union entry_union eu = { .entry.mask = 1 }; 462 union entry_union eu = { .entry.mask = 1 };
492 463
493 spin_lock_irqsave(&ioapic_lock, flags); 464 raw_spin_lock_irqsave(&ioapic_lock, flags);
494 io_apic_write(apic, 0x10 + 2*pin, eu.w1); 465 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
495 io_apic_write(apic, 0x11 + 2*pin, eu.w2); 466 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
496 spin_unlock_irqrestore(&ioapic_lock, flags); 467 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
497} 468}
498 469
499/* 470/*
@@ -555,23 +526,41 @@ static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
555 add_pin_to_irq_node(cfg, node, newapic, newpin); 526 add_pin_to_irq_node(cfg, node, newapic, newpin);
556} 527}
557 528
529static void __io_apic_modify_irq(struct irq_pin_list *entry,
530 int mask_and, int mask_or,
531 void (*final)(struct irq_pin_list *entry))
532{
533 unsigned int reg, pin;
534
535 pin = entry->pin;
536 reg = io_apic_read(entry->apic, 0x10 + pin * 2);
537 reg &= mask_and;
538 reg |= mask_or;
539 io_apic_modify(entry->apic, 0x10 + pin * 2, reg);
540 if (final)
541 final(entry);
542}
543
558static void io_apic_modify_irq(struct irq_cfg *cfg, 544static void io_apic_modify_irq(struct irq_cfg *cfg,
559 int mask_and, int mask_or, 545 int mask_and, int mask_or,
560 void (*final)(struct irq_pin_list *entry)) 546 void (*final)(struct irq_pin_list *entry))
561{ 547{
562 int pin;
563 struct irq_pin_list *entry; 548 struct irq_pin_list *entry;
564 549
565 for_each_irq_pin(entry, cfg->irq_2_pin) { 550 for_each_irq_pin(entry, cfg->irq_2_pin)
566 unsigned int reg; 551 __io_apic_modify_irq(entry, mask_and, mask_or, final);
567 pin = entry->pin; 552}
568 reg = io_apic_read(entry->apic, 0x10 + pin * 2); 553
569 reg &= mask_and; 554static void __mask_and_edge_IO_APIC_irq(struct irq_pin_list *entry)
570 reg |= mask_or; 555{
571 io_apic_modify(entry->apic, 0x10 + pin * 2, reg); 556 __io_apic_modify_irq(entry, ~IO_APIC_REDIR_LEVEL_TRIGGER,
572 if (final) 557 IO_APIC_REDIR_MASKED, NULL);
573 final(entry); 558}
574 } 559
560static void __unmask_and_level_IO_APIC_irq(struct irq_pin_list *entry)
561{
562 __io_apic_modify_irq(entry, ~IO_APIC_REDIR_MASKED,
563 IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
575} 564}
576 565
577static void __unmask_IO_APIC_irq(struct irq_cfg *cfg) 566static void __unmask_IO_APIC_irq(struct irq_cfg *cfg)
@@ -595,18 +584,6 @@ static void __mask_IO_APIC_irq(struct irq_cfg *cfg)
595 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync); 584 io_apic_modify_irq(cfg, ~0, IO_APIC_REDIR_MASKED, &io_apic_sync);
596} 585}
597 586
598static void __mask_and_edge_IO_APIC_irq(struct irq_cfg *cfg)
599{
600 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_LEVEL_TRIGGER,
601 IO_APIC_REDIR_MASKED, NULL);
602}
603
604static void __unmask_and_level_IO_APIC_irq(struct irq_cfg *cfg)
605{
606 io_apic_modify_irq(cfg, ~IO_APIC_REDIR_MASKED,
607 IO_APIC_REDIR_LEVEL_TRIGGER, NULL);
608}
609
610static void mask_IO_APIC_irq_desc(struct irq_desc *desc) 587static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
611{ 588{
612 struct irq_cfg *cfg = desc->chip_data; 589 struct irq_cfg *cfg = desc->chip_data;
@@ -614,9 +591,9 @@ static void mask_IO_APIC_irq_desc(struct irq_desc *desc)
614 591
615 BUG_ON(!cfg); 592 BUG_ON(!cfg);
616 593
617 spin_lock_irqsave(&ioapic_lock, flags); 594 raw_spin_lock_irqsave(&ioapic_lock, flags);
618 __mask_IO_APIC_irq(cfg); 595 __mask_IO_APIC_irq(cfg);
619 spin_unlock_irqrestore(&ioapic_lock, flags); 596 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
620} 597}
621 598
622static void unmask_IO_APIC_irq_desc(struct irq_desc *desc) 599static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
@@ -624,9 +601,9 @@ static void unmask_IO_APIC_irq_desc(struct irq_desc *desc)
624 struct irq_cfg *cfg = desc->chip_data; 601 struct irq_cfg *cfg = desc->chip_data;
625 unsigned long flags; 602 unsigned long flags;
626 603
627 spin_lock_irqsave(&ioapic_lock, flags); 604 raw_spin_lock_irqsave(&ioapic_lock, flags);
628 __unmask_IO_APIC_irq(cfg); 605 __unmask_IO_APIC_irq(cfg);
629 spin_unlock_irqrestore(&ioapic_lock, flags); 606 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
630} 607}
631 608
632static void mask_IO_APIC_irq(unsigned int irq) 609static void mask_IO_APIC_irq(unsigned int irq)
@@ -875,7 +852,7 @@ static int __init find_isa_irq_apic(int irq, int type)
875 */ 852 */
876static int EISA_ELCR(unsigned int irq) 853static int EISA_ELCR(unsigned int irq)
877{ 854{
878 if (irq < nr_legacy_irqs) { 855 if (irq < legacy_pic->nr_legacy_irqs) {
879 unsigned int port = 0x4d0 + (irq >> 3); 856 unsigned int port = 0x4d0 + (irq >> 3);
880 return (inb(port) >> (irq & 7)) & 1; 857 return (inb(port) >> (irq & 7)) & 1;
881 } 858 }
@@ -1150,12 +1127,12 @@ void lock_vector_lock(void)
1150 /* Used to the online set of cpus does not change 1127 /* Used to the online set of cpus does not change
1151 * during assign_irq_vector. 1128 * during assign_irq_vector.
1152 */ 1129 */
1153 spin_lock(&vector_lock); 1130 raw_spin_lock(&vector_lock);
1154} 1131}
1155 1132
1156void unlock_vector_lock(void) 1133void unlock_vector_lock(void)
1157{ 1134{
1158 spin_unlock(&vector_lock); 1135 raw_spin_unlock(&vector_lock);
1159} 1136}
1160 1137
1161static int 1138static int
@@ -1172,12 +1149,13 @@ __assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1172 * Also, we've got to be careful not to trash gate 1149 * Also, we've got to be careful not to trash gate
1173 * 0x80, because int 0x80 is hm, kind of importantish. ;) 1150 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1174 */ 1151 */
1175 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0; 1152 static int current_vector = FIRST_EXTERNAL_VECTOR + VECTOR_OFFSET_START;
1153 static int current_offset = VECTOR_OFFSET_START % 8;
1176 unsigned int old_vector; 1154 unsigned int old_vector;
1177 int cpu, err; 1155 int cpu, err;
1178 cpumask_var_t tmp_mask; 1156 cpumask_var_t tmp_mask;
1179 1157
1180 if ((cfg->move_in_progress) || cfg->move_cleanup_count) 1158 if (cfg->move_in_progress)
1181 return -EBUSY; 1159 return -EBUSY;
1182 1160
1183 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC)) 1161 if (!alloc_cpumask_var(&tmp_mask, GFP_ATOMIC))
@@ -1208,7 +1186,7 @@ next:
1208 if (vector >= first_system_vector) { 1186 if (vector >= first_system_vector) {
1209 /* If out of vectors on large boxen, must share them. */ 1187 /* If out of vectors on large boxen, must share them. */
1210 offset = (offset + 1) % 8; 1188 offset = (offset + 1) % 8;
1211 vector = FIRST_DEVICE_VECTOR + offset; 1189 vector = FIRST_EXTERNAL_VECTOR + offset;
1212 } 1190 }
1213 if (unlikely(current_vector == vector)) 1191 if (unlikely(current_vector == vector))
1214 continue; 1192 continue;
@@ -1237,15 +1215,14 @@ next:
1237 return err; 1215 return err;
1238} 1216}
1239 1217
1240static int 1218int assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1241assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask)
1242{ 1219{
1243 int err; 1220 int err;
1244 unsigned long flags; 1221 unsigned long flags;
1245 1222
1246 spin_lock_irqsave(&vector_lock, flags); 1223 raw_spin_lock_irqsave(&vector_lock, flags);
1247 err = __assign_irq_vector(irq, cfg, mask); 1224 err = __assign_irq_vector(irq, cfg, mask);
1248 spin_unlock_irqrestore(&vector_lock, flags); 1225 raw_spin_unlock_irqrestore(&vector_lock, flags);
1249 return err; 1226 return err;
1250} 1227}
1251 1228
@@ -1279,14 +1256,27 @@ static void __clear_irq_vector(int irq, struct irq_cfg *cfg)
1279void __setup_vector_irq(int cpu) 1256void __setup_vector_irq(int cpu)
1280{ 1257{
1281 /* Initialize vector_irq on a new cpu */ 1258 /* Initialize vector_irq on a new cpu */
1282 /* This function must be called with vector_lock held */
1283 int irq, vector; 1259 int irq, vector;
1284 struct irq_cfg *cfg; 1260 struct irq_cfg *cfg;
1285 struct irq_desc *desc; 1261 struct irq_desc *desc;
1286 1262
1263 /*
1264 * vector_lock will make sure that we don't run into irq vector
1265 * assignments that might be happening on another cpu in parallel,
1266 * while we setup our initial vector to irq mappings.
1267 */
1268 raw_spin_lock(&vector_lock);
1287 /* Mark the inuse vectors */ 1269 /* Mark the inuse vectors */
1288 for_each_irq_desc(irq, desc) { 1270 for_each_irq_desc(irq, desc) {
1289 cfg = desc->chip_data; 1271 cfg = desc->chip_data;
1272
1273 /*
1274 * If it is a legacy IRQ handled by the legacy PIC, this cpu
1275 * will be part of the irq_cfg's domain.
1276 */
1277 if (irq < legacy_pic->nr_legacy_irqs && !IO_APIC_IRQ(irq))
1278 cpumask_set_cpu(cpu, cfg->domain);
1279
1290 if (!cpumask_test_cpu(cpu, cfg->domain)) 1280 if (!cpumask_test_cpu(cpu, cfg->domain))
1291 continue; 1281 continue;
1292 vector = cfg->vector; 1282 vector = cfg->vector;
@@ -1302,6 +1292,7 @@ void __setup_vector_irq(int cpu)
1302 if (!cpumask_test_cpu(cpu, cfg->domain)) 1292 if (!cpumask_test_cpu(cpu, cfg->domain))
1303 per_cpu(vector_irq, cpu)[vector] = -1; 1293 per_cpu(vector_irq, cpu)[vector] = -1;
1304 } 1294 }
1295 raw_spin_unlock(&vector_lock);
1305} 1296}
1306 1297
1307static struct irq_chip ioapic_chip; 1298static struct irq_chip ioapic_chip;
@@ -1451,6 +1442,14 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq
1451 1442
1452 cfg = desc->chip_data; 1443 cfg = desc->chip_data;
1453 1444
1445 /*
1446 * For legacy irqs, cfg->domain starts with cpu 0 for legacy
1447 * controllers like 8259. Now that IO-APIC can handle this irq, update
1448 * the cfg->domain.
1449 */
1450 if (irq < legacy_pic->nr_legacy_irqs && cpumask_test_cpu(0, cfg->domain))
1451 apic->vector_allocation_domain(0, cfg->domain);
1452
1454 if (assign_irq_vector(irq, cfg, apic->target_cpus())) 1453 if (assign_irq_vector(irq, cfg, apic->target_cpus()))
1455 return; 1454 return;
1456 1455
@@ -1472,8 +1471,8 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq
1472 } 1471 }
1473 1472
1474 ioapic_register_intr(irq, desc, trigger); 1473 ioapic_register_intr(irq, desc, trigger);
1475 if (irq < nr_legacy_irqs) 1474 if (irq < legacy_pic->nr_legacy_irqs)
1476 disable_8259A_irq(irq); 1475 legacy_pic->chip->mask(irq);
1477 1476
1478 ioapic_write_entry(apic_id, pin, entry); 1477 ioapic_write_entry(apic_id, pin, entry);
1479} 1478}
@@ -1484,7 +1483,7 @@ static struct {
1484 1483
1485static void __init setup_IO_APIC_irqs(void) 1484static void __init setup_IO_APIC_irqs(void)
1486{ 1485{
1487 int apic_id = 0, pin, idx, irq; 1486 int apic_id, pin, idx, irq;
1488 int notcon = 0; 1487 int notcon = 0;
1489 struct irq_desc *desc; 1488 struct irq_desc *desc;
1490 struct irq_cfg *cfg; 1489 struct irq_cfg *cfg;
@@ -1492,14 +1491,7 @@ static void __init setup_IO_APIC_irqs(void)
1492 1491
1493 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); 1492 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1494 1493
1495#ifdef CONFIG_ACPI 1494 for (apic_id = 0; apic_id < nr_ioapics; apic_id++)
1496 if (!acpi_disabled && acpi_ioapic) {
1497 apic_id = mp_find_ioapic(0);
1498 if (apic_id < 0)
1499 apic_id = 0;
1500 }
1501#endif
1502
1503 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) { 1495 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) {
1504 idx = find_irq_entry(apic_id, pin, mp_INT); 1496 idx = find_irq_entry(apic_id, pin, mp_INT);
1505 if (idx == -1) { 1497 if (idx == -1) {
@@ -1521,6 +1513,9 @@ static void __init setup_IO_APIC_irqs(void)
1521 1513
1522 irq = pin_2_irq(idx, apic_id, pin); 1514 irq = pin_2_irq(idx, apic_id, pin);
1523 1515
1516 if ((apic_id > 0) && (irq > 16))
1517 continue;
1518
1524 /* 1519 /*
1525 * Skip the timer IRQ if there's a quirk handler 1520 * Skip the timer IRQ if there's a quirk handler
1526 * installed and if it returns 1: 1521 * installed and if it returns 1:
@@ -1550,6 +1545,56 @@ static void __init setup_IO_APIC_irqs(void)
1550} 1545}
1551 1546
1552/* 1547/*
1548 * for the gsit that is not in first ioapic
1549 * but could not use acpi_register_gsi()
1550 * like some special sci in IBM x3330
1551 */
1552void setup_IO_APIC_irq_extra(u32 gsi)
1553{
1554 int apic_id = 0, pin, idx, irq;
1555 int node = cpu_to_node(boot_cpu_id);
1556 struct irq_desc *desc;
1557 struct irq_cfg *cfg;
1558
1559 /*
1560 * Convert 'gsi' to 'ioapic.pin'.
1561 */
1562 apic_id = mp_find_ioapic(gsi);
1563 if (apic_id < 0)
1564 return;
1565
1566 pin = mp_find_ioapic_pin(apic_id, gsi);
1567 idx = find_irq_entry(apic_id, pin, mp_INT);
1568 if (idx == -1)
1569 return;
1570
1571 irq = pin_2_irq(idx, apic_id, pin);
1572#ifdef CONFIG_SPARSE_IRQ
1573 desc = irq_to_desc(irq);
1574 if (desc)
1575 return;
1576#endif
1577 desc = irq_to_desc_alloc_node(irq, node);
1578 if (!desc) {
1579 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
1580 return;
1581 }
1582
1583 cfg = desc->chip_data;
1584 add_pin_to_irq_node(cfg, node, apic_id, pin);
1585
1586 if (test_bit(pin, mp_ioapic_routing[apic_id].pin_programmed)) {
1587 pr_debug("Pin %d-%d already programmed\n",
1588 mp_ioapics[apic_id].apicid, pin);
1589 return;
1590 }
1591 set_bit(pin, mp_ioapic_routing[apic_id].pin_programmed);
1592
1593 setup_IO_APIC_irq(apic_id, pin, irq, desc,
1594 irq_trigger(idx), irq_polarity(idx));
1595}
1596
1597/*
1553 * Set up the timer pin, possibly with the 8259A-master behind. 1598 * Set up the timer pin, possibly with the 8259A-master behind.
1554 */ 1599 */
1555static void __init setup_timer_IRQ0_pin(unsigned int apic_id, unsigned int pin, 1600static void __init setup_timer_IRQ0_pin(unsigned int apic_id, unsigned int pin,
@@ -1599,9 +1644,6 @@ __apicdebuginit(void) print_IO_APIC(void)
1599 struct irq_desc *desc; 1644 struct irq_desc *desc;
1600 unsigned int irq; 1645 unsigned int irq;
1601 1646
1602 if (apic_verbosity == APIC_QUIET)
1603 return;
1604
1605 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries); 1647 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1606 for (i = 0; i < nr_ioapics; i++) 1648 for (i = 0; i < nr_ioapics; i++)
1607 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n", 1649 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
@@ -1615,14 +1657,14 @@ __apicdebuginit(void) print_IO_APIC(void)
1615 1657
1616 for (apic = 0; apic < nr_ioapics; apic++) { 1658 for (apic = 0; apic < nr_ioapics; apic++) {
1617 1659
1618 spin_lock_irqsave(&ioapic_lock, flags); 1660 raw_spin_lock_irqsave(&ioapic_lock, flags);
1619 reg_00.raw = io_apic_read(apic, 0); 1661 reg_00.raw = io_apic_read(apic, 0);
1620 reg_01.raw = io_apic_read(apic, 1); 1662 reg_01.raw = io_apic_read(apic, 1);
1621 if (reg_01.bits.version >= 0x10) 1663 if (reg_01.bits.version >= 0x10)
1622 reg_02.raw = io_apic_read(apic, 2); 1664 reg_02.raw = io_apic_read(apic, 2);
1623 if (reg_01.bits.version >= 0x20) 1665 if (reg_01.bits.version >= 0x20)
1624 reg_03.raw = io_apic_read(apic, 3); 1666 reg_03.raw = io_apic_read(apic, 3);
1625 spin_unlock_irqrestore(&ioapic_lock, flags); 1667 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1626 1668
1627 printk("\n"); 1669 printk("\n");
1628 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid); 1670 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].apicid);
@@ -1661,7 +1703,7 @@ __apicdebuginit(void) print_IO_APIC(void)
1661 printk(KERN_DEBUG ".... IRQ redirection table:\n"); 1703 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1662 1704
1663 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol" 1705 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1664 " Stat Dmod Deli Vect: \n"); 1706 " Stat Dmod Deli Vect:\n");
1665 1707
1666 for (i = 0; i <= reg_01.bits.entries; i++) { 1708 for (i = 0; i <= reg_01.bits.entries; i++) {
1667 struct IO_APIC_route_entry entry; 1709 struct IO_APIC_route_entry entry;
@@ -1708,9 +1750,6 @@ __apicdebuginit(void) print_APIC_field(int base)
1708{ 1750{
1709 int i; 1751 int i;
1710 1752
1711 if (apic_verbosity == APIC_QUIET)
1712 return;
1713
1714 printk(KERN_DEBUG); 1753 printk(KERN_DEBUG);
1715 1754
1716 for (i = 0; i < 8; i++) 1755 for (i = 0; i < 8; i++)
@@ -1724,9 +1763,6 @@ __apicdebuginit(void) print_local_APIC(void *dummy)
1724 unsigned int i, v, ver, maxlvt; 1763 unsigned int i, v, ver, maxlvt;
1725 u64 icr; 1764 u64 icr;
1726 1765
1727 if (apic_verbosity == APIC_QUIET)
1728 return;
1729
1730 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n", 1766 printk(KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1731 smp_processor_id(), hard_smp_processor_id()); 1767 smp_processor_id(), hard_smp_processor_id());
1732 v = apic_read(APIC_ID); 1768 v = apic_read(APIC_ID);
@@ -1824,13 +1860,19 @@ __apicdebuginit(void) print_local_APIC(void *dummy)
1824 printk("\n"); 1860 printk("\n");
1825} 1861}
1826 1862
1827__apicdebuginit(void) print_all_local_APICs(void) 1863__apicdebuginit(void) print_local_APICs(int maxcpu)
1828{ 1864{
1829 int cpu; 1865 int cpu;
1830 1866
1867 if (!maxcpu)
1868 return;
1869
1831 preempt_disable(); 1870 preempt_disable();
1832 for_each_online_cpu(cpu) 1871 for_each_online_cpu(cpu) {
1872 if (cpu >= maxcpu)
1873 break;
1833 smp_call_function_single(cpu, print_local_APIC, NULL, 1); 1874 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1875 }
1834 preempt_enable(); 1876 preempt_enable();
1835} 1877}
1836 1878
@@ -1839,12 +1881,12 @@ __apicdebuginit(void) print_PIC(void)
1839 unsigned int v; 1881 unsigned int v;
1840 unsigned long flags; 1882 unsigned long flags;
1841 1883
1842 if (apic_verbosity == APIC_QUIET || !nr_legacy_irqs) 1884 if (!legacy_pic->nr_legacy_irqs)
1843 return; 1885 return;
1844 1886
1845 printk(KERN_DEBUG "\nprinting PIC contents\n"); 1887 printk(KERN_DEBUG "\nprinting PIC contents\n");
1846 1888
1847 spin_lock_irqsave(&i8259A_lock, flags); 1889 raw_spin_lock_irqsave(&i8259A_lock, flags);
1848 1890
1849 v = inb(0xa1) << 8 | inb(0x21); 1891 v = inb(0xa1) << 8 | inb(0x21);
1850 printk(KERN_DEBUG "... PIC IMR: %04x\n", v); 1892 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
@@ -1858,7 +1900,7 @@ __apicdebuginit(void) print_PIC(void)
1858 outb(0x0a,0xa0); 1900 outb(0x0a,0xa0);
1859 outb(0x0a,0x20); 1901 outb(0x0a,0x20);
1860 1902
1861 spin_unlock_irqrestore(&i8259A_lock, flags); 1903 raw_spin_unlock_irqrestore(&i8259A_lock, flags);
1862 1904
1863 printk(KERN_DEBUG "... PIC ISR: %04x\n", v); 1905 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1864 1906
@@ -1866,21 +1908,41 @@ __apicdebuginit(void) print_PIC(void)
1866 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v); 1908 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1867} 1909}
1868 1910
1869__apicdebuginit(int) print_all_ICs(void) 1911static int __initdata show_lapic = 1;
1912static __init int setup_show_lapic(char *arg)
1913{
1914 int num = -1;
1915
1916 if (strcmp(arg, "all") == 0) {
1917 show_lapic = CONFIG_NR_CPUS;
1918 } else {
1919 get_option(&arg, &num);
1920 if (num >= 0)
1921 show_lapic = num;
1922 }
1923
1924 return 1;
1925}
1926__setup("show_lapic=", setup_show_lapic);
1927
1928__apicdebuginit(int) print_ICs(void)
1870{ 1929{
1930 if (apic_verbosity == APIC_QUIET)
1931 return 0;
1932
1871 print_PIC(); 1933 print_PIC();
1872 1934
1873 /* don't print out if apic is not there */ 1935 /* don't print out if apic is not there */
1874 if (!cpu_has_apic && !apic_from_smp_config()) 1936 if (!cpu_has_apic && !apic_from_smp_config())
1875 return 0; 1937 return 0;
1876 1938
1877 print_all_local_APICs(); 1939 print_local_APICs(show_lapic);
1878 print_IO_APIC(); 1940 print_IO_APIC();
1879 1941
1880 return 0; 1942 return 0;
1881} 1943}
1882 1944
1883fs_initcall(print_all_ICs); 1945fs_initcall(print_ICs);
1884 1946
1885 1947
1886/* Where if anywhere is the i8259 connect in external int mode */ 1948/* Where if anywhere is the i8259 connect in external int mode */
@@ -1897,13 +1959,13 @@ void __init enable_IO_APIC(void)
1897 * The number of IO-APIC IRQ registers (== #pins): 1959 * The number of IO-APIC IRQ registers (== #pins):
1898 */ 1960 */
1899 for (apic = 0; apic < nr_ioapics; apic++) { 1961 for (apic = 0; apic < nr_ioapics; apic++) {
1900 spin_lock_irqsave(&ioapic_lock, flags); 1962 raw_spin_lock_irqsave(&ioapic_lock, flags);
1901 reg_01.raw = io_apic_read(apic, 1); 1963 reg_01.raw = io_apic_read(apic, 1);
1902 spin_unlock_irqrestore(&ioapic_lock, flags); 1964 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
1903 nr_ioapic_registers[apic] = reg_01.bits.entries+1; 1965 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
1904 } 1966 }
1905 1967
1906 if (!nr_legacy_irqs) 1968 if (!legacy_pic->nr_legacy_irqs)
1907 return; 1969 return;
1908 1970
1909 for(apic = 0; apic < nr_ioapics; apic++) { 1971 for(apic = 0; apic < nr_ioapics; apic++) {
@@ -1960,7 +2022,7 @@ void disable_IO_APIC(void)
1960 */ 2022 */
1961 clear_IO_APIC(); 2023 clear_IO_APIC();
1962 2024
1963 if (!nr_legacy_irqs) 2025 if (!legacy_pic->nr_legacy_irqs)
1964 return; 2026 return;
1965 2027
1966 /* 2028 /*
@@ -2031,7 +2093,7 @@ void __init setup_ioapic_ids_from_mpc(void)
2031 * This is broken; anything with a real cpu count has to 2093 * This is broken; anything with a real cpu count has to
2032 * circumvent this idiocy regardless. 2094 * circumvent this idiocy regardless.
2033 */ 2095 */
2034 phys_id_present_map = apic->ioapic_phys_id_map(phys_cpu_present_map); 2096 apic->ioapic_phys_id_map(&phys_cpu_present_map, &phys_id_present_map);
2035 2097
2036 /* 2098 /*
2037 * Set the IOAPIC ID to the value stored in the MPC table. 2099 * Set the IOAPIC ID to the value stored in the MPC table.
@@ -2039,9 +2101,9 @@ void __init setup_ioapic_ids_from_mpc(void)
2039 for (apic_id = 0; apic_id < nr_ioapics; apic_id++) { 2101 for (apic_id = 0; apic_id < nr_ioapics; apic_id++) {
2040 2102
2041 /* Read the register 0 value */ 2103 /* Read the register 0 value */
2042 spin_lock_irqsave(&ioapic_lock, flags); 2104 raw_spin_lock_irqsave(&ioapic_lock, flags);
2043 reg_00.raw = io_apic_read(apic_id, 0); 2105 reg_00.raw = io_apic_read(apic_id, 0);
2044 spin_unlock_irqrestore(&ioapic_lock, flags); 2106 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2045 2107
2046 old_id = mp_ioapics[apic_id].apicid; 2108 old_id = mp_ioapics[apic_id].apicid;
2047 2109
@@ -2058,7 +2120,7 @@ void __init setup_ioapic_ids_from_mpc(void)
2058 * system must have a unique ID or we get lots of nice 2120 * system must have a unique ID or we get lots of nice
2059 * 'stuck on smp_invalidate_needed IPI wait' messages. 2121 * 'stuck on smp_invalidate_needed IPI wait' messages.
2060 */ 2122 */
2061 if (apic->check_apicid_used(phys_id_present_map, 2123 if (apic->check_apicid_used(&phys_id_present_map,
2062 mp_ioapics[apic_id].apicid)) { 2124 mp_ioapics[apic_id].apicid)) {
2063 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n", 2125 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
2064 apic_id, mp_ioapics[apic_id].apicid); 2126 apic_id, mp_ioapics[apic_id].apicid);
@@ -2073,7 +2135,7 @@ void __init setup_ioapic_ids_from_mpc(void)
2073 mp_ioapics[apic_id].apicid = i; 2135 mp_ioapics[apic_id].apicid = i;
2074 } else { 2136 } else {
2075 physid_mask_t tmp; 2137 physid_mask_t tmp;
2076 tmp = apic->apicid_to_cpu_present(mp_ioapics[apic_id].apicid); 2138 apic->apicid_to_cpu_present(mp_ioapics[apic_id].apicid, &tmp);
2077 apic_printk(APIC_VERBOSE, "Setting %d in the " 2139 apic_printk(APIC_VERBOSE, "Setting %d in the "
2078 "phys_id_present_map\n", 2140 "phys_id_present_map\n",
2079 mp_ioapics[apic_id].apicid); 2141 mp_ioapics[apic_id].apicid);
@@ -2100,16 +2162,16 @@ void __init setup_ioapic_ids_from_mpc(void)
2100 mp_ioapics[apic_id].apicid); 2162 mp_ioapics[apic_id].apicid);
2101 2163
2102 reg_00.bits.ID = mp_ioapics[apic_id].apicid; 2164 reg_00.bits.ID = mp_ioapics[apic_id].apicid;
2103 spin_lock_irqsave(&ioapic_lock, flags); 2165 raw_spin_lock_irqsave(&ioapic_lock, flags);
2104 io_apic_write(apic_id, 0, reg_00.raw); 2166 io_apic_write(apic_id, 0, reg_00.raw);
2105 spin_unlock_irqrestore(&ioapic_lock, flags); 2167 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2106 2168
2107 /* 2169 /*
2108 * Sanity check 2170 * Sanity check
2109 */ 2171 */
2110 spin_lock_irqsave(&ioapic_lock, flags); 2172 raw_spin_lock_irqsave(&ioapic_lock, flags);
2111 reg_00.raw = io_apic_read(apic_id, 0); 2173 reg_00.raw = io_apic_read(apic_id, 0);
2112 spin_unlock_irqrestore(&ioapic_lock, flags); 2174 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2113 if (reg_00.bits.ID != mp_ioapics[apic_id].apicid) 2175 if (reg_00.bits.ID != mp_ioapics[apic_id].apicid)
2114 printk("could not set ID!\n"); 2176 printk("could not set ID!\n");
2115 else 2177 else
@@ -2192,15 +2254,15 @@ static unsigned int startup_ioapic_irq(unsigned int irq)
2192 unsigned long flags; 2254 unsigned long flags;
2193 struct irq_cfg *cfg; 2255 struct irq_cfg *cfg;
2194 2256
2195 spin_lock_irqsave(&ioapic_lock, flags); 2257 raw_spin_lock_irqsave(&ioapic_lock, flags);
2196 if (irq < nr_legacy_irqs) { 2258 if (irq < legacy_pic->nr_legacy_irqs) {
2197 disable_8259A_irq(irq); 2259 legacy_pic->chip->mask(irq);
2198 if (i8259A_irq_pending(irq)) 2260 if (legacy_pic->irq_pending(irq))
2199 was_pending = 1; 2261 was_pending = 1;
2200 } 2262 }
2201 cfg = irq_cfg(irq); 2263 cfg = irq_cfg(irq);
2202 __unmask_IO_APIC_irq(cfg); 2264 __unmask_IO_APIC_irq(cfg);
2203 spin_unlock_irqrestore(&ioapic_lock, flags); 2265 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2204 2266
2205 return was_pending; 2267 return was_pending;
2206} 2268}
@@ -2211,9 +2273,9 @@ static int ioapic_retrigger_irq(unsigned int irq)
2211 struct irq_cfg *cfg = irq_cfg(irq); 2273 struct irq_cfg *cfg = irq_cfg(irq);
2212 unsigned long flags; 2274 unsigned long flags;
2213 2275
2214 spin_lock_irqsave(&vector_lock, flags); 2276 raw_spin_lock_irqsave(&vector_lock, flags);
2215 apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector); 2277 apic->send_IPI_mask(cpumask_of(cpumask_first(cfg->domain)), cfg->vector);
2216 spin_unlock_irqrestore(&vector_lock, flags); 2278 raw_spin_unlock_irqrestore(&vector_lock, flags);
2217 2279
2218 return 1; 2280 return 1;
2219} 2281}
@@ -2228,20 +2290,16 @@ static int ioapic_retrigger_irq(unsigned int irq)
2228 */ 2290 */
2229 2291
2230#ifdef CONFIG_SMP 2292#ifdef CONFIG_SMP
2231static void send_cleanup_vector(struct irq_cfg *cfg) 2293void send_cleanup_vector(struct irq_cfg *cfg)
2232{ 2294{
2233 cpumask_var_t cleanup_mask; 2295 cpumask_var_t cleanup_mask;
2234 2296
2235 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) { 2297 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2236 unsigned int i; 2298 unsigned int i;
2237 cfg->move_cleanup_count = 0;
2238 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2239 cfg->move_cleanup_count++;
2240 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask) 2299 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2241 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR); 2300 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2242 } else { 2301 } else {
2243 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask); 2302 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2244 cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
2245 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR); 2303 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2246 free_cpumask_var(cleanup_mask); 2304 free_cpumask_var(cleanup_mask);
2247 } 2305 }
@@ -2272,31 +2330,30 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq
2272 } 2330 }
2273} 2331}
2274 2332
2275static int
2276assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
2277
2278/* 2333/*
2279 * Either sets desc->affinity to a valid value, and returns 2334 * Either sets desc->affinity to a valid value, and returns
2280 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and 2335 * ->cpu_mask_to_apicid of that in dest_id, or returns -1 and
2281 * leaves desc->affinity untouched. 2336 * leaves desc->affinity untouched.
2282 */ 2337 */
2283static unsigned int 2338unsigned int
2284set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask) 2339set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask,
2340 unsigned int *dest_id)
2285{ 2341{
2286 struct irq_cfg *cfg; 2342 struct irq_cfg *cfg;
2287 unsigned int irq; 2343 unsigned int irq;
2288 2344
2289 if (!cpumask_intersects(mask, cpu_online_mask)) 2345 if (!cpumask_intersects(mask, cpu_online_mask))
2290 return BAD_APICID; 2346 return -1;
2291 2347
2292 irq = desc->irq; 2348 irq = desc->irq;
2293 cfg = desc->chip_data; 2349 cfg = desc->chip_data;
2294 if (assign_irq_vector(irq, cfg, mask)) 2350 if (assign_irq_vector(irq, cfg, mask))
2295 return BAD_APICID; 2351 return -1;
2296 2352
2297 cpumask_copy(desc->affinity, mask); 2353 cpumask_copy(desc->affinity, mask);
2298 2354
2299 return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain); 2355 *dest_id = apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
2356 return 0;
2300} 2357}
2301 2358
2302static int 2359static int
@@ -2311,15 +2368,14 @@ set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2311 irq = desc->irq; 2368 irq = desc->irq;
2312 cfg = desc->chip_data; 2369 cfg = desc->chip_data;
2313 2370
2314 spin_lock_irqsave(&ioapic_lock, flags); 2371 raw_spin_lock_irqsave(&ioapic_lock, flags);
2315 dest = set_desc_affinity(desc, mask); 2372 ret = set_desc_affinity(desc, mask, &dest);
2316 if (dest != BAD_APICID) { 2373 if (!ret) {
2317 /* Only the high 8 bits are valid. */ 2374 /* Only the high 8 bits are valid. */
2318 dest = SET_APIC_LOGICAL_ID(dest); 2375 dest = SET_APIC_LOGICAL_ID(dest);
2319 __target_IO_APIC_irq(irq, dest, cfg); 2376 __target_IO_APIC_irq(irq, dest, cfg);
2320 ret = 0;
2321 } 2377 }
2322 spin_unlock_irqrestore(&ioapic_lock, flags); 2378 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2323 2379
2324 return ret; 2380 return ret;
2325} 2381}
@@ -2432,8 +2488,13 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void)
2432 continue; 2488 continue;
2433 2489
2434 cfg = irq_cfg(irq); 2490 cfg = irq_cfg(irq);
2435 spin_lock(&desc->lock); 2491 raw_spin_lock(&desc->lock);
2436 if (!cfg->move_cleanup_count) 2492
2493 /*
2494 * Check if the irq migration is in progress. If so, we
2495 * haven't received the cleanup request yet for this irq.
2496 */
2497 if (cfg->move_in_progress)
2437 goto unlock; 2498 goto unlock;
2438 2499
2439 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) 2500 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
@@ -2452,29 +2513,43 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void)
2452 goto unlock; 2513 goto unlock;
2453 } 2514 }
2454 __get_cpu_var(vector_irq)[vector] = -1; 2515 __get_cpu_var(vector_irq)[vector] = -1;
2455 cfg->move_cleanup_count--;
2456unlock: 2516unlock:
2457 spin_unlock(&desc->lock); 2517 raw_spin_unlock(&desc->lock);
2458 } 2518 }
2459 2519
2460 irq_exit(); 2520 irq_exit();
2461} 2521}
2462 2522
2463static void irq_complete_move(struct irq_desc **descp) 2523static void __irq_complete_move(struct irq_desc **descp, unsigned vector)
2464{ 2524{
2465 struct irq_desc *desc = *descp; 2525 struct irq_desc *desc = *descp;
2466 struct irq_cfg *cfg = desc->chip_data; 2526 struct irq_cfg *cfg = desc->chip_data;
2467 unsigned vector, me; 2527 unsigned me;
2468 2528
2469 if (likely(!cfg->move_in_progress)) 2529 if (likely(!cfg->move_in_progress))
2470 return; 2530 return;
2471 2531
2472 vector = ~get_irq_regs()->orig_ax;
2473 me = smp_processor_id(); 2532 me = smp_processor_id();
2474 2533
2475 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) 2534 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2476 send_cleanup_vector(cfg); 2535 send_cleanup_vector(cfg);
2477} 2536}
2537
2538static void irq_complete_move(struct irq_desc **descp)
2539{
2540 __irq_complete_move(descp, ~get_irq_regs()->orig_ax);
2541}
2542
2543void irq_force_complete_move(int irq)
2544{
2545 struct irq_desc *desc = irq_to_desc(irq);
2546 struct irq_cfg *cfg = desc->chip_data;
2547
2548 if (!cfg)
2549 return;
2550
2551 __irq_complete_move(&desc, cfg->vector);
2552}
2478#else 2553#else
2479static inline void irq_complete_move(struct irq_desc **descp) {} 2554static inline void irq_complete_move(struct irq_desc **descp) {}
2480#endif 2555#endif
@@ -2490,6 +2565,59 @@ static void ack_apic_edge(unsigned int irq)
2490 2565
2491atomic_t irq_mis_count; 2566atomic_t irq_mis_count;
2492 2567
2568/*
2569 * IO-APIC versions below 0x20 don't support EOI register.
2570 * For the record, here is the information about various versions:
2571 * 0Xh 82489DX
2572 * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant
2573 * 2Xh I/O(x)APIC which is PCI 2.2 Compliant
2574 * 30h-FFh Reserved
2575 *
2576 * Some of the Intel ICH Specs (ICH2 to ICH5) documents the io-apic
2577 * version as 0x2. This is an error with documentation and these ICH chips
2578 * use io-apic's of version 0x20.
2579 *
2580 * For IO-APIC's with EOI register, we use that to do an explicit EOI.
2581 * Otherwise, we simulate the EOI message manually by changing the trigger
2582 * mode to edge and then back to level, with RTE being masked during this.
2583*/
2584static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2585{
2586 struct irq_pin_list *entry;
2587
2588 for_each_irq_pin(entry, cfg->irq_2_pin) {
2589 if (mp_ioapics[entry->apic].apicver >= 0x20) {
2590 /*
2591 * Intr-remapping uses pin number as the virtual vector
2592 * in the RTE. Actual vector is programmed in
2593 * intr-remapping table entry. Hence for the io-apic
2594 * EOI we use the pin number.
2595 */
2596 if (irq_remapped(irq))
2597 io_apic_eoi(entry->apic, entry->pin);
2598 else
2599 io_apic_eoi(entry->apic, cfg->vector);
2600 } else {
2601 __mask_and_edge_IO_APIC_irq(entry);
2602 __unmask_and_level_IO_APIC_irq(entry);
2603 }
2604 }
2605}
2606
2607static void eoi_ioapic_irq(struct irq_desc *desc)
2608{
2609 struct irq_cfg *cfg;
2610 unsigned long flags;
2611 unsigned int irq;
2612
2613 irq = desc->irq;
2614 cfg = desc->chip_data;
2615
2616 raw_spin_lock_irqsave(&ioapic_lock, flags);
2617 __eoi_ioapic_irq(irq, cfg);
2618 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
2619}
2620
2493static void ack_apic_level(unsigned int irq) 2621static void ack_apic_level(unsigned int irq)
2494{ 2622{
2495 struct irq_desc *desc = irq_to_desc(irq); 2623 struct irq_desc *desc = irq_to_desc(irq);
@@ -2525,6 +2653,19 @@ static void ack_apic_level(unsigned int irq)
2525 * level-triggered interrupt. We mask the source for the time of the 2653 * level-triggered interrupt. We mask the source for the time of the
2526 * operation to prevent an edge-triggered interrupt escaping meanwhile. 2654 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2527 * The idea is from Manfred Spraul. --macro 2655 * The idea is from Manfred Spraul. --macro
2656 *
2657 * Also in the case when cpu goes offline, fixup_irqs() will forward
2658 * any unhandled interrupt on the offlined cpu to the new cpu
2659 * destination that is handling the corresponding interrupt. This
2660 * interrupt forwarding is done via IPI's. Hence, in this case also
2661 * level-triggered io-apic interrupt will be seen as an edge
2662 * interrupt in the IRR. And we can't rely on the cpu's EOI
2663 * to be broadcasted to the IO-APIC's which will clear the remoteIRR
2664 * corresponding to the level-triggered interrupt. Hence on IO-APIC's
2665 * supporting EOI register, we do an explicit EOI to clear the
2666 * remote IRR and on IO-APIC's which don't have an EOI register,
2667 * we use the above logic (mask+edge followed by unmask+level) from
2668 * Manfred Spraul to clear the remote IRR.
2528 */ 2669 */
2529 cfg = desc->chip_data; 2670 cfg = desc->chip_data;
2530 i = cfg->vector; 2671 i = cfg->vector;
@@ -2536,6 +2677,19 @@ static void ack_apic_level(unsigned int irq)
2536 */ 2677 */
2537 ack_APIC_irq(); 2678 ack_APIC_irq();
2538 2679
2680 /*
2681 * Tail end of clearing remote IRR bit (either by delivering the EOI
2682 * message via io-apic EOI register write or simulating it using
2683 * mask+edge followed by unnask+level logic) manually when the
2684 * level triggered interrupt is seen as the edge triggered interrupt
2685 * at the cpu.
2686 */
2687 if (!(v & (1 << (i & 0x1f)))) {
2688 atomic_inc(&irq_mis_count);
2689
2690 eoi_ioapic_irq(desc);
2691 }
2692
2539 /* Now we can move and renable the irq */ 2693 /* Now we can move and renable the irq */
2540 if (unlikely(do_unmask_irq)) { 2694 if (unlikely(do_unmask_irq)) {
2541 /* Only migrate the irq if the ack has been received. 2695 /* Only migrate the irq if the ack has been received.
@@ -2569,41 +2723,9 @@ static void ack_apic_level(unsigned int irq)
2569 move_masked_irq(irq); 2723 move_masked_irq(irq);
2570 unmask_IO_APIC_irq_desc(desc); 2724 unmask_IO_APIC_irq_desc(desc);
2571 } 2725 }
2572
2573 /* Tail end of version 0x11 I/O APIC bug workaround */
2574 if (!(v & (1 << (i & 0x1f)))) {
2575 atomic_inc(&irq_mis_count);
2576 spin_lock(&ioapic_lock);
2577 __mask_and_edge_IO_APIC_irq(cfg);
2578 __unmask_and_level_IO_APIC_irq(cfg);
2579 spin_unlock(&ioapic_lock);
2580 }
2581} 2726}
2582 2727
2583#ifdef CONFIG_INTR_REMAP 2728#ifdef CONFIG_INTR_REMAP
2584static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2585{
2586 struct irq_pin_list *entry;
2587
2588 for_each_irq_pin(entry, cfg->irq_2_pin)
2589 io_apic_eoi(entry->apic, entry->pin);
2590}
2591
2592static void
2593eoi_ioapic_irq(struct irq_desc *desc)
2594{
2595 struct irq_cfg *cfg;
2596 unsigned long flags;
2597 unsigned int irq;
2598
2599 irq = desc->irq;
2600 cfg = desc->chip_data;
2601
2602 spin_lock_irqsave(&ioapic_lock, flags);
2603 __eoi_ioapic_irq(irq, cfg);
2604 spin_unlock_irqrestore(&ioapic_lock, flags);
2605}
2606
2607static void ir_ack_apic_edge(unsigned int irq) 2729static void ir_ack_apic_edge(unsigned int irq)
2608{ 2730{
2609 ack_APIC_irq(); 2731 ack_APIC_irq();
@@ -2671,8 +2793,8 @@ static inline void init_IO_APIC_traps(void)
2671 * so default to an old-fashioned 8259 2793 * so default to an old-fashioned 8259
2672 * interrupt if we can.. 2794 * interrupt if we can..
2673 */ 2795 */
2674 if (irq < nr_legacy_irqs) 2796 if (irq < legacy_pic->nr_legacy_irqs)
2675 make_8259A_irq(irq); 2797 legacy_pic->make_irq(irq);
2676 else 2798 else
2677 /* Strange. Oh, well.. */ 2799 /* Strange. Oh, well.. */
2678 desc->chip = &no_irq_chip; 2800 desc->chip = &no_irq_chip;
@@ -2829,7 +2951,7 @@ static inline void __init check_timer(void)
2829 /* 2951 /*
2830 * get/set the timer IRQ vector: 2952 * get/set the timer IRQ vector:
2831 */ 2953 */
2832 disable_8259A_irq(0); 2954 legacy_pic->chip->mask(0);
2833 assign_irq_vector(0, cfg, apic->target_cpus()); 2955 assign_irq_vector(0, cfg, apic->target_cpus());
2834 2956
2835 /* 2957 /*
@@ -2842,7 +2964,7 @@ static inline void __init check_timer(void)
2842 * automatically. 2964 * automatically.
2843 */ 2965 */
2844 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT); 2966 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
2845 init_8259A(1); 2967 legacy_pic->init(1);
2846#ifdef CONFIG_X86_32 2968#ifdef CONFIG_X86_32
2847 { 2969 {
2848 unsigned int ver; 2970 unsigned int ver;
@@ -2901,7 +3023,7 @@ static inline void __init check_timer(void)
2901 if (timer_irq_works()) { 3023 if (timer_irq_works()) {
2902 if (nmi_watchdog == NMI_IO_APIC) { 3024 if (nmi_watchdog == NMI_IO_APIC) {
2903 setup_nmi(); 3025 setup_nmi();
2904 enable_8259A_irq(0); 3026 legacy_pic->chip->unmask(0);
2905 } 3027 }
2906 if (disable_timer_pin_1 > 0) 3028 if (disable_timer_pin_1 > 0)
2907 clear_IO_APIC_pin(0, pin1); 3029 clear_IO_APIC_pin(0, pin1);
@@ -2924,14 +3046,14 @@ static inline void __init check_timer(void)
2924 */ 3046 */
2925 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2); 3047 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
2926 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector); 3048 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
2927 enable_8259A_irq(0); 3049 legacy_pic->chip->unmask(0);
2928 if (timer_irq_works()) { 3050 if (timer_irq_works()) {
2929 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n"); 3051 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
2930 timer_through_8259 = 1; 3052 timer_through_8259 = 1;
2931 if (nmi_watchdog == NMI_IO_APIC) { 3053 if (nmi_watchdog == NMI_IO_APIC) {
2932 disable_8259A_irq(0); 3054 legacy_pic->chip->mask(0);
2933 setup_nmi(); 3055 setup_nmi();
2934 enable_8259A_irq(0); 3056 legacy_pic->chip->unmask(0);
2935 } 3057 }
2936 goto out; 3058 goto out;
2937 } 3059 }
@@ -2939,7 +3061,7 @@ static inline void __init check_timer(void)
2939 * Cleanup, just in case ... 3061 * Cleanup, just in case ...
2940 */ 3062 */
2941 local_irq_disable(); 3063 local_irq_disable();
2942 disable_8259A_irq(0); 3064 legacy_pic->chip->mask(0);
2943 clear_IO_APIC_pin(apic2, pin2); 3065 clear_IO_APIC_pin(apic2, pin2);
2944 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n"); 3066 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
2945 } 3067 }
@@ -2958,22 +3080,22 @@ static inline void __init check_timer(void)
2958 3080
2959 lapic_register_intr(0, desc); 3081 lapic_register_intr(0, desc);
2960 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */ 3082 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2961 enable_8259A_irq(0); 3083 legacy_pic->chip->unmask(0);
2962 3084
2963 if (timer_irq_works()) { 3085 if (timer_irq_works()) {
2964 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n"); 3086 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
2965 goto out; 3087 goto out;
2966 } 3088 }
2967 local_irq_disable(); 3089 local_irq_disable();
2968 disable_8259A_irq(0); 3090 legacy_pic->chip->mask(0);
2969 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector); 3091 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
2970 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n"); 3092 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
2971 3093
2972 apic_printk(APIC_QUIET, KERN_INFO 3094 apic_printk(APIC_QUIET, KERN_INFO
2973 "...trying to set up timer as ExtINT IRQ...\n"); 3095 "...trying to set up timer as ExtINT IRQ...\n");
2974 3096
2975 init_8259A(0); 3097 legacy_pic->init(0);
2976 make_8259A_irq(0); 3098 legacy_pic->make_irq(0);
2977 apic_write(APIC_LVT0, APIC_DM_EXTINT); 3099 apic_write(APIC_LVT0, APIC_DM_EXTINT);
2978 3100
2979 unlock_ExtINT_logic(); 3101 unlock_ExtINT_logic();
@@ -3015,7 +3137,7 @@ void __init setup_IO_APIC(void)
3015 /* 3137 /*
3016 * calling enable_IO_APIC() is moved to setup_local_APIC for BP 3138 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
3017 */ 3139 */
3018 io_apic_irqs = nr_legacy_irqs ? ~PIC_IRQS : ~0UL; 3140 io_apic_irqs = legacy_pic->nr_legacy_irqs ? ~PIC_IRQS : ~0UL;
3019 3141
3020 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n"); 3142 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
3021 /* 3143 /*
@@ -3026,7 +3148,7 @@ void __init setup_IO_APIC(void)
3026 sync_Arb_IDs(); 3148 sync_Arb_IDs();
3027 setup_IO_APIC_irqs(); 3149 setup_IO_APIC_irqs();
3028 init_IO_APIC_traps(); 3150 init_IO_APIC_traps();
3029 if (nr_legacy_irqs) 3151 if (legacy_pic->nr_legacy_irqs)
3030 check_timer(); 3152 check_timer();
3031} 3153}
3032 3154
@@ -3075,13 +3197,13 @@ static int ioapic_resume(struct sys_device *dev)
3075 data = container_of(dev, struct sysfs_ioapic_data, dev); 3197 data = container_of(dev, struct sysfs_ioapic_data, dev);
3076 entry = data->entry; 3198 entry = data->entry;
3077 3199
3078 spin_lock_irqsave(&ioapic_lock, flags); 3200 raw_spin_lock_irqsave(&ioapic_lock, flags);
3079 reg_00.raw = io_apic_read(dev->id, 0); 3201 reg_00.raw = io_apic_read(dev->id, 0);
3080 if (reg_00.bits.ID != mp_ioapics[dev->id].apicid) { 3202 if (reg_00.bits.ID != mp_ioapics[dev->id].apicid) {
3081 reg_00.bits.ID = mp_ioapics[dev->id].apicid; 3203 reg_00.bits.ID = mp_ioapics[dev->id].apicid;
3082 io_apic_write(dev->id, 0, reg_00.raw); 3204 io_apic_write(dev->id, 0, reg_00.raw);
3083 } 3205 }
3084 spin_unlock_irqrestore(&ioapic_lock, flags); 3206 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3085 for (i = 0; i < nr_ioapic_registers[dev->id]; i++) 3207 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
3086 ioapic_write_entry(dev->id, i, entry[i]); 3208 ioapic_write_entry(dev->id, i, entry[i]);
3087 3209
@@ -3144,7 +3266,7 @@ unsigned int create_irq_nr(unsigned int irq_want, int node)
3144 if (irq_want < nr_irqs_gsi) 3266 if (irq_want < nr_irqs_gsi)
3145 irq_want = nr_irqs_gsi; 3267 irq_want = nr_irqs_gsi;
3146 3268
3147 spin_lock_irqsave(&vector_lock, flags); 3269 raw_spin_lock_irqsave(&vector_lock, flags);
3148 for (new = irq_want; new < nr_irqs; new++) { 3270 for (new = irq_want; new < nr_irqs; new++) {
3149 desc_new = irq_to_desc_alloc_node(new, node); 3271 desc_new = irq_to_desc_alloc_node(new, node);
3150 if (!desc_new) { 3272 if (!desc_new) {
@@ -3157,19 +3279,17 @@ unsigned int create_irq_nr(unsigned int irq_want, int node)
3157 continue; 3279 continue;
3158 3280
3159 desc_new = move_irq_desc(desc_new, node); 3281 desc_new = move_irq_desc(desc_new, node);
3282 cfg_new = desc_new->chip_data;
3160 3283
3161 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0) 3284 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0)
3162 irq = new; 3285 irq = new;
3163 break; 3286 break;
3164 } 3287 }
3165 spin_unlock_irqrestore(&vector_lock, flags); 3288 raw_spin_unlock_irqrestore(&vector_lock, flags);
3289
3290 if (irq > 0)
3291 dynamic_irq_init_keep_chip_data(irq);
3166 3292
3167 if (irq > 0) {
3168 dynamic_irq_init(irq);
3169 /* restore it, in case dynamic_irq_init clear it */
3170 if (desc_new)
3171 desc_new->chip_data = cfg_new;
3172 }
3173 return irq; 3293 return irq;
3174} 3294}
3175 3295
@@ -3191,27 +3311,21 @@ int create_irq(void)
3191void destroy_irq(unsigned int irq) 3311void destroy_irq(unsigned int irq)
3192{ 3312{
3193 unsigned long flags; 3313 unsigned long flags;
3194 struct irq_cfg *cfg;
3195 struct irq_desc *desc;
3196 3314
3197 /* store it, in case dynamic_irq_cleanup clear it */ 3315 dynamic_irq_cleanup_keep_chip_data(irq);
3198 desc = irq_to_desc(irq);
3199 cfg = desc->chip_data;
3200 dynamic_irq_cleanup(irq);
3201 /* connect back irq_cfg */
3202 desc->chip_data = cfg;
3203 3316
3204 free_irte(irq); 3317 free_irte(irq);
3205 spin_lock_irqsave(&vector_lock, flags); 3318 raw_spin_lock_irqsave(&vector_lock, flags);
3206 __clear_irq_vector(irq, cfg); 3319 __clear_irq_vector(irq, get_irq_chip_data(irq));
3207 spin_unlock_irqrestore(&vector_lock, flags); 3320 raw_spin_unlock_irqrestore(&vector_lock, flags);
3208} 3321}
3209 3322
3210/* 3323/*
3211 * MSI message composition 3324 * MSI message composition
3212 */ 3325 */
3213#ifdef CONFIG_PCI_MSI 3326#ifdef CONFIG_PCI_MSI
3214static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg) 3327static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq,
3328 struct msi_msg *msg, u8 hpet_id)
3215{ 3329{
3216 struct irq_cfg *cfg; 3330 struct irq_cfg *cfg;
3217 int err; 3331 int err;
@@ -3245,7 +3359,10 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
3245 irte.dest_id = IRTE_DEST(dest); 3359 irte.dest_id = IRTE_DEST(dest);
3246 3360
3247 /* Set source-id of interrupt request */ 3361 /* Set source-id of interrupt request */
3248 set_msi_sid(&irte, pdev); 3362 if (pdev)
3363 set_msi_sid(&irte, pdev);
3364 else
3365 set_hpet_sid(&irte, hpet_id);
3249 3366
3250 modify_irte(irq, &irte); 3367 modify_irte(irq, &irte);
3251 3368
@@ -3291,8 +3408,7 @@ static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3291 struct msi_msg msg; 3408 struct msi_msg msg;
3292 unsigned int dest; 3409 unsigned int dest;
3293 3410
3294 dest = set_desc_affinity(desc, mask); 3411 if (set_desc_affinity(desc, mask, &dest))
3295 if (dest == BAD_APICID)
3296 return -1; 3412 return -1;
3297 3413
3298 cfg = desc->chip_data; 3414 cfg = desc->chip_data;
@@ -3324,8 +3440,7 @@ ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3324 if (get_irte(irq, &irte)) 3440 if (get_irte(irq, &irte))
3325 return -1; 3441 return -1;
3326 3442
3327 dest = set_desc_affinity(desc, mask); 3443 if (set_desc_affinity(desc, mask, &dest))
3328 if (dest == BAD_APICID)
3329 return -1; 3444 return -1;
3330 3445
3331 irte.vector = cfg->vector; 3446 irte.vector = cfg->vector;
@@ -3410,7 +3525,7 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq)
3410 int ret; 3525 int ret;
3411 struct msi_msg msg; 3526 struct msi_msg msg;
3412 3527
3413 ret = msi_compose_msg(dev, irq, &msg); 3528 ret = msi_compose_msg(dev, irq, &msg, -1);
3414 if (ret < 0) 3529 if (ret < 0)
3415 return ret; 3530 return ret;
3416 3531
@@ -3507,8 +3622,7 @@ static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3507 struct msi_msg msg; 3622 struct msi_msg msg;
3508 unsigned int dest; 3623 unsigned int dest;
3509 3624
3510 dest = set_desc_affinity(desc, mask); 3625 if (set_desc_affinity(desc, mask, &dest))
3511 if (dest == BAD_APICID)
3512 return -1; 3626 return -1;
3513 3627
3514 cfg = desc->chip_data; 3628 cfg = desc->chip_data;
@@ -3543,7 +3657,7 @@ int arch_setup_dmar_msi(unsigned int irq)
3543 int ret; 3657 int ret;
3544 struct msi_msg msg; 3658 struct msi_msg msg;
3545 3659
3546 ret = msi_compose_msg(NULL, irq, &msg); 3660 ret = msi_compose_msg(NULL, irq, &msg, -1);
3547 if (ret < 0) 3661 if (ret < 0)
3548 return ret; 3662 return ret;
3549 dmar_msi_write(irq, &msg); 3663 dmar_msi_write(irq, &msg);
@@ -3563,8 +3677,7 @@ static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3563 struct msi_msg msg; 3677 struct msi_msg msg;
3564 unsigned int dest; 3678 unsigned int dest;
3565 3679
3566 dest = set_desc_affinity(desc, mask); 3680 if (set_desc_affinity(desc, mask, &dest))
3567 if (dest == BAD_APICID)
3568 return -1; 3681 return -1;
3569 3682
3570 cfg = desc->chip_data; 3683 cfg = desc->chip_data;
@@ -3583,6 +3696,19 @@ static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3583 3696
3584#endif /* CONFIG_SMP */ 3697#endif /* CONFIG_SMP */
3585 3698
3699static struct irq_chip ir_hpet_msi_type = {
3700 .name = "IR-HPET_MSI",
3701 .unmask = hpet_msi_unmask,
3702 .mask = hpet_msi_mask,
3703#ifdef CONFIG_INTR_REMAP
3704 .ack = ir_ack_apic_edge,
3705#ifdef CONFIG_SMP
3706 .set_affinity = ir_set_msi_irq_affinity,
3707#endif
3708#endif
3709 .retrigger = ioapic_retrigger_irq,
3710};
3711
3586static struct irq_chip hpet_msi_type = { 3712static struct irq_chip hpet_msi_type = {
3587 .name = "HPET_MSI", 3713 .name = "HPET_MSI",
3588 .unmask = hpet_msi_unmask, 3714 .unmask = hpet_msi_unmask,
@@ -3594,20 +3720,36 @@ static struct irq_chip hpet_msi_type = {
3594 .retrigger = ioapic_retrigger_irq, 3720 .retrigger = ioapic_retrigger_irq,
3595}; 3721};
3596 3722
3597int arch_setup_hpet_msi(unsigned int irq) 3723int arch_setup_hpet_msi(unsigned int irq, unsigned int id)
3598{ 3724{
3599 int ret; 3725 int ret;
3600 struct msi_msg msg; 3726 struct msi_msg msg;
3601 struct irq_desc *desc = irq_to_desc(irq); 3727 struct irq_desc *desc = irq_to_desc(irq);
3602 3728
3603 ret = msi_compose_msg(NULL, irq, &msg); 3729 if (intr_remapping_enabled) {
3730 struct intel_iommu *iommu = map_hpet_to_ir(id);
3731 int index;
3732
3733 if (!iommu)
3734 return -1;
3735
3736 index = alloc_irte(iommu, irq, 1);
3737 if (index < 0)
3738 return -1;
3739 }
3740
3741 ret = msi_compose_msg(NULL, irq, &msg, id);
3604 if (ret < 0) 3742 if (ret < 0)
3605 return ret; 3743 return ret;
3606 3744
3607 hpet_msi_write(irq, &msg); 3745 hpet_msi_write(irq, &msg);
3608 desc->status |= IRQ_MOVE_PCNTXT; 3746 desc->status |= IRQ_MOVE_PCNTXT;
3609 set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq, 3747 if (irq_remapped(irq))
3610 "edge"); 3748 set_irq_chip_and_handler_name(irq, &ir_hpet_msi_type,
3749 handle_edge_irq, "edge");
3750 else
3751 set_irq_chip_and_handler_name(irq, &hpet_msi_type,
3752 handle_edge_irq, "edge");
3611 3753
3612 return 0; 3754 return 0;
3613} 3755}
@@ -3641,8 +3783,7 @@ static int set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3641 struct irq_cfg *cfg; 3783 struct irq_cfg *cfg;
3642 unsigned int dest; 3784 unsigned int dest;
3643 3785
3644 dest = set_desc_affinity(desc, mask); 3786 if (set_desc_affinity(desc, mask, &dest))
3645 if (dest == BAD_APICID)
3646 return -1; 3787 return -1;
3647 3788
3648 cfg = desc->chip_data; 3789 cfg = desc->chip_data;
@@ -3708,83 +3849,14 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3708} 3849}
3709#endif /* CONFIG_HT_IRQ */ 3850#endif /* CONFIG_HT_IRQ */
3710 3851
3711#ifdef CONFIG_X86_UV
3712/*
3713 * Re-target the irq to the specified CPU and enable the specified MMR located
3714 * on the specified blade to allow the sending of MSIs to the specified CPU.
3715 */
3716int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3717 unsigned long mmr_offset)
3718{
3719 const struct cpumask *eligible_cpu = cpumask_of(cpu);
3720 struct irq_cfg *cfg;
3721 int mmr_pnode;
3722 unsigned long mmr_value;
3723 struct uv_IO_APIC_route_entry *entry;
3724 unsigned long flags;
3725 int err;
3726
3727 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3728
3729 cfg = irq_cfg(irq);
3730
3731 err = assign_irq_vector(irq, cfg, eligible_cpu);
3732 if (err != 0)
3733 return err;
3734
3735 spin_lock_irqsave(&vector_lock, flags);
3736 set_irq_chip_and_handler_name(irq, &uv_irq_chip, handle_percpu_irq,
3737 irq_name);
3738 spin_unlock_irqrestore(&vector_lock, flags);
3739
3740 mmr_value = 0;
3741 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3742 entry->vector = cfg->vector;
3743 entry->delivery_mode = apic->irq_delivery_mode;
3744 entry->dest_mode = apic->irq_dest_mode;
3745 entry->polarity = 0;
3746 entry->trigger = 0;
3747 entry->mask = 0;
3748 entry->dest = apic->cpu_mask_to_apicid(eligible_cpu);
3749
3750 mmr_pnode = uv_blade_to_pnode(mmr_blade);
3751 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
3752
3753 if (cfg->move_in_progress)
3754 send_cleanup_vector(cfg);
3755
3756 return irq;
3757}
3758
3759/*
3760 * Disable the specified MMR located on the specified blade so that MSIs are
3761 * longer allowed to be sent.
3762 */
3763void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
3764{
3765 unsigned long mmr_value;
3766 struct uv_IO_APIC_route_entry *entry;
3767 int mmr_pnode;
3768
3769 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3770
3771 mmr_value = 0;
3772 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3773 entry->mask = 1;
3774
3775 mmr_pnode = uv_blade_to_pnode(mmr_blade);
3776 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
3777}
3778#endif /* CONFIG_X86_64 */
3779
3780int __init io_apic_get_redir_entries (int ioapic) 3852int __init io_apic_get_redir_entries (int ioapic)
3781{ 3853{
3782 union IO_APIC_reg_01 reg_01; 3854 union IO_APIC_reg_01 reg_01;
3783 unsigned long flags; 3855 unsigned long flags;
3784 3856
3785 spin_lock_irqsave(&ioapic_lock, flags); 3857 raw_spin_lock_irqsave(&ioapic_lock, flags);
3786 reg_01.raw = io_apic_read(ioapic, 1); 3858 reg_01.raw = io_apic_read(ioapic, 1);
3787 spin_unlock_irqrestore(&ioapic_lock, flags); 3859 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3788 3860
3789 return reg_01.bits.entries; 3861 return reg_01.bits.entries;
3790} 3862}
@@ -3867,7 +3939,7 @@ static int __io_apic_set_pci_routing(struct device *dev, int irq,
3867 /* 3939 /*
3868 * IRQs < 16 are already in the irq_2_pin[] map 3940 * IRQs < 16 are already in the irq_2_pin[] map
3869 */ 3941 */
3870 if (irq >= nr_legacy_irqs) { 3942 if (irq >= legacy_pic->nr_legacy_irqs) {
3871 cfg = desc->chip_data; 3943 cfg = desc->chip_data;
3872 if (add_pin_to_irq_node_nopanic(cfg, node, ioapic, pin)) { 3944 if (add_pin_to_irq_node_nopanic(cfg, node, ioapic, pin)) {
3873 printk(KERN_INFO "can not add pin %d for irq %d\n", 3945 printk(KERN_INFO "can not add pin %d for irq %d\n",
@@ -3944,11 +4016,11 @@ int __init io_apic_get_unique_id(int ioapic, int apic_id)
3944 */ 4016 */
3945 4017
3946 if (physids_empty(apic_id_map)) 4018 if (physids_empty(apic_id_map))
3947 apic_id_map = apic->ioapic_phys_id_map(phys_cpu_present_map); 4019 apic->ioapic_phys_id_map(&phys_cpu_present_map, &apic_id_map);
3948 4020
3949 spin_lock_irqsave(&ioapic_lock, flags); 4021 raw_spin_lock_irqsave(&ioapic_lock, flags);
3950 reg_00.raw = io_apic_read(ioapic, 0); 4022 reg_00.raw = io_apic_read(ioapic, 0);
3951 spin_unlock_irqrestore(&ioapic_lock, flags); 4023 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3952 4024
3953 if (apic_id >= get_physical_broadcast()) { 4025 if (apic_id >= get_physical_broadcast()) {
3954 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying " 4026 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
@@ -3960,10 +4032,10 @@ int __init io_apic_get_unique_id(int ioapic, int apic_id)
3960 * Every APIC in a system must have a unique ID or we get lots of nice 4032 * Every APIC in a system must have a unique ID or we get lots of nice
3961 * 'stuck on smp_invalidate_needed IPI wait' messages. 4033 * 'stuck on smp_invalidate_needed IPI wait' messages.
3962 */ 4034 */
3963 if (apic->check_apicid_used(apic_id_map, apic_id)) { 4035 if (apic->check_apicid_used(&apic_id_map, apic_id)) {
3964 4036
3965 for (i = 0; i < get_physical_broadcast(); i++) { 4037 for (i = 0; i < get_physical_broadcast(); i++) {
3966 if (!apic->check_apicid_used(apic_id_map, i)) 4038 if (!apic->check_apicid_used(&apic_id_map, i))
3967 break; 4039 break;
3968 } 4040 }
3969 4041
@@ -3976,16 +4048,16 @@ int __init io_apic_get_unique_id(int ioapic, int apic_id)
3976 apic_id = i; 4048 apic_id = i;
3977 } 4049 }
3978 4050
3979 tmp = apic->apicid_to_cpu_present(apic_id); 4051 apic->apicid_to_cpu_present(apic_id, &tmp);
3980 physids_or(apic_id_map, apic_id_map, tmp); 4052 physids_or(apic_id_map, apic_id_map, tmp);
3981 4053
3982 if (reg_00.bits.ID != apic_id) { 4054 if (reg_00.bits.ID != apic_id) {
3983 reg_00.bits.ID = apic_id; 4055 reg_00.bits.ID = apic_id;
3984 4056
3985 spin_lock_irqsave(&ioapic_lock, flags); 4057 raw_spin_lock_irqsave(&ioapic_lock, flags);
3986 io_apic_write(ioapic, 0, reg_00.raw); 4058 io_apic_write(ioapic, 0, reg_00.raw);
3987 reg_00.raw = io_apic_read(ioapic, 0); 4059 reg_00.raw = io_apic_read(ioapic, 0);
3988 spin_unlock_irqrestore(&ioapic_lock, flags); 4060 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
3989 4061
3990 /* Sanity check */ 4062 /* Sanity check */
3991 if (reg_00.bits.ID != apic_id) { 4063 if (reg_00.bits.ID != apic_id) {
@@ -4006,9 +4078,9 @@ int __init io_apic_get_version(int ioapic)
4006 union IO_APIC_reg_01 reg_01; 4078 union IO_APIC_reg_01 reg_01;
4007 unsigned long flags; 4079 unsigned long flags;
4008 4080
4009 spin_lock_irqsave(&ioapic_lock, flags); 4081 raw_spin_lock_irqsave(&ioapic_lock, flags);
4010 reg_01.raw = io_apic_read(ioapic, 1); 4082 reg_01.raw = io_apic_read(ioapic, 1);
4011 spin_unlock_irqrestore(&ioapic_lock, flags); 4083 raw_spin_unlock_irqrestore(&ioapic_lock, flags);
4012 4084
4013 return reg_01.bits.version; 4085 return reg_01.bits.version;
4014} 4086}
@@ -4040,27 +4112,23 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
4040#ifdef CONFIG_SMP 4112#ifdef CONFIG_SMP
4041void __init setup_ioapic_dest(void) 4113void __init setup_ioapic_dest(void)
4042{ 4114{
4043 int pin, ioapic = 0, irq, irq_entry; 4115 int pin, ioapic, irq, irq_entry;
4044 struct irq_desc *desc; 4116 struct irq_desc *desc;
4045 const struct cpumask *mask; 4117 const struct cpumask *mask;
4046 4118
4047 if (skip_ioapic_setup == 1) 4119 if (skip_ioapic_setup == 1)
4048 return; 4120 return;
4049 4121
4050#ifdef CONFIG_ACPI 4122 for (ioapic = 0; ioapic < nr_ioapics; ioapic++)
4051 if (!acpi_disabled && acpi_ioapic) {
4052 ioapic = mp_find_ioapic(0);
4053 if (ioapic < 0)
4054 ioapic = 0;
4055 }
4056#endif
4057
4058 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) { 4123 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
4059 irq_entry = find_irq_entry(ioapic, pin, mp_INT); 4124 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
4060 if (irq_entry == -1) 4125 if (irq_entry == -1)
4061 continue; 4126 continue;
4062 irq = pin_2_irq(irq_entry, ioapic, pin); 4127 irq = pin_2_irq(irq_entry, ioapic, pin);
4063 4128
4129 if ((ioapic > 0) && (irq > 16))
4130 continue;
4131
4064 desc = irq_to_desc(irq); 4132 desc = irq_to_desc(irq);
4065 4133
4066 /* 4134 /*
@@ -4106,7 +4174,7 @@ static struct resource * __init ioapic_setup_resources(int nr_ioapics)
4106 for (i = 0; i < nr_ioapics; i++) { 4174 for (i = 0; i < nr_ioapics; i++) {
4107 res[i].name = mem; 4175 res[i].name = mem;
4108 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY; 4176 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
4109 sprintf(mem, "IOAPIC %u", i); 4177 snprintf(mem, IOAPIC_RESOURCE_NAME_SIZE, "IOAPIC %u", i);
4110 mem += IOAPIC_RESOURCE_NAME_SIZE; 4178 mem += IOAPIC_RESOURCE_NAME_SIZE;
4111 } 4179 }
4112 4180
@@ -4140,18 +4208,17 @@ void __init ioapic_init_mappings(void)
4140#ifdef CONFIG_X86_32 4208#ifdef CONFIG_X86_32
4141fake_ioapic_page: 4209fake_ioapic_page:
4142#endif 4210#endif
4143 ioapic_phys = (unsigned long) 4211 ioapic_phys = (unsigned long)alloc_bootmem_pages(PAGE_SIZE);
4144 alloc_bootmem_pages(PAGE_SIZE);
4145 ioapic_phys = __pa(ioapic_phys); 4212 ioapic_phys = __pa(ioapic_phys);
4146 } 4213 }
4147 set_fixmap_nocache(idx, ioapic_phys); 4214 set_fixmap_nocache(idx, ioapic_phys);
4148 apic_printk(APIC_VERBOSE, 4215 apic_printk(APIC_VERBOSE, "mapped IOAPIC to %08lx (%08lx)\n",
4149 "mapped IOAPIC to %08lx (%08lx)\n", 4216 __fix_to_virt(idx) + (ioapic_phys & ~PAGE_MASK),
4150 __fix_to_virt(idx), ioapic_phys); 4217 ioapic_phys);
4151 idx++; 4218 idx++;
4152 4219
4153 ioapic_res->start = ioapic_phys; 4220 ioapic_res->start = ioapic_phys;
4154 ioapic_res->end = ioapic_phys + (4 * 1024) - 1; 4221 ioapic_res->end = ioapic_phys + IO_APIC_SLOT_SIZE - 1;
4155 ioapic_res++; 4222 ioapic_res++;
4156 } 4223 }
4157} 4224}
@@ -4246,3 +4313,24 @@ void __init mp_register_ioapic(int id, u32 address, u32 gsi_base)
4246 4313
4247 nr_ioapics++; 4314 nr_ioapics++;
4248} 4315}
4316
4317/* Enable IOAPIC early just for system timer */
4318void __init pre_init_apic_IRQ0(void)
4319{
4320 struct irq_cfg *cfg;
4321 struct irq_desc *desc;
4322
4323 printk(KERN_INFO "Early APIC setup for system timer0\n");
4324#ifndef CONFIG_SMP
4325 phys_cpu_present_map = physid_mask_of_physid(boot_cpu_physical_apicid);
4326#endif
4327 desc = irq_to_desc_alloc_node(0, 0);
4328
4329 setup_local_APIC();
4330
4331 cfg = irq_cfg(0);
4332 add_pin_to_irq_node(cfg, 0, 0, 0);
4333 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
4334
4335 setup_IO_APIC_irq(0, 0, 0, desc, 0, 0);
4336}
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-05 16:20:40 -0500