aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/apic/io_apic.c
diff options
context:
space:
mode:
authorH. Peter Anvin <hpa@zytor.com>2009-06-01 18:13:02 -0400
committerH. Peter Anvin <hpa@zytor.com>2009-06-01 18:25:31 -0400
commit48b1fddbb100a64f3983ca9768b8ea629a09aa20 (patch)
tree7cf552c623cb0aa0dd945df55b7eaf5a0697e277 /arch/x86/kernel/apic/io_apic.c
parent38736072d45488fd45f076388b6570419bbbc682 (diff)
parentee4c24a5c9b530481394132c8dbc10572d57c075 (diff)
Merge branch 'irq/numa' into x86/mce3
Merge reason: arch/x86/kernel/irqinit_{32,64}.c unified in irq/numa and modified in x86/mce3; this merge resolves the conflict. Conflicts: arch/x86/kernel/irqinit.c Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Diffstat (limited to 'arch/x86/kernel/apic/io_apic.c')
-rw-r--r--arch/x86/kernel/apic/io_apic.c899
1 files changed, 446 insertions, 453 deletions
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 30da617d18e4..ac7f3b6ad583 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -129,12 +129,9 @@ struct irq_pin_list {
129 struct irq_pin_list *next; 129 struct irq_pin_list *next;
130}; 130};
131 131
132static struct irq_pin_list *get_one_free_irq_2_pin(int cpu) 132static struct irq_pin_list *get_one_free_irq_2_pin(int node)
133{ 133{
134 struct irq_pin_list *pin; 134 struct irq_pin_list *pin;
135 int node;
136
137 node = cpu_to_node(cpu);
138 135
139 pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node); 136 pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
140 137
@@ -148,9 +145,6 @@ struct irq_cfg {
148 unsigned move_cleanup_count; 145 unsigned move_cleanup_count;
149 u8 vector; 146 u8 vector;
150 u8 move_in_progress : 1; 147 u8 move_in_progress : 1;
151#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
152 u8 move_desc_pending : 1;
153#endif
154}; 148};
155 149
156/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ 150/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
@@ -212,12 +206,9 @@ static struct irq_cfg *irq_cfg(unsigned int irq)
212 return cfg; 206 return cfg;
213} 207}
214 208
215static struct irq_cfg *get_one_free_irq_cfg(int cpu) 209static struct irq_cfg *get_one_free_irq_cfg(int node)
216{ 210{
217 struct irq_cfg *cfg; 211 struct irq_cfg *cfg;
218 int node;
219
220 node = cpu_to_node(cpu);
221 212
222 cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node); 213 cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
223 if (cfg) { 214 if (cfg) {
@@ -238,13 +229,13 @@ static struct irq_cfg *get_one_free_irq_cfg(int cpu)
238 return cfg; 229 return cfg;
239} 230}
240 231
241int arch_init_chip_data(struct irq_desc *desc, int cpu) 232int arch_init_chip_data(struct irq_desc *desc, int node)
242{ 233{
243 struct irq_cfg *cfg; 234 struct irq_cfg *cfg;
244 235
245 cfg = desc->chip_data; 236 cfg = desc->chip_data;
246 if (!cfg) { 237 if (!cfg) {
247 desc->chip_data = get_one_free_irq_cfg(cpu); 238 desc->chip_data = get_one_free_irq_cfg(node);
248 if (!desc->chip_data) { 239 if (!desc->chip_data) {
249 printk(KERN_ERR "can not alloc irq_cfg\n"); 240 printk(KERN_ERR "can not alloc irq_cfg\n");
250 BUG_ON(1); 241 BUG_ON(1);
@@ -254,10 +245,9 @@ int arch_init_chip_data(struct irq_desc *desc, int cpu)
254 return 0; 245 return 0;
255} 246}
256 247
257#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC 248/* for move_irq_desc */
258
259static void 249static void
260init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu) 250init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int node)
261{ 251{
262 struct irq_pin_list *old_entry, *head, *tail, *entry; 252 struct irq_pin_list *old_entry, *head, *tail, *entry;
263 253
@@ -266,7 +256,7 @@ init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
266 if (!old_entry) 256 if (!old_entry)
267 return; 257 return;
268 258
269 entry = get_one_free_irq_2_pin(cpu); 259 entry = get_one_free_irq_2_pin(node);
270 if (!entry) 260 if (!entry)
271 return; 261 return;
272 262
@@ -276,7 +266,7 @@ init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
276 tail = entry; 266 tail = entry;
277 old_entry = old_entry->next; 267 old_entry = old_entry->next;
278 while (old_entry) { 268 while (old_entry) {
279 entry = get_one_free_irq_2_pin(cpu); 269 entry = get_one_free_irq_2_pin(node);
280 if (!entry) { 270 if (!entry) {
281 entry = head; 271 entry = head;
282 while (entry) { 272 while (entry) {
@@ -316,12 +306,12 @@ static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
316} 306}
317 307
318void arch_init_copy_chip_data(struct irq_desc *old_desc, 308void arch_init_copy_chip_data(struct irq_desc *old_desc,
319 struct irq_desc *desc, int cpu) 309 struct irq_desc *desc, int node)
320{ 310{
321 struct irq_cfg *cfg; 311 struct irq_cfg *cfg;
322 struct irq_cfg *old_cfg; 312 struct irq_cfg *old_cfg;
323 313
324 cfg = get_one_free_irq_cfg(cpu); 314 cfg = get_one_free_irq_cfg(node);
325 315
326 if (!cfg) 316 if (!cfg)
327 return; 317 return;
@@ -332,7 +322,7 @@ void arch_init_copy_chip_data(struct irq_desc *old_desc,
332 322
333 memcpy(cfg, old_cfg, sizeof(struct irq_cfg)); 323 memcpy(cfg, old_cfg, sizeof(struct irq_cfg));
334 324
335 init_copy_irq_2_pin(old_cfg, cfg, cpu); 325 init_copy_irq_2_pin(old_cfg, cfg, node);
336} 326}
337 327
338static void free_irq_cfg(struct irq_cfg *old_cfg) 328static void free_irq_cfg(struct irq_cfg *old_cfg)
@@ -356,19 +346,7 @@ void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
356 old_desc->chip_data = NULL; 346 old_desc->chip_data = NULL;
357 } 347 }
358} 348}
359 349/* end for move_irq_desc */
360static void
361set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
362{
363 struct irq_cfg *cfg = desc->chip_data;
364
365 if (!cfg->move_in_progress) {
366 /* it means that domain is not changed */
367 if (!cpumask_intersects(desc->affinity, mask))
368 cfg->move_desc_pending = 1;
369 }
370}
371#endif
372 350
373#else 351#else
374static struct irq_cfg *irq_cfg(unsigned int irq) 352static struct irq_cfg *irq_cfg(unsigned int irq)
@@ -378,13 +356,6 @@ static struct irq_cfg *irq_cfg(unsigned int irq)
378 356
379#endif 357#endif
380 358
381#ifndef CONFIG_NUMA_MIGRATE_IRQ_DESC
382static inline void
383set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
384{
385}
386#endif
387
388struct io_apic { 359struct io_apic {
389 unsigned int index; 360 unsigned int index;
390 unsigned int unused[3]; 361 unsigned int unused[3];
@@ -518,132 +489,18 @@ static void ioapic_mask_entry(int apic, int pin)
518 spin_unlock_irqrestore(&ioapic_lock, flags); 489 spin_unlock_irqrestore(&ioapic_lock, flags);
519} 490}
520 491
521#ifdef CONFIG_SMP
522static void send_cleanup_vector(struct irq_cfg *cfg)
523{
524 cpumask_var_t cleanup_mask;
525
526 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
527 unsigned int i;
528 cfg->move_cleanup_count = 0;
529 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
530 cfg->move_cleanup_count++;
531 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
532 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
533 } else {
534 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
535 cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
536 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
537 free_cpumask_var(cleanup_mask);
538 }
539 cfg->move_in_progress = 0;
540}
541
542static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
543{
544 int apic, pin;
545 struct irq_pin_list *entry;
546 u8 vector = cfg->vector;
547
548 entry = cfg->irq_2_pin;
549 for (;;) {
550 unsigned int reg;
551
552 if (!entry)
553 break;
554
555 apic = entry->apic;
556 pin = entry->pin;
557 /*
558 * With interrupt-remapping, destination information comes
559 * from interrupt-remapping table entry.
560 */
561 if (!irq_remapped(irq))
562 io_apic_write(apic, 0x11 + pin*2, dest);
563 reg = io_apic_read(apic, 0x10 + pin*2);
564 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
565 reg |= vector;
566 io_apic_modify(apic, 0x10 + pin*2, reg);
567 if (!entry->next)
568 break;
569 entry = entry->next;
570 }
571}
572
573static int
574assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
575
576/*
577 * Either sets desc->affinity to a valid value, and returns
578 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and
579 * leaves desc->affinity untouched.
580 */
581static unsigned int
582set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
583{
584 struct irq_cfg *cfg;
585 unsigned int irq;
586
587 if (!cpumask_intersects(mask, cpu_online_mask))
588 return BAD_APICID;
589
590 irq = desc->irq;
591 cfg = desc->chip_data;
592 if (assign_irq_vector(irq, cfg, mask))
593 return BAD_APICID;
594
595 /* check that before desc->addinity get updated */
596 set_extra_move_desc(desc, mask);
597
598 cpumask_copy(desc->affinity, mask);
599
600 return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
601}
602
603static void
604set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
605{
606 struct irq_cfg *cfg;
607 unsigned long flags;
608 unsigned int dest;
609 unsigned int irq;
610
611 irq = desc->irq;
612 cfg = desc->chip_data;
613
614 spin_lock_irqsave(&ioapic_lock, flags);
615 dest = set_desc_affinity(desc, mask);
616 if (dest != BAD_APICID) {
617 /* Only the high 8 bits are valid. */
618 dest = SET_APIC_LOGICAL_ID(dest);
619 __target_IO_APIC_irq(irq, dest, cfg);
620 }
621 spin_unlock_irqrestore(&ioapic_lock, flags);
622}
623
624static void
625set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
626{
627 struct irq_desc *desc;
628
629 desc = irq_to_desc(irq);
630
631 set_ioapic_affinity_irq_desc(desc, mask);
632}
633#endif /* CONFIG_SMP */
634
635/* 492/*
636 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are 493 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
637 * shared ISA-space IRQs, so we have to support them. We are super 494 * shared ISA-space IRQs, so we have to support them. We are super
638 * fast in the common case, and fast for shared ISA-space IRQs. 495 * fast in the common case, and fast for shared ISA-space IRQs.
639 */ 496 */
640static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin) 497static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
641{ 498{
642 struct irq_pin_list *entry; 499 struct irq_pin_list *entry;
643 500
644 entry = cfg->irq_2_pin; 501 entry = cfg->irq_2_pin;
645 if (!entry) { 502 if (!entry) {
646 entry = get_one_free_irq_2_pin(cpu); 503 entry = get_one_free_irq_2_pin(node);
647 if (!entry) { 504 if (!entry) {
648 printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n", 505 printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
649 apic, pin); 506 apic, pin);
@@ -663,7 +520,7 @@ static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
663 entry = entry->next; 520 entry = entry->next;
664 } 521 }
665 522
666 entry->next = get_one_free_irq_2_pin(cpu); 523 entry->next = get_one_free_irq_2_pin(node);
667 entry = entry->next; 524 entry = entry->next;
668 entry->apic = apic; 525 entry->apic = apic;
669 entry->pin = pin; 526 entry->pin = pin;
@@ -672,7 +529,7 @@ static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
672/* 529/*
673 * Reroute an IRQ to a different pin. 530 * Reroute an IRQ to a different pin.
674 */ 531 */
675static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu, 532static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
676 int oldapic, int oldpin, 533 int oldapic, int oldpin,
677 int newapic, int newpin) 534 int newapic, int newpin)
678{ 535{
@@ -692,7 +549,7 @@ static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu,
692 549
693 /* why? call replace before add? */ 550 /* why? call replace before add? */
694 if (!replaced) 551 if (!replaced)
695 add_pin_to_irq_cpu(cfg, cpu, newapic, newpin); 552 add_pin_to_irq_node(cfg, node, newapic, newpin);
696} 553}
697 554
698static inline void io_apic_modify_irq(struct irq_cfg *cfg, 555static inline void io_apic_modify_irq(struct irq_cfg *cfg,
@@ -850,7 +707,6 @@ static int __init ioapic_pirq_setup(char *str)
850__setup("pirq=", ioapic_pirq_setup); 707__setup("pirq=", ioapic_pirq_setup);
851#endif /* CONFIG_X86_32 */ 708#endif /* CONFIG_X86_32 */
852 709
853#ifdef CONFIG_INTR_REMAP
854struct IO_APIC_route_entry **alloc_ioapic_entries(void) 710struct IO_APIC_route_entry **alloc_ioapic_entries(void)
855{ 711{
856 int apic; 712 int apic;
@@ -948,20 +804,6 @@ int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
948 return 0; 804 return 0;
949} 805}
950 806
951void reinit_intr_remapped_IO_APIC(int intr_remapping,
952 struct IO_APIC_route_entry **ioapic_entries)
953
954{
955 /*
956 * for now plain restore of previous settings.
957 * TBD: In the case of OS enabling interrupt-remapping,
958 * IO-APIC RTE's need to be setup to point to interrupt-remapping
959 * table entries. for now, do a plain restore, and wait for
960 * the setup_IO_APIC_irqs() to do proper initialization.
961 */
962 restore_IO_APIC_setup(ioapic_entries);
963}
964
965void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries) 807void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
966{ 808{
967 int apic; 809 int apic;
@@ -971,7 +813,6 @@ void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
971 813
972 kfree(ioapic_entries); 814 kfree(ioapic_entries);
973} 815}
974#endif
975 816
976/* 817/*
977 * Find the IRQ entry number of a certain pin. 818 * Find the IRQ entry number of a certain pin.
@@ -1032,54 +873,6 @@ static int __init find_isa_irq_apic(int irq, int type)
1032 return -1; 873 return -1;
1033} 874}
1034 875
1035/*
1036 * Find a specific PCI IRQ entry.
1037 * Not an __init, possibly needed by modules
1038 */
1039static int pin_2_irq(int idx, int apic, int pin);
1040
1041int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
1042{
1043 int apic, i, best_guess = -1;
1044
1045 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1046 bus, slot, pin);
1047 if (test_bit(bus, mp_bus_not_pci)) {
1048 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1049 return -1;
1050 }
1051 for (i = 0; i < mp_irq_entries; i++) {
1052 int lbus = mp_irqs[i].srcbus;
1053
1054 for (apic = 0; apic < nr_ioapics; apic++)
1055 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
1056 mp_irqs[i].dstapic == MP_APIC_ALL)
1057 break;
1058
1059 if (!test_bit(lbus, mp_bus_not_pci) &&
1060 !mp_irqs[i].irqtype &&
1061 (bus == lbus) &&
1062 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1063 int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
1064
1065 if (!(apic || IO_APIC_IRQ(irq)))
1066 continue;
1067
1068 if (pin == (mp_irqs[i].srcbusirq & 3))
1069 return irq;
1070 /*
1071 * Use the first all-but-pin matching entry as a
1072 * best-guess fuzzy result for broken mptables.
1073 */
1074 if (best_guess < 0)
1075 best_guess = irq;
1076 }
1077 }
1078 return best_guess;
1079}
1080
1081EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1082
1083#if defined(CONFIG_EISA) || defined(CONFIG_MCA) 876#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
1084/* 877/*
1085 * EISA Edge/Level control register, ELCR 878 * EISA Edge/Level control register, ELCR
@@ -1298,6 +1091,64 @@ static int pin_2_irq(int idx, int apic, int pin)
1298 return irq; 1091 return irq;
1299} 1092}
1300 1093
1094/*
1095 * Find a specific PCI IRQ entry.
1096 * Not an __init, possibly needed by modules
1097 */
1098int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1099 struct io_apic_irq_attr *irq_attr)
1100{
1101 int apic, i, best_guess = -1;
1102
1103 apic_printk(APIC_DEBUG,
1104 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1105 bus, slot, pin);
1106 if (test_bit(bus, mp_bus_not_pci)) {
1107 apic_printk(APIC_VERBOSE,
1108 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1109 return -1;
1110 }
1111 for (i = 0; i < mp_irq_entries; i++) {
1112 int lbus = mp_irqs[i].srcbus;
1113
1114 for (apic = 0; apic < nr_ioapics; apic++)
1115 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
1116 mp_irqs[i].dstapic == MP_APIC_ALL)
1117 break;
1118
1119 if (!test_bit(lbus, mp_bus_not_pci) &&
1120 !mp_irqs[i].irqtype &&
1121 (bus == lbus) &&
1122 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1123 int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
1124
1125 if (!(apic || IO_APIC_IRQ(irq)))
1126 continue;
1127
1128 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1129 set_io_apic_irq_attr(irq_attr, apic,
1130 mp_irqs[i].dstirq,
1131 irq_trigger(i),
1132 irq_polarity(i));
1133 return irq;
1134 }
1135 /*
1136 * Use the first all-but-pin matching entry as a
1137 * best-guess fuzzy result for broken mptables.
1138 */
1139 if (best_guess < 0) {
1140 set_io_apic_irq_attr(irq_attr, apic,
1141 mp_irqs[i].dstirq,
1142 irq_trigger(i),
1143 irq_polarity(i));
1144 best_guess = irq;
1145 }
1146 }
1147 }
1148 return best_guess;
1149}
1150EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1151
1301void lock_vector_lock(void) 1152void lock_vector_lock(void)
1302{ 1153{
1303 /* Used to the online set of cpus does not change 1154 /* Used to the online set of cpus does not change
@@ -1628,58 +1479,70 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq
1628 ioapic_write_entry(apic_id, pin, entry); 1479 ioapic_write_entry(apic_id, pin, entry);
1629} 1480}
1630 1481
1482static struct {
1483 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
1484} mp_ioapic_routing[MAX_IO_APICS];
1485
1631static void __init setup_IO_APIC_irqs(void) 1486static void __init setup_IO_APIC_irqs(void)
1632{ 1487{
1633 int apic_id, pin, idx, irq; 1488 int apic_id = 0, pin, idx, irq;
1634 int notcon = 0; 1489 int notcon = 0;
1635 struct irq_desc *desc; 1490 struct irq_desc *desc;
1636 struct irq_cfg *cfg; 1491 struct irq_cfg *cfg;
1637 int cpu = boot_cpu_id; 1492 int node = cpu_to_node(boot_cpu_id);
1638 1493
1639 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); 1494 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1640 1495
1641 for (apic_id = 0; apic_id < nr_ioapics; apic_id++) { 1496#ifdef CONFIG_ACPI
1642 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) { 1497 if (!acpi_disabled && acpi_ioapic) {
1643 1498 apic_id = mp_find_ioapic(0);
1644 idx = find_irq_entry(apic_id, pin, mp_INT); 1499 if (apic_id < 0)
1645 if (idx == -1) { 1500 apic_id = 0;
1646 if (!notcon) { 1501 }
1647 notcon = 1; 1502#endif
1648 apic_printk(APIC_VERBOSE,
1649 KERN_DEBUG " %d-%d",
1650 mp_ioapics[apic_id].apicid, pin);
1651 } else
1652 apic_printk(APIC_VERBOSE, " %d-%d",
1653 mp_ioapics[apic_id].apicid, pin);
1654 continue;
1655 }
1656 if (notcon) {
1657 apic_printk(APIC_VERBOSE,
1658 " (apicid-pin) not connected\n");
1659 notcon = 0;
1660 }
1661 1503
1662 irq = pin_2_irq(idx, apic_id, pin); 1504 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) {
1505 idx = find_irq_entry(apic_id, pin, mp_INT);
1506 if (idx == -1) {
1507 if (!notcon) {
1508 notcon = 1;
1509 apic_printk(APIC_VERBOSE,
1510 KERN_DEBUG " %d-%d",
1511 mp_ioapics[apic_id].apicid, pin);
1512 } else
1513 apic_printk(APIC_VERBOSE, " %d-%d",
1514 mp_ioapics[apic_id].apicid, pin);
1515 continue;
1516 }
1517 if (notcon) {
1518 apic_printk(APIC_VERBOSE,
1519 " (apicid-pin) not connected\n");
1520 notcon = 0;
1521 }
1663 1522
1664 /* 1523 irq = pin_2_irq(idx, apic_id, pin);
1665 * Skip the timer IRQ if there's a quirk handler
1666 * installed and if it returns 1:
1667 */
1668 if (apic->multi_timer_check &&
1669 apic->multi_timer_check(apic_id, irq))
1670 continue;
1671 1524
1672 desc = irq_to_desc_alloc_cpu(irq, cpu); 1525 /*
1673 if (!desc) { 1526 * Skip the timer IRQ if there's a quirk handler
1674 printk(KERN_INFO "can not get irq_desc for %d\n", irq); 1527 * installed and if it returns 1:
1675 continue; 1528 */
1676 } 1529 if (apic->multi_timer_check &&
1677 cfg = desc->chip_data; 1530 apic->multi_timer_check(apic_id, irq))
1678 add_pin_to_irq_cpu(cfg, cpu, apic_id, pin); 1531 continue;
1679 1532
1680 setup_IO_APIC_irq(apic_id, pin, irq, desc, 1533 desc = irq_to_desc_alloc_node(irq, node);
1681 irq_trigger(idx), irq_polarity(idx)); 1534 if (!desc) {
1535 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
1536 continue;
1682 } 1537 }
1538 cfg = desc->chip_data;
1539 add_pin_to_irq_node(cfg, node, apic_id, pin);
1540 /*
1541 * don't mark it in pin_programmed, so later acpi could
1542 * set it correctly when irq < 16
1543 */
1544 setup_IO_APIC_irq(apic_id, pin, irq, desc,
1545 irq_trigger(idx), irq_polarity(idx));
1683 } 1546 }
1684 1547
1685 if (notcon) 1548 if (notcon)
@@ -1869,7 +1732,7 @@ __apicdebuginit(void) print_APIC_bitfield(int base)
1869 1732
1870__apicdebuginit(void) print_local_APIC(void *dummy) 1733__apicdebuginit(void) print_local_APIC(void *dummy)
1871{ 1734{
1872 unsigned int v, ver, maxlvt; 1735 unsigned int i, v, ver, maxlvt;
1873 u64 icr; 1736 u64 icr;
1874 1737
1875 if (apic_verbosity == APIC_QUIET) 1738 if (apic_verbosity == APIC_QUIET)
@@ -1957,6 +1820,18 @@ __apicdebuginit(void) print_local_APIC(void *dummy)
1957 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v); 1820 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1958 v = apic_read(APIC_TDCR); 1821 v = apic_read(APIC_TDCR);
1959 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v); 1822 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1823
1824 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1825 v = apic_read(APIC_EFEAT);
1826 maxlvt = (v >> 16) & 0xff;
1827 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1828 v = apic_read(APIC_ECTRL);
1829 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1830 for (i = 0; i < maxlvt; i++) {
1831 v = apic_read(APIC_EILVTn(i));
1832 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1833 }
1834 }
1960 printk("\n"); 1835 printk("\n");
1961} 1836}
1962 1837
@@ -2005,6 +1880,11 @@ __apicdebuginit(void) print_PIC(void)
2005__apicdebuginit(int) print_all_ICs(void) 1880__apicdebuginit(int) print_all_ICs(void)
2006{ 1881{
2007 print_PIC(); 1882 print_PIC();
1883
1884 /* don't print out if apic is not there */
1885 if (!cpu_has_apic || disable_apic)
1886 return 0;
1887
2008 print_all_local_APICs(); 1888 print_all_local_APICs();
2009 print_IO_APIC(); 1889 print_IO_APIC();
2010 1890
@@ -2360,6 +2240,118 @@ static int ioapic_retrigger_irq(unsigned int irq)
2360 */ 2240 */
2361 2241
2362#ifdef CONFIG_SMP 2242#ifdef CONFIG_SMP
2243static void send_cleanup_vector(struct irq_cfg *cfg)
2244{
2245 cpumask_var_t cleanup_mask;
2246
2247 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2248 unsigned int i;
2249 cfg->move_cleanup_count = 0;
2250 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2251 cfg->move_cleanup_count++;
2252 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2253 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2254 } else {
2255 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2256 cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
2257 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2258 free_cpumask_var(cleanup_mask);
2259 }
2260 cfg->move_in_progress = 0;
2261}
2262
2263static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2264{
2265 int apic, pin;
2266 struct irq_pin_list *entry;
2267 u8 vector = cfg->vector;
2268
2269 entry = cfg->irq_2_pin;
2270 for (;;) {
2271 unsigned int reg;
2272
2273 if (!entry)
2274 break;
2275
2276 apic = entry->apic;
2277 pin = entry->pin;
2278 /*
2279 * With interrupt-remapping, destination information comes
2280 * from interrupt-remapping table entry.
2281 */
2282 if (!irq_remapped(irq))
2283 io_apic_write(apic, 0x11 + pin*2, dest);
2284 reg = io_apic_read(apic, 0x10 + pin*2);
2285 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2286 reg |= vector;
2287 io_apic_modify(apic, 0x10 + pin*2, reg);
2288 if (!entry->next)
2289 break;
2290 entry = entry->next;
2291 }
2292}
2293
2294static int
2295assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
2296
2297/*
2298 * Either sets desc->affinity to a valid value, and returns
2299 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and
2300 * leaves desc->affinity untouched.
2301 */
2302static unsigned int
2303set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
2304{
2305 struct irq_cfg *cfg;
2306 unsigned int irq;
2307
2308 if (!cpumask_intersects(mask, cpu_online_mask))
2309 return BAD_APICID;
2310
2311 irq = desc->irq;
2312 cfg = desc->chip_data;
2313 if (assign_irq_vector(irq, cfg, mask))
2314 return BAD_APICID;
2315
2316 cpumask_copy(desc->affinity, mask);
2317
2318 return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
2319}
2320
2321static int
2322set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2323{
2324 struct irq_cfg *cfg;
2325 unsigned long flags;
2326 unsigned int dest;
2327 unsigned int irq;
2328 int ret = -1;
2329
2330 irq = desc->irq;
2331 cfg = desc->chip_data;
2332
2333 spin_lock_irqsave(&ioapic_lock, flags);
2334 dest = set_desc_affinity(desc, mask);
2335 if (dest != BAD_APICID) {
2336 /* Only the high 8 bits are valid. */
2337 dest = SET_APIC_LOGICAL_ID(dest);
2338 __target_IO_APIC_irq(irq, dest, cfg);
2339 ret = 0;
2340 }
2341 spin_unlock_irqrestore(&ioapic_lock, flags);
2342
2343 return ret;
2344}
2345
2346static int
2347set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
2348{
2349 struct irq_desc *desc;
2350
2351 desc = irq_to_desc(irq);
2352
2353 return set_ioapic_affinity_irq_desc(desc, mask);
2354}
2363 2355
2364#ifdef CONFIG_INTR_REMAP 2356#ifdef CONFIG_INTR_REMAP
2365 2357
@@ -2374,26 +2366,25 @@ static int ioapic_retrigger_irq(unsigned int irq)
2374 * Real vector that is used for interrupting cpu will be coming from 2366 * Real vector that is used for interrupting cpu will be coming from
2375 * the interrupt-remapping table entry. 2367 * the interrupt-remapping table entry.
2376 */ 2368 */
2377static void 2369static int
2378migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask) 2370migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2379{ 2371{
2380 struct irq_cfg *cfg; 2372 struct irq_cfg *cfg;
2381 struct irte irte; 2373 struct irte irte;
2382 unsigned int dest; 2374 unsigned int dest;
2383 unsigned int irq; 2375 unsigned int irq;
2376 int ret = -1;
2384 2377
2385 if (!cpumask_intersects(mask, cpu_online_mask)) 2378 if (!cpumask_intersects(mask, cpu_online_mask))
2386 return; 2379 return ret;
2387 2380
2388 irq = desc->irq; 2381 irq = desc->irq;
2389 if (get_irte(irq, &irte)) 2382 if (get_irte(irq, &irte))
2390 return; 2383 return ret;
2391 2384
2392 cfg = desc->chip_data; 2385 cfg = desc->chip_data;
2393 if (assign_irq_vector(irq, cfg, mask)) 2386 if (assign_irq_vector(irq, cfg, mask))
2394 return; 2387 return ret;
2395
2396 set_extra_move_desc(desc, mask);
2397 2388
2398 dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); 2389 dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
2399 2390
@@ -2409,27 +2400,30 @@ migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2409 send_cleanup_vector(cfg); 2400 send_cleanup_vector(cfg);
2410 2401
2411 cpumask_copy(desc->affinity, mask); 2402 cpumask_copy(desc->affinity, mask);
2403
2404 return 0;
2412} 2405}
2413 2406
2414/* 2407/*
2415 * Migrates the IRQ destination in the process context. 2408 * Migrates the IRQ destination in the process context.
2416 */ 2409 */
2417static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, 2410static int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2418 const struct cpumask *mask) 2411 const struct cpumask *mask)
2419{ 2412{
2420 migrate_ioapic_irq_desc(desc, mask); 2413 return migrate_ioapic_irq_desc(desc, mask);
2421} 2414}
2422static void set_ir_ioapic_affinity_irq(unsigned int irq, 2415static int set_ir_ioapic_affinity_irq(unsigned int irq,
2423 const struct cpumask *mask) 2416 const struct cpumask *mask)
2424{ 2417{
2425 struct irq_desc *desc = irq_to_desc(irq); 2418 struct irq_desc *desc = irq_to_desc(irq);
2426 2419
2427 set_ir_ioapic_affinity_irq_desc(desc, mask); 2420 return set_ir_ioapic_affinity_irq_desc(desc, mask);
2428} 2421}
2429#else 2422#else
2430static inline void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, 2423static inline int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2431 const struct cpumask *mask) 2424 const struct cpumask *mask)
2432{ 2425{
2426 return 0;
2433} 2427}
2434#endif 2428#endif
2435 2429
@@ -2491,86 +2485,19 @@ static void irq_complete_move(struct irq_desc **descp)
2491 struct irq_cfg *cfg = desc->chip_data; 2485 struct irq_cfg *cfg = desc->chip_data;
2492 unsigned vector, me; 2486 unsigned vector, me;
2493 2487
2494 if (likely(!cfg->move_in_progress)) { 2488 if (likely(!cfg->move_in_progress))
2495#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
2496 if (likely(!cfg->move_desc_pending))
2497 return;
2498
2499 /* domain has not changed, but affinity did */
2500 me = smp_processor_id();
2501 if (cpumask_test_cpu(me, desc->affinity)) {
2502 *descp = desc = move_irq_desc(desc, me);
2503 /* get the new one */
2504 cfg = desc->chip_data;
2505 cfg->move_desc_pending = 0;
2506 }
2507#endif
2508 return; 2489 return;
2509 }
2510 2490
2511 vector = ~get_irq_regs()->orig_ax; 2491 vector = ~get_irq_regs()->orig_ax;
2512 me = smp_processor_id(); 2492 me = smp_processor_id();
2513 2493
2514 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) { 2494 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2515#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
2516 *descp = desc = move_irq_desc(desc, me);
2517 /* get the new one */
2518 cfg = desc->chip_data;
2519#endif
2520 send_cleanup_vector(cfg); 2495 send_cleanup_vector(cfg);
2521 }
2522} 2496}
2523#else 2497#else
2524static inline void irq_complete_move(struct irq_desc **descp) {} 2498static inline void irq_complete_move(struct irq_desc **descp) {}
2525#endif 2499#endif
2526 2500
2527static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2528{
2529 int apic, pin;
2530 struct irq_pin_list *entry;
2531
2532 entry = cfg->irq_2_pin;
2533 for (;;) {
2534
2535 if (!entry)
2536 break;
2537
2538 apic = entry->apic;
2539 pin = entry->pin;
2540 io_apic_eoi(apic, pin);
2541 entry = entry->next;
2542 }
2543}
2544
2545static void
2546eoi_ioapic_irq(struct irq_desc *desc)
2547{
2548 struct irq_cfg *cfg;
2549 unsigned long flags;
2550 unsigned int irq;
2551
2552 irq = desc->irq;
2553 cfg = desc->chip_data;
2554
2555 spin_lock_irqsave(&ioapic_lock, flags);
2556 __eoi_ioapic_irq(irq, cfg);
2557 spin_unlock_irqrestore(&ioapic_lock, flags);
2558}
2559
2560#ifdef CONFIG_X86_X2APIC
2561static void ack_x2apic_level(unsigned int irq)
2562{
2563 struct irq_desc *desc = irq_to_desc(irq);
2564 ack_x2APIC_irq();
2565 eoi_ioapic_irq(desc);
2566}
2567
2568static void ack_x2apic_edge(unsigned int irq)
2569{
2570 ack_x2APIC_irq();
2571}
2572#endif
2573
2574static void ack_apic_edge(unsigned int irq) 2501static void ack_apic_edge(unsigned int irq)
2575{ 2502{
2576 struct irq_desc *desc = irq_to_desc(irq); 2503 struct irq_desc *desc = irq_to_desc(irq);
@@ -2634,9 +2561,6 @@ static void ack_apic_level(unsigned int irq)
2634 */ 2561 */
2635 ack_APIC_irq(); 2562 ack_APIC_irq();
2636 2563
2637 if (irq_remapped(irq))
2638 eoi_ioapic_irq(desc);
2639
2640 /* Now we can move and renable the irq */ 2564 /* Now we can move and renable the irq */
2641 if (unlikely(do_unmask_irq)) { 2565 if (unlikely(do_unmask_irq)) {
2642 /* Only migrate the irq if the ack has been received. 2566 /* Only migrate the irq if the ack has been received.
@@ -2683,22 +2607,50 @@ static void ack_apic_level(unsigned int irq)
2683} 2607}
2684 2608
2685#ifdef CONFIG_INTR_REMAP 2609#ifdef CONFIG_INTR_REMAP
2610static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2611{
2612 int apic, pin;
2613 struct irq_pin_list *entry;
2614
2615 entry = cfg->irq_2_pin;
2616 for (;;) {
2617
2618 if (!entry)
2619 break;
2620
2621 apic = entry->apic;
2622 pin = entry->pin;
2623 io_apic_eoi(apic, pin);
2624 entry = entry->next;
2625 }
2626}
2627
2628static void
2629eoi_ioapic_irq(struct irq_desc *desc)
2630{
2631 struct irq_cfg *cfg;
2632 unsigned long flags;
2633 unsigned int irq;
2634
2635 irq = desc->irq;
2636 cfg = desc->chip_data;
2637
2638 spin_lock_irqsave(&ioapic_lock, flags);
2639 __eoi_ioapic_irq(irq, cfg);
2640 spin_unlock_irqrestore(&ioapic_lock, flags);
2641}
2642
2686static void ir_ack_apic_edge(unsigned int irq) 2643static void ir_ack_apic_edge(unsigned int irq)
2687{ 2644{
2688#ifdef CONFIG_X86_X2APIC 2645 ack_APIC_irq();
2689 if (x2apic_enabled())
2690 return ack_x2apic_edge(irq);
2691#endif
2692 return ack_apic_edge(irq);
2693} 2646}
2694 2647
2695static void ir_ack_apic_level(unsigned int irq) 2648static void ir_ack_apic_level(unsigned int irq)
2696{ 2649{
2697#ifdef CONFIG_X86_X2APIC 2650 struct irq_desc *desc = irq_to_desc(irq);
2698 if (x2apic_enabled()) 2651
2699 return ack_x2apic_level(irq); 2652 ack_APIC_irq();
2700#endif 2653 eoi_ioapic_irq(desc);
2701 return ack_apic_level(irq);
2702} 2654}
2703#endif /* CONFIG_INTR_REMAP */ 2655#endif /* CONFIG_INTR_REMAP */
2704 2656
@@ -2903,7 +2855,7 @@ static inline void __init check_timer(void)
2903{ 2855{
2904 struct irq_desc *desc = irq_to_desc(0); 2856 struct irq_desc *desc = irq_to_desc(0);
2905 struct irq_cfg *cfg = desc->chip_data; 2857 struct irq_cfg *cfg = desc->chip_data;
2906 int cpu = boot_cpu_id; 2858 int node = cpu_to_node(boot_cpu_id);
2907 int apic1, pin1, apic2, pin2; 2859 int apic1, pin1, apic2, pin2;
2908 unsigned long flags; 2860 unsigned long flags;
2909 int no_pin1 = 0; 2861 int no_pin1 = 0;
@@ -2969,7 +2921,7 @@ static inline void __init check_timer(void)
2969 * Ok, does IRQ0 through the IOAPIC work? 2921 * Ok, does IRQ0 through the IOAPIC work?
2970 */ 2922 */
2971 if (no_pin1) { 2923 if (no_pin1) {
2972 add_pin_to_irq_cpu(cfg, cpu, apic1, pin1); 2924 add_pin_to_irq_node(cfg, node, apic1, pin1);
2973 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector); 2925 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2974 } else { 2926 } else {
2975 /* for edge trigger, setup_IO_APIC_irq already 2927 /* for edge trigger, setup_IO_APIC_irq already
@@ -3006,7 +2958,7 @@ static inline void __init check_timer(void)
3006 /* 2958 /*
3007 * legacy devices should be connected to IO APIC #0 2959 * legacy devices should be connected to IO APIC #0
3008 */ 2960 */
3009 replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2); 2961 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
3010 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector); 2962 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
3011 enable_8259A_irq(0); 2963 enable_8259A_irq(0);
3012 if (timer_irq_works()) { 2964 if (timer_irq_works()) {
@@ -3218,14 +3170,13 @@ static int nr_irqs_gsi = NR_IRQS_LEGACY;
3218/* 3170/*
3219 * Dynamic irq allocate and deallocation 3171 * Dynamic irq allocate and deallocation
3220 */ 3172 */
3221unsigned int create_irq_nr(unsigned int irq_want) 3173unsigned int create_irq_nr(unsigned int irq_want, int node)
3222{ 3174{
3223 /* Allocate an unused irq */ 3175 /* Allocate an unused irq */
3224 unsigned int irq; 3176 unsigned int irq;
3225 unsigned int new; 3177 unsigned int new;
3226 unsigned long flags; 3178 unsigned long flags;
3227 struct irq_cfg *cfg_new = NULL; 3179 struct irq_cfg *cfg_new = NULL;
3228 int cpu = boot_cpu_id;
3229 struct irq_desc *desc_new = NULL; 3180 struct irq_desc *desc_new = NULL;
3230 3181
3231 irq = 0; 3182 irq = 0;
@@ -3234,7 +3185,7 @@ unsigned int create_irq_nr(unsigned int irq_want)
3234 3185
3235 spin_lock_irqsave(&vector_lock, flags); 3186 spin_lock_irqsave(&vector_lock, flags);
3236 for (new = irq_want; new < nr_irqs; new++) { 3187 for (new = irq_want; new < nr_irqs; new++) {
3237 desc_new = irq_to_desc_alloc_cpu(new, cpu); 3188 desc_new = irq_to_desc_alloc_node(new, node);
3238 if (!desc_new) { 3189 if (!desc_new) {
3239 printk(KERN_INFO "can not get irq_desc for %d\n", new); 3190 printk(KERN_INFO "can not get irq_desc for %d\n", new);
3240 continue; 3191 continue;
@@ -3243,6 +3194,9 @@ unsigned int create_irq_nr(unsigned int irq_want)
3243 3194
3244 if (cfg_new->vector != 0) 3195 if (cfg_new->vector != 0)
3245 continue; 3196 continue;
3197
3198 desc_new = move_irq_desc(desc_new, node);
3199
3246 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0) 3200 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0)
3247 irq = new; 3201 irq = new;
3248 break; 3202 break;
@@ -3260,11 +3214,12 @@ unsigned int create_irq_nr(unsigned int irq_want)
3260 3214
3261int create_irq(void) 3215int create_irq(void)
3262{ 3216{
3217 int node = cpu_to_node(boot_cpu_id);
3263 unsigned int irq_want; 3218 unsigned int irq_want;
3264 int irq; 3219 int irq;
3265 3220
3266 irq_want = nr_irqs_gsi; 3221 irq_want = nr_irqs_gsi;
3267 irq = create_irq_nr(irq_want); 3222 irq = create_irq_nr(irq_want, node);
3268 3223
3269 if (irq == 0) 3224 if (irq == 0)
3270 irq = -1; 3225 irq = -1;
@@ -3366,7 +3321,7 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
3366} 3321}
3367 3322
3368#ifdef CONFIG_SMP 3323#ifdef CONFIG_SMP
3369static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask) 3324static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3370{ 3325{
3371 struct irq_desc *desc = irq_to_desc(irq); 3326 struct irq_desc *desc = irq_to_desc(irq);
3372 struct irq_cfg *cfg; 3327 struct irq_cfg *cfg;
@@ -3375,7 +3330,7 @@ static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3375 3330
3376 dest = set_desc_affinity(desc, mask); 3331 dest = set_desc_affinity(desc, mask);
3377 if (dest == BAD_APICID) 3332 if (dest == BAD_APICID)
3378 return; 3333 return -1;
3379 3334
3380 cfg = desc->chip_data; 3335 cfg = desc->chip_data;
3381 3336
@@ -3387,13 +3342,15 @@ static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3387 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3342 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3388 3343
3389 write_msi_msg_desc(desc, &msg); 3344 write_msi_msg_desc(desc, &msg);
3345
3346 return 0;
3390} 3347}
3391#ifdef CONFIG_INTR_REMAP 3348#ifdef CONFIG_INTR_REMAP
3392/* 3349/*
3393 * Migrate the MSI irq to another cpumask. This migration is 3350 * Migrate the MSI irq to another cpumask. This migration is
3394 * done in the process context using interrupt-remapping hardware. 3351 * done in the process context using interrupt-remapping hardware.
3395 */ 3352 */
3396static void 3353static int
3397ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask) 3354ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3398{ 3355{
3399 struct irq_desc *desc = irq_to_desc(irq); 3356 struct irq_desc *desc = irq_to_desc(irq);
@@ -3402,11 +3359,11 @@ ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3402 struct irte irte; 3359 struct irte irte;
3403 3360
3404 if (get_irte(irq, &irte)) 3361 if (get_irte(irq, &irte))
3405 return; 3362 return -1;
3406 3363
3407 dest = set_desc_affinity(desc, mask); 3364 dest = set_desc_affinity(desc, mask);
3408 if (dest == BAD_APICID) 3365 if (dest == BAD_APICID)
3409 return; 3366 return -1;
3410 3367
3411 irte.vector = cfg->vector; 3368 irte.vector = cfg->vector;
3412 irte.dest_id = IRTE_DEST(dest); 3369 irte.dest_id = IRTE_DEST(dest);
@@ -3423,6 +3380,8 @@ ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3423 */ 3380 */
3424 if (cfg->move_in_progress) 3381 if (cfg->move_in_progress)
3425 send_cleanup_vector(cfg); 3382 send_cleanup_vector(cfg);
3383
3384 return 0;
3426} 3385}
3427 3386
3428#endif 3387#endif
@@ -3518,15 +3477,17 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3518 unsigned int irq_want; 3477 unsigned int irq_want;
3519 struct intel_iommu *iommu = NULL; 3478 struct intel_iommu *iommu = NULL;
3520 int index = 0; 3479 int index = 0;
3480 int node;
3521 3481
3522 /* x86 doesn't support multiple MSI yet */ 3482 /* x86 doesn't support multiple MSI yet */
3523 if (type == PCI_CAP_ID_MSI && nvec > 1) 3483 if (type == PCI_CAP_ID_MSI && nvec > 1)
3524 return 1; 3484 return 1;
3525 3485
3486 node = dev_to_node(&dev->dev);
3526 irq_want = nr_irqs_gsi; 3487 irq_want = nr_irqs_gsi;
3527 sub_handle = 0; 3488 sub_handle = 0;
3528 list_for_each_entry(msidesc, &dev->msi_list, list) { 3489 list_for_each_entry(msidesc, &dev->msi_list, list) {
3529 irq = create_irq_nr(irq_want); 3490 irq = create_irq_nr(irq_want, node);
3530 if (irq == 0) 3491 if (irq == 0)
3531 return -1; 3492 return -1;
3532 irq_want = irq + 1; 3493 irq_want = irq + 1;
@@ -3576,7 +3537,7 @@ void arch_teardown_msi_irq(unsigned int irq)
3576 3537
3577#if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP) 3538#if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP)
3578#ifdef CONFIG_SMP 3539#ifdef CONFIG_SMP
3579static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask) 3540static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3580{ 3541{
3581 struct irq_desc *desc = irq_to_desc(irq); 3542 struct irq_desc *desc = irq_to_desc(irq);
3582 struct irq_cfg *cfg; 3543 struct irq_cfg *cfg;
@@ -3585,7 +3546,7 @@ static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3585 3546
3586 dest = set_desc_affinity(desc, mask); 3547 dest = set_desc_affinity(desc, mask);
3587 if (dest == BAD_APICID) 3548 if (dest == BAD_APICID)
3588 return; 3549 return -1;
3589 3550
3590 cfg = desc->chip_data; 3551 cfg = desc->chip_data;
3591 3552
@@ -3597,6 +3558,8 @@ static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3597 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3558 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3598 3559
3599 dmar_msi_write(irq, &msg); 3560 dmar_msi_write(irq, &msg);
3561
3562 return 0;
3600} 3563}
3601 3564
3602#endif /* CONFIG_SMP */ 3565#endif /* CONFIG_SMP */
@@ -3630,7 +3593,7 @@ int arch_setup_dmar_msi(unsigned int irq)
3630#ifdef CONFIG_HPET_TIMER 3593#ifdef CONFIG_HPET_TIMER
3631 3594
3632#ifdef CONFIG_SMP 3595#ifdef CONFIG_SMP
3633static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask) 3596static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3634{ 3597{
3635 struct irq_desc *desc = irq_to_desc(irq); 3598 struct irq_desc *desc = irq_to_desc(irq);
3636 struct irq_cfg *cfg; 3599 struct irq_cfg *cfg;
@@ -3639,7 +3602,7 @@ static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3639 3602
3640 dest = set_desc_affinity(desc, mask); 3603 dest = set_desc_affinity(desc, mask);
3641 if (dest == BAD_APICID) 3604 if (dest == BAD_APICID)
3642 return; 3605 return -1;
3643 3606
3644 cfg = desc->chip_data; 3607 cfg = desc->chip_data;
3645 3608
@@ -3651,6 +3614,8 @@ static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3651 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3614 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3652 3615
3653 hpet_msi_write(irq, &msg); 3616 hpet_msi_write(irq, &msg);
3617
3618 return 0;
3654} 3619}
3655 3620
3656#endif /* CONFIG_SMP */ 3621#endif /* CONFIG_SMP */
@@ -3707,7 +3672,7 @@ static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3707 write_ht_irq_msg(irq, &msg); 3672 write_ht_irq_msg(irq, &msg);
3708} 3673}
3709 3674
3710static void set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask) 3675static int set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3711{ 3676{
3712 struct irq_desc *desc = irq_to_desc(irq); 3677 struct irq_desc *desc = irq_to_desc(irq);
3713 struct irq_cfg *cfg; 3678 struct irq_cfg *cfg;
@@ -3715,11 +3680,13 @@ static void set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3715 3680
3716 dest = set_desc_affinity(desc, mask); 3681 dest = set_desc_affinity(desc, mask);
3717 if (dest == BAD_APICID) 3682 if (dest == BAD_APICID)
3718 return; 3683 return -1;
3719 3684
3720 cfg = desc->chip_data; 3685 cfg = desc->chip_data;
3721 3686
3722 target_ht_irq(irq, dest, cfg->vector); 3687 target_ht_irq(irq, dest, cfg->vector);
3688
3689 return 0;
3723} 3690}
3724 3691
3725#endif 3692#endif
@@ -3794,6 +3761,8 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3794 unsigned long flags; 3761 unsigned long flags;
3795 int err; 3762 int err;
3796 3763
3764 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3765
3797 cfg = irq_cfg(irq); 3766 cfg = irq_cfg(irq);
3798 3767
3799 err = assign_irq_vector(irq, cfg, eligible_cpu); 3768 err = assign_irq_vector(irq, cfg, eligible_cpu);
@@ -3807,15 +3776,13 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3807 3776
3808 mmr_value = 0; 3777 mmr_value = 0;
3809 entry = (struct uv_IO_APIC_route_entry *)&mmr_value; 3778 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3810 BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long)); 3779 entry->vector = cfg->vector;
3811 3780 entry->delivery_mode = apic->irq_delivery_mode;
3812 entry->vector = cfg->vector; 3781 entry->dest_mode = apic->irq_dest_mode;
3813 entry->delivery_mode = apic->irq_delivery_mode; 3782 entry->polarity = 0;
3814 entry->dest_mode = apic->irq_dest_mode; 3783 entry->trigger = 0;
3815 entry->polarity = 0; 3784 entry->mask = 0;
3816 entry->trigger = 0; 3785 entry->dest = apic->cpu_mask_to_apicid(eligible_cpu);
3817 entry->mask = 0;
3818 entry->dest = apic->cpu_mask_to_apicid(eligible_cpu);
3819 3786
3820 mmr_pnode = uv_blade_to_pnode(mmr_blade); 3787 mmr_pnode = uv_blade_to_pnode(mmr_blade);
3821 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value); 3788 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
@@ -3833,10 +3800,10 @@ void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
3833 struct uv_IO_APIC_route_entry *entry; 3800 struct uv_IO_APIC_route_entry *entry;
3834 int mmr_pnode; 3801 int mmr_pnode;
3835 3802
3803 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3804
3836 mmr_value = 0; 3805 mmr_value = 0;
3837 entry = (struct uv_IO_APIC_route_entry *)&mmr_value; 3806 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3838 BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3839
3840 entry->mask = 1; 3807 entry->mask = 1;
3841 3808
3842 mmr_pnode = uv_blade_to_pnode(mmr_blade); 3809 mmr_pnode = uv_blade_to_pnode(mmr_blade);
@@ -3900,6 +3867,71 @@ int __init arch_probe_nr_irqs(void)
3900} 3867}
3901#endif 3868#endif
3902 3869
3870static int __io_apic_set_pci_routing(struct device *dev, int irq,
3871 struct io_apic_irq_attr *irq_attr)
3872{
3873 struct irq_desc *desc;
3874 struct irq_cfg *cfg;
3875 int node;
3876 int ioapic, pin;
3877 int trigger, polarity;
3878
3879 ioapic = irq_attr->ioapic;
3880 if (!IO_APIC_IRQ(irq)) {
3881 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3882 ioapic);
3883 return -EINVAL;
3884 }
3885
3886 if (dev)
3887 node = dev_to_node(dev);
3888 else
3889 node = cpu_to_node(boot_cpu_id);
3890
3891 desc = irq_to_desc_alloc_node(irq, node);
3892 if (!desc) {
3893 printk(KERN_INFO "can not get irq_desc %d\n", irq);
3894 return 0;
3895 }
3896
3897 pin = irq_attr->ioapic_pin;
3898 trigger = irq_attr->trigger;
3899 polarity = irq_attr->polarity;
3900
3901 /*
3902 * IRQs < 16 are already in the irq_2_pin[] map
3903 */
3904 if (irq >= NR_IRQS_LEGACY) {
3905 cfg = desc->chip_data;
3906 add_pin_to_irq_node(cfg, node, ioapic, pin);
3907 }
3908
3909 setup_IO_APIC_irq(ioapic, pin, irq, desc, trigger, polarity);
3910
3911 return 0;
3912}
3913
3914int io_apic_set_pci_routing(struct device *dev, int irq,
3915 struct io_apic_irq_attr *irq_attr)
3916{
3917 int ioapic, pin;
3918 /*
3919 * Avoid pin reprogramming. PRTs typically include entries
3920 * with redundant pin->gsi mappings (but unique PCI devices);
3921 * we only program the IOAPIC on the first.
3922 */
3923 ioapic = irq_attr->ioapic;
3924 pin = irq_attr->ioapic_pin;
3925 if (test_bit(pin, mp_ioapic_routing[ioapic].pin_programmed)) {
3926 pr_debug("Pin %d-%d already programmed\n",
3927 mp_ioapics[ioapic].apicid, pin);
3928 return 0;
3929 }
3930 set_bit(pin, mp_ioapic_routing[ioapic].pin_programmed);
3931
3932 return __io_apic_set_pci_routing(dev, irq, irq_attr);
3933}
3934
3903/* -------------------------------------------------------------------------- 3935/* --------------------------------------------------------------------------
3904 ACPI-based IOAPIC Configuration 3936 ACPI-based IOAPIC Configuration
3905 -------------------------------------------------------------------------- */ 3937 -------------------------------------------------------------------------- */
@@ -3994,38 +4026,6 @@ int __init io_apic_get_version(int ioapic)
3994} 4026}
3995#endif 4027#endif
3996 4028
3997int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
3998{
3999 struct irq_desc *desc;
4000 struct irq_cfg *cfg;
4001 int cpu = boot_cpu_id;
4002
4003 if (!IO_APIC_IRQ(irq)) {
4004 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
4005 ioapic);
4006 return -EINVAL;
4007 }
4008
4009 desc = irq_to_desc_alloc_cpu(irq, cpu);
4010 if (!desc) {
4011 printk(KERN_INFO "can not get irq_desc %d\n", irq);
4012 return 0;
4013 }
4014
4015 /*
4016 * IRQs < 16 are already in the irq_2_pin[] map
4017 */
4018 if (irq >= NR_IRQS_LEGACY) {
4019 cfg = desc->chip_data;
4020 add_pin_to_irq_cpu(cfg, cpu, ioapic, pin);
4021 }
4022
4023 setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity);
4024
4025 return 0;
4026}
4027
4028
4029int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity) 4029int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
4030{ 4030{
4031 int i; 4031 int i;
@@ -4055,51 +4055,44 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
4055#ifdef CONFIG_SMP 4055#ifdef CONFIG_SMP
4056void __init setup_ioapic_dest(void) 4056void __init setup_ioapic_dest(void)
4057{ 4057{
4058 int pin, ioapic, irq, irq_entry; 4058 int pin, ioapic = 0, irq, irq_entry;
4059 struct irq_desc *desc; 4059 struct irq_desc *desc;
4060 struct irq_cfg *cfg;
4061 const struct cpumask *mask; 4060 const struct cpumask *mask;
4062 4061
4063 if (skip_ioapic_setup == 1) 4062 if (skip_ioapic_setup == 1)
4064 return; 4063 return;
4065 4064
4066 for (ioapic = 0; ioapic < nr_ioapics; ioapic++) { 4065#ifdef CONFIG_ACPI
4067 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) { 4066 if (!acpi_disabled && acpi_ioapic) {
4068 irq_entry = find_irq_entry(ioapic, pin, mp_INT); 4067 ioapic = mp_find_ioapic(0);
4069 if (irq_entry == -1) 4068 if (ioapic < 0)
4070 continue; 4069 ioapic = 0;
4071 irq = pin_2_irq(irq_entry, ioapic, pin); 4070 }
4072 4071#endif
4073 /* setup_IO_APIC_irqs could fail to get vector for some device
4074 * when you have too many devices, because at that time only boot
4075 * cpu is online.
4076 */
4077 desc = irq_to_desc(irq);
4078 cfg = desc->chip_data;
4079 if (!cfg->vector) {
4080 setup_IO_APIC_irq(ioapic, pin, irq, desc,
4081 irq_trigger(irq_entry),
4082 irq_polarity(irq_entry));
4083 continue;
4084 4072
4085 } 4073 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
4074 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
4075 if (irq_entry == -1)
4076 continue;
4077 irq = pin_2_irq(irq_entry, ioapic, pin);
4086 4078
4087 /* 4079 desc = irq_to_desc(irq);
4088 * Honour affinities which have been set in early boot
4089 */
4090 if (desc->status &
4091 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
4092 mask = desc->affinity;
4093 else
4094 mask = apic->target_cpus();
4095 4080
4096 if (intr_remapping_enabled) 4081 /*
4097 set_ir_ioapic_affinity_irq_desc(desc, mask); 4082 * Honour affinities which have been set in early boot
4098 else 4083 */
4099 set_ioapic_affinity_irq_desc(desc, mask); 4084 if (desc->status &
4100 } 4085 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
4086 mask = desc->affinity;
4087 else
4088 mask = apic->target_cpus();
4101 4089
4090 if (intr_remapping_enabled)
4091 set_ir_ioapic_affinity_irq_desc(desc, mask);
4092 else
4093 set_ioapic_affinity_irq_desc(desc, mask);
4102 } 4094 }
4095
4103} 4096}
4104#endif 4097#endif
4105 4098