aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/apic/io_apic.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel/apic/io_apic.c')
-rw-r--r--arch/x86/kernel/apic/io_apic.c902
1 files changed, 448 insertions, 454 deletions
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 30da617d18e4..1946fac42ab3 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -59,6 +59,7 @@
59#include <asm/setup.h> 59#include <asm/setup.h>
60#include <asm/irq_remapping.h> 60#include <asm/irq_remapping.h>
61#include <asm/hpet.h> 61#include <asm/hpet.h>
62#include <asm/hw_irq.h>
62#include <asm/uv/uv_hub.h> 63#include <asm/uv/uv_hub.h>
63#include <asm/uv/uv_irq.h> 64#include <asm/uv/uv_irq.h>
64 65
@@ -129,12 +130,9 @@ struct irq_pin_list {
129 struct irq_pin_list *next; 130 struct irq_pin_list *next;
130}; 131};
131 132
132static struct irq_pin_list *get_one_free_irq_2_pin(int cpu) 133static struct irq_pin_list *get_one_free_irq_2_pin(int node)
133{ 134{
134 struct irq_pin_list *pin; 135 struct irq_pin_list *pin;
135 int node;
136
137 node = cpu_to_node(cpu);
138 136
139 pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node); 137 pin = kzalloc_node(sizeof(*pin), GFP_ATOMIC, node);
140 138
@@ -148,9 +146,6 @@ struct irq_cfg {
148 unsigned move_cleanup_count; 146 unsigned move_cleanup_count;
149 u8 vector; 147 u8 vector;
150 u8 move_in_progress : 1; 148 u8 move_in_progress : 1;
151#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
152 u8 move_desc_pending : 1;
153#endif
154}; 149};
155 150
156/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */ 151/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
@@ -212,12 +207,9 @@ static struct irq_cfg *irq_cfg(unsigned int irq)
212 return cfg; 207 return cfg;
213} 208}
214 209
215static struct irq_cfg *get_one_free_irq_cfg(int cpu) 210static struct irq_cfg *get_one_free_irq_cfg(int node)
216{ 211{
217 struct irq_cfg *cfg; 212 struct irq_cfg *cfg;
218 int node;
219
220 node = cpu_to_node(cpu);
221 213
222 cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node); 214 cfg = kzalloc_node(sizeof(*cfg), GFP_ATOMIC, node);
223 if (cfg) { 215 if (cfg) {
@@ -238,13 +230,13 @@ static struct irq_cfg *get_one_free_irq_cfg(int cpu)
238 return cfg; 230 return cfg;
239} 231}
240 232
241int arch_init_chip_data(struct irq_desc *desc, int cpu) 233int arch_init_chip_data(struct irq_desc *desc, int node)
242{ 234{
243 struct irq_cfg *cfg; 235 struct irq_cfg *cfg;
244 236
245 cfg = desc->chip_data; 237 cfg = desc->chip_data;
246 if (!cfg) { 238 if (!cfg) {
247 desc->chip_data = get_one_free_irq_cfg(cpu); 239 desc->chip_data = get_one_free_irq_cfg(node);
248 if (!desc->chip_data) { 240 if (!desc->chip_data) {
249 printk(KERN_ERR "can not alloc irq_cfg\n"); 241 printk(KERN_ERR "can not alloc irq_cfg\n");
250 BUG_ON(1); 242 BUG_ON(1);
@@ -254,10 +246,9 @@ int arch_init_chip_data(struct irq_desc *desc, int cpu)
254 return 0; 246 return 0;
255} 247}
256 248
257#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC 249/* for move_irq_desc */
258
259static void 250static void
260init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu) 251init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int node)
261{ 252{
262 struct irq_pin_list *old_entry, *head, *tail, *entry; 253 struct irq_pin_list *old_entry, *head, *tail, *entry;
263 254
@@ -266,7 +257,7 @@ init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
266 if (!old_entry) 257 if (!old_entry)
267 return; 258 return;
268 259
269 entry = get_one_free_irq_2_pin(cpu); 260 entry = get_one_free_irq_2_pin(node);
270 if (!entry) 261 if (!entry)
271 return; 262 return;
272 263
@@ -276,7 +267,7 @@ init_copy_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg, int cpu)
276 tail = entry; 267 tail = entry;
277 old_entry = old_entry->next; 268 old_entry = old_entry->next;
278 while (old_entry) { 269 while (old_entry) {
279 entry = get_one_free_irq_2_pin(cpu); 270 entry = get_one_free_irq_2_pin(node);
280 if (!entry) { 271 if (!entry) {
281 entry = head; 272 entry = head;
282 while (entry) { 273 while (entry) {
@@ -316,12 +307,12 @@ static void free_irq_2_pin(struct irq_cfg *old_cfg, struct irq_cfg *cfg)
316} 307}
317 308
318void arch_init_copy_chip_data(struct irq_desc *old_desc, 309void arch_init_copy_chip_data(struct irq_desc *old_desc,
319 struct irq_desc *desc, int cpu) 310 struct irq_desc *desc, int node)
320{ 311{
321 struct irq_cfg *cfg; 312 struct irq_cfg *cfg;
322 struct irq_cfg *old_cfg; 313 struct irq_cfg *old_cfg;
323 314
324 cfg = get_one_free_irq_cfg(cpu); 315 cfg = get_one_free_irq_cfg(node);
325 316
326 if (!cfg) 317 if (!cfg)
327 return; 318 return;
@@ -332,7 +323,7 @@ void arch_init_copy_chip_data(struct irq_desc *old_desc,
332 323
333 memcpy(cfg, old_cfg, sizeof(struct irq_cfg)); 324 memcpy(cfg, old_cfg, sizeof(struct irq_cfg));
334 325
335 init_copy_irq_2_pin(old_cfg, cfg, cpu); 326 init_copy_irq_2_pin(old_cfg, cfg, node);
336} 327}
337 328
338static void free_irq_cfg(struct irq_cfg *old_cfg) 329static void free_irq_cfg(struct irq_cfg *old_cfg)
@@ -356,19 +347,7 @@ void arch_free_chip_data(struct irq_desc *old_desc, struct irq_desc *desc)
356 old_desc->chip_data = NULL; 347 old_desc->chip_data = NULL;
357 } 348 }
358} 349}
359 350/* end for move_irq_desc */
360static void
361set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
362{
363 struct irq_cfg *cfg = desc->chip_data;
364
365 if (!cfg->move_in_progress) {
366 /* it means that domain is not changed */
367 if (!cpumask_intersects(desc->affinity, mask))
368 cfg->move_desc_pending = 1;
369 }
370}
371#endif
372 351
373#else 352#else
374static struct irq_cfg *irq_cfg(unsigned int irq) 353static struct irq_cfg *irq_cfg(unsigned int irq)
@@ -378,13 +357,6 @@ static struct irq_cfg *irq_cfg(unsigned int irq)
378 357
379#endif 358#endif
380 359
381#ifndef CONFIG_NUMA_MIGRATE_IRQ_DESC
382static inline void
383set_extra_move_desc(struct irq_desc *desc, const struct cpumask *mask)
384{
385}
386#endif
387
388struct io_apic { 360struct io_apic {
389 unsigned int index; 361 unsigned int index;
390 unsigned int unused[3]; 362 unsigned int unused[3];
@@ -518,132 +490,18 @@ static void ioapic_mask_entry(int apic, int pin)
518 spin_unlock_irqrestore(&ioapic_lock, flags); 490 spin_unlock_irqrestore(&ioapic_lock, flags);
519} 491}
520 492
521#ifdef CONFIG_SMP
522static void send_cleanup_vector(struct irq_cfg *cfg)
523{
524 cpumask_var_t cleanup_mask;
525
526 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
527 unsigned int i;
528 cfg->move_cleanup_count = 0;
529 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
530 cfg->move_cleanup_count++;
531 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
532 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
533 } else {
534 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
535 cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
536 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
537 free_cpumask_var(cleanup_mask);
538 }
539 cfg->move_in_progress = 0;
540}
541
542static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
543{
544 int apic, pin;
545 struct irq_pin_list *entry;
546 u8 vector = cfg->vector;
547
548 entry = cfg->irq_2_pin;
549 for (;;) {
550 unsigned int reg;
551
552 if (!entry)
553 break;
554
555 apic = entry->apic;
556 pin = entry->pin;
557 /*
558 * With interrupt-remapping, destination information comes
559 * from interrupt-remapping table entry.
560 */
561 if (!irq_remapped(irq))
562 io_apic_write(apic, 0x11 + pin*2, dest);
563 reg = io_apic_read(apic, 0x10 + pin*2);
564 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
565 reg |= vector;
566 io_apic_modify(apic, 0x10 + pin*2, reg);
567 if (!entry->next)
568 break;
569 entry = entry->next;
570 }
571}
572
573static int
574assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
575
576/*
577 * Either sets desc->affinity to a valid value, and returns
578 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and
579 * leaves desc->affinity untouched.
580 */
581static unsigned int
582set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
583{
584 struct irq_cfg *cfg;
585 unsigned int irq;
586
587 if (!cpumask_intersects(mask, cpu_online_mask))
588 return BAD_APICID;
589
590 irq = desc->irq;
591 cfg = desc->chip_data;
592 if (assign_irq_vector(irq, cfg, mask))
593 return BAD_APICID;
594
595 /* check that before desc->addinity get updated */
596 set_extra_move_desc(desc, mask);
597
598 cpumask_copy(desc->affinity, mask);
599
600 return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
601}
602
603static void
604set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
605{
606 struct irq_cfg *cfg;
607 unsigned long flags;
608 unsigned int dest;
609 unsigned int irq;
610
611 irq = desc->irq;
612 cfg = desc->chip_data;
613
614 spin_lock_irqsave(&ioapic_lock, flags);
615 dest = set_desc_affinity(desc, mask);
616 if (dest != BAD_APICID) {
617 /* Only the high 8 bits are valid. */
618 dest = SET_APIC_LOGICAL_ID(dest);
619 __target_IO_APIC_irq(irq, dest, cfg);
620 }
621 spin_unlock_irqrestore(&ioapic_lock, flags);
622}
623
624static void
625set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
626{
627 struct irq_desc *desc;
628
629 desc = irq_to_desc(irq);
630
631 set_ioapic_affinity_irq_desc(desc, mask);
632}
633#endif /* CONFIG_SMP */
634
635/* 493/*
636 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are 494 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
637 * shared ISA-space IRQs, so we have to support them. We are super 495 * shared ISA-space IRQs, so we have to support them. We are super
638 * fast in the common case, and fast for shared ISA-space IRQs. 496 * fast in the common case, and fast for shared ISA-space IRQs.
639 */ 497 */
640static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin) 498static void add_pin_to_irq_node(struct irq_cfg *cfg, int node, int apic, int pin)
641{ 499{
642 struct irq_pin_list *entry; 500 struct irq_pin_list *entry;
643 501
644 entry = cfg->irq_2_pin; 502 entry = cfg->irq_2_pin;
645 if (!entry) { 503 if (!entry) {
646 entry = get_one_free_irq_2_pin(cpu); 504 entry = get_one_free_irq_2_pin(node);
647 if (!entry) { 505 if (!entry) {
648 printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n", 506 printk(KERN_ERR "can not alloc irq_2_pin to add %d - %d\n",
649 apic, pin); 507 apic, pin);
@@ -663,7 +521,7 @@ static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
663 entry = entry->next; 521 entry = entry->next;
664 } 522 }
665 523
666 entry->next = get_one_free_irq_2_pin(cpu); 524 entry->next = get_one_free_irq_2_pin(node);
667 entry = entry->next; 525 entry = entry->next;
668 entry->apic = apic; 526 entry->apic = apic;
669 entry->pin = pin; 527 entry->pin = pin;
@@ -672,7 +530,7 @@ static void add_pin_to_irq_cpu(struct irq_cfg *cfg, int cpu, int apic, int pin)
672/* 530/*
673 * Reroute an IRQ to a different pin. 531 * Reroute an IRQ to a different pin.
674 */ 532 */
675static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu, 533static void __init replace_pin_at_irq_node(struct irq_cfg *cfg, int node,
676 int oldapic, int oldpin, 534 int oldapic, int oldpin,
677 int newapic, int newpin) 535 int newapic, int newpin)
678{ 536{
@@ -692,7 +550,7 @@ static void __init replace_pin_at_irq_cpu(struct irq_cfg *cfg, int cpu,
692 550
693 /* why? call replace before add? */ 551 /* why? call replace before add? */
694 if (!replaced) 552 if (!replaced)
695 add_pin_to_irq_cpu(cfg, cpu, newapic, newpin); 553 add_pin_to_irq_node(cfg, node, newapic, newpin);
696} 554}
697 555
698static inline void io_apic_modify_irq(struct irq_cfg *cfg, 556static inline void io_apic_modify_irq(struct irq_cfg *cfg,
@@ -850,7 +708,6 @@ static int __init ioapic_pirq_setup(char *str)
850__setup("pirq=", ioapic_pirq_setup); 708__setup("pirq=", ioapic_pirq_setup);
851#endif /* CONFIG_X86_32 */ 709#endif /* CONFIG_X86_32 */
852 710
853#ifdef CONFIG_INTR_REMAP
854struct IO_APIC_route_entry **alloc_ioapic_entries(void) 711struct IO_APIC_route_entry **alloc_ioapic_entries(void)
855{ 712{
856 int apic; 713 int apic;
@@ -948,20 +805,6 @@ int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
948 return 0; 805 return 0;
949} 806}
950 807
951void reinit_intr_remapped_IO_APIC(int intr_remapping,
952 struct IO_APIC_route_entry **ioapic_entries)
953
954{
955 /*
956 * for now plain restore of previous settings.
957 * TBD: In the case of OS enabling interrupt-remapping,
958 * IO-APIC RTE's need to be setup to point to interrupt-remapping
959 * table entries. for now, do a plain restore, and wait for
960 * the setup_IO_APIC_irqs() to do proper initialization.
961 */
962 restore_IO_APIC_setup(ioapic_entries);
963}
964
965void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries) 808void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
966{ 809{
967 int apic; 810 int apic;
@@ -971,7 +814,6 @@ void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
971 814
972 kfree(ioapic_entries); 815 kfree(ioapic_entries);
973} 816}
974#endif
975 817
976/* 818/*
977 * Find the IRQ entry number of a certain pin. 819 * Find the IRQ entry number of a certain pin.
@@ -1032,54 +874,6 @@ static int __init find_isa_irq_apic(int irq, int type)
1032 return -1; 874 return -1;
1033} 875}
1034 876
1035/*
1036 * Find a specific PCI IRQ entry.
1037 * Not an __init, possibly needed by modules
1038 */
1039static int pin_2_irq(int idx, int apic, int pin);
1040
1041int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
1042{
1043 int apic, i, best_guess = -1;
1044
1045 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1046 bus, slot, pin);
1047 if (test_bit(bus, mp_bus_not_pci)) {
1048 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1049 return -1;
1050 }
1051 for (i = 0; i < mp_irq_entries; i++) {
1052 int lbus = mp_irqs[i].srcbus;
1053
1054 for (apic = 0; apic < nr_ioapics; apic++)
1055 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
1056 mp_irqs[i].dstapic == MP_APIC_ALL)
1057 break;
1058
1059 if (!test_bit(lbus, mp_bus_not_pci) &&
1060 !mp_irqs[i].irqtype &&
1061 (bus == lbus) &&
1062 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1063 int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
1064
1065 if (!(apic || IO_APIC_IRQ(irq)))
1066 continue;
1067
1068 if (pin == (mp_irqs[i].srcbusirq & 3))
1069 return irq;
1070 /*
1071 * Use the first all-but-pin matching entry as a
1072 * best-guess fuzzy result for broken mptables.
1073 */
1074 if (best_guess < 0)
1075 best_guess = irq;
1076 }
1077 }
1078 return best_guess;
1079}
1080
1081EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1082
1083#if defined(CONFIG_EISA) || defined(CONFIG_MCA) 877#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
1084/* 878/*
1085 * EISA Edge/Level control register, ELCR 879 * EISA Edge/Level control register, ELCR
@@ -1298,6 +1092,64 @@ static int pin_2_irq(int idx, int apic, int pin)
1298 return irq; 1092 return irq;
1299} 1093}
1300 1094
1095/*
1096 * Find a specific PCI IRQ entry.
1097 * Not an __init, possibly needed by modules
1098 */
1099int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin,
1100 struct io_apic_irq_attr *irq_attr)
1101{
1102 int apic, i, best_guess = -1;
1103
1104 apic_printk(APIC_DEBUG,
1105 "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
1106 bus, slot, pin);
1107 if (test_bit(bus, mp_bus_not_pci)) {
1108 apic_printk(APIC_VERBOSE,
1109 "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1110 return -1;
1111 }
1112 for (i = 0; i < mp_irq_entries; i++) {
1113 int lbus = mp_irqs[i].srcbus;
1114
1115 for (apic = 0; apic < nr_ioapics; apic++)
1116 if (mp_ioapics[apic].apicid == mp_irqs[i].dstapic ||
1117 mp_irqs[i].dstapic == MP_APIC_ALL)
1118 break;
1119
1120 if (!test_bit(lbus, mp_bus_not_pci) &&
1121 !mp_irqs[i].irqtype &&
1122 (bus == lbus) &&
1123 (slot == ((mp_irqs[i].srcbusirq >> 2) & 0x1f))) {
1124 int irq = pin_2_irq(i, apic, mp_irqs[i].dstirq);
1125
1126 if (!(apic || IO_APIC_IRQ(irq)))
1127 continue;
1128
1129 if (pin == (mp_irqs[i].srcbusirq & 3)) {
1130 set_io_apic_irq_attr(irq_attr, apic,
1131 mp_irqs[i].dstirq,
1132 irq_trigger(i),
1133 irq_polarity(i));
1134 return irq;
1135 }
1136 /*
1137 * Use the first all-but-pin matching entry as a
1138 * best-guess fuzzy result for broken mptables.
1139 */
1140 if (best_guess < 0) {
1141 set_io_apic_irq_attr(irq_attr, apic,
1142 mp_irqs[i].dstirq,
1143 irq_trigger(i),
1144 irq_polarity(i));
1145 best_guess = irq;
1146 }
1147 }
1148 }
1149 return best_guess;
1150}
1151EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1152
1301void lock_vector_lock(void) 1153void lock_vector_lock(void)
1302{ 1154{
1303 /* Used to the online set of cpus does not change 1155 /* Used to the online set of cpus does not change
@@ -1628,58 +1480,70 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq
1628 ioapic_write_entry(apic_id, pin, entry); 1480 ioapic_write_entry(apic_id, pin, entry);
1629} 1481}
1630 1482
1483static struct {
1484 DECLARE_BITMAP(pin_programmed, MP_MAX_IOAPIC_PIN + 1);
1485} mp_ioapic_routing[MAX_IO_APICS];
1486
1631static void __init setup_IO_APIC_irqs(void) 1487static void __init setup_IO_APIC_irqs(void)
1632{ 1488{
1633 int apic_id, pin, idx, irq; 1489 int apic_id = 0, pin, idx, irq;
1634 int notcon = 0; 1490 int notcon = 0;
1635 struct irq_desc *desc; 1491 struct irq_desc *desc;
1636 struct irq_cfg *cfg; 1492 struct irq_cfg *cfg;
1637 int cpu = boot_cpu_id; 1493 int node = cpu_to_node(boot_cpu_id);
1638 1494
1639 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n"); 1495 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1640 1496
1641 for (apic_id = 0; apic_id < nr_ioapics; apic_id++) { 1497#ifdef CONFIG_ACPI
1642 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) { 1498 if (!acpi_disabled && acpi_ioapic) {
1643 1499 apic_id = mp_find_ioapic(0);
1644 idx = find_irq_entry(apic_id, pin, mp_INT); 1500 if (apic_id < 0)
1645 if (idx == -1) { 1501 apic_id = 0;
1646 if (!notcon) { 1502 }
1647 notcon = 1; 1503#endif
1648 apic_printk(APIC_VERBOSE,
1649 KERN_DEBUG " %d-%d",
1650 mp_ioapics[apic_id].apicid, pin);
1651 } else
1652 apic_printk(APIC_VERBOSE, " %d-%d",
1653 mp_ioapics[apic_id].apicid, pin);
1654 continue;
1655 }
1656 if (notcon) {
1657 apic_printk(APIC_VERBOSE,
1658 " (apicid-pin) not connected\n");
1659 notcon = 0;
1660 }
1661 1504
1662 irq = pin_2_irq(idx, apic_id, pin); 1505 for (pin = 0; pin < nr_ioapic_registers[apic_id]; pin++) {
1506 idx = find_irq_entry(apic_id, pin, mp_INT);
1507 if (idx == -1) {
1508 if (!notcon) {
1509 notcon = 1;
1510 apic_printk(APIC_VERBOSE,
1511 KERN_DEBUG " %d-%d",
1512 mp_ioapics[apic_id].apicid, pin);
1513 } else
1514 apic_printk(APIC_VERBOSE, " %d-%d",
1515 mp_ioapics[apic_id].apicid, pin);
1516 continue;
1517 }
1518 if (notcon) {
1519 apic_printk(APIC_VERBOSE,
1520 " (apicid-pin) not connected\n");
1521 notcon = 0;
1522 }
1663 1523
1664 /* 1524 irq = pin_2_irq(idx, apic_id, pin);
1665 * Skip the timer IRQ if there's a quirk handler
1666 * installed and if it returns 1:
1667 */
1668 if (apic->multi_timer_check &&
1669 apic->multi_timer_check(apic_id, irq))
1670 continue;
1671 1525
1672 desc = irq_to_desc_alloc_cpu(irq, cpu); 1526 /*
1673 if (!desc) { 1527 * Skip the timer IRQ if there's a quirk handler
1674 printk(KERN_INFO "can not get irq_desc for %d\n", irq); 1528 * installed and if it returns 1:
1675 continue; 1529 */
1676 } 1530 if (apic->multi_timer_check &&
1677 cfg = desc->chip_data; 1531 apic->multi_timer_check(apic_id, irq))
1678 add_pin_to_irq_cpu(cfg, cpu, apic_id, pin); 1532 continue;
1679 1533
1680 setup_IO_APIC_irq(apic_id, pin, irq, desc, 1534 desc = irq_to_desc_alloc_node(irq, node);
1681 irq_trigger(idx), irq_polarity(idx)); 1535 if (!desc) {
1536 printk(KERN_INFO "can not get irq_desc for %d\n", irq);
1537 continue;
1682 } 1538 }
1539 cfg = desc->chip_data;
1540 add_pin_to_irq_node(cfg, node, apic_id, pin);
1541 /*
1542 * don't mark it in pin_programmed, so later acpi could
1543 * set it correctly when irq < 16
1544 */
1545 setup_IO_APIC_irq(apic_id, pin, irq, desc,
1546 irq_trigger(idx), irq_polarity(idx));
1683 } 1547 }
1684 1548
1685 if (notcon) 1549 if (notcon)
@@ -1869,7 +1733,7 @@ __apicdebuginit(void) print_APIC_bitfield(int base)
1869 1733
1870__apicdebuginit(void) print_local_APIC(void *dummy) 1734__apicdebuginit(void) print_local_APIC(void *dummy)
1871{ 1735{
1872 unsigned int v, ver, maxlvt; 1736 unsigned int i, v, ver, maxlvt;
1873 u64 icr; 1737 u64 icr;
1874 1738
1875 if (apic_verbosity == APIC_QUIET) 1739 if (apic_verbosity == APIC_QUIET)
@@ -1957,6 +1821,18 @@ __apicdebuginit(void) print_local_APIC(void *dummy)
1957 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v); 1821 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1958 v = apic_read(APIC_TDCR); 1822 v = apic_read(APIC_TDCR);
1959 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v); 1823 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1824
1825 if (boot_cpu_has(X86_FEATURE_EXTAPIC)) {
1826 v = apic_read(APIC_EFEAT);
1827 maxlvt = (v >> 16) & 0xff;
1828 printk(KERN_DEBUG "... APIC EFEAT: %08x\n", v);
1829 v = apic_read(APIC_ECTRL);
1830 printk(KERN_DEBUG "... APIC ECTRL: %08x\n", v);
1831 for (i = 0; i < maxlvt; i++) {
1832 v = apic_read(APIC_EILVTn(i));
1833 printk(KERN_DEBUG "... APIC EILVT%d: %08x\n", i, v);
1834 }
1835 }
1960 printk("\n"); 1836 printk("\n");
1961} 1837}
1962 1838
@@ -2005,6 +1881,11 @@ __apicdebuginit(void) print_PIC(void)
2005__apicdebuginit(int) print_all_ICs(void) 1881__apicdebuginit(int) print_all_ICs(void)
2006{ 1882{
2007 print_PIC(); 1883 print_PIC();
1884
1885 /* don't print out if apic is not there */
1886 if (!cpu_has_apic || disable_apic)
1887 return 0;
1888
2008 print_all_local_APICs(); 1889 print_all_local_APICs();
2009 print_IO_APIC(); 1890 print_IO_APIC();
2010 1891
@@ -2360,6 +2241,118 @@ static int ioapic_retrigger_irq(unsigned int irq)
2360 */ 2241 */
2361 2242
2362#ifdef CONFIG_SMP 2243#ifdef CONFIG_SMP
2244static void send_cleanup_vector(struct irq_cfg *cfg)
2245{
2246 cpumask_var_t cleanup_mask;
2247
2248 if (unlikely(!alloc_cpumask_var(&cleanup_mask, GFP_ATOMIC))) {
2249 unsigned int i;
2250 cfg->move_cleanup_count = 0;
2251 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2252 cfg->move_cleanup_count++;
2253 for_each_cpu_and(i, cfg->old_domain, cpu_online_mask)
2254 apic->send_IPI_mask(cpumask_of(i), IRQ_MOVE_CLEANUP_VECTOR);
2255 } else {
2256 cpumask_and(cleanup_mask, cfg->old_domain, cpu_online_mask);
2257 cfg->move_cleanup_count = cpumask_weight(cleanup_mask);
2258 apic->send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2259 free_cpumask_var(cleanup_mask);
2260 }
2261 cfg->move_in_progress = 0;
2262}
2263
2264static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq_cfg *cfg)
2265{
2266 int apic, pin;
2267 struct irq_pin_list *entry;
2268 u8 vector = cfg->vector;
2269
2270 entry = cfg->irq_2_pin;
2271 for (;;) {
2272 unsigned int reg;
2273
2274 if (!entry)
2275 break;
2276
2277 apic = entry->apic;
2278 pin = entry->pin;
2279 /*
2280 * With interrupt-remapping, destination information comes
2281 * from interrupt-remapping table entry.
2282 */
2283 if (!irq_remapped(irq))
2284 io_apic_write(apic, 0x11 + pin*2, dest);
2285 reg = io_apic_read(apic, 0x10 + pin*2);
2286 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
2287 reg |= vector;
2288 io_apic_modify(apic, 0x10 + pin*2, reg);
2289 if (!entry->next)
2290 break;
2291 entry = entry->next;
2292 }
2293}
2294
2295static int
2296assign_irq_vector(int irq, struct irq_cfg *cfg, const struct cpumask *mask);
2297
2298/*
2299 * Either sets desc->affinity to a valid value, and returns
2300 * ->cpu_mask_to_apicid of that, or returns BAD_APICID and
2301 * leaves desc->affinity untouched.
2302 */
2303static unsigned int
2304set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask)
2305{
2306 struct irq_cfg *cfg;
2307 unsigned int irq;
2308
2309 if (!cpumask_intersects(mask, cpu_online_mask))
2310 return BAD_APICID;
2311
2312 irq = desc->irq;
2313 cfg = desc->chip_data;
2314 if (assign_irq_vector(irq, cfg, mask))
2315 return BAD_APICID;
2316
2317 cpumask_copy(desc->affinity, mask);
2318
2319 return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain);
2320}
2321
2322static int
2323set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2324{
2325 struct irq_cfg *cfg;
2326 unsigned long flags;
2327 unsigned int dest;
2328 unsigned int irq;
2329 int ret = -1;
2330
2331 irq = desc->irq;
2332 cfg = desc->chip_data;
2333
2334 spin_lock_irqsave(&ioapic_lock, flags);
2335 dest = set_desc_affinity(desc, mask);
2336 if (dest != BAD_APICID) {
2337 /* Only the high 8 bits are valid. */
2338 dest = SET_APIC_LOGICAL_ID(dest);
2339 __target_IO_APIC_irq(irq, dest, cfg);
2340 ret = 0;
2341 }
2342 spin_unlock_irqrestore(&ioapic_lock, flags);
2343
2344 return ret;
2345}
2346
2347static int
2348set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
2349{
2350 struct irq_desc *desc;
2351
2352 desc = irq_to_desc(irq);
2353
2354 return set_ioapic_affinity_irq_desc(desc, mask);
2355}
2363 2356
2364#ifdef CONFIG_INTR_REMAP 2357#ifdef CONFIG_INTR_REMAP
2365 2358
@@ -2374,26 +2367,25 @@ static int ioapic_retrigger_irq(unsigned int irq)
2374 * Real vector that is used for interrupting cpu will be coming from 2367 * Real vector that is used for interrupting cpu will be coming from
2375 * the interrupt-remapping table entry. 2368 * the interrupt-remapping table entry.
2376 */ 2369 */
2377static void 2370static int
2378migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask) 2371migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2379{ 2372{
2380 struct irq_cfg *cfg; 2373 struct irq_cfg *cfg;
2381 struct irte irte; 2374 struct irte irte;
2382 unsigned int dest; 2375 unsigned int dest;
2383 unsigned int irq; 2376 unsigned int irq;
2377 int ret = -1;
2384 2378
2385 if (!cpumask_intersects(mask, cpu_online_mask)) 2379 if (!cpumask_intersects(mask, cpu_online_mask))
2386 return; 2380 return ret;
2387 2381
2388 irq = desc->irq; 2382 irq = desc->irq;
2389 if (get_irte(irq, &irte)) 2383 if (get_irte(irq, &irte))
2390 return; 2384 return ret;
2391 2385
2392 cfg = desc->chip_data; 2386 cfg = desc->chip_data;
2393 if (assign_irq_vector(irq, cfg, mask)) 2387 if (assign_irq_vector(irq, cfg, mask))
2394 return; 2388 return ret;
2395
2396 set_extra_move_desc(desc, mask);
2397 2389
2398 dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); 2390 dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask);
2399 2391
@@ -2409,27 +2401,30 @@ migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
2409 send_cleanup_vector(cfg); 2401 send_cleanup_vector(cfg);
2410 2402
2411 cpumask_copy(desc->affinity, mask); 2403 cpumask_copy(desc->affinity, mask);
2404
2405 return 0;
2412} 2406}
2413 2407
2414/* 2408/*
2415 * Migrates the IRQ destination in the process context. 2409 * Migrates the IRQ destination in the process context.
2416 */ 2410 */
2417static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, 2411static int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2418 const struct cpumask *mask) 2412 const struct cpumask *mask)
2419{ 2413{
2420 migrate_ioapic_irq_desc(desc, mask); 2414 return migrate_ioapic_irq_desc(desc, mask);
2421} 2415}
2422static void set_ir_ioapic_affinity_irq(unsigned int irq, 2416static int set_ir_ioapic_affinity_irq(unsigned int irq,
2423 const struct cpumask *mask) 2417 const struct cpumask *mask)
2424{ 2418{
2425 struct irq_desc *desc = irq_to_desc(irq); 2419 struct irq_desc *desc = irq_to_desc(irq);
2426 2420
2427 set_ir_ioapic_affinity_irq_desc(desc, mask); 2421 return set_ir_ioapic_affinity_irq_desc(desc, mask);
2428} 2422}
2429#else 2423#else
2430static inline void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, 2424static inline int set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc,
2431 const struct cpumask *mask) 2425 const struct cpumask *mask)
2432{ 2426{
2427 return 0;
2433} 2428}
2434#endif 2429#endif
2435 2430
@@ -2491,86 +2486,19 @@ static void irq_complete_move(struct irq_desc **descp)
2491 struct irq_cfg *cfg = desc->chip_data; 2486 struct irq_cfg *cfg = desc->chip_data;
2492 unsigned vector, me; 2487 unsigned vector, me;
2493 2488
2494 if (likely(!cfg->move_in_progress)) { 2489 if (likely(!cfg->move_in_progress))
2495#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
2496 if (likely(!cfg->move_desc_pending))
2497 return;
2498
2499 /* domain has not changed, but affinity did */
2500 me = smp_processor_id();
2501 if (cpumask_test_cpu(me, desc->affinity)) {
2502 *descp = desc = move_irq_desc(desc, me);
2503 /* get the new one */
2504 cfg = desc->chip_data;
2505 cfg->move_desc_pending = 0;
2506 }
2507#endif
2508 return; 2490 return;
2509 }
2510 2491
2511 vector = ~get_irq_regs()->orig_ax; 2492 vector = ~get_irq_regs()->orig_ax;
2512 me = smp_processor_id(); 2493 me = smp_processor_id();
2513 2494
2514 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) { 2495 if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
2515#ifdef CONFIG_NUMA_MIGRATE_IRQ_DESC
2516 *descp = desc = move_irq_desc(desc, me);
2517 /* get the new one */
2518 cfg = desc->chip_data;
2519#endif
2520 send_cleanup_vector(cfg); 2496 send_cleanup_vector(cfg);
2521 }
2522} 2497}
2523#else 2498#else
2524static inline void irq_complete_move(struct irq_desc **descp) {} 2499static inline void irq_complete_move(struct irq_desc **descp) {}
2525#endif 2500#endif
2526 2501
2527static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2528{
2529 int apic, pin;
2530 struct irq_pin_list *entry;
2531
2532 entry = cfg->irq_2_pin;
2533 for (;;) {
2534
2535 if (!entry)
2536 break;
2537
2538 apic = entry->apic;
2539 pin = entry->pin;
2540 io_apic_eoi(apic, pin);
2541 entry = entry->next;
2542 }
2543}
2544
2545static void
2546eoi_ioapic_irq(struct irq_desc *desc)
2547{
2548 struct irq_cfg *cfg;
2549 unsigned long flags;
2550 unsigned int irq;
2551
2552 irq = desc->irq;
2553 cfg = desc->chip_data;
2554
2555 spin_lock_irqsave(&ioapic_lock, flags);
2556 __eoi_ioapic_irq(irq, cfg);
2557 spin_unlock_irqrestore(&ioapic_lock, flags);
2558}
2559
2560#ifdef CONFIG_X86_X2APIC
2561static void ack_x2apic_level(unsigned int irq)
2562{
2563 struct irq_desc *desc = irq_to_desc(irq);
2564 ack_x2APIC_irq();
2565 eoi_ioapic_irq(desc);
2566}
2567
2568static void ack_x2apic_edge(unsigned int irq)
2569{
2570 ack_x2APIC_irq();
2571}
2572#endif
2573
2574static void ack_apic_edge(unsigned int irq) 2502static void ack_apic_edge(unsigned int irq)
2575{ 2503{
2576 struct irq_desc *desc = irq_to_desc(irq); 2504 struct irq_desc *desc = irq_to_desc(irq);
@@ -2634,9 +2562,6 @@ static void ack_apic_level(unsigned int irq)
2634 */ 2562 */
2635 ack_APIC_irq(); 2563 ack_APIC_irq();
2636 2564
2637 if (irq_remapped(irq))
2638 eoi_ioapic_irq(desc);
2639
2640 /* Now we can move and renable the irq */ 2565 /* Now we can move and renable the irq */
2641 if (unlikely(do_unmask_irq)) { 2566 if (unlikely(do_unmask_irq)) {
2642 /* Only migrate the irq if the ack has been received. 2567 /* Only migrate the irq if the ack has been received.
@@ -2683,22 +2608,50 @@ static void ack_apic_level(unsigned int irq)
2683} 2608}
2684 2609
2685#ifdef CONFIG_INTR_REMAP 2610#ifdef CONFIG_INTR_REMAP
2611static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2612{
2613 int apic, pin;
2614 struct irq_pin_list *entry;
2615
2616 entry = cfg->irq_2_pin;
2617 for (;;) {
2618
2619 if (!entry)
2620 break;
2621
2622 apic = entry->apic;
2623 pin = entry->pin;
2624 io_apic_eoi(apic, pin);
2625 entry = entry->next;
2626 }
2627}
2628
2629static void
2630eoi_ioapic_irq(struct irq_desc *desc)
2631{
2632 struct irq_cfg *cfg;
2633 unsigned long flags;
2634 unsigned int irq;
2635
2636 irq = desc->irq;
2637 cfg = desc->chip_data;
2638
2639 spin_lock_irqsave(&ioapic_lock, flags);
2640 __eoi_ioapic_irq(irq, cfg);
2641 spin_unlock_irqrestore(&ioapic_lock, flags);
2642}
2643
2686static void ir_ack_apic_edge(unsigned int irq) 2644static void ir_ack_apic_edge(unsigned int irq)
2687{ 2645{
2688#ifdef CONFIG_X86_X2APIC 2646 ack_APIC_irq();
2689 if (x2apic_enabled())
2690 return ack_x2apic_edge(irq);
2691#endif
2692 return ack_apic_edge(irq);
2693} 2647}
2694 2648
2695static void ir_ack_apic_level(unsigned int irq) 2649static void ir_ack_apic_level(unsigned int irq)
2696{ 2650{
2697#ifdef CONFIG_X86_X2APIC 2651 struct irq_desc *desc = irq_to_desc(irq);
2698 if (x2apic_enabled()) 2652
2699 return ack_x2apic_level(irq); 2653 ack_APIC_irq();
2700#endif 2654 eoi_ioapic_irq(desc);
2701 return ack_apic_level(irq);
2702} 2655}
2703#endif /* CONFIG_INTR_REMAP */ 2656#endif /* CONFIG_INTR_REMAP */
2704 2657
@@ -2903,7 +2856,7 @@ static inline void __init check_timer(void)
2903{ 2856{
2904 struct irq_desc *desc = irq_to_desc(0); 2857 struct irq_desc *desc = irq_to_desc(0);
2905 struct irq_cfg *cfg = desc->chip_data; 2858 struct irq_cfg *cfg = desc->chip_data;
2906 int cpu = boot_cpu_id; 2859 int node = cpu_to_node(boot_cpu_id);
2907 int apic1, pin1, apic2, pin2; 2860 int apic1, pin1, apic2, pin2;
2908 unsigned long flags; 2861 unsigned long flags;
2909 int no_pin1 = 0; 2862 int no_pin1 = 0;
@@ -2969,7 +2922,7 @@ static inline void __init check_timer(void)
2969 * Ok, does IRQ0 through the IOAPIC work? 2922 * Ok, does IRQ0 through the IOAPIC work?
2970 */ 2923 */
2971 if (no_pin1) { 2924 if (no_pin1) {
2972 add_pin_to_irq_cpu(cfg, cpu, apic1, pin1); 2925 add_pin_to_irq_node(cfg, node, apic1, pin1);
2973 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector); 2926 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
2974 } else { 2927 } else {
2975 /* for edge trigger, setup_IO_APIC_irq already 2928 /* for edge trigger, setup_IO_APIC_irq already
@@ -3006,7 +2959,7 @@ static inline void __init check_timer(void)
3006 /* 2959 /*
3007 * legacy devices should be connected to IO APIC #0 2960 * legacy devices should be connected to IO APIC #0
3008 */ 2961 */
3009 replace_pin_at_irq_cpu(cfg, cpu, apic1, pin1, apic2, pin2); 2962 replace_pin_at_irq_node(cfg, node, apic1, pin1, apic2, pin2);
3010 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector); 2963 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
3011 enable_8259A_irq(0); 2964 enable_8259A_irq(0);
3012 if (timer_irq_works()) { 2965 if (timer_irq_works()) {
@@ -3218,14 +3171,13 @@ static int nr_irqs_gsi = NR_IRQS_LEGACY;
3218/* 3171/*
3219 * Dynamic irq allocate and deallocation 3172 * Dynamic irq allocate and deallocation
3220 */ 3173 */
3221unsigned int create_irq_nr(unsigned int irq_want) 3174unsigned int create_irq_nr(unsigned int irq_want, int node)
3222{ 3175{
3223 /* Allocate an unused irq */ 3176 /* Allocate an unused irq */
3224 unsigned int irq; 3177 unsigned int irq;
3225 unsigned int new; 3178 unsigned int new;
3226 unsigned long flags; 3179 unsigned long flags;
3227 struct irq_cfg *cfg_new = NULL; 3180 struct irq_cfg *cfg_new = NULL;
3228 int cpu = boot_cpu_id;
3229 struct irq_desc *desc_new = NULL; 3181 struct irq_desc *desc_new = NULL;
3230 3182
3231 irq = 0; 3183 irq = 0;
@@ -3234,7 +3186,7 @@ unsigned int create_irq_nr(unsigned int irq_want)
3234 3186
3235 spin_lock_irqsave(&vector_lock, flags); 3187 spin_lock_irqsave(&vector_lock, flags);
3236 for (new = irq_want; new < nr_irqs; new++) { 3188 for (new = irq_want; new < nr_irqs; new++) {
3237 desc_new = irq_to_desc_alloc_cpu(new, cpu); 3189 desc_new = irq_to_desc_alloc_node(new, node);
3238 if (!desc_new) { 3190 if (!desc_new) {
3239 printk(KERN_INFO "can not get irq_desc for %d\n", new); 3191 printk(KERN_INFO "can not get irq_desc for %d\n", new);
3240 continue; 3192 continue;
@@ -3243,6 +3195,9 @@ unsigned int create_irq_nr(unsigned int irq_want)
3243 3195
3244 if (cfg_new->vector != 0) 3196 if (cfg_new->vector != 0)
3245 continue; 3197 continue;
3198
3199 desc_new = move_irq_desc(desc_new, node);
3200
3246 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0) 3201 if (__assign_irq_vector(new, cfg_new, apic->target_cpus()) == 0)
3247 irq = new; 3202 irq = new;
3248 break; 3203 break;
@@ -3260,11 +3215,12 @@ unsigned int create_irq_nr(unsigned int irq_want)
3260 3215
3261int create_irq(void) 3216int create_irq(void)
3262{ 3217{
3218 int node = cpu_to_node(boot_cpu_id);
3263 unsigned int irq_want; 3219 unsigned int irq_want;
3264 int irq; 3220 int irq;
3265 3221
3266 irq_want = nr_irqs_gsi; 3222 irq_want = nr_irqs_gsi;
3267 irq = create_irq_nr(irq_want); 3223 irq = create_irq_nr(irq_want, node);
3268 3224
3269 if (irq == 0) 3225 if (irq == 0)
3270 irq = -1; 3226 irq = -1;
@@ -3366,7 +3322,7 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms
3366} 3322}
3367 3323
3368#ifdef CONFIG_SMP 3324#ifdef CONFIG_SMP
3369static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask) 3325static int set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3370{ 3326{
3371 struct irq_desc *desc = irq_to_desc(irq); 3327 struct irq_desc *desc = irq_to_desc(irq);
3372 struct irq_cfg *cfg; 3328 struct irq_cfg *cfg;
@@ -3375,7 +3331,7 @@ static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3375 3331
3376 dest = set_desc_affinity(desc, mask); 3332 dest = set_desc_affinity(desc, mask);
3377 if (dest == BAD_APICID) 3333 if (dest == BAD_APICID)
3378 return; 3334 return -1;
3379 3335
3380 cfg = desc->chip_data; 3336 cfg = desc->chip_data;
3381 3337
@@ -3387,13 +3343,15 @@ static void set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3387 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3343 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3388 3344
3389 write_msi_msg_desc(desc, &msg); 3345 write_msi_msg_desc(desc, &msg);
3346
3347 return 0;
3390} 3348}
3391#ifdef CONFIG_INTR_REMAP 3349#ifdef CONFIG_INTR_REMAP
3392/* 3350/*
3393 * Migrate the MSI irq to another cpumask. This migration is 3351 * Migrate the MSI irq to another cpumask. This migration is
3394 * done in the process context using interrupt-remapping hardware. 3352 * done in the process context using interrupt-remapping hardware.
3395 */ 3353 */
3396static void 3354static int
3397ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask) 3355ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3398{ 3356{
3399 struct irq_desc *desc = irq_to_desc(irq); 3357 struct irq_desc *desc = irq_to_desc(irq);
@@ -3402,11 +3360,11 @@ ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3402 struct irte irte; 3360 struct irte irte;
3403 3361
3404 if (get_irte(irq, &irte)) 3362 if (get_irte(irq, &irte))
3405 return; 3363 return -1;
3406 3364
3407 dest = set_desc_affinity(desc, mask); 3365 dest = set_desc_affinity(desc, mask);
3408 if (dest == BAD_APICID) 3366 if (dest == BAD_APICID)
3409 return; 3367 return -1;
3410 3368
3411 irte.vector = cfg->vector; 3369 irte.vector = cfg->vector;
3412 irte.dest_id = IRTE_DEST(dest); 3370 irte.dest_id = IRTE_DEST(dest);
@@ -3423,6 +3381,8 @@ ir_set_msi_irq_affinity(unsigned int irq, const struct cpumask *mask)
3423 */ 3381 */
3424 if (cfg->move_in_progress) 3382 if (cfg->move_in_progress)
3425 send_cleanup_vector(cfg); 3383 send_cleanup_vector(cfg);
3384
3385 return 0;
3426} 3386}
3427 3387
3428#endif 3388#endif
@@ -3518,15 +3478,17 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3518 unsigned int irq_want; 3478 unsigned int irq_want;
3519 struct intel_iommu *iommu = NULL; 3479 struct intel_iommu *iommu = NULL;
3520 int index = 0; 3480 int index = 0;
3481 int node;
3521 3482
3522 /* x86 doesn't support multiple MSI yet */ 3483 /* x86 doesn't support multiple MSI yet */
3523 if (type == PCI_CAP_ID_MSI && nvec > 1) 3484 if (type == PCI_CAP_ID_MSI && nvec > 1)
3524 return 1; 3485 return 1;
3525 3486
3487 node = dev_to_node(&dev->dev);
3526 irq_want = nr_irqs_gsi; 3488 irq_want = nr_irqs_gsi;
3527 sub_handle = 0; 3489 sub_handle = 0;
3528 list_for_each_entry(msidesc, &dev->msi_list, list) { 3490 list_for_each_entry(msidesc, &dev->msi_list, list) {
3529 irq = create_irq_nr(irq_want); 3491 irq = create_irq_nr(irq_want, node);
3530 if (irq == 0) 3492 if (irq == 0)
3531 return -1; 3493 return -1;
3532 irq_want = irq + 1; 3494 irq_want = irq + 1;
@@ -3576,7 +3538,7 @@ void arch_teardown_msi_irq(unsigned int irq)
3576 3538
3577#if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP) 3539#if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP)
3578#ifdef CONFIG_SMP 3540#ifdef CONFIG_SMP
3579static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask) 3541static int dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3580{ 3542{
3581 struct irq_desc *desc = irq_to_desc(irq); 3543 struct irq_desc *desc = irq_to_desc(irq);
3582 struct irq_cfg *cfg; 3544 struct irq_cfg *cfg;
@@ -3585,7 +3547,7 @@ static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3585 3547
3586 dest = set_desc_affinity(desc, mask); 3548 dest = set_desc_affinity(desc, mask);
3587 if (dest == BAD_APICID) 3549 if (dest == BAD_APICID)
3588 return; 3550 return -1;
3589 3551
3590 cfg = desc->chip_data; 3552 cfg = desc->chip_data;
3591 3553
@@ -3597,6 +3559,8 @@ static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3597 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3559 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3598 3560
3599 dmar_msi_write(irq, &msg); 3561 dmar_msi_write(irq, &msg);
3562
3563 return 0;
3600} 3564}
3601 3565
3602#endif /* CONFIG_SMP */ 3566#endif /* CONFIG_SMP */
@@ -3630,7 +3594,7 @@ int arch_setup_dmar_msi(unsigned int irq)
3630#ifdef CONFIG_HPET_TIMER 3594#ifdef CONFIG_HPET_TIMER
3631 3595
3632#ifdef CONFIG_SMP 3596#ifdef CONFIG_SMP
3633static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask) 3597static int hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3634{ 3598{
3635 struct irq_desc *desc = irq_to_desc(irq); 3599 struct irq_desc *desc = irq_to_desc(irq);
3636 struct irq_cfg *cfg; 3600 struct irq_cfg *cfg;
@@ -3639,7 +3603,7 @@ static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3639 3603
3640 dest = set_desc_affinity(desc, mask); 3604 dest = set_desc_affinity(desc, mask);
3641 if (dest == BAD_APICID) 3605 if (dest == BAD_APICID)
3642 return; 3606 return -1;
3643 3607
3644 cfg = desc->chip_data; 3608 cfg = desc->chip_data;
3645 3609
@@ -3651,6 +3615,8 @@ static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask)
3651 msg.address_lo |= MSI_ADDR_DEST_ID(dest); 3615 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3652 3616
3653 hpet_msi_write(irq, &msg); 3617 hpet_msi_write(irq, &msg);
3618
3619 return 0;
3654} 3620}
3655 3621
3656#endif /* CONFIG_SMP */ 3622#endif /* CONFIG_SMP */
@@ -3707,7 +3673,7 @@ static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
3707 write_ht_irq_msg(irq, &msg); 3673 write_ht_irq_msg(irq, &msg);
3708} 3674}
3709 3675
3710static void set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask) 3676static int set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3711{ 3677{
3712 struct irq_desc *desc = irq_to_desc(irq); 3678 struct irq_desc *desc = irq_to_desc(irq);
3713 struct irq_cfg *cfg; 3679 struct irq_cfg *cfg;
@@ -3715,11 +3681,13 @@ static void set_ht_irq_affinity(unsigned int irq, const struct cpumask *mask)
3715 3681
3716 dest = set_desc_affinity(desc, mask); 3682 dest = set_desc_affinity(desc, mask);
3717 if (dest == BAD_APICID) 3683 if (dest == BAD_APICID)
3718 return; 3684 return -1;
3719 3685
3720 cfg = desc->chip_data; 3686 cfg = desc->chip_data;
3721 3687
3722 target_ht_irq(irq, dest, cfg->vector); 3688 target_ht_irq(irq, dest, cfg->vector);
3689
3690 return 0;
3723} 3691}
3724 3692
3725#endif 3693#endif
@@ -3794,6 +3762,8 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3794 unsigned long flags; 3762 unsigned long flags;
3795 int err; 3763 int err;
3796 3764
3765 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3766
3797 cfg = irq_cfg(irq); 3767 cfg = irq_cfg(irq);
3798 3768
3799 err = assign_irq_vector(irq, cfg, eligible_cpu); 3769 err = assign_irq_vector(irq, cfg, eligible_cpu);
@@ -3807,15 +3777,13 @@ int arch_enable_uv_irq(char *irq_name, unsigned int irq, int cpu, int mmr_blade,
3807 3777
3808 mmr_value = 0; 3778 mmr_value = 0;
3809 entry = (struct uv_IO_APIC_route_entry *)&mmr_value; 3779 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3810 BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long)); 3780 entry->vector = cfg->vector;
3811 3781 entry->delivery_mode = apic->irq_delivery_mode;
3812 entry->vector = cfg->vector; 3782 entry->dest_mode = apic->irq_dest_mode;
3813 entry->delivery_mode = apic->irq_delivery_mode; 3783 entry->polarity = 0;
3814 entry->dest_mode = apic->irq_dest_mode; 3784 entry->trigger = 0;
3815 entry->polarity = 0; 3785 entry->mask = 0;
3816 entry->trigger = 0; 3786 entry->dest = apic->cpu_mask_to_apicid(eligible_cpu);
3817 entry->mask = 0;
3818 entry->dest = apic->cpu_mask_to_apicid(eligible_cpu);
3819 3787
3820 mmr_pnode = uv_blade_to_pnode(mmr_blade); 3788 mmr_pnode = uv_blade_to_pnode(mmr_blade);
3821 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value); 3789 uv_write_global_mmr64(mmr_pnode, mmr_offset, mmr_value);
@@ -3833,10 +3801,10 @@ void arch_disable_uv_irq(int mmr_blade, unsigned long mmr_offset)
3833 struct uv_IO_APIC_route_entry *entry; 3801 struct uv_IO_APIC_route_entry *entry;
3834 int mmr_pnode; 3802 int mmr_pnode;
3835 3803
3804 BUILD_BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3805
3836 mmr_value = 0; 3806 mmr_value = 0;
3837 entry = (struct uv_IO_APIC_route_entry *)&mmr_value; 3807 entry = (struct uv_IO_APIC_route_entry *)&mmr_value;
3838 BUG_ON(sizeof(struct uv_IO_APIC_route_entry) != sizeof(unsigned long));
3839
3840 entry->mask = 1; 3808 entry->mask = 1;
3841 3809
3842 mmr_pnode = uv_blade_to_pnode(mmr_blade); 3810 mmr_pnode = uv_blade_to_pnode(mmr_blade);
@@ -3900,6 +3868,71 @@ int __init arch_probe_nr_irqs(void)
3900} 3868}
3901#endif 3869#endif
3902 3870
3871static int __io_apic_set_pci_routing(struct device *dev, int irq,
3872 struct io_apic_irq_attr *irq_attr)
3873{
3874 struct irq_desc *desc;
3875 struct irq_cfg *cfg;
3876 int node;
3877 int ioapic, pin;
3878 int trigger, polarity;
3879
3880 ioapic = irq_attr->ioapic;
3881 if (!IO_APIC_IRQ(irq)) {
3882 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
3883 ioapic);
3884 return -EINVAL;
3885 }
3886
3887 if (dev)
3888 node = dev_to_node(dev);
3889 else
3890 node = cpu_to_node(boot_cpu_id);
3891
3892 desc = irq_to_desc_alloc_node(irq, node);
3893 if (!desc) {
3894 printk(KERN_INFO "can not get irq_desc %d\n", irq);
3895 return 0;
3896 }
3897
3898 pin = irq_attr->ioapic_pin;
3899 trigger = irq_attr->trigger;
3900 polarity = irq_attr->polarity;
3901
3902 /*
3903 * IRQs < 16 are already in the irq_2_pin[] map
3904 */
3905 if (irq >= NR_IRQS_LEGACY) {
3906 cfg = desc->chip_data;
3907 add_pin_to_irq_node(cfg, node, ioapic, pin);
3908 }
3909
3910 setup_IO_APIC_irq(ioapic, pin, irq, desc, trigger, polarity);
3911
3912 return 0;
3913}
3914
3915int io_apic_set_pci_routing(struct device *dev, int irq,
3916 struct io_apic_irq_attr *irq_attr)
3917{
3918 int ioapic, pin;
3919 /*
3920 * Avoid pin reprogramming. PRTs typically include entries
3921 * with redundant pin->gsi mappings (but unique PCI devices);
3922 * we only program the IOAPIC on the first.
3923 */
3924 ioapic = irq_attr->ioapic;
3925 pin = irq_attr->ioapic_pin;
3926 if (test_bit(pin, mp_ioapic_routing[ioapic].pin_programmed)) {
3927 pr_debug("Pin %d-%d already programmed\n",
3928 mp_ioapics[ioapic].apicid, pin);
3929 return 0;
3930 }
3931 set_bit(pin, mp_ioapic_routing[ioapic].pin_programmed);
3932
3933 return __io_apic_set_pci_routing(dev, irq, irq_attr);
3934}
3935
3903/* -------------------------------------------------------------------------- 3936/* --------------------------------------------------------------------------
3904 ACPI-based IOAPIC Configuration 3937 ACPI-based IOAPIC Configuration
3905 -------------------------------------------------------------------------- */ 3938 -------------------------------------------------------------------------- */
@@ -3980,6 +4013,7 @@ int __init io_apic_get_unique_id(int ioapic, int apic_id)
3980 4013
3981 return apic_id; 4014 return apic_id;
3982} 4015}
4016#endif
3983 4017
3984int __init io_apic_get_version(int ioapic) 4018int __init io_apic_get_version(int ioapic)
3985{ 4019{
@@ -3992,39 +4026,6 @@ int __init io_apic_get_version(int ioapic)
3992 4026
3993 return reg_01.bits.version; 4027 return reg_01.bits.version;
3994} 4028}
3995#endif
3996
3997int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
3998{
3999 struct irq_desc *desc;
4000 struct irq_cfg *cfg;
4001 int cpu = boot_cpu_id;
4002
4003 if (!IO_APIC_IRQ(irq)) {
4004 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
4005 ioapic);
4006 return -EINVAL;
4007 }
4008
4009 desc = irq_to_desc_alloc_cpu(irq, cpu);
4010 if (!desc) {
4011 printk(KERN_INFO "can not get irq_desc %d\n", irq);
4012 return 0;
4013 }
4014
4015 /*
4016 * IRQs < 16 are already in the irq_2_pin[] map
4017 */
4018 if (irq >= NR_IRQS_LEGACY) {
4019 cfg = desc->chip_data;
4020 add_pin_to_irq_cpu(cfg, cpu, ioapic, pin);
4021 }
4022
4023 setup_IO_APIC_irq(ioapic, pin, irq, desc, triggering, polarity);
4024
4025 return 0;
4026}
4027
4028 4029
4029int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity) 4030int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
4030{ 4031{
@@ -4055,51 +4056,44 @@ int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
4055#ifdef CONFIG_SMP 4056#ifdef CONFIG_SMP
4056void __init setup_ioapic_dest(void) 4057void __init setup_ioapic_dest(void)
4057{ 4058{
4058 int pin, ioapic, irq, irq_entry; 4059 int pin, ioapic = 0, irq, irq_entry;
4059 struct irq_desc *desc; 4060 struct irq_desc *desc;
4060 struct irq_cfg *cfg;
4061 const struct cpumask *mask; 4061 const struct cpumask *mask;
4062 4062
4063 if (skip_ioapic_setup == 1) 4063 if (skip_ioapic_setup == 1)
4064 return; 4064 return;
4065 4065
4066 for (ioapic = 0; ioapic < nr_ioapics; ioapic++) { 4066#ifdef CONFIG_ACPI
4067 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) { 4067 if (!acpi_disabled && acpi_ioapic) {
4068 irq_entry = find_irq_entry(ioapic, pin, mp_INT); 4068 ioapic = mp_find_ioapic(0);
4069 if (irq_entry == -1) 4069 if (ioapic < 0)
4070 continue; 4070 ioapic = 0;
4071 irq = pin_2_irq(irq_entry, ioapic, pin); 4071 }
4072 4072#endif
4073 /* setup_IO_APIC_irqs could fail to get vector for some device
4074 * when you have too many devices, because at that time only boot
4075 * cpu is online.
4076 */
4077 desc = irq_to_desc(irq);
4078 cfg = desc->chip_data;
4079 if (!cfg->vector) {
4080 setup_IO_APIC_irq(ioapic, pin, irq, desc,
4081 irq_trigger(irq_entry),
4082 irq_polarity(irq_entry));
4083 continue;
4084 4073
4085 } 4074 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
4075 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
4076 if (irq_entry == -1)
4077 continue;
4078 irq = pin_2_irq(irq_entry, ioapic, pin);
4086 4079
4087 /* 4080 desc = irq_to_desc(irq);
4088 * Honour affinities which have been set in early boot
4089 */
4090 if (desc->status &
4091 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
4092 mask = desc->affinity;
4093 else
4094 mask = apic->target_cpus();
4095 4081
4096 if (intr_remapping_enabled) 4082 /*
4097 set_ir_ioapic_affinity_irq_desc(desc, mask); 4083 * Honour affinities which have been set in early boot
4098 else 4084 */
4099 set_ioapic_affinity_irq_desc(desc, mask); 4085 if (desc->status &
4100 } 4086 (IRQ_NO_BALANCING | IRQ_AFFINITY_SET))
4087 mask = desc->affinity;
4088 else
4089 mask = apic->target_cpus();
4101 4090
4091 if (intr_remapping_enabled)
4092 set_ir_ioapic_affinity_irq_desc(desc, mask);
4093 else
4094 set_ioapic_affinity_irq_desc(desc, mask);
4102 } 4095 }
4096
4103} 4097}
4104#endif 4098#endif
4105 4099
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 12:20:00 -0500