diff options
Diffstat (limited to 'arch/x86/kernel/apic/io_apic.c')
-rw-r--r-- | arch/x86/kernel/apic/io_apic.c | 292 |
1 files changed, 142 insertions, 150 deletions
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 00e6071cefc4..da99ffcdfde6 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c | |||
@@ -389,6 +389,8 @@ struct io_apic { | |||
389 | unsigned int index; | 389 | unsigned int index; |
390 | unsigned int unused[3]; | 390 | unsigned int unused[3]; |
391 | unsigned int data; | 391 | unsigned int data; |
392 | unsigned int unused2[11]; | ||
393 | unsigned int eoi; | ||
392 | }; | 394 | }; |
393 | 395 | ||
394 | static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) | 396 | static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) |
@@ -397,6 +399,12 @@ static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx) | |||
397 | + (mp_ioapics[idx].apicaddr & ~PAGE_MASK); | 399 | + (mp_ioapics[idx].apicaddr & ~PAGE_MASK); |
398 | } | 400 | } |
399 | 401 | ||
402 | static inline void io_apic_eoi(unsigned int apic, unsigned int vector) | ||
403 | { | ||
404 | struct io_apic __iomem *io_apic = io_apic_base(apic); | ||
405 | writel(vector, &io_apic->eoi); | ||
406 | } | ||
407 | |||
400 | static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg) | 408 | static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg) |
401 | { | 409 | { |
402 | struct io_apic __iomem *io_apic = io_apic_base(apic); | 410 | struct io_apic __iomem *io_apic = io_apic_base(apic); |
@@ -546,16 +554,12 @@ static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, struct irq | |||
546 | 554 | ||
547 | apic = entry->apic; | 555 | apic = entry->apic; |
548 | pin = entry->pin; | 556 | pin = entry->pin; |
549 | #ifdef CONFIG_INTR_REMAP | ||
550 | /* | 557 | /* |
551 | * With interrupt-remapping, destination information comes | 558 | * With interrupt-remapping, destination information comes |
552 | * from interrupt-remapping table entry. | 559 | * from interrupt-remapping table entry. |
553 | */ | 560 | */ |
554 | if (!irq_remapped(irq)) | 561 | if (!irq_remapped(irq)) |
555 | io_apic_write(apic, 0x11 + pin*2, dest); | 562 | io_apic_write(apic, 0x11 + pin*2, dest); |
556 | #else | ||
557 | io_apic_write(apic, 0x11 + pin*2, dest); | ||
558 | #endif | ||
559 | reg = io_apic_read(apic, 0x10 + pin*2); | 563 | reg = io_apic_read(apic, 0x10 + pin*2); |
560 | reg &= ~IO_APIC_REDIR_VECTOR_MASK; | 564 | reg &= ~IO_APIC_REDIR_VECTOR_MASK; |
561 | reg |= vector; | 565 | reg |= vector; |
@@ -588,10 +592,12 @@ set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask) | |||
588 | if (assign_irq_vector(irq, cfg, mask)) | 592 | if (assign_irq_vector(irq, cfg, mask)) |
589 | return BAD_APICID; | 593 | return BAD_APICID; |
590 | 594 | ||
591 | cpumask_and(desc->affinity, cfg->domain, mask); | 595 | /* check that before desc->addinity get updated */ |
592 | set_extra_move_desc(desc, mask); | 596 | set_extra_move_desc(desc, mask); |
593 | 597 | ||
594 | return apic->cpu_mask_to_apicid_and(desc->affinity, cpu_online_mask); | 598 | cpumask_copy(desc->affinity, mask); |
599 | |||
600 | return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain); | ||
595 | } | 601 | } |
596 | 602 | ||
597 | static void | 603 | static void |
@@ -849,9 +855,9 @@ __setup("pirq=", ioapic_pirq_setup); | |||
849 | static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS]; | 855 | static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS]; |
850 | 856 | ||
851 | /* | 857 | /* |
852 | * Saves and masks all the unmasked IO-APIC RTE's | 858 | * Saves all the IO-APIC RTE's |
853 | */ | 859 | */ |
854 | int save_mask_IO_APIC_setup(void) | 860 | int save_IO_APIC_setup(void) |
855 | { | 861 | { |
856 | union IO_APIC_reg_01 reg_01; | 862 | union IO_APIC_reg_01 reg_01; |
857 | unsigned long flags; | 863 | unsigned long flags; |
@@ -876,16 +882,9 @@ int save_mask_IO_APIC_setup(void) | |||
876 | } | 882 | } |
877 | 883 | ||
878 | for (apic = 0; apic < nr_ioapics; apic++) | 884 | for (apic = 0; apic < nr_ioapics; apic++) |
879 | for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { | 885 | for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) |
880 | struct IO_APIC_route_entry entry; | 886 | early_ioapic_entries[apic][pin] = |
881 | |||
882 | entry = early_ioapic_entries[apic][pin] = | ||
883 | ioapic_read_entry(apic, pin); | 887 | ioapic_read_entry(apic, pin); |
884 | if (!entry.mask) { | ||
885 | entry.mask = 1; | ||
886 | ioapic_write_entry(apic, pin, entry); | ||
887 | } | ||
888 | } | ||
889 | 888 | ||
890 | return 0; | 889 | return 0; |
891 | 890 | ||
@@ -898,6 +897,25 @@ nomem: | |||
898 | return -ENOMEM; | 897 | return -ENOMEM; |
899 | } | 898 | } |
900 | 899 | ||
900 | void mask_IO_APIC_setup(void) | ||
901 | { | ||
902 | int apic, pin; | ||
903 | |||
904 | for (apic = 0; apic < nr_ioapics; apic++) { | ||
905 | if (!early_ioapic_entries[apic]) | ||
906 | break; | ||
907 | for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { | ||
908 | struct IO_APIC_route_entry entry; | ||
909 | |||
910 | entry = early_ioapic_entries[apic][pin]; | ||
911 | if (!entry.mask) { | ||
912 | entry.mask = 1; | ||
913 | ioapic_write_entry(apic, pin, entry); | ||
914 | } | ||
915 | } | ||
916 | } | ||
917 | } | ||
918 | |||
901 | void restore_IO_APIC_setup(void) | 919 | void restore_IO_APIC_setup(void) |
902 | { | 920 | { |
903 | int apic, pin; | 921 | int apic, pin; |
@@ -1411,9 +1429,7 @@ void __setup_vector_irq(int cpu) | |||
1411 | } | 1429 | } |
1412 | 1430 | ||
1413 | static struct irq_chip ioapic_chip; | 1431 | static struct irq_chip ioapic_chip; |
1414 | #ifdef CONFIG_INTR_REMAP | ||
1415 | static struct irq_chip ir_ioapic_chip; | 1432 | static struct irq_chip ir_ioapic_chip; |
1416 | #endif | ||
1417 | 1433 | ||
1418 | #define IOAPIC_AUTO -1 | 1434 | #define IOAPIC_AUTO -1 |
1419 | #define IOAPIC_EDGE 0 | 1435 | #define IOAPIC_EDGE 0 |
@@ -1452,7 +1468,6 @@ static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long t | |||
1452 | else | 1468 | else |
1453 | desc->status &= ~IRQ_LEVEL; | 1469 | desc->status &= ~IRQ_LEVEL; |
1454 | 1470 | ||
1455 | #ifdef CONFIG_INTR_REMAP | ||
1456 | if (irq_remapped(irq)) { | 1471 | if (irq_remapped(irq)) { |
1457 | desc->status |= IRQ_MOVE_PCNTXT; | 1472 | desc->status |= IRQ_MOVE_PCNTXT; |
1458 | if (trigger) | 1473 | if (trigger) |
@@ -1464,7 +1479,7 @@ static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long t | |||
1464 | handle_edge_irq, "edge"); | 1479 | handle_edge_irq, "edge"); |
1465 | return; | 1480 | return; |
1466 | } | 1481 | } |
1467 | #endif | 1482 | |
1468 | if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || | 1483 | if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || |
1469 | trigger == IOAPIC_LEVEL) | 1484 | trigger == IOAPIC_LEVEL) |
1470 | set_irq_chip_and_handler_name(irq, &ioapic_chip, | 1485 | set_irq_chip_and_handler_name(irq, &ioapic_chip, |
@@ -1478,14 +1493,13 @@ static void ioapic_register_intr(int irq, struct irq_desc *desc, unsigned long t | |||
1478 | int setup_ioapic_entry(int apic_id, int irq, | 1493 | int setup_ioapic_entry(int apic_id, int irq, |
1479 | struct IO_APIC_route_entry *entry, | 1494 | struct IO_APIC_route_entry *entry, |
1480 | unsigned int destination, int trigger, | 1495 | unsigned int destination, int trigger, |
1481 | int polarity, int vector) | 1496 | int polarity, int vector, int pin) |
1482 | { | 1497 | { |
1483 | /* | 1498 | /* |
1484 | * add it to the IO-APIC irq-routing table: | 1499 | * add it to the IO-APIC irq-routing table: |
1485 | */ | 1500 | */ |
1486 | memset(entry,0,sizeof(*entry)); | 1501 | memset(entry,0,sizeof(*entry)); |
1487 | 1502 | ||
1488 | #ifdef CONFIG_INTR_REMAP | ||
1489 | if (intr_remapping_enabled) { | 1503 | if (intr_remapping_enabled) { |
1490 | struct intel_iommu *iommu = map_ioapic_to_ir(apic_id); | 1504 | struct intel_iommu *iommu = map_ioapic_to_ir(apic_id); |
1491 | struct irte irte; | 1505 | struct irte irte; |
@@ -1504,7 +1518,14 @@ int setup_ioapic_entry(int apic_id, int irq, | |||
1504 | 1518 | ||
1505 | irte.present = 1; | 1519 | irte.present = 1; |
1506 | irte.dst_mode = apic->irq_dest_mode; | 1520 | irte.dst_mode = apic->irq_dest_mode; |
1507 | irte.trigger_mode = trigger; | 1521 | /* |
1522 | * Trigger mode in the IRTE will always be edge, and the | ||
1523 | * actual level or edge trigger will be setup in the IO-APIC | ||
1524 | * RTE. This will help simplify level triggered irq migration. | ||
1525 | * For more details, see the comments above explainig IO-APIC | ||
1526 | * irq migration in the presence of interrupt-remapping. | ||
1527 | */ | ||
1528 | irte.trigger_mode = 0; | ||
1508 | irte.dlvry_mode = apic->irq_delivery_mode; | 1529 | irte.dlvry_mode = apic->irq_delivery_mode; |
1509 | irte.vector = vector; | 1530 | irte.vector = vector; |
1510 | irte.dest_id = IRTE_DEST(destination); | 1531 | irte.dest_id = IRTE_DEST(destination); |
@@ -1515,18 +1536,21 @@ int setup_ioapic_entry(int apic_id, int irq, | |||
1515 | ir_entry->zero = 0; | 1536 | ir_entry->zero = 0; |
1516 | ir_entry->format = 1; | 1537 | ir_entry->format = 1; |
1517 | ir_entry->index = (index & 0x7fff); | 1538 | ir_entry->index = (index & 0x7fff); |
1518 | } else | 1539 | /* |
1519 | #endif | 1540 | * IO-APIC RTE will be configured with virtual vector. |
1520 | { | 1541 | * irq handler will do the explicit EOI to the io-apic. |
1542 | */ | ||
1543 | ir_entry->vector = pin; | ||
1544 | } else { | ||
1521 | entry->delivery_mode = apic->irq_delivery_mode; | 1545 | entry->delivery_mode = apic->irq_delivery_mode; |
1522 | entry->dest_mode = apic->irq_dest_mode; | 1546 | entry->dest_mode = apic->irq_dest_mode; |
1523 | entry->dest = destination; | 1547 | entry->dest = destination; |
1548 | entry->vector = vector; | ||
1524 | } | 1549 | } |
1525 | 1550 | ||
1526 | entry->mask = 0; /* enable IRQ */ | 1551 | entry->mask = 0; /* enable IRQ */ |
1527 | entry->trigger = trigger; | 1552 | entry->trigger = trigger; |
1528 | entry->polarity = polarity; | 1553 | entry->polarity = polarity; |
1529 | entry->vector = vector; | ||
1530 | 1554 | ||
1531 | /* Mask level triggered irqs. | 1555 | /* Mask level triggered irqs. |
1532 | * Use IRQ_DELAYED_DISABLE for edge triggered irqs. | 1556 | * Use IRQ_DELAYED_DISABLE for edge triggered irqs. |
@@ -1561,7 +1585,7 @@ static void setup_IO_APIC_irq(int apic_id, int pin, unsigned int irq, struct irq | |||
1561 | 1585 | ||
1562 | 1586 | ||
1563 | if (setup_ioapic_entry(mp_ioapics[apic_id].apicid, irq, &entry, | 1587 | if (setup_ioapic_entry(mp_ioapics[apic_id].apicid, irq, &entry, |
1564 | dest, trigger, polarity, cfg->vector)) { | 1588 | dest, trigger, polarity, cfg->vector, pin)) { |
1565 | printk("Failed to setup ioapic entry for ioapic %d, pin %d\n", | 1589 | printk("Failed to setup ioapic entry for ioapic %d, pin %d\n", |
1566 | mp_ioapics[apic_id].apicid, pin); | 1590 | mp_ioapics[apic_id].apicid, pin); |
1567 | __clear_irq_vector(irq, cfg); | 1591 | __clear_irq_vector(irq, cfg); |
@@ -1642,10 +1666,8 @@ static void __init setup_timer_IRQ0_pin(unsigned int apic_id, unsigned int pin, | |||
1642 | { | 1666 | { |
1643 | struct IO_APIC_route_entry entry; | 1667 | struct IO_APIC_route_entry entry; |
1644 | 1668 | ||
1645 | #ifdef CONFIG_INTR_REMAP | ||
1646 | if (intr_remapping_enabled) | 1669 | if (intr_remapping_enabled) |
1647 | return; | 1670 | return; |
1648 | #endif | ||
1649 | 1671 | ||
1650 | memset(&entry, 0, sizeof(entry)); | 1672 | memset(&entry, 0, sizeof(entry)); |
1651 | 1673 | ||
@@ -2040,8 +2062,13 @@ void disable_IO_APIC(void) | |||
2040 | * If the i8259 is routed through an IOAPIC | 2062 | * If the i8259 is routed through an IOAPIC |
2041 | * Put that IOAPIC in virtual wire mode | 2063 | * Put that IOAPIC in virtual wire mode |
2042 | * so legacy interrupts can be delivered. | 2064 | * so legacy interrupts can be delivered. |
2065 | * | ||
2066 | * With interrupt-remapping, for now we will use virtual wire A mode, | ||
2067 | * as virtual wire B is little complex (need to configure both | ||
2068 | * IOAPIC RTE aswell as interrupt-remapping table entry). | ||
2069 | * As this gets called during crash dump, keep this simple for now. | ||
2043 | */ | 2070 | */ |
2044 | if (ioapic_i8259.pin != -1) { | 2071 | if (ioapic_i8259.pin != -1 && !intr_remapping_enabled) { |
2045 | struct IO_APIC_route_entry entry; | 2072 | struct IO_APIC_route_entry entry; |
2046 | 2073 | ||
2047 | memset(&entry, 0, sizeof(entry)); | 2074 | memset(&entry, 0, sizeof(entry)); |
@@ -2061,7 +2088,10 @@ void disable_IO_APIC(void) | |||
2061 | ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry); | 2088 | ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry); |
2062 | } | 2089 | } |
2063 | 2090 | ||
2064 | disconnect_bsp_APIC(ioapic_i8259.pin != -1); | 2091 | /* |
2092 | * Use virtual wire A mode when interrupt remapping is enabled. | ||
2093 | */ | ||
2094 | disconnect_bsp_APIC(!intr_remapping_enabled && ioapic_i8259.pin != -1); | ||
2065 | } | 2095 | } |
2066 | 2096 | ||
2067 | #ifdef CONFIG_X86_32 | 2097 | #ifdef CONFIG_X86_32 |
@@ -2303,37 +2333,24 @@ static int ioapic_retrigger_irq(unsigned int irq) | |||
2303 | #ifdef CONFIG_SMP | 2333 | #ifdef CONFIG_SMP |
2304 | 2334 | ||
2305 | #ifdef CONFIG_INTR_REMAP | 2335 | #ifdef CONFIG_INTR_REMAP |
2306 | static void ir_irq_migration(struct work_struct *work); | ||
2307 | |||
2308 | static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration); | ||
2309 | 2336 | ||
2310 | /* | 2337 | /* |
2311 | * Migrate the IO-APIC irq in the presence of intr-remapping. | 2338 | * Migrate the IO-APIC irq in the presence of intr-remapping. |
2312 | * | 2339 | * |
2313 | * For edge triggered, irq migration is a simple atomic update(of vector | 2340 | * For both level and edge triggered, irq migration is a simple atomic |
2314 | * and cpu destination) of IRTE and flush the hardware cache. | 2341 | * update(of vector and cpu destination) of IRTE and flush the hardware cache. |
2315 | * | ||
2316 | * For level triggered, we need to modify the io-apic RTE aswell with the update | ||
2317 | * vector information, along with modifying IRTE with vector and destination. | ||
2318 | * So irq migration for level triggered is little bit more complex compared to | ||
2319 | * edge triggered migration. But the good news is, we use the same algorithm | ||
2320 | * for level triggered migration as we have today, only difference being, | ||
2321 | * we now initiate the irq migration from process context instead of the | ||
2322 | * interrupt context. | ||
2323 | * | 2342 | * |
2324 | * In future, when we do a directed EOI (combined with cpu EOI broadcast | 2343 | * For level triggered, we eliminate the io-apic RTE modification (with the |
2325 | * suppression) to the IO-APIC, level triggered irq migration will also be | 2344 | * updated vector information), by using a virtual vector (io-apic pin number). |
2326 | * as simple as edge triggered migration and we can do the irq migration | 2345 | * Real vector that is used for interrupting cpu will be coming from |
2327 | * with a simple atomic update to IO-APIC RTE. | 2346 | * the interrupt-remapping table entry. |
2328 | */ | 2347 | */ |
2329 | static void | 2348 | static void |
2330 | migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask) | 2349 | migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask) |
2331 | { | 2350 | { |
2332 | struct irq_cfg *cfg; | 2351 | struct irq_cfg *cfg; |
2333 | struct irte irte; | 2352 | struct irte irte; |
2334 | int modify_ioapic_rte; | ||
2335 | unsigned int dest; | 2353 | unsigned int dest; |
2336 | unsigned long flags; | ||
2337 | unsigned int irq; | 2354 | unsigned int irq; |
2338 | 2355 | ||
2339 | if (!cpumask_intersects(mask, cpu_online_mask)) | 2356 | if (!cpumask_intersects(mask, cpu_online_mask)) |
@@ -2351,13 +2368,6 @@ migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask) | |||
2351 | 2368 | ||
2352 | dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); | 2369 | dest = apic->cpu_mask_to_apicid_and(cfg->domain, mask); |
2353 | 2370 | ||
2354 | modify_ioapic_rte = desc->status & IRQ_LEVEL; | ||
2355 | if (modify_ioapic_rte) { | ||
2356 | spin_lock_irqsave(&ioapic_lock, flags); | ||
2357 | __target_IO_APIC_irq(irq, dest, cfg); | ||
2358 | spin_unlock_irqrestore(&ioapic_lock, flags); | ||
2359 | } | ||
2360 | |||
2361 | irte.vector = cfg->vector; | 2371 | irte.vector = cfg->vector; |
2362 | irte.dest_id = IRTE_DEST(dest); | 2372 | irte.dest_id = IRTE_DEST(dest); |
2363 | 2373 | ||
@@ -2372,73 +2382,12 @@ migrate_ioapic_irq_desc(struct irq_desc *desc, const struct cpumask *mask) | |||
2372 | cpumask_copy(desc->affinity, mask); | 2382 | cpumask_copy(desc->affinity, mask); |
2373 | } | 2383 | } |
2374 | 2384 | ||
2375 | static int migrate_irq_remapped_level_desc(struct irq_desc *desc) | ||
2376 | { | ||
2377 | int ret = -1; | ||
2378 | struct irq_cfg *cfg = desc->chip_data; | ||
2379 | |||
2380 | mask_IO_APIC_irq_desc(desc); | ||
2381 | |||
2382 | if (io_apic_level_ack_pending(cfg)) { | ||
2383 | /* | ||
2384 | * Interrupt in progress. Migrating irq now will change the | ||
2385 | * vector information in the IO-APIC RTE and that will confuse | ||
2386 | * the EOI broadcast performed by cpu. | ||
2387 | * So, delay the irq migration to the next instance. | ||
2388 | */ | ||
2389 | schedule_delayed_work(&ir_migration_work, 1); | ||
2390 | goto unmask; | ||
2391 | } | ||
2392 | |||
2393 | /* everthing is clear. we have right of way */ | ||
2394 | migrate_ioapic_irq_desc(desc, desc->pending_mask); | ||
2395 | |||
2396 | ret = 0; | ||
2397 | desc->status &= ~IRQ_MOVE_PENDING; | ||
2398 | cpumask_clear(desc->pending_mask); | ||
2399 | |||
2400 | unmask: | ||
2401 | unmask_IO_APIC_irq_desc(desc); | ||
2402 | |||
2403 | return ret; | ||
2404 | } | ||
2405 | |||
2406 | static void ir_irq_migration(struct work_struct *work) | ||
2407 | { | ||
2408 | unsigned int irq; | ||
2409 | struct irq_desc *desc; | ||
2410 | |||
2411 | for_each_irq_desc(irq, desc) { | ||
2412 | if (desc->status & IRQ_MOVE_PENDING) { | ||
2413 | unsigned long flags; | ||
2414 | |||
2415 | spin_lock_irqsave(&desc->lock, flags); | ||
2416 | if (!desc->chip->set_affinity || | ||
2417 | !(desc->status & IRQ_MOVE_PENDING)) { | ||
2418 | desc->status &= ~IRQ_MOVE_PENDING; | ||
2419 | spin_unlock_irqrestore(&desc->lock, flags); | ||
2420 | continue; | ||
2421 | } | ||
2422 | |||
2423 | desc->chip->set_affinity(irq, desc->pending_mask); | ||
2424 | spin_unlock_irqrestore(&desc->lock, flags); | ||
2425 | } | ||
2426 | } | ||
2427 | } | ||
2428 | |||
2429 | /* | 2385 | /* |
2430 | * Migrates the IRQ destination in the process context. | 2386 | * Migrates the IRQ destination in the process context. |
2431 | */ | 2387 | */ |
2432 | static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, | 2388 | static void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, |
2433 | const struct cpumask *mask) | 2389 | const struct cpumask *mask) |
2434 | { | 2390 | { |
2435 | if (desc->status & IRQ_LEVEL) { | ||
2436 | desc->status |= IRQ_MOVE_PENDING; | ||
2437 | cpumask_copy(desc->pending_mask, mask); | ||
2438 | migrate_irq_remapped_level_desc(desc); | ||
2439 | return; | ||
2440 | } | ||
2441 | |||
2442 | migrate_ioapic_irq_desc(desc, mask); | 2391 | migrate_ioapic_irq_desc(desc, mask); |
2443 | } | 2392 | } |
2444 | static void set_ir_ioapic_affinity_irq(unsigned int irq, | 2393 | static void set_ir_ioapic_affinity_irq(unsigned int irq, |
@@ -2448,6 +2397,11 @@ static void set_ir_ioapic_affinity_irq(unsigned int irq, | |||
2448 | 2397 | ||
2449 | set_ir_ioapic_affinity_irq_desc(desc, mask); | 2398 | set_ir_ioapic_affinity_irq_desc(desc, mask); |
2450 | } | 2399 | } |
2400 | #else | ||
2401 | static inline void set_ir_ioapic_affinity_irq_desc(struct irq_desc *desc, | ||
2402 | const struct cpumask *mask) | ||
2403 | { | ||
2404 | } | ||
2451 | #endif | 2405 | #endif |
2452 | 2406 | ||
2453 | asmlinkage void smp_irq_move_cleanup_interrupt(void) | 2407 | asmlinkage void smp_irq_move_cleanup_interrupt(void) |
@@ -2461,6 +2415,7 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void) | |||
2461 | me = smp_processor_id(); | 2415 | me = smp_processor_id(); |
2462 | for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { | 2416 | for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) { |
2463 | unsigned int irq; | 2417 | unsigned int irq; |
2418 | unsigned int irr; | ||
2464 | struct irq_desc *desc; | 2419 | struct irq_desc *desc; |
2465 | struct irq_cfg *cfg; | 2420 | struct irq_cfg *cfg; |
2466 | irq = __get_cpu_var(vector_irq)[vector]; | 2421 | irq = __get_cpu_var(vector_irq)[vector]; |
@@ -2480,6 +2435,18 @@ asmlinkage void smp_irq_move_cleanup_interrupt(void) | |||
2480 | if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) | 2435 | if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain)) |
2481 | goto unlock; | 2436 | goto unlock; |
2482 | 2437 | ||
2438 | irr = apic_read(APIC_IRR + (vector / 32 * 0x10)); | ||
2439 | /* | ||
2440 | * Check if the vector that needs to be cleanedup is | ||
2441 | * registered at the cpu's IRR. If so, then this is not | ||
2442 | * the best time to clean it up. Lets clean it up in the | ||
2443 | * next attempt by sending another IRQ_MOVE_CLEANUP_VECTOR | ||
2444 | * to myself. | ||
2445 | */ | ||
2446 | if (irr & (1 << (vector % 32))) { | ||
2447 | apic->send_IPI_self(IRQ_MOVE_CLEANUP_VECTOR); | ||
2448 | goto unlock; | ||
2449 | } | ||
2483 | __get_cpu_var(vector_irq)[vector] = -1; | 2450 | __get_cpu_var(vector_irq)[vector] = -1; |
2484 | cfg->move_cleanup_count--; | 2451 | cfg->move_cleanup_count--; |
2485 | unlock: | 2452 | unlock: |
@@ -2529,9 +2496,44 @@ static inline void irq_complete_move(struct irq_desc **descp) {} | |||
2529 | #endif | 2496 | #endif |
2530 | 2497 | ||
2531 | #ifdef CONFIG_INTR_REMAP | 2498 | #ifdef CONFIG_INTR_REMAP |
2499 | static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg) | ||
2500 | { | ||
2501 | int apic, pin; | ||
2502 | struct irq_pin_list *entry; | ||
2503 | |||
2504 | entry = cfg->irq_2_pin; | ||
2505 | for (;;) { | ||
2506 | |||
2507 | if (!entry) | ||
2508 | break; | ||
2509 | |||
2510 | apic = entry->apic; | ||
2511 | pin = entry->pin; | ||
2512 | io_apic_eoi(apic, pin); | ||
2513 | entry = entry->next; | ||
2514 | } | ||
2515 | } | ||
2516 | |||
2517 | static void | ||
2518 | eoi_ioapic_irq(struct irq_desc *desc) | ||
2519 | { | ||
2520 | struct irq_cfg *cfg; | ||
2521 | unsigned long flags; | ||
2522 | unsigned int irq; | ||
2523 | |||
2524 | irq = desc->irq; | ||
2525 | cfg = desc->chip_data; | ||
2526 | |||
2527 | spin_lock_irqsave(&ioapic_lock, flags); | ||
2528 | __eoi_ioapic_irq(irq, cfg); | ||
2529 | spin_unlock_irqrestore(&ioapic_lock, flags); | ||
2530 | } | ||
2531 | |||
2532 | static void ack_x2apic_level(unsigned int irq) | 2532 | static void ack_x2apic_level(unsigned int irq) |
2533 | { | 2533 | { |
2534 | struct irq_desc *desc = irq_to_desc(irq); | ||
2534 | ack_x2APIC_irq(); | 2535 | ack_x2APIC_irq(); |
2536 | eoi_ioapic_irq(desc); | ||
2535 | } | 2537 | } |
2536 | 2538 | ||
2537 | static void ack_x2apic_edge(unsigned int irq) | 2539 | static void ack_x2apic_edge(unsigned int irq) |
@@ -2662,20 +2664,20 @@ static struct irq_chip ioapic_chip __read_mostly = { | |||
2662 | .retrigger = ioapic_retrigger_irq, | 2664 | .retrigger = ioapic_retrigger_irq, |
2663 | }; | 2665 | }; |
2664 | 2666 | ||
2665 | #ifdef CONFIG_INTR_REMAP | ||
2666 | static struct irq_chip ir_ioapic_chip __read_mostly = { | 2667 | static struct irq_chip ir_ioapic_chip __read_mostly = { |
2667 | .name = "IR-IO-APIC", | 2668 | .name = "IR-IO-APIC", |
2668 | .startup = startup_ioapic_irq, | 2669 | .startup = startup_ioapic_irq, |
2669 | .mask = mask_IO_APIC_irq, | 2670 | .mask = mask_IO_APIC_irq, |
2670 | .unmask = unmask_IO_APIC_irq, | 2671 | .unmask = unmask_IO_APIC_irq, |
2672 | #ifdef CONFIG_INTR_REMAP | ||
2671 | .ack = ack_x2apic_edge, | 2673 | .ack = ack_x2apic_edge, |
2672 | .eoi = ack_x2apic_level, | 2674 | .eoi = ack_x2apic_level, |
2673 | #ifdef CONFIG_SMP | 2675 | #ifdef CONFIG_SMP |
2674 | .set_affinity = set_ir_ioapic_affinity_irq, | 2676 | .set_affinity = set_ir_ioapic_affinity_irq, |
2675 | #endif | 2677 | #endif |
2678 | #endif | ||
2676 | .retrigger = ioapic_retrigger_irq, | 2679 | .retrigger = ioapic_retrigger_irq, |
2677 | }; | 2680 | }; |
2678 | #endif | ||
2679 | 2681 | ||
2680 | static inline void init_IO_APIC_traps(void) | 2682 | static inline void init_IO_APIC_traps(void) |
2681 | { | 2683 | { |
@@ -2901,10 +2903,8 @@ static inline void __init check_timer(void) | |||
2901 | * 8259A. | 2903 | * 8259A. |
2902 | */ | 2904 | */ |
2903 | if (pin1 == -1) { | 2905 | if (pin1 == -1) { |
2904 | #ifdef CONFIG_INTR_REMAP | ||
2905 | if (intr_remapping_enabled) | 2906 | if (intr_remapping_enabled) |
2906 | panic("BIOS bug: timer not connected to IO-APIC"); | 2907 | panic("BIOS bug: timer not connected to IO-APIC"); |
2907 | #endif | ||
2908 | pin1 = pin2; | 2908 | pin1 = pin2; |
2909 | apic1 = apic2; | 2909 | apic1 = apic2; |
2910 | no_pin1 = 1; | 2910 | no_pin1 = 1; |
@@ -2940,10 +2940,8 @@ static inline void __init check_timer(void) | |||
2940 | clear_IO_APIC_pin(0, pin1); | 2940 | clear_IO_APIC_pin(0, pin1); |
2941 | goto out; | 2941 | goto out; |
2942 | } | 2942 | } |
2943 | #ifdef CONFIG_INTR_REMAP | ||
2944 | if (intr_remapping_enabled) | 2943 | if (intr_remapping_enabled) |
2945 | panic("timer doesn't work through Interrupt-remapped IO-APIC"); | 2944 | panic("timer doesn't work through Interrupt-remapped IO-APIC"); |
2946 | #endif | ||
2947 | local_irq_disable(); | 2945 | local_irq_disable(); |
2948 | clear_IO_APIC_pin(apic1, pin1); | 2946 | clear_IO_APIC_pin(apic1, pin1); |
2949 | if (!no_pin1) | 2947 | if (!no_pin1) |
@@ -3237,9 +3235,7 @@ void destroy_irq(unsigned int irq) | |||
3237 | if (desc) | 3235 | if (desc) |
3238 | desc->chip_data = cfg; | 3236 | desc->chip_data = cfg; |
3239 | 3237 | ||
3240 | #ifdef CONFIG_INTR_REMAP | ||
3241 | free_irte(irq); | 3238 | free_irte(irq); |
3242 | #endif | ||
3243 | spin_lock_irqsave(&vector_lock, flags); | 3239 | spin_lock_irqsave(&vector_lock, flags); |
3244 | __clear_irq_vector(irq, cfg); | 3240 | __clear_irq_vector(irq, cfg); |
3245 | spin_unlock_irqrestore(&vector_lock, flags); | 3241 | spin_unlock_irqrestore(&vector_lock, flags); |
@@ -3265,7 +3261,6 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms | |||
3265 | 3261 | ||
3266 | dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus()); | 3262 | dest = apic->cpu_mask_to_apicid_and(cfg->domain, apic->target_cpus()); |
3267 | 3263 | ||
3268 | #ifdef CONFIG_INTR_REMAP | ||
3269 | if (irq_remapped(irq)) { | 3264 | if (irq_remapped(irq)) { |
3270 | struct irte irte; | 3265 | struct irte irte; |
3271 | int ir_index; | 3266 | int ir_index; |
@@ -3291,10 +3286,13 @@ static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_ms | |||
3291 | MSI_ADDR_IR_SHV | | 3286 | MSI_ADDR_IR_SHV | |
3292 | MSI_ADDR_IR_INDEX1(ir_index) | | 3287 | MSI_ADDR_IR_INDEX1(ir_index) | |
3293 | MSI_ADDR_IR_INDEX2(ir_index); | 3288 | MSI_ADDR_IR_INDEX2(ir_index); |
3294 | } else | 3289 | } else { |
3295 | #endif | 3290 | if (x2apic_enabled()) |
3296 | { | 3291 | msg->address_hi = MSI_ADDR_BASE_HI | |
3297 | msg->address_hi = MSI_ADDR_BASE_HI; | 3292 | MSI_ADDR_EXT_DEST_ID(dest); |
3293 | else | ||
3294 | msg->address_hi = MSI_ADDR_BASE_HI; | ||
3295 | |||
3298 | msg->address_lo = | 3296 | msg->address_lo = |
3299 | MSI_ADDR_BASE_LO | | 3297 | MSI_ADDR_BASE_LO | |
3300 | ((apic->irq_dest_mode == 0) ? | 3298 | ((apic->irq_dest_mode == 0) ? |
@@ -3394,15 +3392,16 @@ static struct irq_chip msi_chip = { | |||
3394 | .retrigger = ioapic_retrigger_irq, | 3392 | .retrigger = ioapic_retrigger_irq, |
3395 | }; | 3393 | }; |
3396 | 3394 | ||
3397 | #ifdef CONFIG_INTR_REMAP | ||
3398 | static struct irq_chip msi_ir_chip = { | 3395 | static struct irq_chip msi_ir_chip = { |
3399 | .name = "IR-PCI-MSI", | 3396 | .name = "IR-PCI-MSI", |
3400 | .unmask = unmask_msi_irq, | 3397 | .unmask = unmask_msi_irq, |
3401 | .mask = mask_msi_irq, | 3398 | .mask = mask_msi_irq, |
3399 | #ifdef CONFIG_INTR_REMAP | ||
3402 | .ack = ack_x2apic_edge, | 3400 | .ack = ack_x2apic_edge, |
3403 | #ifdef CONFIG_SMP | 3401 | #ifdef CONFIG_SMP |
3404 | .set_affinity = ir_set_msi_irq_affinity, | 3402 | .set_affinity = ir_set_msi_irq_affinity, |
3405 | #endif | 3403 | #endif |
3404 | #endif | ||
3406 | .retrigger = ioapic_retrigger_irq, | 3405 | .retrigger = ioapic_retrigger_irq, |
3407 | }; | 3406 | }; |
3408 | 3407 | ||
@@ -3432,7 +3431,6 @@ static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec) | |||
3432 | } | 3431 | } |
3433 | return index; | 3432 | return index; |
3434 | } | 3433 | } |
3435 | #endif | ||
3436 | 3434 | ||
3437 | static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq) | 3435 | static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq) |
3438 | { | 3436 | { |
@@ -3446,7 +3444,6 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq) | |||
3446 | set_irq_msi(irq, msidesc); | 3444 | set_irq_msi(irq, msidesc); |
3447 | write_msi_msg(irq, &msg); | 3445 | write_msi_msg(irq, &msg); |
3448 | 3446 | ||
3449 | #ifdef CONFIG_INTR_REMAP | ||
3450 | if (irq_remapped(irq)) { | 3447 | if (irq_remapped(irq)) { |
3451 | struct irq_desc *desc = irq_to_desc(irq); | 3448 | struct irq_desc *desc = irq_to_desc(irq); |
3452 | /* | 3449 | /* |
@@ -3455,7 +3452,6 @@ static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *msidesc, int irq) | |||
3455 | desc->status |= IRQ_MOVE_PCNTXT; | 3452 | desc->status |= IRQ_MOVE_PCNTXT; |
3456 | set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge"); | 3453 | set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge"); |
3457 | } else | 3454 | } else |
3458 | #endif | ||
3459 | set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge"); | 3455 | set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge"); |
3460 | 3456 | ||
3461 | dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq); | 3457 | dev_printk(KERN_DEBUG, &dev->dev, "irq %d for MSI/MSI-X\n", irq); |
@@ -3469,11 +3465,8 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3469 | int ret, sub_handle; | 3465 | int ret, sub_handle; |
3470 | struct msi_desc *msidesc; | 3466 | struct msi_desc *msidesc; |
3471 | unsigned int irq_want; | 3467 | unsigned int irq_want; |
3472 | 3468 | struct intel_iommu *iommu = NULL; | |
3473 | #ifdef CONFIG_INTR_REMAP | ||
3474 | struct intel_iommu *iommu = 0; | ||
3475 | int index = 0; | 3469 | int index = 0; |
3476 | #endif | ||
3477 | 3470 | ||
3478 | irq_want = nr_irqs_gsi; | 3471 | irq_want = nr_irqs_gsi; |
3479 | sub_handle = 0; | 3472 | sub_handle = 0; |
@@ -3482,7 +3475,6 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3482 | if (irq == 0) | 3475 | if (irq == 0) |
3483 | return -1; | 3476 | return -1; |
3484 | irq_want = irq + 1; | 3477 | irq_want = irq + 1; |
3485 | #ifdef CONFIG_INTR_REMAP | ||
3486 | if (!intr_remapping_enabled) | 3478 | if (!intr_remapping_enabled) |
3487 | goto no_ir; | 3479 | goto no_ir; |
3488 | 3480 | ||
@@ -3510,7 +3502,6 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3510 | set_irte_irq(irq, iommu, index, sub_handle); | 3502 | set_irte_irq(irq, iommu, index, sub_handle); |
3511 | } | 3503 | } |
3512 | no_ir: | 3504 | no_ir: |
3513 | #endif | ||
3514 | ret = setup_msi_irq(dev, msidesc, irq); | 3505 | ret = setup_msi_irq(dev, msidesc, irq); |
3515 | if (ret < 0) | 3506 | if (ret < 0) |
3516 | goto error; | 3507 | goto error; |
@@ -3528,7 +3519,7 @@ void arch_teardown_msi_irq(unsigned int irq) | |||
3528 | destroy_irq(irq); | 3519 | destroy_irq(irq); |
3529 | } | 3520 | } |
3530 | 3521 | ||
3531 | #ifdef CONFIG_DMAR | 3522 | #if defined (CONFIG_DMAR) || defined (CONFIG_INTR_REMAP) |
3532 | #ifdef CONFIG_SMP | 3523 | #ifdef CONFIG_SMP |
3533 | static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask) | 3524 | static void dmar_msi_set_affinity(unsigned int irq, const struct cpumask *mask) |
3534 | { | 3525 | { |
@@ -3609,7 +3600,7 @@ static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask) | |||
3609 | 3600 | ||
3610 | #endif /* CONFIG_SMP */ | 3601 | #endif /* CONFIG_SMP */ |
3611 | 3602 | ||
3612 | struct irq_chip hpet_msi_type = { | 3603 | static struct irq_chip hpet_msi_type = { |
3613 | .name = "HPET_MSI", | 3604 | .name = "HPET_MSI", |
3614 | .unmask = hpet_msi_unmask, | 3605 | .unmask = hpet_msi_unmask, |
3615 | .mask = hpet_msi_mask, | 3606 | .mask = hpet_msi_mask, |
@@ -4045,11 +4036,9 @@ void __init setup_ioapic_dest(void) | |||
4045 | else | 4036 | else |
4046 | mask = apic->target_cpus(); | 4037 | mask = apic->target_cpus(); |
4047 | 4038 | ||
4048 | #ifdef CONFIG_INTR_REMAP | ||
4049 | if (intr_remapping_enabled) | 4039 | if (intr_remapping_enabled) |
4050 | set_ir_ioapic_affinity_irq_desc(desc, mask); | 4040 | set_ir_ioapic_affinity_irq_desc(desc, mask); |
4051 | else | 4041 | else |
4052 | #endif | ||
4053 | set_ioapic_affinity_irq_desc(desc, mask); | 4042 | set_ioapic_affinity_irq_desc(desc, mask); |
4054 | } | 4043 | } |
4055 | 4044 | ||
@@ -4142,9 +4131,12 @@ static int __init ioapic_insert_resources(void) | |||
4142 | struct resource *r = ioapic_resources; | 4131 | struct resource *r = ioapic_resources; |
4143 | 4132 | ||
4144 | if (!r) { | 4133 | if (!r) { |
4145 | printk(KERN_ERR | 4134 | if (nr_ioapics > 0) { |
4146 | "IO APIC resources could be not be allocated.\n"); | 4135 | printk(KERN_ERR |
4147 | return -1; | 4136 | "IO APIC resources couldn't be allocated.\n"); |
4137 | return -1; | ||
4138 | } | ||
4139 | return 0; | ||
4148 | } | 4140 | } |
4149 | 4141 | ||
4150 | for (i = 0; i < nr_ioapics; i++) { | 4142 | for (i = 0; i < nr_ioapics; i++) { |