aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86/Kconfig2
-rw-r--r--arch/x86/include/asm/apic.h3
-rw-r--r--arch/x86/include/asm/io_apic.h11
-rw-r--r--arch/x86/kernel/apic/apic.c70
-rw-r--r--arch/x86/kernel/apic/io_apic.c140
-rw-r--r--drivers/pci/dmar.c71
-rw-r--r--drivers/pci/intel-iommu.c239
-rw-r--r--drivers/pci/intr_remapping.c84
-rw-r--r--include/linux/dmar.h11
-rw-r--r--include/linux/intel-iommu.h13
10 files changed, 539 insertions, 105 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 5b2196ab8168..4b3408206091 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -253,6 +253,7 @@ config SMP
253config X86_X2APIC 253config X86_X2APIC
254 bool "Support x2apic" 254 bool "Support x2apic"
255 depends on X86_LOCAL_APIC && X86_64 255 depends on X86_LOCAL_APIC && X86_64
256 select INTR_REMAP
256 ---help--- 257 ---help---
257 This enables x2apic support on CPUs that have this feature. 258 This enables x2apic support on CPUs that have this feature.
258 259
@@ -1881,7 +1882,6 @@ config DMAR_FLOPPY_WA
1881config INTR_REMAP 1882config INTR_REMAP
1882 bool "Support for Interrupt Remapping (EXPERIMENTAL)" 1883 bool "Support for Interrupt Remapping (EXPERIMENTAL)"
1883 depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL 1884 depends on X86_64 && X86_IO_APIC && PCI_MSI && ACPI && EXPERIMENTAL
1884 select X86_X2APIC
1885 ---help--- 1885 ---help---
1886 Supports Interrupt remapping for IO-APIC and MSI devices. 1886 Supports Interrupt remapping for IO-APIC and MSI devices.
1887 To use x2apic mode in the CPU's which support x2APIC enhancements or 1887 To use x2apic mode in the CPU's which support x2APIC enhancements or
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index df8a300dfe6c..42f2f8377422 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -107,6 +107,9 @@ extern u32 native_safe_apic_wait_icr_idle(void);
107extern void native_apic_icr_write(u32 low, u32 id); 107extern void native_apic_icr_write(u32 low, u32 id);
108extern u64 native_apic_icr_read(void); 108extern u64 native_apic_icr_read(void);
109 109
110#define EIM_8BIT_APIC_ID 0
111#define EIM_32BIT_APIC_ID 1
112
110#ifdef CONFIG_X86_X2APIC 113#ifdef CONFIG_X86_X2APIC
111/* 114/*
112 * Make previous memory operations globally visible before 115 * Make previous memory operations globally visible before
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 373cc2bbcad2..9d826e436010 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -162,10 +162,13 @@ extern int (*ioapic_renumber_irq)(int ioapic, int irq);
162extern void ioapic_init_mappings(void); 162extern void ioapic_init_mappings(void);
163 163
164#ifdef CONFIG_X86_64 164#ifdef CONFIG_X86_64
165extern int save_IO_APIC_setup(void); 165extern struct IO_APIC_route_entry **alloc_ioapic_entries(void);
166extern void mask_IO_APIC_setup(void); 166extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries);
167extern void restore_IO_APIC_setup(void); 167extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
168extern void reinit_intr_remapped_IO_APIC(int); 168extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
169extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
170extern void reinit_intr_remapped_IO_APIC(int intr_remapping,
171 struct IO_APIC_route_entry **ioapic_entries);
169#endif 172#endif
170 173
171extern void probe_nr_irqs_gsi(void); 174extern void probe_nr_irqs_gsi(void);
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 85eb8e100818..098ec84b8c00 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1304,6 +1304,7 @@ void __init enable_IR_x2apic(void)
1304#ifdef CONFIG_INTR_REMAP 1304#ifdef CONFIG_INTR_REMAP
1305 int ret; 1305 int ret;
1306 unsigned long flags; 1306 unsigned long flags;
1307 struct IO_APIC_route_entry **ioapic_entries = NULL;
1307 1308
1308 if (!cpu_has_x2apic) 1309 if (!cpu_has_x2apic)
1309 return; 1310 return;
@@ -1334,17 +1335,23 @@ void __init enable_IR_x2apic(void)
1334 return; 1335 return;
1335 } 1336 }
1336 1337
1337 ret = save_IO_APIC_setup(); 1338 ioapic_entries = alloc_ioapic_entries();
1339 if (!ioapic_entries) {
1340 pr_info("Allocate ioapic_entries failed: %d\n", ret);
1341 goto end;
1342 }
1343
1344 ret = save_IO_APIC_setup(ioapic_entries);
1338 if (ret) { 1345 if (ret) {
1339 pr_info("Saving IO-APIC state failed: %d\n", ret); 1346 pr_info("Saving IO-APIC state failed: %d\n", ret);
1340 goto end; 1347 goto end;
1341 } 1348 }
1342 1349
1343 local_irq_save(flags); 1350 local_irq_save(flags);
1344 mask_IO_APIC_setup(); 1351 mask_IO_APIC_setup(ioapic_entries);
1345 mask_8259A(); 1352 mask_8259A();
1346 1353
1347 ret = enable_intr_remapping(1); 1354 ret = enable_intr_remapping(EIM_32BIT_APIC_ID);
1348 1355
1349 if (ret && x2apic_preenabled) { 1356 if (ret && x2apic_preenabled) {
1350 local_irq_restore(flags); 1357 local_irq_restore(flags);
@@ -1364,9 +1371,9 @@ end_restore:
1364 /* 1371 /*
1365 * IR enabling failed 1372 * IR enabling failed
1366 */ 1373 */
1367 restore_IO_APIC_setup(); 1374 restore_IO_APIC_setup(ioapic_entries);
1368 else 1375 else
1369 reinit_intr_remapped_IO_APIC(x2apic_preenabled); 1376 reinit_intr_remapped_IO_APIC(x2apic_preenabled, ioapic_entries);
1370 1377
1371 unmask_8259A(); 1378 unmask_8259A();
1372 local_irq_restore(flags); 1379 local_irq_restore(flags);
@@ -1379,6 +1386,8 @@ end:
1379 pr_info("Enabled Interrupt-remapping\n"); 1386 pr_info("Enabled Interrupt-remapping\n");
1380 } else 1387 } else
1381 pr_err("Failed to enable Interrupt-remapping and x2apic\n"); 1388 pr_err("Failed to enable Interrupt-remapping and x2apic\n");
1389 if (ioapic_entries)
1390 free_ioapic_entries(ioapic_entries);
1382#else 1391#else
1383 if (!cpu_has_x2apic) 1392 if (!cpu_has_x2apic)
1384 return; 1393 return;
@@ -1954,6 +1963,10 @@ static int lapic_suspend(struct sys_device *dev, pm_message_t state)
1954 1963
1955 local_irq_save(flags); 1964 local_irq_save(flags);
1956 disable_local_APIC(); 1965 disable_local_APIC();
1966#ifdef CONFIG_INTR_REMAP
1967 if (intr_remapping_enabled)
1968 disable_intr_remapping();
1969#endif
1957 local_irq_restore(flags); 1970 local_irq_restore(flags);
1958 return 0; 1971 return 0;
1959} 1972}
@@ -1964,15 +1977,41 @@ static int lapic_resume(struct sys_device *dev)
1964 unsigned long flags; 1977 unsigned long flags;
1965 int maxlvt; 1978 int maxlvt;
1966 1979
1980#ifdef CONFIG_INTR_REMAP
1981 int ret;
1982 struct IO_APIC_route_entry **ioapic_entries = NULL;
1983
1967 if (!apic_pm_state.active) 1984 if (!apic_pm_state.active)
1968 return 0; 1985 return 0;
1969 1986
1970 maxlvt = lapic_get_maxlvt();
1971
1972 local_irq_save(flags); 1987 local_irq_save(flags);
1988 if (x2apic) {
1989 ioapic_entries = alloc_ioapic_entries();
1990 if (!ioapic_entries) {
1991 WARN(1, "Alloc ioapic_entries in lapic resume failed.");
1992 return -ENOMEM;
1993 }
1994
1995 ret = save_IO_APIC_setup(ioapic_entries);
1996 if (ret) {
1997 WARN(1, "Saving IO-APIC state failed: %d\n", ret);
1998 free_ioapic_entries(ioapic_entries);
1999 return ret;
2000 }
2001
2002 mask_IO_APIC_setup(ioapic_entries);
2003 mask_8259A();
2004 enable_x2apic();
2005 }
2006#else
2007 if (!apic_pm_state.active)
2008 return 0;
1973 2009
2010 local_irq_save(flags);
1974 if (x2apic) 2011 if (x2apic)
1975 enable_x2apic(); 2012 enable_x2apic();
2013#endif
2014
1976 else { 2015 else {
1977 /* 2016 /*
1978 * Make sure the APICBASE points to the right address 2017 * Make sure the APICBASE points to the right address
@@ -1986,6 +2025,7 @@ static int lapic_resume(struct sys_device *dev)
1986 wrmsr(MSR_IA32_APICBASE, l, h); 2025 wrmsr(MSR_IA32_APICBASE, l, h);
1987 } 2026 }
1988 2027
2028 maxlvt = lapic_get_maxlvt();
1989 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED); 2029 apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
1990 apic_write(APIC_ID, apic_pm_state.apic_id); 2030 apic_write(APIC_ID, apic_pm_state.apic_id);
1991 apic_write(APIC_DFR, apic_pm_state.apic_dfr); 2031 apic_write(APIC_DFR, apic_pm_state.apic_dfr);
@@ -2009,8 +2049,20 @@ static int lapic_resume(struct sys_device *dev)
2009 apic_write(APIC_ESR, 0); 2049 apic_write(APIC_ESR, 0);
2010 apic_read(APIC_ESR); 2050 apic_read(APIC_ESR);
2011 2051
2052#ifdef CONFIG_INTR_REMAP
2053 if (intr_remapping_enabled)
2054 reenable_intr_remapping(EIM_32BIT_APIC_ID);
2055
2056 if (x2apic) {
2057 unmask_8259A();
2058 restore_IO_APIC_setup(ioapic_entries);
2059 free_ioapic_entries(ioapic_entries);
2060 }
2061#endif
2062
2012 local_irq_restore(flags); 2063 local_irq_restore(flags);
2013 2064
2065
2014 return 0; 2066 return 0;
2015} 2067}
2016 2068
@@ -2048,7 +2100,9 @@ static int __init init_lapic_sysfs(void)
2048 error = sysdev_register(&device_lapic); 2100 error = sysdev_register(&device_lapic);
2049 return error; 2101 return error;
2050} 2102}
2051device_initcall(init_lapic_sysfs); 2103
2104/* local apic needs to resume before other devices access its registers. */
2105core_initcall(init_lapic_sysfs);
2052 2106
2053#else /* CONFIG_PM */ 2107#else /* CONFIG_PM */
2054 2108
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 1bb5c6cee3eb..767fe7e46d68 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -851,63 +851,74 @@ __setup("pirq=", ioapic_pirq_setup);
851#endif /* CONFIG_X86_32 */ 851#endif /* CONFIG_X86_32 */
852 852
853#ifdef CONFIG_INTR_REMAP 853#ifdef CONFIG_INTR_REMAP
854/* I/O APIC RTE contents at the OS boot up */ 854struct IO_APIC_route_entry **alloc_ioapic_entries(void)
855static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS]; 855{
856 int apic;
857 struct IO_APIC_route_entry **ioapic_entries;
858
859 ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
860 GFP_ATOMIC);
861 if (!ioapic_entries)
862 return 0;
863
864 for (apic = 0; apic < nr_ioapics; apic++) {
865 ioapic_entries[apic] =
866 kzalloc(sizeof(struct IO_APIC_route_entry) *
867 nr_ioapic_registers[apic], GFP_ATOMIC);
868 if (!ioapic_entries[apic])
869 goto nomem;
870 }
871
872 return ioapic_entries;
873
874nomem:
875 while (--apic >= 0)
876 kfree(ioapic_entries[apic]);
877 kfree(ioapic_entries);
878
879 return 0;
880}
856 881
857/* 882/*
858 * Saves all the IO-APIC RTE's 883 * Saves all the IO-APIC RTE's
859 */ 884 */
860int save_IO_APIC_setup(void) 885int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
861{ 886{
862 union IO_APIC_reg_01 reg_01;
863 unsigned long flags;
864 int apic, pin; 887 int apic, pin;
865 888
866 /* 889 if (!ioapic_entries)
867 * The number of IO-APIC IRQ registers (== #pins): 890 return -ENOMEM;
868 */
869 for (apic = 0; apic < nr_ioapics; apic++) {
870 spin_lock_irqsave(&ioapic_lock, flags);
871 reg_01.raw = io_apic_read(apic, 1);
872 spin_unlock_irqrestore(&ioapic_lock, flags);
873 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
874 }
875 891
876 for (apic = 0; apic < nr_ioapics; apic++) { 892 for (apic = 0; apic < nr_ioapics; apic++) {
877 early_ioapic_entries[apic] = 893 if (!ioapic_entries[apic])
878 kzalloc(sizeof(struct IO_APIC_route_entry) * 894 return -ENOMEM;
879 nr_ioapic_registers[apic], GFP_KERNEL);
880 if (!early_ioapic_entries[apic])
881 goto nomem;
882 }
883 895
884 for (apic = 0; apic < nr_ioapics; apic++)
885 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) 896 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
886 early_ioapic_entries[apic][pin] = 897 ioapic_entries[apic][pin] =
887 ioapic_read_entry(apic, pin); 898 ioapic_read_entry(apic, pin);
899 }
888 900
889 return 0; 901 return 0;
890
891nomem:
892 while (apic >= 0)
893 kfree(early_ioapic_entries[apic--]);
894 memset(early_ioapic_entries, 0,
895 ARRAY_SIZE(early_ioapic_entries));
896
897 return -ENOMEM;
898} 902}
899 903
900void mask_IO_APIC_setup(void) 904/*
905 * Mask all IO APIC entries.
906 */
907void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
901{ 908{
902 int apic, pin; 909 int apic, pin;
903 910
911 if (!ioapic_entries)
912 return;
913
904 for (apic = 0; apic < nr_ioapics; apic++) { 914 for (apic = 0; apic < nr_ioapics; apic++) {
905 if (!early_ioapic_entries[apic]) 915 if (!ioapic_entries[apic])
906 break; 916 break;
917
907 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) { 918 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
908 struct IO_APIC_route_entry entry; 919 struct IO_APIC_route_entry entry;
909 920
910 entry = early_ioapic_entries[apic][pin]; 921 entry = ioapic_entries[apic][pin];
911 if (!entry.mask) { 922 if (!entry.mask) {
912 entry.mask = 1; 923 entry.mask = 1;
913 ioapic_write_entry(apic, pin, entry); 924 ioapic_write_entry(apic, pin, entry);
@@ -916,22 +927,30 @@ void mask_IO_APIC_setup(void)
916 } 927 }
917} 928}
918 929
919void restore_IO_APIC_setup(void) 930/*
931 * Restore IO APIC entries which was saved in ioapic_entries.
932 */
933int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
920{ 934{
921 int apic, pin; 935 int apic, pin;
922 936
937 if (!ioapic_entries)
938 return -ENOMEM;
939
923 for (apic = 0; apic < nr_ioapics; apic++) { 940 for (apic = 0; apic < nr_ioapics; apic++) {
924 if (!early_ioapic_entries[apic]) 941 if (!ioapic_entries[apic])
925 break; 942 return -ENOMEM;
943
926 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) 944 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
927 ioapic_write_entry(apic, pin, 945 ioapic_write_entry(apic, pin,
928 early_ioapic_entries[apic][pin]); 946 ioapic_entries[apic][pin]);
929 kfree(early_ioapic_entries[apic]);
930 early_ioapic_entries[apic] = NULL;
931 } 947 }
948 return 0;
932} 949}
933 950
934void reinit_intr_remapped_IO_APIC(int intr_remapping) 951void reinit_intr_remapped_IO_APIC(int intr_remapping,
952 struct IO_APIC_route_entry **ioapic_entries)
953
935{ 954{
936 /* 955 /*
937 * for now plain restore of previous settings. 956 * for now plain restore of previous settings.
@@ -940,7 +959,17 @@ void reinit_intr_remapped_IO_APIC(int intr_remapping)
940 * table entries. for now, do a plain restore, and wait for 959 * table entries. for now, do a plain restore, and wait for
941 * the setup_IO_APIC_irqs() to do proper initialization. 960 * the setup_IO_APIC_irqs() to do proper initialization.
942 */ 961 */
943 restore_IO_APIC_setup(); 962 restore_IO_APIC_setup(ioapic_entries);
963}
964
965void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
966{
967 int apic;
968
969 for (apic = 0; apic < nr_ioapics; apic++)
970 kfree(ioapic_entries[apic]);
971
972 kfree(ioapic_entries);
944} 973}
945#endif 974#endif
946 975
@@ -2495,7 +2524,7 @@ static void irq_complete_move(struct irq_desc **descp)
2495static inline void irq_complete_move(struct irq_desc **descp) {} 2524static inline void irq_complete_move(struct irq_desc **descp) {}
2496#endif 2525#endif
2497 2526
2498#ifdef CONFIG_INTR_REMAP 2527#ifdef CONFIG_X86_X2APIC
2499static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg) 2528static void __eoi_ioapic_irq(unsigned int irq, struct irq_cfg *cfg)
2500{ 2529{
2501 int apic, pin; 2530 int apic, pin;
@@ -2540,7 +2569,6 @@ static void ack_x2apic_edge(unsigned int irq)
2540{ 2569{
2541 ack_x2APIC_irq(); 2570 ack_x2APIC_irq();
2542} 2571}
2543
2544#endif 2572#endif
2545 2573
2546static void ack_apic_edge(unsigned int irq) 2574static void ack_apic_edge(unsigned int irq)
@@ -2651,6 +2679,26 @@ static void ack_apic_level(unsigned int irq)
2651#endif 2679#endif
2652} 2680}
2653 2681
2682#ifdef CONFIG_INTR_REMAP
2683static void ir_ack_apic_edge(unsigned int irq)
2684{
2685#ifdef CONFIG_X86_X2APIC
2686 if (x2apic_enabled())
2687 return ack_x2apic_edge(irq);
2688#endif
2689 return ack_apic_edge(irq);
2690}
2691
2692static void ir_ack_apic_level(unsigned int irq)
2693{
2694#ifdef CONFIG_X86_X2APIC
2695 if (x2apic_enabled())
2696 return ack_x2apic_level(irq);
2697#endif
2698 return ack_apic_level(irq);
2699}
2700#endif /* CONFIG_INTR_REMAP */
2701
2654static struct irq_chip ioapic_chip __read_mostly = { 2702static struct irq_chip ioapic_chip __read_mostly = {
2655 .name = "IO-APIC", 2703 .name = "IO-APIC",
2656 .startup = startup_ioapic_irq, 2704 .startup = startup_ioapic_irq,
@@ -2670,8 +2718,8 @@ static struct irq_chip ir_ioapic_chip __read_mostly = {
2670 .mask = mask_IO_APIC_irq, 2718 .mask = mask_IO_APIC_irq,
2671 .unmask = unmask_IO_APIC_irq, 2719 .unmask = unmask_IO_APIC_irq,
2672#ifdef CONFIG_INTR_REMAP 2720#ifdef CONFIG_INTR_REMAP
2673 .ack = ack_x2apic_edge, 2721 .ack = ir_ack_apic_edge,
2674 .eoi = ack_x2apic_level, 2722 .eoi = ir_ack_apic_level,
2675#ifdef CONFIG_SMP 2723#ifdef CONFIG_SMP
2676 .set_affinity = set_ir_ioapic_affinity_irq, 2724 .set_affinity = set_ir_ioapic_affinity_irq,
2677#endif 2725#endif
@@ -3397,7 +3445,7 @@ static struct irq_chip msi_ir_chip = {
3397 .unmask = unmask_msi_irq, 3445 .unmask = unmask_msi_irq,
3398 .mask = mask_msi_irq, 3446 .mask = mask_msi_irq,
3399#ifdef CONFIG_INTR_REMAP 3447#ifdef CONFIG_INTR_REMAP
3400 .ack = ack_x2apic_edge, 3448 .ack = ir_ack_apic_edge,
3401#ifdef CONFIG_SMP 3449#ifdef CONFIG_SMP
3402 .set_affinity = ir_set_msi_irq_affinity, 3450 .set_affinity = ir_set_msi_irq_affinity,
3403#endif 3451#endif
diff --git a/drivers/pci/dmar.c b/drivers/pci/dmar.c
index d313039e2fdf..25a00ce4f24d 100644
--- a/drivers/pci/dmar.c
+++ b/drivers/pci/dmar.c
@@ -180,6 +180,7 @@ dmar_parse_one_drhd(struct acpi_dmar_header *header)
180 dmaru->hdr = header; 180 dmaru->hdr = header;
181 drhd = (struct acpi_dmar_hardware_unit *)header; 181 drhd = (struct acpi_dmar_hardware_unit *)header;
182 dmaru->reg_base_addr = drhd->address; 182 dmaru->reg_base_addr = drhd->address;
183 dmaru->segment = drhd->segment;
183 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */ 184 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
184 185
185 ret = alloc_iommu(dmaru); 186 ret = alloc_iommu(dmaru);
@@ -790,14 +791,41 @@ end:
790} 791}
791 792
792/* 793/*
794 * Enable queued invalidation.
795 */
796static void __dmar_enable_qi(struct intel_iommu *iommu)
797{
798 u32 cmd, sts;
799 unsigned long flags;
800 struct q_inval *qi = iommu->qi;
801
802 qi->free_head = qi->free_tail = 0;
803 qi->free_cnt = QI_LENGTH;
804
805 spin_lock_irqsave(&iommu->register_lock, flags);
806
807 /* write zero to the tail reg */
808 writel(0, iommu->reg + DMAR_IQT_REG);
809
810 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
811
812 cmd = iommu->gcmd | DMA_GCMD_QIE;
813 iommu->gcmd |= DMA_GCMD_QIE;
814 writel(cmd, iommu->reg + DMAR_GCMD_REG);
815
816 /* Make sure hardware complete it */
817 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
818
819 spin_unlock_irqrestore(&iommu->register_lock, flags);
820}
821
822/*
793 * Enable Queued Invalidation interface. This is a must to support 823 * Enable Queued Invalidation interface. This is a must to support
794 * interrupt-remapping. Also used by DMA-remapping, which replaces 824 * interrupt-remapping. Also used by DMA-remapping, which replaces
795 * register based IOTLB invalidation. 825 * register based IOTLB invalidation.
796 */ 826 */
797int dmar_enable_qi(struct intel_iommu *iommu) 827int dmar_enable_qi(struct intel_iommu *iommu)
798{ 828{
799 u32 cmd, sts;
800 unsigned long flags;
801 struct q_inval *qi; 829 struct q_inval *qi;
802 830
803 if (!ecap_qis(iommu->ecap)) 831 if (!ecap_qis(iommu->ecap))
@@ -835,19 +863,7 @@ int dmar_enable_qi(struct intel_iommu *iommu)
835 863
836 spin_lock_init(&qi->q_lock); 864 spin_lock_init(&qi->q_lock);
837 865
838 spin_lock_irqsave(&iommu->register_lock, flags); 866 __dmar_enable_qi(iommu);
839 /* write zero to the tail reg */
840 writel(0, iommu->reg + DMAR_IQT_REG);
841
842 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
843
844 cmd = iommu->gcmd | DMA_GCMD_QIE;
845 iommu->gcmd |= DMA_GCMD_QIE;
846 writel(cmd, iommu->reg + DMAR_GCMD_REG);
847
848 /* Make sure hardware complete it */
849 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
850 spin_unlock_irqrestore(&iommu->register_lock, flags);
851 867
852 return 0; 868 return 0;
853} 869}
@@ -1102,3 +1118,28 @@ int __init enable_drhd_fault_handling(void)
1102 1118
1103 return 0; 1119 return 0;
1104} 1120}
1121
1122/*
1123 * Re-enable Queued Invalidation interface.
1124 */
1125int dmar_reenable_qi(struct intel_iommu *iommu)
1126{
1127 if (!ecap_qis(iommu->ecap))
1128 return -ENOENT;
1129
1130 if (!iommu->qi)
1131 return -ENOENT;
1132
1133 /*
1134 * First disable queued invalidation.
1135 */
1136 dmar_disable_qi(iommu);
1137 /*
1138 * Then enable queued invalidation again. Since there is no pending
1139 * invalidation requests now, it's safe to re-enable queued
1140 * invalidation.
1141 */
1142 __dmar_enable_qi(iommu);
1143
1144 return 0;
1145}
diff --git a/drivers/pci/intel-iommu.c b/drivers/pci/intel-iommu.c
index 23e56a564e05..dcda5212f3bb 100644
--- a/drivers/pci/intel-iommu.c
+++ b/drivers/pci/intel-iommu.c
@@ -36,6 +36,7 @@
36#include <linux/iova.h> 36#include <linux/iova.h>
37#include <linux/iommu.h> 37#include <linux/iommu.h>
38#include <linux/intel-iommu.h> 38#include <linux/intel-iommu.h>
39#include <linux/sysdev.h>
39#include <asm/cacheflush.h> 40#include <asm/cacheflush.h>
40#include <asm/iommu.h> 41#include <asm/iommu.h>
41#include "pci.h" 42#include "pci.h"
@@ -247,7 +248,8 @@ struct dmar_domain {
247struct device_domain_info { 248struct device_domain_info {
248 struct list_head link; /* link to domain siblings */ 249 struct list_head link; /* link to domain siblings */
249 struct list_head global; /* link to global list */ 250 struct list_head global; /* link to global list */
250 u8 bus; /* PCI bus numer */ 251 int segment; /* PCI domain */
252 u8 bus; /* PCI bus number */
251 u8 devfn; /* PCI devfn number */ 253 u8 devfn; /* PCI devfn number */
252 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */ 254 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
253 struct dmar_domain *domain; /* pointer to domain */ 255 struct dmar_domain *domain; /* pointer to domain */
@@ -467,7 +469,7 @@ static void domain_update_iommu_cap(struct dmar_domain *domain)
467 domain_update_iommu_snooping(domain); 469 domain_update_iommu_snooping(domain);
468} 470}
469 471
470static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn) 472static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
471{ 473{
472 struct dmar_drhd_unit *drhd = NULL; 474 struct dmar_drhd_unit *drhd = NULL;
473 int i; 475 int i;
@@ -475,12 +477,20 @@ static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn)
475 for_each_drhd_unit(drhd) { 477 for_each_drhd_unit(drhd) {
476 if (drhd->ignored) 478 if (drhd->ignored)
477 continue; 479 continue;
480 if (segment != drhd->segment)
481 continue;
478 482
479 for (i = 0; i < drhd->devices_cnt; i++) 483 for (i = 0; i < drhd->devices_cnt; i++) {
480 if (drhd->devices[i] && 484 if (drhd->devices[i] &&
481 drhd->devices[i]->bus->number == bus && 485 drhd->devices[i]->bus->number == bus &&
482 drhd->devices[i]->devfn == devfn) 486 drhd->devices[i]->devfn == devfn)
483 return drhd->iommu; 487 return drhd->iommu;
488 if (drhd->devices[i] &&
489 drhd->devices[i]->subordinate &&
490 drhd->devices[i]->subordinate->number <= bus &&
491 drhd->devices[i]->subordinate->subordinate >= bus)
492 return drhd->iommu;
493 }
484 494
485 if (drhd->include_all) 495 if (drhd->include_all)
486 return drhd->iommu; 496 return drhd->iommu;
@@ -1312,7 +1322,7 @@ static void domain_exit(struct dmar_domain *domain)
1312} 1322}
1313 1323
1314static int domain_context_mapping_one(struct dmar_domain *domain, 1324static int domain_context_mapping_one(struct dmar_domain *domain,
1315 u8 bus, u8 devfn) 1325 int segment, u8 bus, u8 devfn)
1316{ 1326{
1317 struct context_entry *context; 1327 struct context_entry *context;
1318 unsigned long flags; 1328 unsigned long flags;
@@ -1327,7 +1337,7 @@ static int domain_context_mapping_one(struct dmar_domain *domain,
1327 bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); 1337 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1328 BUG_ON(!domain->pgd); 1338 BUG_ON(!domain->pgd);
1329 1339
1330 iommu = device_to_iommu(bus, devfn); 1340 iommu = device_to_iommu(segment, bus, devfn);
1331 if (!iommu) 1341 if (!iommu)
1332 return -ENODEV; 1342 return -ENODEV;
1333 1343
@@ -1417,8 +1427,8 @@ domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1417 int ret; 1427 int ret;
1418 struct pci_dev *tmp, *parent; 1428 struct pci_dev *tmp, *parent;
1419 1429
1420 ret = domain_context_mapping_one(domain, pdev->bus->number, 1430 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1421 pdev->devfn); 1431 pdev->bus->number, pdev->devfn);
1422 if (ret) 1432 if (ret)
1423 return ret; 1433 return ret;
1424 1434
@@ -1429,18 +1439,23 @@ domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1429 /* Secondary interface's bus number and devfn 0 */ 1439 /* Secondary interface's bus number and devfn 0 */
1430 parent = pdev->bus->self; 1440 parent = pdev->bus->self;
1431 while (parent != tmp) { 1441 while (parent != tmp) {
1432 ret = domain_context_mapping_one(domain, parent->bus->number, 1442 ret = domain_context_mapping_one(domain,
1433 parent->devfn); 1443 pci_domain_nr(parent->bus),
1444 parent->bus->number,
1445 parent->devfn);
1434 if (ret) 1446 if (ret)
1435 return ret; 1447 return ret;
1436 parent = parent->bus->self; 1448 parent = parent->bus->self;
1437 } 1449 }
1438 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */ 1450 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1439 return domain_context_mapping_one(domain, 1451 return domain_context_mapping_one(domain,
1440 tmp->subordinate->number, 0); 1452 pci_domain_nr(tmp->subordinate),
1453 tmp->subordinate->number, 0);
1441 else /* this is a legacy PCI bridge */ 1454 else /* this is a legacy PCI bridge */
1442 return domain_context_mapping_one(domain, 1455 return domain_context_mapping_one(domain,
1443 tmp->bus->number, tmp->devfn); 1456 pci_domain_nr(tmp->bus),
1457 tmp->bus->number,
1458 tmp->devfn);
1444} 1459}
1445 1460
1446static int domain_context_mapped(struct pci_dev *pdev) 1461static int domain_context_mapped(struct pci_dev *pdev)
@@ -1449,12 +1464,12 @@ static int domain_context_mapped(struct pci_dev *pdev)
1449 struct pci_dev *tmp, *parent; 1464 struct pci_dev *tmp, *parent;
1450 struct intel_iommu *iommu; 1465 struct intel_iommu *iommu;
1451 1466
1452 iommu = device_to_iommu(pdev->bus->number, pdev->devfn); 1467 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1468 pdev->devfn);
1453 if (!iommu) 1469 if (!iommu)
1454 return -ENODEV; 1470 return -ENODEV;
1455 1471
1456 ret = device_context_mapped(iommu, 1472 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1457 pdev->bus->number, pdev->devfn);
1458 if (!ret) 1473 if (!ret)
1459 return ret; 1474 return ret;
1460 /* dependent device mapping */ 1475 /* dependent device mapping */
@@ -1465,17 +1480,17 @@ static int domain_context_mapped(struct pci_dev *pdev)
1465 parent = pdev->bus->self; 1480 parent = pdev->bus->self;
1466 while (parent != tmp) { 1481 while (parent != tmp) {
1467 ret = device_context_mapped(iommu, parent->bus->number, 1482 ret = device_context_mapped(iommu, parent->bus->number,
1468 parent->devfn); 1483 parent->devfn);
1469 if (!ret) 1484 if (!ret)
1470 return ret; 1485 return ret;
1471 parent = parent->bus->self; 1486 parent = parent->bus->self;
1472 } 1487 }
1473 if (tmp->is_pcie) 1488 if (tmp->is_pcie)
1474 return device_context_mapped(iommu, 1489 return device_context_mapped(iommu, tmp->subordinate->number,
1475 tmp->subordinate->number, 0); 1490 0);
1476 else 1491 else
1477 return device_context_mapped(iommu, 1492 return device_context_mapped(iommu, tmp->bus->number,
1478 tmp->bus->number, tmp->devfn); 1493 tmp->devfn);
1479} 1494}
1480 1495
1481static int 1496static int
@@ -1542,7 +1557,7 @@ static void domain_remove_dev_info(struct dmar_domain *domain)
1542 info->dev->dev.archdata.iommu = NULL; 1557 info->dev->dev.archdata.iommu = NULL;
1543 spin_unlock_irqrestore(&device_domain_lock, flags); 1558 spin_unlock_irqrestore(&device_domain_lock, flags);
1544 1559
1545 iommu = device_to_iommu(info->bus, info->devfn); 1560 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1546 iommu_detach_dev(iommu, info->bus, info->devfn); 1561 iommu_detach_dev(iommu, info->bus, info->devfn);
1547 free_devinfo_mem(info); 1562 free_devinfo_mem(info);
1548 1563
@@ -1577,11 +1592,14 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1577 struct pci_dev *dev_tmp; 1592 struct pci_dev *dev_tmp;
1578 unsigned long flags; 1593 unsigned long flags;
1579 int bus = 0, devfn = 0; 1594 int bus = 0, devfn = 0;
1595 int segment;
1580 1596
1581 domain = find_domain(pdev); 1597 domain = find_domain(pdev);
1582 if (domain) 1598 if (domain)
1583 return domain; 1599 return domain;
1584 1600
1601 segment = pci_domain_nr(pdev->bus);
1602
1585 dev_tmp = pci_find_upstream_pcie_bridge(pdev); 1603 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1586 if (dev_tmp) { 1604 if (dev_tmp) {
1587 if (dev_tmp->is_pcie) { 1605 if (dev_tmp->is_pcie) {
@@ -1593,7 +1611,8 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1593 } 1611 }
1594 spin_lock_irqsave(&device_domain_lock, flags); 1612 spin_lock_irqsave(&device_domain_lock, flags);
1595 list_for_each_entry(info, &device_domain_list, global) { 1613 list_for_each_entry(info, &device_domain_list, global) {
1596 if (info->bus == bus && info->devfn == devfn) { 1614 if (info->segment == segment &&
1615 info->bus == bus && info->devfn == devfn) {
1597 found = info->domain; 1616 found = info->domain;
1598 break; 1617 break;
1599 } 1618 }
@@ -1631,6 +1650,7 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1631 domain_exit(domain); 1650 domain_exit(domain);
1632 goto error; 1651 goto error;
1633 } 1652 }
1653 info->segment = segment;
1634 info->bus = bus; 1654 info->bus = bus;
1635 info->devfn = devfn; 1655 info->devfn = devfn;
1636 info->dev = NULL; 1656 info->dev = NULL;
@@ -1642,7 +1662,8 @@ static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1642 found = NULL; 1662 found = NULL;
1643 spin_lock_irqsave(&device_domain_lock, flags); 1663 spin_lock_irqsave(&device_domain_lock, flags);
1644 list_for_each_entry(tmp, &device_domain_list, global) { 1664 list_for_each_entry(tmp, &device_domain_list, global) {
1645 if (tmp->bus == bus && tmp->devfn == devfn) { 1665 if (tmp->segment == segment &&
1666 tmp->bus == bus && tmp->devfn == devfn) {
1646 found = tmp->domain; 1667 found = tmp->domain;
1647 break; 1668 break;
1648 } 1669 }
@@ -1662,6 +1683,7 @@ found_domain:
1662 info = alloc_devinfo_mem(); 1683 info = alloc_devinfo_mem();
1663 if (!info) 1684 if (!info)
1664 goto error; 1685 goto error;
1686 info->segment = segment;
1665 info->bus = pdev->bus->number; 1687 info->bus = pdev->bus->number;
1666 info->devfn = pdev->devfn; 1688 info->devfn = pdev->devfn;
1667 info->dev = pdev; 1689 info->dev = pdev;
@@ -1946,6 +1968,15 @@ static int __init init_dmars(void)
1946 } 1968 }
1947 } 1969 }
1948 1970
1971#ifdef CONFIG_INTR_REMAP
1972 if (!intr_remapping_enabled) {
1973 ret = enable_intr_remapping(0);
1974 if (ret)
1975 printk(KERN_ERR
1976 "IOMMU: enable interrupt remapping failed\n");
1977 }
1978#endif
1979
1949 /* 1980 /*
1950 * For each rmrr 1981 * For each rmrr
1951 * for each dev attached to rmrr 1982 * for each dev attached to rmrr
@@ -2597,6 +2628,150 @@ static void __init init_no_remapping_devices(void)
2597 } 2628 }
2598} 2629}
2599 2630
2631#ifdef CONFIG_SUSPEND
2632static int init_iommu_hw(void)
2633{
2634 struct dmar_drhd_unit *drhd;
2635 struct intel_iommu *iommu = NULL;
2636
2637 for_each_active_iommu(iommu, drhd)
2638 if (iommu->qi)
2639 dmar_reenable_qi(iommu);
2640
2641 for_each_active_iommu(iommu, drhd) {
2642 iommu_flush_write_buffer(iommu);
2643
2644 iommu_set_root_entry(iommu);
2645
2646 iommu->flush.flush_context(iommu, 0, 0, 0,
2647 DMA_CCMD_GLOBAL_INVL, 0);
2648 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2649 DMA_TLB_GLOBAL_FLUSH, 0);
2650 iommu_disable_protect_mem_regions(iommu);
2651 iommu_enable_translation(iommu);
2652 }
2653
2654 return 0;
2655}
2656
2657static void iommu_flush_all(void)
2658{
2659 struct dmar_drhd_unit *drhd;
2660 struct intel_iommu *iommu;
2661
2662 for_each_active_iommu(iommu, drhd) {
2663 iommu->flush.flush_context(iommu, 0, 0, 0,
2664 DMA_CCMD_GLOBAL_INVL, 0);
2665 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2666 DMA_TLB_GLOBAL_FLUSH, 0);
2667 }
2668}
2669
2670static int iommu_suspend(struct sys_device *dev, pm_message_t state)
2671{
2672 struct dmar_drhd_unit *drhd;
2673 struct intel_iommu *iommu = NULL;
2674 unsigned long flag;
2675
2676 for_each_active_iommu(iommu, drhd) {
2677 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
2678 GFP_ATOMIC);
2679 if (!iommu->iommu_state)
2680 goto nomem;
2681 }
2682
2683 iommu_flush_all();
2684
2685 for_each_active_iommu(iommu, drhd) {
2686 iommu_disable_translation(iommu);
2687
2688 spin_lock_irqsave(&iommu->register_lock, flag);
2689
2690 iommu->iommu_state[SR_DMAR_FECTL_REG] =
2691 readl(iommu->reg + DMAR_FECTL_REG);
2692 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2693 readl(iommu->reg + DMAR_FEDATA_REG);
2694 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2695 readl(iommu->reg + DMAR_FEADDR_REG);
2696 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2697 readl(iommu->reg + DMAR_FEUADDR_REG);
2698
2699 spin_unlock_irqrestore(&iommu->register_lock, flag);
2700 }
2701 return 0;
2702
2703nomem:
2704 for_each_active_iommu(iommu, drhd)
2705 kfree(iommu->iommu_state);
2706
2707 return -ENOMEM;
2708}
2709
2710static int iommu_resume(struct sys_device *dev)
2711{
2712 struct dmar_drhd_unit *drhd;
2713 struct intel_iommu *iommu = NULL;
2714 unsigned long flag;
2715
2716 if (init_iommu_hw()) {
2717 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2718 return -EIO;
2719 }
2720
2721 for_each_active_iommu(iommu, drhd) {
2722
2723 spin_lock_irqsave(&iommu->register_lock, flag);
2724
2725 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2726 iommu->reg + DMAR_FECTL_REG);
2727 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2728 iommu->reg + DMAR_FEDATA_REG);
2729 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2730 iommu->reg + DMAR_FEADDR_REG);
2731 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2732 iommu->reg + DMAR_FEUADDR_REG);
2733
2734 spin_unlock_irqrestore(&iommu->register_lock, flag);
2735 }
2736
2737 for_each_active_iommu(iommu, drhd)
2738 kfree(iommu->iommu_state);
2739
2740 return 0;
2741}
2742
2743static struct sysdev_class iommu_sysclass = {
2744 .name = "iommu",
2745 .resume = iommu_resume,
2746 .suspend = iommu_suspend,
2747};
2748
2749static struct sys_device device_iommu = {
2750 .cls = &iommu_sysclass,
2751};
2752
2753static int __init init_iommu_sysfs(void)
2754{
2755 int error;
2756
2757 error = sysdev_class_register(&iommu_sysclass);
2758 if (error)
2759 return error;
2760
2761 error = sysdev_register(&device_iommu);
2762 if (error)
2763 sysdev_class_unregister(&iommu_sysclass);
2764
2765 return error;
2766}
2767
2768#else
2769static int __init init_iommu_sysfs(void)
2770{
2771 return 0;
2772}
2773#endif /* CONFIG_PM */
2774
2600int __init intel_iommu_init(void) 2775int __init intel_iommu_init(void)
2601{ 2776{
2602 int ret = 0; 2777 int ret = 0;
@@ -2632,6 +2807,7 @@ int __init intel_iommu_init(void)
2632 init_timer(&unmap_timer); 2807 init_timer(&unmap_timer);
2633 force_iommu = 1; 2808 force_iommu = 1;
2634 dma_ops = &intel_dma_ops; 2809 dma_ops = &intel_dma_ops;
2810 init_iommu_sysfs();
2635 2811
2636 register_iommu(&intel_iommu_ops); 2812 register_iommu(&intel_iommu_ops);
2637 2813
@@ -2648,6 +2824,7 @@ static int vm_domain_add_dev_info(struct dmar_domain *domain,
2648 if (!info) 2824 if (!info)
2649 return -ENOMEM; 2825 return -ENOMEM;
2650 2826
2827 info->segment = pci_domain_nr(pdev->bus);
2651 info->bus = pdev->bus->number; 2828 info->bus = pdev->bus->number;
2652 info->devfn = pdev->devfn; 2829 info->devfn = pdev->devfn;
2653 info->dev = pdev; 2830 info->dev = pdev;
@@ -2677,15 +2854,15 @@ static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
2677 parent = pdev->bus->self; 2854 parent = pdev->bus->self;
2678 while (parent != tmp) { 2855 while (parent != tmp) {
2679 iommu_detach_dev(iommu, parent->bus->number, 2856 iommu_detach_dev(iommu, parent->bus->number,
2680 parent->devfn); 2857 parent->devfn);
2681 parent = parent->bus->self; 2858 parent = parent->bus->self;
2682 } 2859 }
2683 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */ 2860 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
2684 iommu_detach_dev(iommu, 2861 iommu_detach_dev(iommu,
2685 tmp->subordinate->number, 0); 2862 tmp->subordinate->number, 0);
2686 else /* this is a legacy PCI bridge */ 2863 else /* this is a legacy PCI bridge */
2687 iommu_detach_dev(iommu, 2864 iommu_detach_dev(iommu, tmp->bus->number,
2688 tmp->bus->number, tmp->devfn); 2865 tmp->devfn);
2689 } 2866 }
2690} 2867}
2691 2868
@@ -2698,13 +2875,15 @@ static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
2698 int found = 0; 2875 int found = 0;
2699 struct list_head *entry, *tmp; 2876 struct list_head *entry, *tmp;
2700 2877
2701 iommu = device_to_iommu(pdev->bus->number, pdev->devfn); 2878 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
2879 pdev->devfn);
2702 if (!iommu) 2880 if (!iommu)
2703 return; 2881 return;
2704 2882
2705 spin_lock_irqsave(&device_domain_lock, flags); 2883 spin_lock_irqsave(&device_domain_lock, flags);
2706 list_for_each_safe(entry, tmp, &domain->devices) { 2884 list_for_each_safe(entry, tmp, &domain->devices) {
2707 info = list_entry(entry, struct device_domain_info, link); 2885 info = list_entry(entry, struct device_domain_info, link);
2886 /* No need to compare PCI domain; it has to be the same */
2708 if (info->bus == pdev->bus->number && 2887 if (info->bus == pdev->bus->number &&
2709 info->devfn == pdev->devfn) { 2888 info->devfn == pdev->devfn) {
2710 list_del(&info->link); 2889 list_del(&info->link);
@@ -2729,7 +2908,8 @@ static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
2729 * owned by this domain, clear this iommu in iommu_bmp 2908 * owned by this domain, clear this iommu in iommu_bmp
2730 * update iommu count and coherency 2909 * update iommu count and coherency
2731 */ 2910 */
2732 if (device_to_iommu(info->bus, info->devfn) == iommu) 2911 if (iommu == device_to_iommu(info->segment, info->bus,
2912 info->devfn))
2733 found = 1; 2913 found = 1;
2734 } 2914 }
2735 2915
@@ -2762,7 +2942,7 @@ static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
2762 2942
2763 spin_unlock_irqrestore(&device_domain_lock, flags1); 2943 spin_unlock_irqrestore(&device_domain_lock, flags1);
2764 2944
2765 iommu = device_to_iommu(info->bus, info->devfn); 2945 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
2766 iommu_detach_dev(iommu, info->bus, info->devfn); 2946 iommu_detach_dev(iommu, info->bus, info->devfn);
2767 iommu_detach_dependent_devices(iommu, info->dev); 2947 iommu_detach_dependent_devices(iommu, info->dev);
2768 2948
@@ -2950,7 +3130,8 @@ static int intel_iommu_attach_device(struct iommu_domain *domain,
2950 } 3130 }
2951 } 3131 }
2952 3132
2953 iommu = device_to_iommu(pdev->bus->number, pdev->devfn); 3133 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3134 pdev->devfn);
2954 if (!iommu) 3135 if (!iommu)
2955 return -ENODEV; 3136 return -ENODEV;
2956 3137
diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c
index b041a409f4a7..f5e0ea724a6f 100644
--- a/drivers/pci/intr_remapping.c
+++ b/drivers/pci/intr_remapping.c
@@ -9,6 +9,7 @@
9#include <asm/cpu.h> 9#include <asm/cpu.h>
10#include <linux/intel-iommu.h> 10#include <linux/intel-iommu.h>
11#include "intr_remapping.h" 11#include "intr_remapping.h"
12#include <acpi/acpi.h>
12 13
13static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; 14static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
14static int ir_ioapic_num; 15static int ir_ioapic_num;
@@ -415,12 +416,27 @@ static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
415 416
416 /* Set interrupt-remapping table pointer */ 417 /* Set interrupt-remapping table pointer */
417 cmd = iommu->gcmd | DMA_GCMD_SIRTP; 418 cmd = iommu->gcmd | DMA_GCMD_SIRTP;
419 iommu->gcmd |= DMA_GCMD_SIRTP;
418 writel(cmd, iommu->reg + DMAR_GCMD_REG); 420 writel(cmd, iommu->reg + DMAR_GCMD_REG);
419 421
420 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, 422 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
421 readl, (sts & DMA_GSTS_IRTPS), sts); 423 readl, (sts & DMA_GSTS_IRTPS), sts);
422 spin_unlock_irqrestore(&iommu->register_lock, flags); 424 spin_unlock_irqrestore(&iommu->register_lock, flags);
423 425
426 if (mode == 0) {
427 spin_lock_irqsave(&iommu->register_lock, flags);
428
429 /* enable comaptiblity format interrupt pass through */
430 cmd = iommu->gcmd | DMA_GCMD_CFI;
431 iommu->gcmd |= DMA_GCMD_CFI;
432 writel(cmd, iommu->reg + DMAR_GCMD_REG);
433
434 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
435 readl, (sts & DMA_GSTS_CFIS), sts);
436
437 spin_unlock_irqrestore(&iommu->register_lock, flags);
438 }
439
424 /* 440 /*
425 * global invalidation of interrupt entry cache before enabling 441 * global invalidation of interrupt entry cache before enabling
426 * interrupt-remapping. 442 * interrupt-remapping.
@@ -470,7 +486,7 @@ static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
470/* 486/*
471 * Disable Interrupt Remapping. 487 * Disable Interrupt Remapping.
472 */ 488 */
473static void disable_intr_remapping(struct intel_iommu *iommu) 489static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
474{ 490{
475 unsigned long flags; 491 unsigned long flags;
476 u32 sts; 492 u32 sts;
@@ -478,6 +494,12 @@ static void disable_intr_remapping(struct intel_iommu *iommu)
478 if (!ecap_ir_support(iommu->ecap)) 494 if (!ecap_ir_support(iommu->ecap))
479 return; 495 return;
480 496
497 /*
498 * global invalidation of interrupt entry cache before disabling
499 * interrupt-remapping.
500 */
501 qi_global_iec(iommu);
502
481 spin_lock_irqsave(&iommu->register_lock, flags); 503 spin_lock_irqsave(&iommu->register_lock, flags);
482 504
483 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); 505 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
@@ -503,6 +525,13 @@ int __init enable_intr_remapping(int eim)
503 struct intel_iommu *iommu = drhd->iommu; 525 struct intel_iommu *iommu = drhd->iommu;
504 526
505 /* 527 /*
528 * If the queued invalidation is already initialized,
529 * shouldn't disable it.
530 */
531 if (iommu->qi)
532 continue;
533
534 /*
506 * Clear previous faults. 535 * Clear previous faults.
507 */ 536 */
508 dmar_fault(-1, iommu); 537 dmar_fault(-1, iommu);
@@ -511,7 +540,7 @@ int __init enable_intr_remapping(int eim)
511 * Disable intr remapping and queued invalidation, if already 540 * Disable intr remapping and queued invalidation, if already
512 * enabled prior to OS handover. 541 * enabled prior to OS handover.
513 */ 542 */
514 disable_intr_remapping(iommu); 543 iommu_disable_intr_remapping(iommu);
515 544
516 dmar_disable_qi(iommu); 545 dmar_disable_qi(iommu);
517 } 546 }
@@ -639,3 +668,54 @@ int __init parse_ioapics_under_ir(void)
639 668
640 return ir_supported; 669 return ir_supported;
641} 670}
671
672void disable_intr_remapping(void)
673{
674 struct dmar_drhd_unit *drhd;
675 struct intel_iommu *iommu = NULL;
676
677 /*
678 * Disable Interrupt-remapping for all the DRHD's now.
679 */
680 for_each_iommu(iommu, drhd) {
681 if (!ecap_ir_support(iommu->ecap))
682 continue;
683
684 iommu_disable_intr_remapping(iommu);
685 }
686}
687
688int reenable_intr_remapping(int eim)
689{
690 struct dmar_drhd_unit *drhd;
691 int setup = 0;
692 struct intel_iommu *iommu = NULL;
693
694 for_each_iommu(iommu, drhd)
695 if (iommu->qi)
696 dmar_reenable_qi(iommu);
697
698 /*
699 * Setup Interrupt-remapping for all the DRHD's now.
700 */
701 for_each_iommu(iommu, drhd) {
702 if (!ecap_ir_support(iommu->ecap))
703 continue;
704
705 /* Set up interrupt remapping for iommu.*/
706 iommu_set_intr_remapping(iommu, eim);
707 setup = 1;
708 }
709
710 if (!setup)
711 goto error;
712
713 return 0;
714
715error:
716 /*
717 * handle error condition gracefully here!
718 */
719 return -1;
720}
721
diff --git a/include/linux/dmar.h b/include/linux/dmar.h
index 2f3427468956..e397dc342cda 100644
--- a/include/linux/dmar.h
+++ b/include/linux/dmar.h
@@ -34,6 +34,7 @@ struct dmar_drhd_unit {
34 u64 reg_base_addr; /* register base address*/ 34 u64 reg_base_addr; /* register base address*/
35 struct pci_dev **devices; /* target device array */ 35 struct pci_dev **devices; /* target device array */
36 int devices_cnt; /* target device count */ 36 int devices_cnt; /* target device count */
37 u16 segment; /* PCI domain */
37 u8 ignored:1; /* ignore drhd */ 38 u8 ignored:1; /* ignore drhd */
38 u8 include_all:1; 39 u8 include_all:1;
39 struct intel_iommu *iommu; 40 struct intel_iommu *iommu;
@@ -44,6 +45,14 @@ extern struct list_head dmar_drhd_units;
44#define for_each_drhd_unit(drhd) \ 45#define for_each_drhd_unit(drhd) \
45 list_for_each_entry(drhd, &dmar_drhd_units, list) 46 list_for_each_entry(drhd, &dmar_drhd_units, list)
46 47
48#define for_each_active_iommu(i, drhd) \
49 list_for_each_entry(drhd, &dmar_drhd_units, list) \
50 if (i=drhd->iommu, drhd->ignored) {} else
51
52#define for_each_iommu(i, drhd) \
53 list_for_each_entry(drhd, &dmar_drhd_units, list) \
54 if (i=drhd->iommu, 0) {} else
55
47extern int dmar_table_init(void); 56extern int dmar_table_init(void);
48extern int dmar_dev_scope_init(void); 57extern int dmar_dev_scope_init(void);
49 58
@@ -100,6 +109,8 @@ struct irte {
100#ifdef CONFIG_INTR_REMAP 109#ifdef CONFIG_INTR_REMAP
101extern int intr_remapping_enabled; 110extern int intr_remapping_enabled;
102extern int enable_intr_remapping(int); 111extern int enable_intr_remapping(int);
112extern void disable_intr_remapping(void);
113extern int reenable_intr_remapping(int);
103 114
104extern int get_irte(int irq, struct irte *entry); 115extern int get_irte(int irq, struct irte *entry);
105extern int modify_irte(int irq, struct irte *irte_modified); 116extern int modify_irte(int irq, struct irte *irte_modified);
diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h
index 77214ead1a36..aa8c53171233 100644
--- a/include/linux/intel-iommu.h
+++ b/include/linux/intel-iommu.h
@@ -164,6 +164,7 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
164#define DMA_GCMD_QIE (((u32)1) << 26) 164#define DMA_GCMD_QIE (((u32)1) << 26)
165#define DMA_GCMD_SIRTP (((u32)1) << 24) 165#define DMA_GCMD_SIRTP (((u32)1) << 24)
166#define DMA_GCMD_IRE (((u32) 1) << 25) 166#define DMA_GCMD_IRE (((u32) 1) << 25)
167#define DMA_GCMD_CFI (((u32) 1) << 23)
167 168
168/* GSTS_REG */ 169/* GSTS_REG */
169#define DMA_GSTS_TES (((u32)1) << 31) 170#define DMA_GSTS_TES (((u32)1) << 31)
@@ -174,6 +175,7 @@ static inline void dmar_writeq(void __iomem *addr, u64 val)
174#define DMA_GSTS_QIES (((u32)1) << 26) 175#define DMA_GSTS_QIES (((u32)1) << 26)
175#define DMA_GSTS_IRTPS (((u32)1) << 24) 176#define DMA_GSTS_IRTPS (((u32)1) << 24)
176#define DMA_GSTS_IRES (((u32)1) << 25) 177#define DMA_GSTS_IRES (((u32)1) << 25)
178#define DMA_GSTS_CFIS (((u32)1) << 23)
177 179
178/* CCMD_REG */ 180/* CCMD_REG */
179#define DMA_CCMD_ICC (((u64)1) << 63) 181#define DMA_CCMD_ICC (((u64)1) << 63)
@@ -284,6 +286,14 @@ struct iommu_flush {
284 unsigned int size_order, u64 type, int non_present_entry_flush); 286 unsigned int size_order, u64 type, int non_present_entry_flush);
285}; 287};
286 288
289enum {
290 SR_DMAR_FECTL_REG,
291 SR_DMAR_FEDATA_REG,
292 SR_DMAR_FEADDR_REG,
293 SR_DMAR_FEUADDR_REG,
294 MAX_SR_DMAR_REGS
295};
296
287struct intel_iommu { 297struct intel_iommu {
288 void __iomem *reg; /* Pointer to hardware regs, virtual addr */ 298 void __iomem *reg; /* Pointer to hardware regs, virtual addr */
289 u64 cap; 299 u64 cap;
@@ -304,6 +314,8 @@ struct intel_iommu {
304 struct iommu_flush flush; 314 struct iommu_flush flush;
305#endif 315#endif
306 struct q_inval *qi; /* Queued invalidation info */ 316 struct q_inval *qi; /* Queued invalidation info */
317 u32 *iommu_state; /* Store iommu states between suspend and resume.*/
318
307#ifdef CONFIG_INTR_REMAP 319#ifdef CONFIG_INTR_REMAP
308 struct ir_table *ir_table; /* Interrupt remapping info */ 320 struct ir_table *ir_table; /* Interrupt remapping info */
309#endif 321#endif
@@ -322,6 +334,7 @@ extern int alloc_iommu(struct dmar_drhd_unit *drhd);
322extern void free_iommu(struct intel_iommu *iommu); 334extern void free_iommu(struct intel_iommu *iommu);
323extern int dmar_enable_qi(struct intel_iommu *iommu); 335extern int dmar_enable_qi(struct intel_iommu *iommu);
324extern void dmar_disable_qi(struct intel_iommu *iommu); 336extern void dmar_disable_qi(struct intel_iommu *iommu);
337extern int dmar_reenable_qi(struct intel_iommu *iommu);
325extern void qi_global_iec(struct intel_iommu *iommu); 338extern void qi_global_iec(struct intel_iommu *iommu);
326 339
327extern int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, 340extern int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid,
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-26 20:44:42 -0500