Intel IOMMU Suspend/Resume Support - Interrupt Remapping

This patch enables suspend/resume for interrupt remapping. During suspend,
interrupt remapping is disabled. When resume, interrupt remapping is enabled
again.

Signed-off-by: Fenghua Yu <fenghua.yu@intel.com>
Acked-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: David Woodhouse <David.Woodhouse@intel.com>

6 files changed, 204 insertions, 55 deletions

diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index df8a300dfe6c..f9f0866ed6f8 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -108,6 +108,10 @@ extern void native_apic_icr_write(u32 low, u32 id);
 extern u64 native_apic_icr_read(void);
 
 #ifdef CONFIG_X86_X2APIC
+
+#define EIM_8BIT_APIC_ID        0
+#define EIM_32BIT_APIC_ID       1
+
 /*
  * Make previous memory operations globally visible before
  * sending the IPI through x2apic wrmsr. We need a serializing instruction or
diff --git a/arch/x86/include/asm/io_apic.h b/arch/x86/include/asm/io_apic.h
index 373cc2bbcad2..9d826e436010 100644
--- a/arch/x86/include/asm/io_apic.h
+++ b/arch/x86/include/asm/io_apic.h
@@ -162,10 +162,13 @@ extern int (*ioapic_renumber_irq)(int ioapic, int irq);
 extern void ioapic_init_mappings(void);
 
 #ifdef CONFIG_X86_64
-extern int save_IO_APIC_setup(void);
-extern void mask_IO_APIC_setup(void);
-extern void restore_IO_APIC_setup(void);
-extern void reinit_intr_remapped_IO_APIC(int);
+extern struct IO_APIC_route_entry **alloc_ioapic_entries(void);
+extern void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries);
+extern int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries);
+extern void reinit_intr_remapped_IO_APIC(int intr_remapping,
+        struct IO_APIC_route_entry **ioapic_entries);
 #endif
 
 extern void probe_nr_irqs_gsi(void);
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 85eb8e100818..098ec84b8c00 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1304,6 +1304,7 @@ void __init enable_IR_x2apic(void)
 #ifdef CONFIG_INTR_REMAP
         int ret;
         unsigned long flags;
+        struct IO_APIC_route_entry **ioapic_entries = NULL;
 
         if (!cpu_has_x2apic)
                 return;
@@ -1334,17 +1335,23 @@ void __init enable_IR_x2apic(void)
                 return;
         }
 
-        ret = save_IO_APIC_setup();
+        ioapic_entries = alloc_ioapic_entries();
+        if (!ioapic_entries) {
+                pr_info("Allocate ioapic_entries failed: %d\n", ret);
+                goto end;
+        }
+
+        ret = save_IO_APIC_setup(ioapic_entries);
         if (ret) {
                 pr_info("Saving IO-APIC state failed: %d\n", ret);
                 goto end;
         }
 
         local_irq_save(flags);
-        mask_IO_APIC_setup();
+        mask_IO_APIC_setup(ioapic_entries);
         mask_8259A();
 
-        ret = enable_intr_remapping(1);
+        ret = enable_intr_remapping(EIM_32BIT_APIC_ID);
 
         if (ret && x2apic_preenabled) {
                 local_irq_restore(flags);
@@ -1364,9 +1371,9 @@ end_restore:
                 /*
                  * IR enabling failed
                  */
-                restore_IO_APIC_setup();
+                restore_IO_APIC_setup(ioapic_entries);
         else
-                reinit_intr_remapped_IO_APIC(x2apic_preenabled);
+                reinit_intr_remapped_IO_APIC(x2apic_preenabled, ioapic_entries);
 
         unmask_8259A();
         local_irq_restore(flags);
@@ -1379,6 +1386,8 @@ end:
                         pr_info("Enabled Interrupt-remapping\n");
         } else
                 pr_err("Failed to enable Interrupt-remapping and x2apic\n");
+        if (ioapic_entries)
+                free_ioapic_entries(ioapic_entries);
 #else
         if (!cpu_has_x2apic)
                 return;
@@ -1954,6 +1963,10 @@ static int lapic_suspend(struct sys_device *dev, pm_message_t state)
 
         local_irq_save(flags);
         disable_local_APIC();
+#ifdef CONFIG_INTR_REMAP
+        if (intr_remapping_enabled)
+                disable_intr_remapping();
+#endif
         local_irq_restore(flags);
         return 0;
 }
@@ -1964,15 +1977,41 @@ static int lapic_resume(struct sys_device *dev)
         unsigned long flags;
         int maxlvt;
 
+#ifdef CONFIG_INTR_REMAP
+        int ret;
+        struct IO_APIC_route_entry **ioapic_entries = NULL;
+
         if (!apic_pm_state.active)
                 return 0;
 
-        maxlvt = lapic_get_maxlvt();
-
         local_irq_save(flags);
+        if (x2apic) {
+                ioapic_entries = alloc_ioapic_entries();
+                if (!ioapic_entries) {
+                        WARN(1, "Alloc ioapic_entries in lapic resume failed.");
+                        return -ENOMEM;
+                }
+
+                ret = save_IO_APIC_setup(ioapic_entries);
+                if (ret) {
+                        WARN(1, "Saving IO-APIC state failed: %d\n", ret);
+                        free_ioapic_entries(ioapic_entries);
+                        return ret;
+                }
+
+                mask_IO_APIC_setup(ioapic_entries);
+                mask_8259A();
+                enable_x2apic();
+        }
+#else
+        if (!apic_pm_state.active)
+                return 0;
 
+        local_irq_save(flags);
         if (x2apic)
                 enable_x2apic();
+#endif
+
         else {
                 /*
                  * Make sure the APICBASE points to the right address
@@ -1986,6 +2025,7 @@ static int lapic_resume(struct sys_device *dev)
                 wrmsr(MSR_IA32_APICBASE, l, h);
         }
 
+        maxlvt = lapic_get_maxlvt();
         apic_write(APIC_LVTERR, ERROR_APIC_VECTOR | APIC_LVT_MASKED);
         apic_write(APIC_ID, apic_pm_state.apic_id);
         apic_write(APIC_DFR, apic_pm_state.apic_dfr);
@@ -2009,8 +2049,20 @@ static int lapic_resume(struct sys_device *dev)
         apic_write(APIC_ESR, 0);
         apic_read(APIC_ESR);
 
+#ifdef CONFIG_INTR_REMAP
+        if (intr_remapping_enabled)
+                reenable_intr_remapping(EIM_32BIT_APIC_ID);
+
+        if (x2apic) {
+                unmask_8259A();
+                restore_IO_APIC_setup(ioapic_entries);
+                free_ioapic_entries(ioapic_entries);
+        }
+#endif
+
         local_irq_restore(flags);
 
+
         return 0;
 }
 
@@ -2048,7 +2100,9 @@ static int __init init_lapic_sysfs(void)
                 error = sysdev_register(&device_lapic);
         return error;
 }
-device_initcall(init_lapic_sysfs);
+
+/* local apic needs to resume before other devices access its registers. */
+core_initcall(init_lapic_sysfs);
 
 #else   /* CONFIG_PM */
 
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 1bb5c6cee3eb..0ad3fe77641a 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -851,63 +851,74 @@ __setup("pirq=", ioapic_pirq_setup);
 #endif /* CONFIG_X86_32 */
 
 #ifdef CONFIG_INTR_REMAP
-/* I/O APIC RTE contents at the OS boot up */
-static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];
+struct IO_APIC_route_entry **alloc_ioapic_entries(void)
+{
+        int apic;
+        struct IO_APIC_route_entry **ioapic_entries;
+
+        ioapic_entries = kzalloc(sizeof(*ioapic_entries) * nr_ioapics,
+                                GFP_ATOMIC);
+        if (!ioapic_entries)
+                return 0;
+
+        for (apic = 0; apic < nr_ioapics; apic++) {
+                ioapic_entries[apic] =
+                        kzalloc(sizeof(struct IO_APIC_route_entry) *
+                                nr_ioapic_registers[apic], GFP_ATOMIC);
+                if (!ioapic_entries[apic])
+                        goto nomem;
+        }
+
+        return ioapic_entries;
+
+nomem:
+        while (--apic >= 0)
+                kfree(ioapic_entries[apic]);
+        kfree(ioapic_entries);
+
+        return 0;
+}
 
 /*
  * Saves all the IO-APIC RTE's
  */
-int save_IO_APIC_setup(void)
+int save_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
 {
-        union IO_APIC_reg_01 reg_01;
-        unsigned long flags;
         int apic, pin;
 
-        /*
-         * The number of IO-APIC IRQ registers (== #pins):
-         */
-        for (apic = 0; apic < nr_ioapics; apic++) {
-                spin_lock_irqsave(&ioapic_lock, flags);
-                reg_01.raw = io_apic_read(apic, 1);
-                spin_unlock_irqrestore(&ioapic_lock, flags);
-                nr_ioapic_registers[apic] = reg_01.bits.entries+1;
-        }
+        if (!ioapic_entries)
+                return -ENOMEM;
 
         for (apic = 0; apic < nr_ioapics; apic++) {
-                early_ioapic_entries[apic] =
-                        kzalloc(sizeof(struct IO_APIC_route_entry) *
-                                nr_ioapic_registers[apic], GFP_KERNEL);
-                if (!early_ioapic_entries[apic])
-                        goto nomem;
-        }
+                if (!ioapic_entries[apic])
+                        return -ENOMEM;
 
-        for (apic = 0; apic < nr_ioapics; apic++)
                 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
-                        early_ioapic_entries[apic][pin] =
+                        ioapic_entries[apic][pin] =
                                 ioapic_read_entry(apic, pin);
+        }
 
         return 0;
-
-nomem:
-        while (apic >= 0)
-                kfree(early_ioapic_entries[apic--]);
-        memset(early_ioapic_entries, 0,
-                ARRAY_SIZE(early_ioapic_entries));
-
-        return -ENOMEM;
 }
 
-void mask_IO_APIC_setup(void)
+/*
+ * Mask all IO APIC entries.
+ */
+void mask_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
 {
         int apic, pin;
 
+        if (!ioapic_entries)
+                return;
+
         for (apic = 0; apic < nr_ioapics; apic++) {
-                if (!early_ioapic_entries[apic])
+                if (!ioapic_entries[apic])
                         break;
+
                 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
                         struct IO_APIC_route_entry entry;
 
-                        entry = early_ioapic_entries[apic][pin];
+                        entry = ioapic_entries[apic][pin];
                         if (!entry.mask) {
                                 entry.mask = 1;
                                 ioapic_write_entry(apic, pin, entry);
@@ -916,22 +927,30 @@ void mask_IO_APIC_setup(void)
         }
 }
 
-void restore_IO_APIC_setup(void)
+/*
+ * Restore IO APIC entries which was saved in ioapic_entries.
+ */
+int restore_IO_APIC_setup(struct IO_APIC_route_entry **ioapic_entries)
 {
         int apic, pin;
 
+        if (!ioapic_entries)
+                return -ENOMEM;
+
         for (apic = 0; apic < nr_ioapics; apic++) {
-                if (!early_ioapic_entries[apic])
-                        break;
+                if (!ioapic_entries[apic])
+                        return -ENOMEM;
+
                 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
                         ioapic_write_entry(apic, pin,
-                                           early_ioapic_entries[apic][pin]);
-                kfree(early_ioapic_entries[apic]);
-                early_ioapic_entries[apic] = NULL;
+                                        ioapic_entries[apic][pin]);
         }
+        return 0;
 }
 
-void reinit_intr_remapped_IO_APIC(int intr_remapping)
+void reinit_intr_remapped_IO_APIC(int intr_remapping,
+        struct IO_APIC_route_entry **ioapic_entries)
+
 {
         /*
          * for now plain restore of previous settings.
@@ -940,7 +959,17 @@ void reinit_intr_remapped_IO_APIC(int intr_remapping)
          * table entries. for now, do a plain restore, and wait for
          * the setup_IO_APIC_irqs() to do proper initialization.
          */
-        restore_IO_APIC_setup();
+        restore_IO_APIC_setup(ioapic_entries);
+}
+
+void free_ioapic_entries(struct IO_APIC_route_entry **ioapic_entries)
+{
+        int apic;
+
+        for (apic = 0; apic < nr_ioapics; apic++)
+                kfree(ioapic_entries[apic]);
+
+        kfree(ioapic_entries);
 }
 #endif
 
diff --git a/drivers/pci/intr_remapping.c b/drivers/pci/intr_remapping.c
index b041a409f4a7..c26633d7e7da 100644
--- a/drivers/pci/intr_remapping.c
+++ b/drivers/pci/intr_remapping.c
@@ -470,7 +470,7 @@ static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
 /*
  * Disable Interrupt Remapping.
  */
-static void disable_intr_remapping(struct intel_iommu *iommu)
+static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
 {
         unsigned long flags;
         u32 sts;
@@ -478,6 +478,12 @@ static void disable_intr_remapping(struct intel_iommu *iommu)
         if (!ecap_ir_support(iommu->ecap))
                 return;
 
+        /*
+         * global invalidation of interrupt entry cache before disabling
+         * interrupt-remapping.
+         */
+        qi_global_iec(iommu);
+
         spin_lock_irqsave(&iommu->register_lock, flags);
 
         sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
@@ -511,7 +517,7 @@ int __init enable_intr_remapping(int eim)
                  * Disable intr remapping and queued invalidation, if already
                  * enabled prior to OS handover.
                  */
-                disable_intr_remapping(iommu);
+                iommu_disable_intr_remapping(iommu);
 
                 dmar_disable_qi(iommu);
         }
@@ -639,3 +645,54 @@ int __init parse_ioapics_under_ir(void)
 
         return ir_supported;
 }
+
+void disable_intr_remapping(void)
+{
+        struct dmar_drhd_unit *drhd;
+        struct intel_iommu *iommu = NULL;
+
+        /*
+         * Disable Interrupt-remapping for all the DRHD's now.
+         */
+        for_each_iommu(iommu, drhd) {
+                if (!ecap_ir_support(iommu->ecap))
+                        continue;
+
+                iommu_disable_intr_remapping(iommu);
+        }
+}
+
+int reenable_intr_remapping(int eim)
+{
+        struct dmar_drhd_unit *drhd;
+        int setup = 0;
+        struct intel_iommu *iommu = NULL;
+
+        for_each_iommu(iommu, drhd)
+                if (iommu->qi)
+                        dmar_reenable_qi(iommu);
+
+        /*
+         * Setup Interrupt-remapping for all the DRHD's now.
+         */
+        for_each_iommu(iommu, drhd) {
+                if (!ecap_ir_support(iommu->ecap))
+                        continue;
+
+                /* Set up interrupt remapping for iommu.*/
+                iommu_set_intr_remapping(iommu, eim);
+                setup = 1;
+        }
+
+        if (!setup)
+                goto error;
+
+        return 0;
+
+error:
+        /*
+         * handle error condition gracefully here!
+         */
+        return -1;
+}
+
diff --git a/include/linux/dmar.h b/include/linux/dmar.h
index 0b4aa8069985..4a0ce6f27e1b 100644
--- a/include/linux/dmar.h
+++ b/include/linux/dmar.h
@@ -108,6 +108,8 @@ struct irte {
 #ifdef CONFIG_INTR_REMAP
 extern int intr_remapping_enabled;
 extern int enable_intr_remapping(int);
+extern void disable_intr_remapping(void);
+extern int reenable_intr_remapping(int);
 
 extern int get_irte(int irq, struct irte *entry);
 extern int modify_irte(int irq, struct irte *irte_modified);