15 files changed, 127 insertions, 86 deletions

diff --git a/include/asm-x86/device.h b/include/asm-x86/device.h
index 87a715367a1b..3c034f48fdb0 100644
--- a/include/asm-x86/device.h
+++ b/include/asm-x86/device.h
@@ -5,6 +5,9 @@ struct dev_archdata {
 #ifdef CONFIG_ACPI
         void    *acpi_handle;
 #endif
+#ifdef CONFIG_X86_64
+struct dma_mapping_ops *dma_ops;
+#endif
 #ifdef CONFIG_DMAR
         void *iommu; /* hook for IOMMU specific extension */
 #endif
diff --git a/include/asm-x86/dma-mapping.h b/include/asm-x86/dma-mapping.h
index c2ddd3d1b883..ad9cd6d49bfc 100644
--- a/include/asm-x86/dma-mapping.h
+++ b/include/asm-x86/dma-mapping.h
@@ -17,7 +17,8 @@ extern int panic_on_overflow;
 extern int force_iommu;
 
 struct dma_mapping_ops {
-        int             (*mapping_error)(dma_addr_t dma_addr);
+        int             (*mapping_error)(struct device *dev,
+                                         dma_addr_t dma_addr);
         void*           (*alloc_coherent)(struct device *dev, size_t size,
                                 dma_addr_t *dma_handle, gfp_t gfp);
         void            (*free_coherent)(struct device *dev, size_t size,
@@ -56,14 +57,32 @@ struct dma_mapping_ops {
         int             is_phys;
 };
 
-extern const struct dma_mapping_ops *dma_ops;
+extern struct dma_mapping_ops *dma_ops;
 
-static inline int dma_mapping_error(dma_addr_t dma_addr)
+static inline struct dma_mapping_ops *get_dma_ops(struct device *dev)
 {
-        if (dma_ops->mapping_error)
-                return dma_ops->mapping_error(dma_addr);
+#ifdef CONFIG_X86_32
+        return dma_ops;
+#else
+        if (unlikely(!dev) || !dev->archdata.dma_ops)
+                return dma_ops;
+        else
+                return dev->archdata.dma_ops;
+#endif
+}
+
+/* Make sure we keep the same behaviour */
+static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+#ifdef CONFIG_X86_32
+        return 0;
+#else
+        struct dma_mapping_ops *ops = get_dma_ops(dev);
+        if (ops->mapping_error)
+                return ops->mapping_error(dev, dma_addr);
 
         return (dma_addr == bad_dma_address);
+#endif
 }
 
 #define dma_alloc_noncoherent(d, s, h, f) dma_alloc_coherent(d, s, h, f)
@@ -83,44 +102,53 @@ static inline dma_addr_t
 dma_map_single(struct device *hwdev, void *ptr, size_t size,
                int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        return dma_ops->map_single(hwdev, virt_to_phys(ptr), size, direction);
+        return ops->map_single(hwdev, virt_to_phys(ptr), size, direction);
 }
 
 static inline void
 dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size,
                  int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(dev);
+
         BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->unmap_single)
-                dma_ops->unmap_single(dev, addr, size, direction);
+        if (ops->unmap_single)
+                ops->unmap_single(dev, addr, size, direction);
 }
 
 static inline int
 dma_map_sg(struct device *hwdev, struct scatterlist *sg,
            int nents, int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        return dma_ops->map_sg(hwdev, sg, nents, direction);
+        return ops->map_sg(hwdev, sg, nents, direction);
 }
 
 static inline void
 dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents,
              int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->unmap_sg)
-                dma_ops->unmap_sg(hwdev, sg, nents, direction);
+        if (ops->unmap_sg)
+                ops->unmap_sg(hwdev, sg, nents, direction);
 }
 
 static inline void
 dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle,
                         size_t size, int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->sync_single_for_cpu)
-                dma_ops->sync_single_for_cpu(hwdev, dma_handle, size,
-                                             direction);
+        if (ops->sync_single_for_cpu)
+                ops->sync_single_for_cpu(hwdev, dma_handle, size, direction);
         flush_write_buffers();
 }
 
@@ -128,10 +156,11 @@ static inline void
 dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle,
                            size_t size, int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->sync_single_for_device)
-                dma_ops->sync_single_for_device(hwdev, dma_handle, size,
-                                                direction);
+        if (ops->sync_single_for_device)
+                ops->sync_single_for_device(hwdev, dma_handle, size, direction);
         flush_write_buffers();
 }
 
@@ -139,11 +168,12 @@ static inline void
 dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle,
                               unsigned long offset, size_t size, int direction)
 {
-        BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->sync_single_range_for_cpu)
-                dma_ops->sync_single_range_for_cpu(hwdev, dma_handle, offset,
-                                                   size, direction);
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
 
+        BUG_ON(!valid_dma_direction(direction));
+        if (ops->sync_single_range_for_cpu)
+                ops->sync_single_range_for_cpu(hwdev, dma_handle, offset,
+                                               size, direction);
         flush_write_buffers();
 }
 
@@ -152,11 +182,12 @@ dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle,
                                  unsigned long offset, size_t size,
                                  int direction)
 {
-        BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->sync_single_range_for_device)
-                dma_ops->sync_single_range_for_device(hwdev, dma_handle,
-                                                      offset, size, direction);
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
 
+        BUG_ON(!valid_dma_direction(direction));
+        if (ops->sync_single_range_for_device)
+                ops->sync_single_range_for_device(hwdev, dma_handle,
+                                                  offset, size, direction);
         flush_write_buffers();
 }
 
@@ -164,9 +195,11 @@ static inline void
 dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
                     int nelems, int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->sync_sg_for_cpu)
-                dma_ops->sync_sg_for_cpu(hwdev, sg, nelems, direction);
+        if (ops->sync_sg_for_cpu)
+                ops->sync_sg_for_cpu(hwdev, sg, nelems, direction);
         flush_write_buffers();
 }
 
@@ -174,9 +207,11 @@ static inline void
 dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
                        int nelems, int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(hwdev);
+
         BUG_ON(!valid_dma_direction(direction));
-        if (dma_ops->sync_sg_for_device)
-                dma_ops->sync_sg_for_device(hwdev, sg, nelems, direction);
+        if (ops->sync_sg_for_device)
+                ops->sync_sg_for_device(hwdev, sg, nelems, direction);
 
         flush_write_buffers();
 }
@@ -185,9 +220,11 @@ static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
                                       size_t offset, size_t size,
                                       int direction)
 {
+        struct dma_mapping_ops *ops = get_dma_ops(dev);
+
         BUG_ON(!valid_dma_direction(direction));
-        return dma_ops->map_single(dev, page_to_phys(page)+offset,
-                                   size, direction);
+        return ops->map_single(dev, page_to_phys(page) + offset,
+                               size, direction);
 }
 
 static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
@@ -212,25 +249,5 @@ static inline int dma_get_cache_alignment(void)
 
 #define dma_is_consistent(d, h) (1)
 
-#ifdef CONFIG_X86_32
-#  define ARCH_HAS_DMA_DECLARE_COHERENT_MEMORY
-struct dma_coherent_mem {
-        void            *virt_base;
-        u32             device_base;
-        int             size;
-        int             flags;
-        unsigned long   *bitmap;
-};
-
-extern int
-dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
-                            dma_addr_t device_addr, size_t size, int flags);
-
-extern void
-dma_release_declared_memory(struct device *dev);
-
-extern void *
-dma_mark_declared_memory_occupied(struct device *dev,
-                                  dma_addr_t device_addr, size_t size);
-#endif /* CONFIG_X86_32 */
+#include <asm-generic/dma-coherent.h>
 #endif
diff --git a/include/asm-x86/efi.h b/include/asm-x86/efi.h
index 7ed2bd7a7f51..d4f2b0abe929 100644
--- a/include/asm-x86/efi.h
+++ b/include/asm-x86/efi.h
@@ -86,7 +86,7 @@ extern u64 efi_call6(void *fp, u64 arg1, u64 arg2, u64 arg3,
         efi_call6((void *)(efi.systab->runtime->f), (u64)(a1), (u64)(a2), \
                   (u64)(a3), (u64)(a4), (u64)(a5), (u64)(a6))
 
-extern void *efi_ioremap(unsigned long addr, unsigned long size);
+extern void __iomem *efi_ioremap(unsigned long addr, unsigned long size);
 
 #endif /* CONFIG_X86_32 */
 
diff --git a/include/asm-x86/gpio.h b/include/asm-x86/gpio.h
index 116e9147fe66..c4c91b37c104 100644
--- a/include/asm-x86/gpio.h
+++ b/include/asm-x86/gpio.h
@@ -16,10 +16,6 @@
 #ifndef _ASM_I386_GPIO_H
 #define _ASM_I386_GPIO_H
 
-#ifdef CONFIG_X86_RDC321X
-#include <gpio.h>
-#else /* CONFIG_X86_RDC321X */
-
 #include <asm-generic/gpio.h>
 
 #ifdef CONFIG_GPIOLIB
@@ -57,6 +53,4 @@ static inline int irq_to_gpio(unsigned int irq)
 
 #endif /* CONFIG_GPIOLIB */
 
-#endif /* CONFIG_X86_RDC321X */
-
 #endif /* _ASM_I386_GPIO_H */
diff --git a/include/asm-x86/hw_irq.h b/include/asm-x86/hw_irq.h
index 77ba51df5668..edd0b95f14d0 100644
--- a/include/asm-x86/hw_irq.h
+++ b/include/asm-x86/hw_irq.h
@@ -98,9 +98,17 @@ extern void (*const interrupt[NR_IRQS])(void);
 #else
 typedef int vector_irq_t[NR_VECTORS];
 DECLARE_PER_CPU(vector_irq_t, vector_irq);
-extern spinlock_t vector_lock;
 #endif
-extern void setup_vector_irq(int cpu);
+
+#if defined(CONFIG_X86_IO_APIC) && defined(CONFIG_X86_64)
+extern void lock_vector_lock(void);
+extern void unlock_vector_lock(void);
+extern void __setup_vector_irq(int cpu);
+#else
+static inline void lock_vector_lock(void) {}
+static inline void unlock_vector_lock(void) {}
+static inline void __setup_vector_irq(int cpu) {}
+#endif
 
 #endif /* !ASSEMBLY_ */
 
diff --git a/include/asm-x86/iommu.h b/include/asm-x86/iommu.h
index d63166fb3ab7..5f888cc5be49 100644
--- a/include/asm-x86/iommu.h
+++ b/include/asm-x86/iommu.h
@@ -3,9 +3,12 @@
 
 extern void pci_iommu_shutdown(void);
 extern void no_iommu_init(void);
+extern struct dma_mapping_ops nommu_dma_ops;
 extern int force_iommu, no_iommu;
 extern int iommu_detected;
 
+extern unsigned long iommu_num_pages(unsigned long addr, unsigned long len);
+
 #ifdef CONFIG_GART_IOMMU
 extern int gart_iommu_aperture;
 extern int gart_iommu_aperture_allowed;
diff --git a/include/asm-x86/irq_vectors.h b/include/asm-x86/irq_vectors.h
index 90b1d1f12f08..b95d167b7fb2 100644
--- a/include/asm-x86/irq_vectors.h
+++ b/include/asm-x86/irq_vectors.h
@@ -109,7 +109,15 @@
 #define LAST_VM86_IRQ           15
 #define invalid_vm86_irq(irq)   ((irq) < 3 || (irq) > 15)
 
-#if !defined(CONFIG_X86_VOYAGER)
+#ifdef CONFIG_X86_64
+# if NR_CPUS < MAX_IO_APICS
+#  define NR_IRQS (NR_VECTORS + (32 * NR_CPUS))
+# else
+#  define NR_IRQS (NR_VECTORS + (32 * MAX_IO_APICS))
+# endif
+# define NR_IRQ_VECTORS NR_IRQS
+
+#elif !defined(CONFIG_X86_VOYAGER)
 
 # if defined(CONFIG_X86_IO_APIC) || defined(CONFIG_PARAVIRT) || defined(CONFIG_X86_VISWS)
 
diff --git a/include/asm-x86/kexec.h b/include/asm-x86/kexec.h
index 8f855a15f64d..c0e52a14fd4d 100644
--- a/include/asm-x86/kexec.h
+++ b/include/asm-x86/kexec.h
@@ -10,14 +10,15 @@
 # define VA_PTE_0               5
 # define PA_PTE_1               6
 # define VA_PTE_1               7
+# define PA_SWAP_PAGE           8
 # ifdef CONFIG_X86_PAE
-#  define PA_PMD_0              8
-#  define VA_PMD_0              9
-#  define PA_PMD_1              10
-#  define VA_PMD_1              11
-#  define PAGES_NR              12
+#  define PA_PMD_0              9
+#  define VA_PMD_0              10
+#  define PA_PMD_1              11
+#  define VA_PMD_1              12
+#  define PAGES_NR              13
 # else
-#  define PAGES_NR              8
+#  define PAGES_NR              9
 # endif
 #else
 # define PA_CONTROL_PAGE        0
@@ -152,11 +153,12 @@ static inline void crash_setup_regs(struct pt_regs *newregs,
 }
 
 #ifdef CONFIG_X86_32
-asmlinkage NORET_TYPE void
+asmlinkage unsigned long
 relocate_kernel(unsigned long indirection_page,
                 unsigned long control_page,
                 unsigned long start_address,
-                unsigned int has_pae) ATTRIB_NORET;
+                unsigned int has_pae,
+                unsigned int preserve_context);
 #else
 NORET_TYPE void
 relocate_kernel(unsigned long indirection_page,
diff --git a/include/asm-x86/kvm_host.h b/include/asm-x86/kvm_host.h
index fdde0bedaa90..0f3c53114614 100644
--- a/include/asm-x86/kvm_host.h
+++ b/include/asm-x86/kvm_host.h
@@ -13,6 +13,7 @@
 
 #include <linux/types.h>
 #include <linux/mm.h>
+#include <linux/mmu_notifier.h>
 
 #include <linux/kvm.h>
 #include <linux/kvm_para.h>
@@ -251,6 +252,7 @@ struct kvm_vcpu_arch {
                 gfn_t gfn;      /* presumed gfn during guest pte update */
                 pfn_t pfn;      /* pfn corresponding to that gfn */
                 int largepage;
+                unsigned long mmu_seq;
         } update_pte;
 
         struct i387_fxsave_struct host_fx_image;
@@ -556,6 +558,7 @@ int kvm_fix_hypercall(struct kvm_vcpu *vcpu);
 int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gva_t gva, u32 error_code);
 
 void kvm_enable_tdp(void);
+void kvm_disable_tdp(void);
 
 int load_pdptrs(struct kvm_vcpu *vcpu, unsigned long cr3);
 int complete_pio(struct kvm_vcpu *vcpu);
@@ -728,4 +731,8 @@ asmlinkage void kvm_handle_fault_on_reboot(void);
         KVM_EX_ENTRY " 666b, 667b \n\t" \
         ".popsection"
 
+#define KVM_ARCH_WANT_MMU_NOTIFIER
+int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
+int kvm_age_hva(struct kvm *kvm, unsigned long hva);
+
 #endif
diff --git a/include/asm-x86/mach-summit/mach_apic.h b/include/asm-x86/mach-summit/mach_apic.h
index 75d2c95005d7..c47e2ab5c5ca 100644
--- a/include/asm-x86/mach-summit/mach_apic.h
+++ b/include/asm-x86/mach-summit/mach_apic.h
@@ -122,7 +122,7 @@ static inline physid_mask_t ioapic_phys_id_map(physid_mask_t phys_id_map)
 
 static inline physid_mask_t apicid_to_cpu_present(int apicid)
 {
-        return physid_mask_of_physid(0);
+        return physid_mask_of_physid(apicid);
 }
 
 static inline void setup_portio_remap(void)
diff --git a/include/asm-x86/mmzone_32.h b/include/asm-x86/mmzone_32.h
index b2298a227567..5862e6460658 100644
--- a/include/asm-x86/mmzone_32.h
+++ b/include/asm-x86/mmzone_32.h
@@ -97,10 +97,16 @@ static inline int pfn_valid(int pfn)
         reserve_bootmem_node(NODE_DATA(0), (addr), (size), (flags))
 #define alloc_bootmem(x) \
         __alloc_bootmem_node(NODE_DATA(0), (x), SMP_CACHE_BYTES, __pa(MAX_DMA_ADDRESS))
+#define alloc_bootmem_nopanic(x) \
+        __alloc_bootmem_node_nopanic(NODE_DATA(0), (x), SMP_CACHE_BYTES, \
+                                __pa(MAX_DMA_ADDRESS))
 #define alloc_bootmem_low(x) \
         __alloc_bootmem_node(NODE_DATA(0), (x), SMP_CACHE_BYTES, 0)
 #define alloc_bootmem_pages(x) \
         __alloc_bootmem_node(NODE_DATA(0), (x), PAGE_SIZE, __pa(MAX_DMA_ADDRESS))
+#define alloc_bootmem_pages_nopanic(x) \
+        __alloc_bootmem_node_nopanic(NODE_DATA(0), (x), PAGE_SIZE, \
+                                __pa(MAX_DMA_ADDRESS))
 #define alloc_bootmem_low_pages(x) \
         __alloc_bootmem_node(NODE_DATA(0), (x), PAGE_SIZE, 0)
 #define alloc_bootmem_node(pgdat, x)                                    \
diff --git a/include/asm-x86/namei.h b/include/asm-x86/namei.h
deleted file mode 100644
index 415ef5d9550e..000000000000
--- a/include/asm-x86/namei.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef _ASM_X86_NAMEI_H
-#define _ASM_X86_NAMEI_H
-
-/* This dummy routine maybe changed to something useful
- * for /usr/gnemul/ emulation stuff.
- * Look at asm-sparc/namei.h for details.
- */
-
-#define __emul_prefix() NULL
-
-#endif /* _ASM_X86_NAMEI_H */
diff --git a/include/asm-x86/pgtable.h b/include/asm-x86/pgtable.h
index 3e5dbc4195f4..04caa2f544df 100644
--- a/include/asm-x86/pgtable.h
+++ b/include/asm-x86/pgtable.h
@@ -18,6 +18,7 @@
 #define _PAGE_BIT_UNUSED2       10
 #define _PAGE_BIT_UNUSED3       11
 #define _PAGE_BIT_PAT_LARGE     12      /* On 2MB or 1GB pages */
+#define _PAGE_BIT_SPECIAL       _PAGE_BIT_UNUSED1
 #define _PAGE_BIT_NX           63       /* No execute: only valid after cpuid check */
 
 #define _PAGE_PRESENT   (_AT(pteval_t, 1) << _PAGE_BIT_PRESENT)
@@ -34,6 +35,8 @@
 #define _PAGE_UNUSED3   (_AT(pteval_t, 1) << _PAGE_BIT_UNUSED3)
 #define _PAGE_PAT       (_AT(pteval_t, 1) << _PAGE_BIT_PAT)
 #define _PAGE_PAT_LARGE (_AT(pteval_t, 1) << _PAGE_BIT_PAT_LARGE)
+#define _PAGE_SPECIAL   (_AT(pteval_t, 1) << _PAGE_BIT_SPECIAL)
+#define __HAVE_ARCH_PTE_SPECIAL
 
 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 #define _PAGE_NX        (_AT(pteval_t, 1) << _PAGE_BIT_NX)
@@ -54,7 +57,7 @@
 
 /* Set of bits not changed in pte_modify */
 #define _PAGE_CHG_MASK  (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT |         \
-                         _PAGE_ACCESSED | _PAGE_DIRTY)
+                         _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY)
 
 #define _PAGE_CACHE_MASK        (_PAGE_PCD | _PAGE_PWT)
 #define _PAGE_CACHE_WB          (0)
@@ -180,7 +183,7 @@ static inline int pte_exec(pte_t pte)
 
 static inline int pte_special(pte_t pte)
 {
-        return 0;
+        return pte_val(pte) & _PAGE_SPECIAL;
 }
 
 static inline int pmd_large(pmd_t pte)
@@ -246,7 +249,7 @@ static inline pte_t pte_clrglobal(pte_t pte)
 
 static inline pte_t pte_mkspecial(pte_t pte)
 {
-        return pte;
+        return __pte(pte_val(pte) | _PAGE_SPECIAL);
 }
 
 extern pteval_t __supported_pte_mask;
diff --git a/include/asm-x86/swiotlb.h b/include/asm-x86/swiotlb.h
index c706a7442633..2730b351afcf 100644
--- a/include/asm-x86/swiotlb.h
+++ b/include/asm-x86/swiotlb.h
@@ -35,7 +35,7 @@ extern int swiotlb_map_sg(struct device *hwdev, struct scatterlist *sg,
                           int nents, int direction);
 extern void swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sg,
                              int nents, int direction);
-extern int swiotlb_dma_mapping_error(dma_addr_t dma_addr);
+extern int swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr);
 extern void swiotlb_free_coherent(struct device *hwdev, size_t size,
                                   void *vaddr, dma_addr_t dma_handle);
 extern int swiotlb_dma_supported(struct device *hwdev, u64 mask);
diff --git a/include/asm-x86/uaccess.h b/include/asm-x86/uaccess.h
index f6fa4d841bbc..5f702d1d5218 100644
--- a/include/asm-x86/uaccess.h
+++ b/include/asm-x86/uaccess.h
@@ -451,3 +451,4 @@ extern struct movsl_mask {
 #endif
 
 #endif
+