Merge branch 'x86/mm' into core/percpu

Conflicts:
	arch/x86/mm/fault.c

9 files changed, 291 insertions, 240 deletions

diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index 8078955845ae..cdf4a9623237 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -307,10 +307,10 @@ config X86_CMPXCHG
 
 config X86_L1_CACHE_SHIFT
         int
-        default "7" if MPENTIUM4 || X86_GENERIC || GENERIC_CPU || MPSC
+        default "7" if MPENTIUM4 || MPSC
         default "4" if X86_ELAN || M486 || M386 || MGEODEGX1
         default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
-        default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MVIAC7
+        default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MVIAC7 || X86_GENERIC || GENERIC_CPU
 
 config X86_XADD
         def_bool y
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index a9f8a814a1f7..4a8e80cdcfa5 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -22,4 +22,3 @@ unifdef-y += unistd_32.h
 unifdef-y += unistd_64.h
 unifdef-y += vm86.h
 unifdef-y += vsyscall.h
-unifdef-y += swab.h
diff --git a/arch/x86/include/asm/byteorder.h b/arch/x86/include/asm/byteorder.h
index 7c49917e3d9d..b13a7a88f3eb 100644
--- a/arch/x86/include/asm/byteorder.h
+++ b/arch/x86/include/asm/byteorder.h
@@ -1,7 +1,6 @@
 #ifndef _ASM_X86_BYTEORDER_H
 #define _ASM_X86_BYTEORDER_H
 
-#include <asm/swab.h>
 #include <linux/byteorder/little_endian.h>
 
 #endif /* _ASM_X86_BYTEORDER_H */
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index 884d985b8b82..e948b28a5a9a 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -446,7 +446,7 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
 static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
                                        struct kprobe_ctlblk *kcb)
 {
-#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PM)
+#if !defined(CONFIG_PREEMPT) || defined(CONFIG_FREEZER)
         if (p->ainsn.boostable == 1 && !p->post_handler) {
                 /* Boost up -- we can execute copied instructions directly */
                 reset_current_kprobe();
diff --git a/arch/x86/kernel/syscall_table_32.S b/arch/x86/kernel/syscall_table_32.S
index d44395ff34c3..e2e86a08f31d 100644
--- a/arch/x86/kernel/syscall_table_32.S
+++ b/arch/x86/kernel/syscall_table_32.S
@@ -88,7 +88,7 @@ ENTRY(sys_call_table)
         .long sys_uselib
         .long sys_swapon
         .long sys_reboot
-        .long old_readdir
+        .long sys_old_readdir
         .long old_mmap          /* 90 */
         .long sys_munmap
         .long sys_truncate
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 37242c405f16..65709a6aa6ee 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -92,8 +92,8 @@ static inline int notify_page_fault(struct pt_regs *regs)
  *
  * Opcode checker based on code by Richard Brunner
  */
-static int is_prefetch(struct pt_regs *regs, unsigned long addr,
-                       unsigned long error_code)
+static int is_prefetch(struct pt_regs *regs, unsigned long error_code,
+                        unsigned long addr)
 {
         unsigned char *instr;
         int scan_more = 1;
@@ -410,15 +410,15 @@ static void show_fault_oops(struct pt_regs *regs, unsigned long error_code,
 }
 
 #ifdef CONFIG_X86_64
-static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
-                                 unsigned long error_code)
+static noinline void pgtable_bad(struct pt_regs *regs,
+                         unsigned long error_code, unsigned long address)
 {
         unsigned long flags = oops_begin();
         int sig = SIGKILL;
-        struct task_struct *tsk;
+        struct task_struct *tsk = current;
 
         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
-               current->comm, address);
+               tsk->comm, address);
         dump_pagetable(address);
         tsk = current;
         tsk->thread.cr2 = address;
@@ -430,6 +430,196 @@ static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 }
 #endif
 
+static noinline void no_context(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        struct task_struct *tsk = current;
+        unsigned long *stackend;
+
+#ifdef CONFIG_X86_64
+        unsigned long flags;
+        int sig;
+#endif
+
+        /* Are we prepared to handle this kernel fault?  */
+        if (fixup_exception(regs))
+                return;
+
+        /*
+         * X86_32
+         * Valid to do another page fault here, because if this fault
+         * had been triggered by is_prefetch fixup_exception would have
+         * handled it.
+         *
+         * X86_64
+         * Hall of shame of CPU/BIOS bugs.
+         */
+        if (is_prefetch(regs, error_code, address))
+                return;
+
+        if (is_errata93(regs, address))
+                return;
+
+        /*
+         * Oops. The kernel tried to access some bad page. We'll have to
+         * terminate things with extreme prejudice.
+         */
+#ifdef CONFIG_X86_32
+        bust_spinlocks(1);
+#else
+        flags = oops_begin();
+#endif
+
+        show_fault_oops(regs, error_code, address);
+
+        stackend = end_of_stack(tsk);
+        if (*stackend != STACK_END_MAGIC)
+                printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
+
+        tsk->thread.cr2 = address;
+        tsk->thread.trap_no = 14;
+        tsk->thread.error_code = error_code;
+
+#ifdef CONFIG_X86_32
+        die("Oops", regs, error_code);
+        bust_spinlocks(0);
+        do_exit(SIGKILL);
+#else
+        sig = SIGKILL;
+        if (__die("Oops", regs, error_code))
+                sig = 0;
+        /* Executive summary in case the body of the oops scrolled away */
+        printk(KERN_EMERG "CR2: %016lx\n", address);
+        oops_end(flags, regs, sig);
+#endif
+}
+
+static void __bad_area_nosemaphore(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address,
+                        int si_code)
+{
+        struct task_struct *tsk = current;
+
+        /* User mode accesses just cause a SIGSEGV */
+        if (error_code & PF_USER) {
+                /*
+                 * It's possible to have interrupts off here.
+                 */
+                local_irq_enable();
+
+                /*
+                 * Valid to do another page fault here because this one came
+                 * from user space.
+                 */
+                if (is_prefetch(regs, error_code, address))
+                        return;
+
+                if (is_errata100(regs, address))
+                        return;
+
+                if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
+                    printk_ratelimit()) {
+                        printk(
+                        "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
+                        task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
+                        tsk->comm, task_pid_nr(tsk), address,
+                        (void *) regs->ip, (void *) regs->sp, error_code);
+                        print_vma_addr(" in ", regs->ip);
+                        printk("\n");
+                }
+
+                tsk->thread.cr2 = address;
+                /* Kernel addresses are always protection faults */
+                tsk->thread.error_code = error_code | (address >= TASK_SIZE);
+                tsk->thread.trap_no = 14;
+                force_sig_info_fault(SIGSEGV, si_code, address, tsk);
+                return;
+        }
+
+        if (is_f00f_bug(regs, address))
+                return;
+
+        no_context(regs, error_code, address);
+}
+
+static noinline void bad_area_nosemaphore(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area_nosemaphore(regs, error_code, address, SEGV_MAPERR);
+}
+
+static void __bad_area(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address,
+                        int si_code)
+{
+        struct mm_struct *mm = current->mm;
+
+        /*
+         * Something tried to access memory that isn't in our memory map..
+         * Fix it, but check if it's kernel or user first..
+         */
+        up_read(&mm->mmap_sem);
+
+        __bad_area_nosemaphore(regs, error_code, address, si_code);
+}
+
+static noinline void bad_area(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area(regs, error_code, address, SEGV_MAPERR);
+}
+
+static noinline void bad_area_access_error(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        __bad_area(regs, error_code, address, SEGV_ACCERR);
+}
+
+/* TODO: fixup for "mm-invoke-oom-killer-from-page-fault.patch" */
+static void out_of_memory(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        /*
+         * We ran out of memory, call the OOM killer, and return the userspace
+         * (which will retry the fault, or kill us if we got oom-killed).
+         */
+        up_read(&current->mm->mmap_sem);
+        pagefault_out_of_memory();
+}
+
+static void do_sigbus(struct pt_regs *regs,
+                        unsigned long error_code, unsigned long address)
+{
+        struct task_struct *tsk = current;
+        struct mm_struct *mm = tsk->mm;
+
+        up_read(&mm->mmap_sem);
+
+        /* Kernel mode? Handle exceptions or die */
+        if (!(error_code & PF_USER))
+                no_context(regs, error_code, address);
+#ifdef CONFIG_X86_32
+        /* User space => ok to do another page fault */
+        if (is_prefetch(regs, error_code, address))
+                return;
+#endif
+        tsk->thread.cr2 = address;
+        tsk->thread.error_code = error_code;
+        tsk->thread.trap_no = 14;
+        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
+}
+
+static noinline void mm_fault_error(struct pt_regs *regs,
+                unsigned long error_code, unsigned long address, unsigned int fault)
+{
+        if (fault & VM_FAULT_OOM)
+                out_of_memory(regs, error_code, address);
+        else if (fault & VM_FAULT_SIGBUS)
+                do_sigbus(regs, error_code, address);
+        else
+                BUG();
+}
+
 static int spurious_fault_check(unsigned long error_code, pte_t *pte)
 {
         if ((error_code & PF_WRITE) && !pte_write(*pte))
@@ -449,8 +639,8 @@ static int spurious_fault_check(unsigned long error_code, pte_t *pte)
  * There are no security implications to leaving a stale TLB when
  * increasing the permissions on a page.
  */
-static int spurious_fault(unsigned long address,
-                          unsigned long error_code)
+static noinline int spurious_fault(unsigned long error_code,
+                                unsigned long address)
 {
         pgd_t *pgd;
         pud_t *pud;
@@ -495,7 +685,7 @@ static int spurious_fault(unsigned long address,
  *
  * This assumes no large pages in there.
  */
-static int vmalloc_fault(unsigned long address)
+static noinline int vmalloc_fault(unsigned long address)
 {
 #ifdef CONFIG_X86_32
         unsigned long pgd_paddr;
@@ -574,6 +764,25 @@ static int vmalloc_fault(unsigned long address)
 
 int show_unhandled_signals = 1;
 
+static inline int access_error(unsigned long error_code, int write,
+                                struct vm_area_struct *vma)
+{
+        if (write) {
+                /* write, present and write, not present */
+                if (unlikely(!(vma->vm_flags & VM_WRITE)))
+                        return 1;
+        } else if (unlikely(error_code & PF_PROT)) {
+                /* read, present */
+                return 1;
+        } else {
+                /* read, not present */
+                if (unlikely(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE))))
+                        return 1;
+        }
+
+        return 0;
+}
+
 /*
  * This routine handles page faults.  It determines the address,
  * and the problem, and then passes it off to one of the appropriate
@@ -584,18 +793,12 @@ asmlinkage
 #endif
 void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 {
+        unsigned long address;
         struct task_struct *tsk;
         struct mm_struct *mm;
         struct vm_area_struct *vma;
-        unsigned long address;
-        int write, si_code;
+        int write;
         int fault;
-        unsigned long *stackend;
-
-#ifdef CONFIG_X86_64
-        unsigned long flags;
-        int sig;
-#endif
 
         tsk = current;
         mm = tsk->mm;
@@ -604,9 +807,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
         /* get the address */
         address = read_cr2();
 
-        si_code = SEGV_MAPERR;
-
-        if (notify_page_fault(regs))
+        if (unlikely(notify_page_fault(regs)))
                 return;
         if (unlikely(kmmio_fault(regs, address)))
                 return;
@@ -634,17 +835,17 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                         return;
 
                 /* Can handle a stale RO->RW TLB */
-                if (spurious_fault(address, error_code))
+                if (spurious_fault(error_code, address))
                         return;
 
                 /*
                  * Don't take the mm semaphore here. If we fixup a prefetch
                  * fault we could otherwise deadlock.
                  */
-                goto bad_area_nosemaphore;
+                bad_area_nosemaphore(regs, error_code, address);
+                return;
         }
 
-
         /*
          * It's safe to allow irq's after cr2 has been saved and the
          * vmalloc fault has been handled.
@@ -660,15 +861,17 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
 
 #ifdef CONFIG_X86_64
         if (unlikely(error_code & PF_RSVD))
-                pgtable_bad(address, regs, error_code);
+                pgtable_bad(regs, error_code, address);
 #endif
 
         /*
          * If we're in an interrupt, have no user context or are running in an
          * atomic region then we must not take the fault.
          */
-        if (unlikely(in_atomic() || !mm))
-                goto bad_area_nosemaphore;
+        if (unlikely(in_atomic() || !mm)) {
+                bad_area_nosemaphore(regs, error_code, address);
+                return;
+        }
 
         /*
          * When running in the kernel we expect faults to occur only to
@@ -686,20 +889,26 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
          * source.  If this is invalid we can skip the address space check,
          * thus avoiding the deadlock.
          */
-        if (!down_read_trylock(&mm->mmap_sem)) {
+        if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
                 if ((error_code & PF_USER) == 0 &&
-                    !search_exception_tables(regs->ip))
-                        goto bad_area_nosemaphore;
+                    !search_exception_tables(regs->ip)) {
+                        bad_area_nosemaphore(regs, error_code, address);
+                        return;
+                }
                 down_read(&mm->mmap_sem);
         }
 
         vma = find_vma(mm, address);
-        if (!vma)
-                goto bad_area;
-        if (vma->vm_start <= address)
+        if (unlikely(!vma)) {
+                bad_area(regs, error_code, address);
+                return;
+        }
+        if (likely(vma->vm_start <= address))
                 goto good_area;
-        if (!(vma->vm_flags & VM_GROWSDOWN))
-                goto bad_area;
+        if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
+                bad_area(regs, error_code, address);
+                return;
+        }
         if (error_code & PF_USER) {
                 /*
                  * Accessing the stack below %sp is always a bug.
@@ -707,31 +916,25 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
                  * and pusha to work.  ("enter $65535,$31" pushes
                  * 32 pointers and then decrements %sp by 65535.)
                  */
-                if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
-                        goto bad_area;
+                if (unlikely(address + 65536 + 32 * sizeof(unsigned long) < regs->sp)) {
+                        bad_area(regs, error_code, address);
+                        return;
+                }
         }
-        if (expand_stack(vma, address))
-                goto bad_area;
-/*
- * Ok, we have a good vm_area for this memory access, so
- * we can handle it..
- */
+        if (unlikely(expand_stack(vma, address))) {
+                bad_area(regs, error_code, address);
+                return;
+        }
+
+        /*
+         * Ok, we have a good vm_area for this memory access, so
+         * we can handle it..
+         */
 good_area:
-        si_code = SEGV_ACCERR;
-        write = 0;
-        switch (error_code & (PF_PROT|PF_WRITE)) {
-        default:        /* 3: write, present */
-                /* fall through */
-        case PF_WRITE:          /* write, not present */
-                if (!(vma->vm_flags & VM_WRITE))
-                        goto bad_area;
-                write++;
-                break;
-        case PF_PROT:           /* read, present */
-                goto bad_area;
-        case 0:                 /* read, not present */
-                if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
-                        goto bad_area;
+        write = error_code & PF_WRITE;
+        if (unlikely(access_error(error_code, write, vma))) {
+                bad_area_access_error(regs, error_code, address);
+                return;
         }
 
         /*
@@ -741,11 +944,8 @@ good_area:
          */
         fault = handle_mm_fault(mm, vma, address, write);
         if (unlikely(fault & VM_FAULT_ERROR)) {
-                if (fault & VM_FAULT_OOM)
-                        goto out_of_memory;
-                else if (fault & VM_FAULT_SIGBUS)
-                        goto do_sigbus;
-                BUG();
+                mm_fault_error(regs, error_code, address, fault);
+                return;
         }
         if (fault & VM_FAULT_MAJOR)
                 tsk->maj_flt++;
@@ -763,132 +963,6 @@ good_area:
         }
 #endif
         up_read(&mm->mmap_sem);
-        return;
-
-/*
- * Something tried to access memory that isn't in our memory map..
- * Fix it, but check if it's kernel or user first..
- */
-bad_area:
-        up_read(&mm->mmap_sem);
-
-bad_area_nosemaphore:
-        /* User mode accesses just cause a SIGSEGV */
-        if (error_code & PF_USER) {
-                /*
-                 * It's possible to have interrupts off here.
-                 */
-                local_irq_enable();
-
-                /*
-                 * Valid to do another page fault here because this one came
-                 * from user space.
-                 */
-                if (is_prefetch(regs, address, error_code))
-                        return;
-
-                if (is_errata100(regs, address))
-                        return;
-
-                if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
-                    printk_ratelimit()) {
-                        printk(
-                        "%s%s[%d]: segfault at %lx ip %p sp %p error %lx",
-                        task_pid_nr(tsk) > 1 ? KERN_INFO : KERN_EMERG,
-                        tsk->comm, task_pid_nr(tsk), address,
-                        (void *) regs->ip, (void *) regs->sp, error_code);
-                        print_vma_addr(" in ", regs->ip);
-                        printk("\n");
-                }
-
-                tsk->thread.cr2 = address;
-                /* Kernel addresses are always protection faults */
-                tsk->thread.error_code = error_code | (address >= TASK_SIZE);
-                tsk->thread.trap_no = 14;
-                force_sig_info_fault(SIGSEGV, si_code, address, tsk);
-                return;
-        }
-
-        if (is_f00f_bug(regs, address))
-                return;
-
-no_context:
-        /* Are we prepared to handle this kernel fault?  */
-        if (fixup_exception(regs))
-                return;
-
-        /*
-         * X86_32
-         * Valid to do another page fault here, because if this fault
-         * had been triggered by is_prefetch fixup_exception would have
-         * handled it.
-         *
-         * X86_64
-         * Hall of shame of CPU/BIOS bugs.
-         */
-        if (is_prefetch(regs, address, error_code))
-                return;
-
-        if (is_errata93(regs, address))
-                return;
-
-/*
- * Oops. The kernel tried to access some bad page. We'll have to
- * terminate things with extreme prejudice.
- */
-#ifdef CONFIG_X86_32
-        bust_spinlocks(1);
-#else
-        flags = oops_begin();
-#endif
-
-        show_fault_oops(regs, error_code, address);
-
-        stackend = end_of_stack(tsk);
-        if (*stackend != STACK_END_MAGIC)
-                printk(KERN_ALERT "Thread overran stack, or stack corrupted\n");
-
-        tsk->thread.cr2 = address;
-        tsk->thread.trap_no = 14;
-        tsk->thread.error_code = error_code;
-
-#ifdef CONFIG_X86_32
-        die("Oops", regs, error_code);
-        bust_spinlocks(0);
-        do_exit(SIGKILL);
-#else
-        sig = SIGKILL;
-        if (__die("Oops", regs, error_code))
-                sig = 0;
-        /* Executive summary in case the body of the oops scrolled away */
-        printk(KERN_EMERG "CR2: %016lx\n", address);
-        oops_end(flags, regs, sig);
-#endif
-
-out_of_memory:
-        /*
-         * We ran out of memory, call the OOM killer, and return the userspace
-         * (which will retry the fault, or kill us if we got oom-killed).
-         */
-        up_read(&mm->mmap_sem);
-        pagefault_out_of_memory();
-        return;
-
-do_sigbus:
-        up_read(&mm->mmap_sem);
-
-        /* Kernel mode? Handle exceptions or die */
-        if (!(error_code & PF_USER))
-                goto no_context;
-#ifdef CONFIG_X86_32
-        /* User space => ok to do another page fault */
-        if (is_prefetch(regs, address, error_code))
-                return;
-#endif
-        tsk->thread.cr2 = address;
-        tsk->thread.error_code = error_code;
-        tsk->thread.trap_no = 14;
-        force_sig_info_fault(SIGBUS, BUS_ADRERR, address, tsk);
 }
 
 DEFINE_SPINLOCK(pgd_lock);
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 4cf30dee8161..e89d24815f26 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -555,12 +555,10 @@ repeat:
         if (!pte_val(old_pte)) {
                 if (!primary)
                         return 0;
-
-                /*
-                 *  Special error value returned, indicating that the mapping
-                 * did not exist at this address.
-                 */
-                return -EFAULT;
+                WARN(1, KERN_WARNING "CPA: called for zero pte. "
+                       "vaddr = %lx cpa->vaddr = %lx\n", address,
+                       *cpa->vaddr);
+                return -EINVAL;
         }
 
         if (level == PG_LEVEL_4K) {
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 3be399013de6..c9488513fd70 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -522,35 +522,6 @@ static inline int range_is_allowed(unsigned long pfn, unsigned long size)
 }
 #endif /* CONFIG_STRICT_DEVMEM */
 
-/*
- * Change the memory type for the physial address range in kernel identity
- * mapping space if that range is a part of identity map.
- */
-static int kernel_map_sync_memtype(u64 base, unsigned long size,
-                                        unsigned long flags)
-{
-        unsigned long id_sz;
-        int ret;
-
-        if (!pat_enabled || base >= __pa(high_memory))
-                return 0;
-
-        id_sz = (__pa(high_memory) < base + size) ?
-                                                __pa(high_memory) - base :
-                                                size;
-
-        ret = ioremap_change_attr((unsigned long)__va(base), id_sz, flags);
-        /*
-         * -EFAULT return means that the addr was not valid and did not have
-         * any identity mapping. That case is a success for
-         * kernel_map_sync_memtype.
-         */
-        if (ret == -EFAULT)
-                ret = 0;
-
-        return ret;
-}
-
 int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
                                 unsigned long size, pgprot_t *vma_prot)
 {
@@ -601,7 +572,9 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
         if (retval < 0)
                 return 0;
 
-        if (kernel_map_sync_memtype(offset, size, flags)) {
+        if (((pfn < max_low_pfn_mapped) ||
+             (pfn >= (1UL<<(32 - PAGE_SHIFT)) && pfn < max_pfn_mapped)) &&
+            ioremap_change_attr((unsigned long)__va(offset), size, flags) < 0) {
                 free_memtype(offset, offset + size);
                 printk(KERN_INFO
                 "%s:%d /dev/mem ioremap_change_attr failed %s for %Lx-%Lx\n",
@@ -649,7 +622,7 @@ static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
                                 int strict_prot)
 {
         int is_ram = 0;
-        int ret;
+        int id_sz, ret;
         unsigned long flags;
         unsigned long want_flags = (pgprot_val(*vma_prot) & _PAGE_CACHE_MASK);
 
@@ -690,7 +663,15 @@ static int reserve_pfn_range(u64 paddr, unsigned long size, pgprot_t *vma_prot,
                                      flags);
         }
 
-        if (kernel_map_sync_memtype(paddr, size, flags)) {
+        /* Need to keep identity mapping in sync */
+        if (paddr >= __pa(high_memory))
+                return 0;
+
+        id_sz = (__pa(high_memory) < paddr + size) ?
+                                __pa(high_memory) - paddr :
+                                size;
+
+        if (ioremap_change_attr((unsigned long)__va(paddr), id_sz, flags) < 0) {
                 free_memtype(paddr, paddr + size);
                 printk(KERN_ERR
                         "%s:%d reserve_pfn_range ioremap_change_attr failed %s "
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index b3ca1b940654..72a6d4ebe34d 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -29,7 +29,7 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
  *      To avoid global state use 8 different call vectors.
  *      Each CPU uses a specific vector to trigger flushes on other
  *      CPUs. Depending on the received vector the target CPUs look into
- *      the right per cpu variable for the flush data.
+ *      the right array slot for the flush data.
  *
  *      With more than 8 CPUs they are hashed to the 8 available
  *      vectors. The limited global vector space forces us to this right now.
@@ -44,13 +44,13 @@ union smp_flush_state {
                 spinlock_t tlbstate_lock;
                 DECLARE_BITMAP(flush_cpumask, NR_CPUS);
         };
-        char pad[SMP_CACHE_BYTES];
-} ____cacheline_aligned;
+        char pad[CONFIG_X86_INTERNODE_CACHE_BYTES];
+} ____cacheline_internodealigned_in_smp;
 
 /* State is put into the per CPU data section, but padded
    to a full cache line because other CPUs can access it and we don't
    want false sharing in the per cpu data segment. */
-static DEFINE_PER_CPU(union smp_flush_state, flush_state);
+static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
 
 /*
  * We cannot call mmdrop() because we are in interrupt context,
@@ -135,7 +135,7 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
          * Use that to determine where the sender put the data.
          */
         sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
-        f = &per_cpu(flush_state, sender);
+        f = &flush_state[sender];
 
         if (!cpumask_test_cpu(cpu, to_cpumask(f->flush_cpumask)))
                 goto out;
@@ -173,7 +173,7 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
 
         /* Caller has disabled preemption */
         sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
-        f = &per_cpu(flush_state, sender);
+        f = &flush_state[sender];
 
         /*
          * Could avoid this lock when
@@ -227,8 +227,8 @@ static int __cpuinit init_smp_flush(void)
 {
         int i;
 
-        for_each_possible_cpu(i)
-                spin_lock_init(&per_cpu(flush_state, i).tlbstate_lock);
+        for (i = 0; i < ARRAY_SIZE(flush_state); i++)
+                spin_lock_init(&flush_state[i].tlbstate_lock);
 
         return 0;
 }