13 files changed, 95 insertions, 34 deletions

diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index cdb1b70ddad0..c887cd944f0c 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -32,6 +32,7 @@ obj-$(CONFIG_X86_32)	+= i386_ksyms_32.o
 obj-$(CONFIG_X86_64)    += sys_x86_64.o x8664_ksyms_64.o
 obj-$(CONFIG_X86_64)    += mcount_64.o
 obj-y                   += syscall_$(BITS).o vsyscall_gtod.o
+obj-$(CONFIG_IA32_EMULATION)    += syscall_32.o
 obj-$(CONFIG_X86_VSYSCALL_EMULATION)    += vsyscall_64.o vsyscall_emu_64.o
 obj-$(CONFIG_X86_ESPFIX64)      += espfix_64.o
 obj-$(CONFIG_SYSFS)     += ksysfs.o
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index b5c8ff5e9dfc..2346c95c6ab1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1396,6 +1396,12 @@ void cpu_init(void)
 
         wait_for_master_cpu(cpu);
 
+        /*
+         * Initialize the CR4 shadow before doing anything that could
+         * try to read it.
+         */
+        cr4_init_shadow();
+
         show_ucode_info_early();
 
         printk(KERN_INFO "Initializing CPU#%d\n", cpu);
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 94d7dcb12145..50163fa9034f 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -565,8 +565,8 @@ static const struct _tlb_table intel_tlb_table[] = {
         { 0xb2, TLB_INST_4K,            64,     " TLB_INST 4KByte pages, 4-way set associative" },
         { 0xb3, TLB_DATA_4K,            128,    " TLB_DATA 4 KByte pages, 4-way set associative" },
         { 0xb4, TLB_DATA_4K,            256,    " TLB_DATA 4 KByte pages, 4-way associative" },
-        { 0xb5, TLB_INST_4K,            64,     " TLB_INST 4 KByte pages, 8-way set ssociative" },
-        { 0xb6, TLB_INST_4K,            128,    " TLB_INST 4 KByte pages, 8-way set ssociative" },
+        { 0xb5, TLB_INST_4K,            64,     " TLB_INST 4 KByte pages, 8-way set associative" },
+        { 0xb6, TLB_INST_4K,            128,    " TLB_INST 4 KByte pages, 8-way set associative" },
         { 0xba, TLB_DATA_4K,            64,     " TLB_DATA 4 KByte pages, 4-way associative" },
         { 0xc0, TLB_DATA_4K_4M,         8,      " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" },
         { 0xc1, STLB_4K_2M,             1024,   " STLB 4 KByte and 2 MByte pages, 8-way associative" },
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index b910577d1ff9..7e0323cc0b7d 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -976,6 +976,9 @@ ENTRY(xen_hypervisor_callback)
 ENTRY(xen_do_upcall)
 1:      mov %esp, %eax
         call xen_evtchn_do_upcall
+#ifndef CONFIG_PREEMPT
+        call xen_maybe_preempt_hcall
+#endif
         jmp  ret_from_intr
         CFI_ENDPROC
 ENDPROC(xen_hypervisor_callback)
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index db13655c3a2a..10074ad9ebf8 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1208,6 +1208,9 @@ ENTRY(xen_do_hypervisor_callback)   # do_hypervisor_callback(struct *pt_regs)
         popq %rsp
         CFI_DEF_CFA_REGISTER rsp
         decl PER_CPU_VAR(irq_count)
+#ifndef CONFIG_PREEMPT
+        call xen_maybe_preempt_hcall
+#endif
         jmp  error_exit
         CFI_ENDPROC
 END(xen_do_hypervisor_callback)
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index f36bd42d6f0c..d031bad9e07e 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -22,6 +22,7 @@
 #include <asm/cpufeature.h>
 #include <asm/percpu.h>
 #include <asm/nops.h>
+#include <asm/bootparam.h>
 
 /* Physical address */
 #define pa(X) ((X) - __PAGE_OFFSET)
@@ -90,7 +91,7 @@ ENTRY(startup_32)
         
         /* test KEEP_SEGMENTS flag to see if the bootloader is asking
                 us to not reload segments */
-        testb $(1<<6), BP_loadflags(%esi)
+        testb $KEEP_SEGMENTS, BP_loadflags(%esi)
         jnz 2f
 
 /*
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 6fd514d9f69a..9a0919678f97 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -1,5 +1,5 @@
 /*
- *  linux/arch/x86_64/kernel/head.S -- start in 32bit and switch to 64bit
+ *  linux/arch/x86/kernel/head_64.S -- start in 32bit and switch to 64bit
  *
  *  Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
  *  Copyright (C) 2000 Pavel Machek <pavel@suse.cz>
@@ -56,7 +56,7 @@ startup_64:
          * %rsi holds a physical pointer to real_mode_data.
          *
          * We come here either directly from a 64bit bootloader, or from
-         * arch/x86_64/boot/compressed/head.S.
+         * arch/x86/boot/compressed/head_64.S.
          *
          * We only come here initially at boot nothing else comes here.
          *
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 6a1146ea4d4d..4e3d5a9621fe 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -223,27 +223,48 @@ static unsigned long
 __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
 {
         struct kprobe *kp;
+        unsigned long faddr;
 
         kp = get_kprobe((void *)addr);
-        /* There is no probe, return original address */
-        if (!kp)
+        faddr = ftrace_location(addr);
+        /*
+         * Addresses inside the ftrace location are refused by
+         * arch_check_ftrace_location(). Something went terribly wrong
+         * if such an address is checked here.
+         */
+        if (WARN_ON(faddr && faddr != addr))
+                return 0UL;
+        /*
+         * Use the current code if it is not modified by Kprobe
+         * and it cannot be modified by ftrace.
+         */
+        if (!kp && !faddr)
                 return addr;
 
         /*
-         *  Basically, kp->ainsn.insn has an original instruction.
-         *  However, RIP-relative instruction can not do single-stepping
-         *  at different place, __copy_instruction() tweaks the displacement of
-         *  that instruction. In that case, we can't recover the instruction
-         *  from the kp->ainsn.insn.
+         * Basically, kp->ainsn.insn has an original instruction.
+         * However, RIP-relative instruction can not do single-stepping
+         * at different place, __copy_instruction() tweaks the displacement of
+         * that instruction. In that case, we can't recover the instruction
+         * from the kp->ainsn.insn.
          *
-         *  On the other hand, kp->opcode has a copy of the first byte of
-         *  the probed instruction, which is overwritten by int3. And
-         *  the instruction at kp->addr is not modified by kprobes except
-         *  for the first byte, we can recover the original instruction
-         *  from it and kp->opcode.
+         * On the other hand, in case on normal Kprobe, kp->opcode has a copy
+         * of the first byte of the probed instruction, which is overwritten
+         * by int3. And the instruction at kp->addr is not modified by kprobes
+         * except for the first byte, we can recover the original instruction
+         * from it and kp->opcode.
+         *
+         * In case of Kprobes using ftrace, we do not have a copy of
+         * the original instruction. In fact, the ftrace location might
+         * be modified at anytime and even could be in an inconsistent state.
+         * Fortunately, we know that the original code is the ideal 5-byte
+         * long NOP.
          */
-        memcpy(buf, kp->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
-        buf[0] = kp->opcode;
+        memcpy(buf, (void *)addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
+        if (faddr)
+                memcpy(buf, ideal_nops[NOP_ATOMIC5], 5);
+        else
+                buf[0] = kp->opcode;
         return (unsigned long)buf;
 }
 
@@ -251,6 +272,7 @@ __recover_probed_insn(kprobe_opcode_t *buf, unsigned long addr)
  * Recover the probed instruction at addr for further analysis.
  * Caller must lock kprobes by kprobe_mutex, or disable preemption
  * for preventing to release referencing kprobes.
+ * Returns zero if the instruction can not get recovered.
  */
 unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
 {
@@ -285,6 +307,8 @@ static int can_probe(unsigned long paddr)
                  * normally used, we just go through if there is no kprobe.
                  */
                 __addr = recover_probed_instruction(buf, addr);
+                if (!__addr)
+                        return 0;
                 kernel_insn_init(&insn, (void *)__addr, MAX_INSN_SIZE);
                 insn_get_length(&insn);
 
@@ -333,6 +357,8 @@ int __copy_instruction(u8 *dest, u8 *src)
         unsigned long recovered_insn =
                 recover_probed_instruction(buf, (unsigned long)src);
 
+        if (!recovered_insn)
+                return 0;
         kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE);
         insn_get_length(&insn);
         /* Another subsystem puts a breakpoint, failed to recover */
diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c
index 0dd8d089c315..7b3b9d15c47a 100644
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -259,6 +259,8 @@ static int can_optimize(unsigned long paddr)
                          */
                         return 0;
                 recovered_insn = recover_probed_instruction(buf, addr);
+                if (!recovered_insn)
+                        return 0;
                 kernel_insn_init(&insn, (void *)recovered_insn, MAX_INSN_SIZE);
                 insn_get_length(&insn);
                 /* Another subsystem puts a breakpoint */
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 67fcc43577d2..854b5981b327 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -207,7 +207,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
          */
         if (clone_flags & CLONE_SETTLS) {
 #ifdef CONFIG_IA32_EMULATION
-                if (test_thread_flag(TIF_IA32))
+                if (is_ia32_task())
                         err = do_set_thread_area(p, -1,
                                 (struct user_desc __user *)childregs->si, 0);
                 else
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index e510618b2e91..1e125817cf9f 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -364,18 +364,12 @@ static int set_segment_reg(struct task_struct *task,
         case offsetof(struct user_regs_struct,cs):
                 if (unlikely(value == 0))
                         return -EIO;
-#ifdef CONFIG_IA32_EMULATION
-                if (test_tsk_thread_flag(task, TIF_IA32))
-                        task_pt_regs(task)->cs = value;
-#endif
+                task_pt_regs(task)->cs = value;
                 break;
         case offsetof(struct user_regs_struct,ss):
                 if (unlikely(value == 0))
                         return -EIO;
-#ifdef CONFIG_IA32_EMULATION
-                if (test_tsk_thread_flag(task, TIF_IA32))
-                        task_pt_regs(task)->ss = value;
-#endif
+                task_pt_regs(task)->ss = value;
                 break;
         }
 
diff --git a/arch/x86/kernel/syscall_32.c b/arch/x86/kernel/syscall_32.c
index e9bcd57d8a9e..3777189c4a19 100644
--- a/arch/x86/kernel/syscall_32.c
+++ b/arch/x86/kernel/syscall_32.c
@@ -5,21 +5,29 @@
 #include <linux/cache.h>
 #include <asm/asm-offsets.h>
 
-#define __SYSCALL_I386(nr, sym, compat) extern asmlinkage void sym(void) ;
+#ifdef CONFIG_IA32_EMULATION
+#define SYM(sym, compat) compat
+#else
+#define SYM(sym, compat) sym
+#define ia32_sys_call_table sys_call_table
+#define __NR_ia32_syscall_max __NR_syscall_max
+#endif
+
+#define __SYSCALL_I386(nr, sym, compat) extern asmlinkage void SYM(sym, compat)(void) ;
 #include <asm/syscalls_32.h>
 #undef __SYSCALL_I386
 
-#define __SYSCALL_I386(nr, sym, compat) [nr] = sym,
+#define __SYSCALL_I386(nr, sym, compat) [nr] = SYM(sym, compat),
 
 typedef asmlinkage void (*sys_call_ptr_t)(void);
 
 extern asmlinkage void sys_ni_syscall(void);
 
-__visible const sys_call_ptr_t sys_call_table[__NR_syscall_max+1] = {
+__visible const sys_call_ptr_t ia32_sys_call_table[__NR_ia32_syscall_max+1] = {
         /*
          * Smells like a compiler bug -- it doesn't work
          * when the & below is removed.
          */
-        [0 ... __NR_syscall_max] = &sys_ni_syscall,
+        [0 ... __NR_ia32_syscall_max] = &sys_ni_syscall,
 #include <asm/syscalls_32.h>
 };
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index 9d2073e2ecc9..42819886be0c 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -925,9 +925,17 @@ dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
 /* Set of traps needed for early debugging. */
 void __init early_trap_init(void)
 {
-        set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
+        /*
+         * Don't set ist to DEBUG_STACK as it doesn't work until TSS is
+         * ready in cpu_init() <-- trap_init(). Before trap_init(), CPU
+         * runs at ring 0 so it is impossible to hit an invalid stack.
+         * Using the original stack works well enough at this early
+         * stage. DEBUG_STACK will be equipped after cpu_init() in
+         * trap_init().
+         */
+        set_intr_gate_ist(X86_TRAP_DB, &debug, 0);
         /* int3 can be called from all */
-        set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
+        set_system_intr_gate_ist(X86_TRAP_BP, &int3, 0);
 #ifdef CONFIG_X86_32
         set_intr_gate(X86_TRAP_PF, page_fault);
 #endif
@@ -1005,6 +1013,15 @@ void __init trap_init(void)
          */
         cpu_init();
 
+        /*
+         * X86_TRAP_DB and X86_TRAP_BP have been set
+         * in early_trap_init(). However, DEBUG_STACK works only after
+         * cpu_init() loads TSS. See comments in early_trap_init().
+         */
+        set_intr_gate_ist(X86_TRAP_DB, &debug, DEBUG_STACK);
+        /* int3 can be called from all */
+        set_system_intr_gate_ist(X86_TRAP_BP, &int3, DEBUG_STACK);
+
         x86_init.irqs.trap_init();
 
 #ifdef CONFIG_X86_64