Merge branch 'x86-extable-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull exception table generation updates from Ingo Molnar:
 "The biggest change here is to allow the build-time sorting of the
  exception table, to speed up booting.  This is achieved by the
  architecture enabling BUILDTIME_EXTABLE_SORT.  This option is enabled
  for x86 and MIPS currently.

  On x86 a number of fixes and changes were needed to allow build-time
  sorting of the exception table, in particular a relocation invariant
  exception table format was needed.  This required the abstracting out
  of exception table protocol and the removal of 20 years of accumulated
  assumptions about the x86 exception table format.

  While at it, this tree also cleans up various other aspects of
  exception handling, such as early(er) exception handling for
  rdmsr_safe() et al.

  All in one, as the result of these changes the x86 exception code is
  now pretty nice and modern.  As an added bonus any regressions in this
  code will be early and violent crashes, so if you see any of those,
  you'll know whom to blame!"

Fix up trivial conflicts in arch/{mips,x86}/Kconfig files due to nearby
modifications of other core architecture options.

* 'x86-extable-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (35 commits)
  Revert "x86, extable: Disable presorted exception table for now"
  scripts/sortextable: Handle relative entries, and other cleanups
  x86, extable: Switch to relative exception table entries
  x86, extable: Disable presorted exception table for now
  x86, extable: Add _ASM_EXTABLE_EX() macro
  x86, extable: Remove open-coded exception table entries in arch/x86/ia32/ia32entry.S
  x86, extable: Remove open-coded exception table entries in arch/x86/include/asm/xsave.h
  x86, extable: Remove open-coded exception table entries in arch/x86/include/asm/kvm_host.h
  x86, extable: Remove the now-unused __ASM_EX_SEC macros
  x86, extable: Remove open-coded exception table entries in arch/x86/xen/xen-asm_32.S
  x86, extable: Remove open-coded exception table entries in arch/x86/um/checksum_32.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/usercopy_32.c
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/putuser.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/getuser.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/csum-copy_64.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/copy_user_nocache_64.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/copy_user_64.S
  x86, extable: Remove open-coded exception table entries in arch/x86/lib/checksum_32.S
  x86, extable: Remove open-coded exception table entries in arch/x86/kernel/test_rodata.c
  x86, extable: Remove open-coded exception table entries in arch/x86/kernel/entry_64.S
  ...

8 files changed, 55 insertions, 46 deletions

diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h
index 9412d6558c88..1c2d247f65ce 100644
--- a/arch/x86/include/asm/asm.h
+++ b/arch/x86/include/asm/asm.h
@@ -4,11 +4,9 @@
 #ifdef __ASSEMBLY__
 # define __ASM_FORM(x)  x
 # define __ASM_FORM_COMMA(x) x,
-# define __ASM_EX_SEC   .section __ex_table, "a"
 #else
 # define __ASM_FORM(x)  " " #x " "
 # define __ASM_FORM_COMMA(x) " " #x ","
-# define __ASM_EX_SEC   " .section __ex_table,\"a\"\n"
 #endif
 
 #ifdef CONFIG_X86_32
@@ -42,17 +40,33 @@
 
 /* Exception table entry */
 #ifdef __ASSEMBLY__
-# define _ASM_EXTABLE(from,to)      \
-        __ASM_EX_SEC ;              \
-        _ASM_ALIGN ;                \
-        _ASM_PTR from , to ;        \
-        .previous
+# define _ASM_EXTABLE(from,to)                                  \
+        .pushsection "__ex_table","a" ;                         \
+        .balign 8 ;                                             \
+        .long (from) - . ;                                      \
+        .long (to) - . ;                                        \
+        .popsection
+
+# define _ASM_EXTABLE_EX(from,to)                               \
+        .pushsection "__ex_table","a" ;                         \
+        .balign 8 ;                                             \
+        .long (from) - . ;                                      \
+        .long (to) - . + 0x7ffffff0 ;                           \
+        .popsection
 #else
-# define _ASM_EXTABLE(from,to) \
-        __ASM_EX_SEC    \
-        _ASM_ALIGN "\n" \
-        _ASM_PTR #from "," #to "\n" \
-        " .previous\n"
+# define _ASM_EXTABLE(from,to)                                  \
+        " .pushsection \"__ex_table\",\"a\"\n"                  \
+        " .balign 8\n"                                          \
+        " .long (" #from ") - .\n"                              \
+        " .long (" #to ") - .\n"                                \
+        " .popsection\n"
+
+# define _ASM_EXTABLE_EX(from,to)                               \
+        " .pushsection \"__ex_table\",\"a\"\n"                  \
+        " .balign 8\n"                                          \
+        " .long (" #from ") - .\n"                              \
+        " .long (" #to ") - . + 0x7ffffff0\n"                   \
+        " .popsection\n"
 #endif
 
 #endif /* _ASM_X86_ASM_H */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index e216ba066e79..e5b97be12d2a 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -27,6 +27,7 @@
 #include <asm/desc.h>
 #include <asm/mtrr.h>
 #include <asm/msr-index.h>
+#include <asm/asm.h>
 
 #define KVM_MAX_VCPUS 254
 #define KVM_SOFT_MAX_VCPUS 160
@@ -921,9 +922,7 @@ extern bool kvm_rebooting;
         __ASM_SIZE(push) " $666b \n\t"        \
         "call kvm_spurious_fault \n\t"        \
         ".popsection \n\t" \
-        ".pushsection __ex_table, \"a\" \n\t" \
-        _ASM_PTR " 666b, 667b \n\t" \
-        ".popsection"
+        _ASM_EXTABLE(666b, 667b)
 
 #define __kvm_handle_fault_on_reboot(insn)              \
         ____kvm_handle_fault_on_reboot(insn, "")
diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 95203d40ffdd..084ef95274cd 100644
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -169,14 +169,7 @@ static inline int wrmsr_safe(unsigned msr, unsigned low, unsigned high)
         return native_write_msr_safe(msr, low, high);
 }
 
-/*
- * rdmsr with exception handling.
- *
- * Please note that the exception handling works only after we've
- * switched to the "smart" #GP handler in trap_init() which knows about
- * exception tables - using this macro earlier than that causes machine
- * hangs on boxes which do not implement the @msr in the first argument.
- */
+/* rdmsr with exception handling */
 #define rdmsr_safe(msr, p1, p2)                                 \
 ({                                                              \
         int __err;                                              \
diff --git a/arch/x86/include/asm/nops.h b/arch/x86/include/asm/nops.h
index 405b4032a60b..aff2b3356101 100644
--- a/arch/x86/include/asm/nops.h
+++ b/arch/x86/include/asm/nops.h
@@ -87,7 +87,11 @@
 #define P6_NOP8 0x0f,0x1f,0x84,0x00,0,0,0,0
 #define P6_NOP5_ATOMIC P6_NOP5
 
+#ifdef __ASSEMBLY__
+#define _ASM_MK_NOP(x) .byte x
+#else
 #define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n"
+#endif
 
 #if defined(CONFIG_MK7)
 #define ASM_NOP1 _ASM_MK_NOP(K7_NOP1)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index aa0f91308367..6cbbabf52707 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -1023,10 +1023,8 @@ extern void default_banner(void);
                   call PARA_INDIRECT(pv_cpu_ops+PV_CPU_swapgs)          \
                  )
 
-#define GET_CR2_INTO_RCX                                \
-        call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2); \
-        movq %rax, %rcx;                                \
-        xorq %rax, %rax;
+#define GET_CR2_INTO_RAX                                \
+        call PARA_INDIRECT(pv_mmu_ops+PV_MMU_read_cr2)
 
 #define PARAVIRT_ADJUST_EXCEPTION_FRAME                                 \
         PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_adjust_exception_frame), \
diff --git a/arch/x86/include/asm/segment.h b/arch/x86/include/asm/segment.h
index 165466233ab0..c48a95035a77 100644
--- a/arch/x86/include/asm/segment.h
+++ b/arch/x86/include/asm/segment.h
@@ -205,13 +205,15 @@
 
 #define IDT_ENTRIES 256
 #define NUM_EXCEPTION_VECTORS 32
+/* Bitmask of exception vectors which push an error code on the stack */
+#define EXCEPTION_ERRCODE_MASK  0x00027d00
 #define GDT_SIZE (GDT_ENTRIES * 8)
 #define GDT_ENTRY_TLS_ENTRIES 3
 #define TLS_SIZE (GDT_ENTRY_TLS_ENTRIES * 8)
 
 #ifdef __KERNEL__
 #ifndef __ASSEMBLY__
-extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][10];
+extern const char early_idt_handlers[NUM_EXCEPTION_VECTORS][2+2+5];
 
 /*
  * Load a segment. Fall back on loading the zero
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index e0544597cfe7..851fe0dc13bc 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -79,11 +79,12 @@
 #define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
 
 /*
- * The exception table consists of pairs of addresses: the first is the
- * address of an instruction that is allowed to fault, and the second is
- * the address at which the program should continue.  No registers are
- * modified, so it is entirely up to the continuation code to figure out
- * what to do.
+ * The exception table consists of pairs of addresses relative to the
+ * exception table enty itself: the first is the address of an
+ * instruction that is allowed to fault, and the second is the address
+ * at which the program should continue.  No registers are modified,
+ * so it is entirely up to the continuation code to figure out what to
+ * do.
  *
  * All the routines below use bits of fixup code that are out of line
  * with the main instruction path.  This means when everything is well,
@@ -92,10 +93,14 @@
  */
 
 struct exception_table_entry {
-        unsigned long insn, fixup;
+        int insn, fixup;
 };
+/* This is not the generic standard exception_table_entry format */
+#define ARCH_HAS_SORT_EXTABLE
+#define ARCH_HAS_SEARCH_EXTABLE
 
 extern int fixup_exception(struct pt_regs *regs);
+extern int early_fixup_exception(unsigned long *ip);
 
 /*
  * These are the main single-value transfer routines.  They automatically
@@ -202,8 +207,8 @@ extern int __get_user_bad(void);
         asm volatile("1:        movl %%eax,0(%1)\n"                     \
                      "2:        movl %%edx,4(%1)\n"                     \
                      "3:\n"                                             \
-                     _ASM_EXTABLE(1b, 2b - 1b)                          \
-                     _ASM_EXTABLE(2b, 3b - 2b)                          \
+                     _ASM_EXTABLE_EX(1b, 2b)                            \
+                     _ASM_EXTABLE_EX(2b, 3b)                            \
                      : : "A" (x), "r" (addr))
 
 #define __put_user_x8(x, ptr, __ret_pu)                         \
@@ -408,7 +413,7 @@ do {									\
 #define __get_user_asm_ex(x, addr, itype, rtype, ltype)                 \
         asm volatile("1:        mov"itype" %1,%"rtype"0\n"              \
                      "2:\n"                                             \
-                     _ASM_EXTABLE(1b, 2b - 1b)                          \
+                     _ASM_EXTABLE_EX(1b, 2b)                            \
                      : ltype(x) : "m" (__m(addr)))
 
 #define __put_user_nocheck(x, ptr, size)                        \
@@ -450,7 +455,7 @@ struct __large_struct { unsigned long buf[100]; };
 #define __put_user_asm_ex(x, addr, itype, rtype, ltype)                 \
         asm volatile("1:        mov"itype" %"rtype"0,%1\n"              \
                      "2:\n"                                             \
-                     _ASM_EXTABLE(1b, 2b - 1b)                          \
+                     _ASM_EXTABLE_EX(1b, 2b)                            \
                      : : ltype(x), "m" (__m(addr)))
 
 /*
diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h
index c6ce2452f10c..8a1b6f9b594a 100644
--- a/arch/x86/include/asm/xsave.h
+++ b/arch/x86/include/asm/xsave.h
@@ -80,10 +80,7 @@ static inline int xsave_user(struct xsave_struct __user *buf)
                              "3:  movl $-1,%[err]\n"
                              "    jmp  2b\n"
                              ".previous\n"
-                             ".section __ex_table,\"a\"\n"
-                             _ASM_ALIGN "\n"
-                             _ASM_PTR "1b,3b\n"
-                             ".previous"
+                             _ASM_EXTABLE(1b,3b)
                              : [err] "=r" (err)
                              : "D" (buf), "a" (-1), "d" (-1), "0" (0)
                              : "memory");
@@ -106,10 +103,7 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)
                              "3:  movl $-1,%[err]\n"
                              "    jmp  2b\n"
                              ".previous\n"
-                             ".section __ex_table,\"a\"\n"
-                             _ASM_ALIGN "\n"
-                             _ASM_PTR "1b,3b\n"
-                             ".previous"
+                             _ASM_EXTABLE(1b,3b)
                              : [err] "=r" (err)
                              : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0)
                              : "memory");       /* memory required? */