Merge branch 'x86-boot-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 boot and percpu updates from Ingo Molnar:
 "This tree contains a bootable images documentation update plus three
  slightly misplaced x86/asm percpu changes/optimizations"

* 'x86-boot-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86-64: Use RIP-relative addressing for most per-CPU accesses
  x86-64: Handle PC-relative relocations on per-CPU data
  x86: Convert a few more per-CPU items to read-mostly ones
  x86, boot: Document intermediates more clearly

2 files changed, 47 insertions, 18 deletions

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index fd472181a1d0..e0ba66ca68c6 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -64,7 +64,7 @@
 #define __percpu_prefix         ""
 #endif
 
-#define __percpu_arg(x)         __percpu_prefix "%P" #x
+#define __percpu_arg(x)         __percpu_prefix "%" #x
 
 /*
  * Initialized pointers to per-cpu variables needed for the boot
@@ -179,29 +179,58 @@ do {									\
         }                                                               \
 } while (0)
 
-#define percpu_from_op(op, var, constraint)             \
+#define percpu_from_op(op, var)                         \
 ({                                                      \
         typeof(var) pfo_ret__;                          \
         switch (sizeof(var)) {                          \
         case 1:                                         \
                 asm(op "b "__percpu_arg(1)",%0"         \
                     : "=q" (pfo_ret__)                  \
-                    : constraint);                      \
+                    : "m" (var));                       \
                 break;                                  \
         case 2:                                         \
                 asm(op "w "__percpu_arg(1)",%0"         \
                     : "=r" (pfo_ret__)                  \
-                    : constraint);                      \
+                    : "m" (var));                       \
                 break;                                  \
         case 4:                                         \
                 asm(op "l "__percpu_arg(1)",%0"         \
                     : "=r" (pfo_ret__)                  \
-                    : constraint);                      \
+                    : "m" (var));                       \
                 break;                                  \
         case 8:                                         \
                 asm(op "q "__percpu_arg(1)",%0"         \
                     : "=r" (pfo_ret__)                  \
-                    : constraint);                      \
+                    : "m" (var));                       \
+                break;                                  \
+        default: __bad_percpu_size();                   \
+        }                                               \
+        pfo_ret__;                                      \
+})
+
+#define percpu_stable_op(op, var)                       \
+({                                                      \
+        typeof(var) pfo_ret__;                          \
+        switch (sizeof(var)) {                          \
+        case 1:                                         \
+                asm(op "b "__percpu_arg(P1)",%0"        \
+                    : "=q" (pfo_ret__)                  \
+                    : "p" (&(var)));                    \
+                break;                                  \
+        case 2:                                         \
+                asm(op "w "__percpu_arg(P1)",%0"        \
+                    : "=r" (pfo_ret__)                  \
+                    : "p" (&(var)));                    \
+                break;                                  \
+        case 4:                                         \
+                asm(op "l "__percpu_arg(P1)",%0"        \
+                    : "=r" (pfo_ret__)                  \
+                    : "p" (&(var)));                    \
+                break;                                  \
+        case 8:                                         \
+                asm(op "q "__percpu_arg(P1)",%0"        \
+                    : "=r" (pfo_ret__)                  \
+                    : "p" (&(var)));                    \
                 break;                                  \
         default: __bad_percpu_size();                   \
         }                                               \
@@ -359,11 +388,11 @@ do {									\
  * per-thread variables implemented as per-cpu variables and thus
  * stable for the duration of the respective task.
  */
-#define this_cpu_read_stable(var)       percpu_from_op("mov", var, "p" (&(var)))
+#define this_cpu_read_stable(var)       percpu_stable_op("mov", var)
 
-#define raw_cpu_read_1(pcp)             percpu_from_op("mov", (pcp), "m"(pcp))
-#define raw_cpu_read_2(pcp)             percpu_from_op("mov", (pcp), "m"(pcp))
-#define raw_cpu_read_4(pcp)             percpu_from_op("mov", (pcp), "m"(pcp))
+#define raw_cpu_read_1(pcp)             percpu_from_op("mov", pcp)
+#define raw_cpu_read_2(pcp)             percpu_from_op("mov", pcp)
+#define raw_cpu_read_4(pcp)             percpu_from_op("mov", pcp)
 
 #define raw_cpu_write_1(pcp, val)       percpu_to_op("mov", (pcp), val)
 #define raw_cpu_write_2(pcp, val)       percpu_to_op("mov", (pcp), val)
@@ -381,9 +410,9 @@ do {									\
 #define raw_cpu_xchg_2(pcp, val)        percpu_xchg_op(pcp, val)
 #define raw_cpu_xchg_4(pcp, val)        percpu_xchg_op(pcp, val)
 
-#define this_cpu_read_1(pcp)            percpu_from_op("mov", (pcp), "m"(pcp))
-#define this_cpu_read_2(pcp)            percpu_from_op("mov", (pcp), "m"(pcp))
-#define this_cpu_read_4(pcp)            percpu_from_op("mov", (pcp), "m"(pcp))
+#define this_cpu_read_1(pcp)            percpu_from_op("mov", pcp)
+#define this_cpu_read_2(pcp)            percpu_from_op("mov", pcp)
+#define this_cpu_read_4(pcp)            percpu_from_op("mov", pcp)
 #define this_cpu_write_1(pcp, val)      percpu_to_op("mov", (pcp), val)
 #define this_cpu_write_2(pcp, val)      percpu_to_op("mov", (pcp), val)
 #define this_cpu_write_4(pcp, val)      percpu_to_op("mov", (pcp), val)
@@ -435,7 +464,7 @@ do {									\
  * 32 bit must fall back to generic operations.
  */
 #ifdef CONFIG_X86_64
-#define raw_cpu_read_8(pcp)                     percpu_from_op("mov", (pcp), "m"(pcp))
+#define raw_cpu_read_8(pcp)                     percpu_from_op("mov", pcp)
 #define raw_cpu_write_8(pcp, val)               percpu_to_op("mov", (pcp), val)
 #define raw_cpu_add_8(pcp, val)                 percpu_add_op((pcp), val)
 #define raw_cpu_and_8(pcp, val)                 percpu_to_op("and", (pcp), val)
@@ -444,7 +473,7 @@ do {									\
 #define raw_cpu_xchg_8(pcp, nval)               percpu_xchg_op(pcp, nval)
 #define raw_cpu_cmpxchg_8(pcp, oval, nval)      percpu_cmpxchg_op(pcp, oval, nval)
 
-#define this_cpu_read_8(pcp)                    percpu_from_op("mov", (pcp), "m"(pcp))
+#define this_cpu_read_8(pcp)                    percpu_from_op("mov", pcp)
 #define this_cpu_write_8(pcp, val)              percpu_to_op("mov", (pcp), val)
 #define this_cpu_add_8(pcp, val)                percpu_add_op((pcp), val)
 #define this_cpu_and_8(pcp, val)                percpu_to_op("and", (pcp), val)
@@ -522,7 +551,7 @@ static inline int x86_this_cpu_variable_test_bit(int nr,
 #include <asm-generic/percpu.h>
 
 /* We can use this directly for local CPU (faster). */
-DECLARE_PER_CPU(unsigned long, this_cpu_off);
+DECLARE_PER_CPU_READ_MOSTLY(unsigned long, this_cpu_off);
 
 #endif /* !__ASSEMBLY__ */
 
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 9617a1716813..25b8de0f21c0 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -127,7 +127,7 @@ struct cpuinfo_x86 {
         /* Index into per_cpu list: */
         u16                     cpu_index;
         u32                     microcode;
-} __attribute__((__aligned__(SMP_CACHE_BYTES)));
+};
 
 #define X86_VENDOR_INTEL        0
 #define X86_VENDOR_CYRIX        1
@@ -151,7 +151,7 @@ extern __u32			cpu_caps_cleared[NCAPINTS];
 extern __u32                    cpu_caps_set[NCAPINTS];
 
 #ifdef CONFIG_SMP
-DECLARE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
+DECLARE_PER_CPU_READ_MOSTLY(struct cpuinfo_x86, cpu_info);
 #define cpu_data(cpu)           per_cpu(cpu_info, cpu)
 #else
 #define cpu_info                boot_cpu_data