x86: Define early read-mostly per-cpu macros

Some read-mostly per-cpu data may need to be declared or defined
early, so it can be initialized and accessed before per_cpu
areas are allocated.

Only the data that resides in the per_cpu areas should be
read-mostly, as there is little benefit in optimizing cache
lines on initialization.

Signed-off-by: Ido Yariv <ido@wizery.com>
[ Added the missing declarations in !SMP code. ]
Signed-off-by: Vlad Zolotarov <vlad@scalemp.com>
Acked-by: Shai Fultheim <shai@scalemp.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Link: http://lkml.kernel.org/r/46188571.ddB8aVQYWo@vlad
Signed-off-by: Ingo Molnar <mingo@kernel.org>

1 files changed, 17 insertions, 0 deletions

diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d9b8e3f7f42a..1104afaba52b 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -551,6 +551,12 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
                                 { [0 ... NR_CPUS-1] = _initvalue };     \
         __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
 
+#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue)      \
+        DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue;          \
+        __typeof__(_type) _name##_early_map[NR_CPUS] __initdata =       \
+                                { [0 ... NR_CPUS-1] = _initvalue };     \
+        __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
+
 #define EXPORT_EARLY_PER_CPU_SYMBOL(_name)                      \
         EXPORT_PER_CPU_SYMBOL(_name)
 
@@ -559,6 +565,11 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
         extern __typeof__(_type) *_name##_early_ptr;            \
         extern __typeof__(_type)  _name##_early_map[]
 
+#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name)         \
+        DECLARE_PER_CPU_READ_MOSTLY(_type, _name);              \
+        extern __typeof__(_type) *_name##_early_ptr;            \
+        extern __typeof__(_type)  _name##_early_map[]
+
 #define early_per_cpu_ptr(_name) (_name##_early_ptr)
 #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
 #define early_per_cpu(_name, _cpu)                              \
@@ -570,12 +581,18 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
 #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue)          \
         DEFINE_PER_CPU(_type, _name) = _initvalue
 
+#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue)      \
+        DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue
+
 #define EXPORT_EARLY_PER_CPU_SYMBOL(_name)                      \
         EXPORT_PER_CPU_SYMBOL(_name)
 
 #define DECLARE_EARLY_PER_CPU(_type, _name)                     \
         DECLARE_PER_CPU(_type, _name)
 
+#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name)         \
+        DECLARE_PER_CPU_READ_MOSTLY(_type, _name)
+
 #define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
 #define early_per_cpu_ptr(_name) NULL
 /* no early_per_cpu_map() */