percpu: add percpu locked infrastructure

RT needs per cpu data structures protected by per cpu locks instead of
disabling preemption. Add the infrastructure for per cpu locked data.

Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

2 files changed, 41 insertions, 0 deletions

diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h
index 68438e18fff4..c02051de18d4 100644
--- a/include/linux/percpu-defs.h
+++ b/include/linux/percpu-defs.h
@@ -38,6 +38,22 @@
         DEFINE_PER_CPU_SECTION(type, name, "")
 
 /*
+ * next two added for RT patch
+ * (wonder if we need corresponding DECLARE_*'s?) (clrkwllms)
+ */
+#define DEFINE_PER_CPU_SPINLOCK(name, section)                          \
+        __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
+        PER_CPU_ATTRIBUTES __DEFINE_SPINLOCK(per_cpu__lock_##name##_locked);
+
+#define DECLARE_PER_CPU_LOCKED(type, name)                              \
+        extern PER_CPU_ATTRIBUTES spinlock_t __per_cpu_var_lock(name);  \
+        extern PER_CPU_ATTRIBUTES __typeof__(type) __per_cpu_var_lock_var(name)
+
+#define DEFINE_PER_CPU_LOCKED(type, name)                               \
+        DEFINE_PER_CPU_SPINLOCK(name, "")                               \
+        DEFINE_PER_CPU_SECTION(type, name##_locked, "")
+
+/*
  * Declaration/definition used for per-CPU variables that must come first in
  * the set of variables.
  */
@@ -79,7 +95,9 @@
  * Intermodule exports for per-CPU variables.
  */
 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
+#define EXPORT_PER_CPU_LOCKED_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var##_locked)
 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
+#define EXPORT_PER_CPU_LOCKED_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var##_locked)
 
 
 #endif /* _LINUX_PERCPU_DEFS_H */
diff --git a/include/linux/percpu.h b/include/linux/percpu.h
index 26fd9d12f050..39c4f01d2cd4 100644
--- a/include/linux/percpu.h
+++ b/include/linux/percpu.h
@@ -32,6 +32,29 @@
         &__get_cpu_var(var); }))
 #define put_cpu_var(var) preempt_enable()
 
+/*
+ * Per-CPU data structures with an additional lock - useful for
+ * PREEMPT_RT code that wants to reschedule but also wants
+ * per-CPU data structures.
+ *
+ * 'cpu' gets updated with the CPU the task is currently executing on.
+ *
+ * NOTE: on normal !PREEMPT_RT kernels these per-CPU variables
+ * are the same as the normal per-CPU variables, so there no
+ * runtime overhead.
+ */
+#define get_cpu_var_locked(var, cpuptr)                 \
+(*({                                                    \
+        int __cpu = raw_smp_processor_id();             \
+                                                        \
+        *(cpuptr) = __cpu;                              \
+        spin_lock(&__get_cpu_lock(var, __cpu));         \
+        &__get_cpu_var_locked(var, __cpu);              \
+}))
+
+#define put_cpu_var_locked(var, cpu) \
+         do { (void)cpu; spin_unlock(&__get_cpu_lock(var, cpu)); } while (0)
+
 #ifdef CONFIG_SMP
 
 #ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA