jump label: Introduce static_branch() interface

Introduce:

static __always_inline bool static_branch(struct jump_label_key *key);

instead of the old JUMP_LABEL(key, label) macro.

In this way, jump labels become really easy to use:

Define:

        struct jump_label_key jump_key;

Can be used as:

        if (static_branch(&jump_key))
                do unlikely code

enable/disale via:

        jump_label_inc(&jump_key);
        jump_label_dec(&jump_key);

that's it!

For the jump labels disabled case, the static_branch() becomes an
atomic_read(), and jump_label_inc()/dec() are simply atomic_inc(),
atomic_dec() operations. We show testing results for this change below.

Thanks to H. Peter Anvin for suggesting the 'static_branch()' construct.

Since we now require a 'struct jump_label_key *key', we can store a pointer into
the jump table addresses. In this way, we can enable/disable jump labels, in
basically constant time. This change allows us to completely remove the previous
hashtable scheme. Thanks to Peter Zijlstra for this re-write.

Testing:

I ran a series of 'tbench 20' runs 5 times (with reboots) for 3
configurations, where tracepoints were disabled.

jump label configured in
avg: 815.6

jump label *not* configured in (using atomic reads)
avg: 800.1

jump label *not* configured in (regular reads)
avg: 803.4

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
LKML-Reference: <20110316212947.GA8792@redhat.com>
Signed-off-by: Jason Baron <jbaron@redhat.com>
Suggested-by: H. Peter Anvin <hpa@linux.intel.com>
Tested-by: David Daney <ddaney@caviumnetworks.com>
Acked-by: Ralf Baechle <ralf@linux-mips.org>
Acked-by: David S. Miller <davem@davemloft.net>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>

6 files changed, 83 insertions, 114 deletions

diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
index 32c45e5fe0a..79522166d7f 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -170,6 +170,10 @@
         STRUCT_ALIGN();                                                 \
         *(__tracepoints)                                                \
         /* implement dynamic printk debug */                            \
+        . = ALIGN(8);                                                   \
+        VMLINUX_SYMBOL(__start___jump_table) = .;                       \
+        *(__jump_table)                                                 \
+        VMLINUX_SYMBOL(__stop___jump_table) = .;                        \
         . = ALIGN(8);                                                   \
         VMLINUX_SYMBOL(__start___verbose) = .;                          \
         *(__verbose)                                                    \
@@ -228,8 +232,6 @@
                                                                         \
         BUG_TABLE                                                       \
                                                                         \
-        JUMP_TABLE                                                      \
-                                                                        \
         /* PCI quirks */                                                \
         .pci_fixup        : AT(ADDR(.pci_fixup) - LOAD_OFFSET) {        \
                 VMLINUX_SYMBOL(__start_pci_fixups_early) = .;           \
@@ -589,14 +591,6 @@
 #define BUG_TABLE
 #endif
 
-#define JUMP_TABLE                                                      \
-        . = ALIGN(8);                                                   \
-        __jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) {           \
-                VMLINUX_SYMBOL(__start___jump_table) = .;               \
-                *(__jump_table)                                         \
-                VMLINUX_SYMBOL(__stop___jump_table) = .;                \
-        }
-
 #ifdef CONFIG_PM_TRACE
 #define TRACEDATA                                                       \
         . = ALIGN(4);                                                   \
diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h
index 0c9653f11c1..e747ecd48e1 100644
--- a/include/linux/dynamic_debug.h
+++ b/include/linux/dynamic_debug.h
@@ -1,8 +1,6 @@
 #ifndef _DYNAMIC_DEBUG_H
 #define _DYNAMIC_DEBUG_H
 
-#include <linux/jump_label.h>
-
 /* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
  * bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
  * use independent hash functions, to reduce the chance of false positives.
diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h
index 7880f18e4b8..83e745f3ead 100644
--- a/include/linux/jump_label.h
+++ b/include/linux/jump_label.h
@@ -1,20 +1,43 @@
 #ifndef _LINUX_JUMP_LABEL_H
 #define _LINUX_JUMP_LABEL_H
 
+#include <linux/types.h>
+#include <linux/compiler.h>
+
 #if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
+
+struct jump_label_key {
+        atomic_t enabled;
+        struct jump_entry *entries;
+#ifdef CONFIG_MODULES
+        struct jump_label_mod *next;
+#endif
+};
+
 # include <asm/jump_label.h>
 # define HAVE_JUMP_LABEL
 #endif
 
 enum jump_label_type {
+        JUMP_LABEL_DISABLE = 0,
         JUMP_LABEL_ENABLE,
-        JUMP_LABEL_DISABLE
 };
 
 struct module;
 
 #ifdef HAVE_JUMP_LABEL
 
+#ifdef CONFIG_MODULES
+#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
+#else
+#define JUMP_LABEL_INIT {{ 0 }, NULL}
+#endif
+
+static __always_inline bool static_branch(struct jump_label_key *key)
+{
+        return arch_static_branch(key);
+}
+
 extern struct jump_entry __start___jump_table[];
 extern struct jump_entry __stop___jump_table[];
 
@@ -23,37 +46,37 @@ extern void jump_label_unlock(void);
 extern void arch_jump_label_transform(struct jump_entry *entry,
                                  enum jump_label_type type);
 extern void arch_jump_label_text_poke_early(jump_label_t addr);
-extern void jump_label_update(unsigned long key, enum jump_label_type type);
-extern void jump_label_apply_nops(struct module *mod);
 extern int jump_label_text_reserved(void *start, void *end);
+extern void jump_label_inc(struct jump_label_key *key);
+extern void jump_label_dec(struct jump_label_key *key);
+extern bool jump_label_enabled(struct jump_label_key *key);
+extern void jump_label_apply_nops(struct module *mod);
 
-#define jump_label_enable(key) \
-        jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
+#else
 
-#define jump_label_disable(key) \
-        jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
+#include <asm/atomic.h>
 
-#else
+#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
 
-#define JUMP_LABEL(key, label)                  \
-do {                                            \
-        if (unlikely(*key))                     \
-                goto label;                     \
-} while (0)
+struct jump_label_key {
+        atomic_t enabled;
+};
 
-#define jump_label_enable(cond_var)     \
-do {                                    \
-       *(cond_var) = 1;                 \
-} while (0)
+static __always_inline bool static_branch(struct jump_label_key *key)
+{
+        if (unlikely(atomic_read(&key->enabled)))
+                return true;
+        return false;
+}
 
-#define jump_label_disable(cond_var)    \
-do {                                    \
-       *(cond_var) = 0;                 \
-} while (0)
+static inline void jump_label_inc(struct jump_label_key *key)
+{
+        atomic_inc(&key->enabled);
+}
 
-static inline int jump_label_apply_nops(struct module *mod)
+static inline void jump_label_dec(struct jump_label_key *key)
 {
-        return 0;
+        atomic_dec(&key->enabled);
 }
 
 static inline int jump_label_text_reserved(void *start, void *end)
@@ -64,16 +87,16 @@ static inline int jump_label_text_reserved(void *start, void *end)
 static inline void jump_label_lock(void) {}
 static inline void jump_label_unlock(void) {}
 
-#endif
+static inline bool jump_label_enabled(struct jump_label_key *key)
+{
+        return !!atomic_read(&key->enabled);
+}
 
-#define COND_STMT(key, stmt)                                    \
-do {                                                            \
-        __label__ jl_enabled;                                   \
-        JUMP_LABEL(key, jl_enabled);                            \
-        if (0) {                                                \
-jl_enabled:                                                     \
-                stmt;                                           \
-        }                                                       \
-} while (0)
+static inline int jump_label_apply_nops(struct module *mod)
+{
+        return 0;
+}
+
+#endif
 
 #endif
diff --git a/include/linux/jump_label_ref.h b/include/linux/jump_label_ref.h
deleted file mode 100644
index e5d012ad92c..00000000000
--- a/include/linux/jump_label_ref.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef _LINUX_JUMP_LABEL_REF_H
-#define _LINUX_JUMP_LABEL_REF_H
-
-#include <linux/jump_label.h>
-#include <asm/atomic.h>
-
-#ifdef HAVE_JUMP_LABEL
-
-static inline void jump_label_inc(atomic_t *key)
-{
-        if (atomic_add_return(1, key) == 1)
-                jump_label_enable(key);
-}
-
-static inline void jump_label_dec(atomic_t *key)
-{
-        if (atomic_dec_and_test(key))
-                jump_label_disable(key);
-}
-
-#else /* !HAVE_JUMP_LABEL */
-
-static inline void jump_label_inc(atomic_t *key)
-{
-        atomic_inc(key);
-}
-
-static inline void jump_label_dec(atomic_t *key)
-{
-        atomic_dec(key);
-}
-
-#undef JUMP_LABEL
-#define JUMP_LABEL(key, label)                                          \
-do {                                                                    \
-        if (unlikely(__builtin_choose_expr(                             \
-              __builtin_types_compatible_p(typeof(key), atomic_t *),    \
-              atomic_read((atomic_t *)(key)), *(key))))                 \
-                goto label;                                             \
-} while (0)
-
-#endif /* HAVE_JUMP_LABEL */
-
-#endif /* _LINUX_JUMP_LABEL_REF_H */
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 311b4dc785a..730b7821690 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -505,7 +505,7 @@ struct perf_guest_info_callbacks {
 #include <linux/ftrace.h>
 #include <linux/cpu.h>
 #include <linux/irq_work.h>
-#include <linux/jump_label_ref.h>
+#include <linux/jump_label.h>
 #include <asm/atomic.h>
 #include <asm/local.h>
 
@@ -1034,7 +1034,7 @@ static inline int is_software_event(struct perf_event *event)
         return event->pmu->task_ctx_nr == perf_sw_context;
 }
 
-extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
+extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
 
 extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64);
 
@@ -1063,22 +1063,21 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
 {
         struct pt_regs hot_regs;
 
-        JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
-        return;
-
-have_event:
-        if (!regs) {
-                perf_fetch_caller_regs(&hot_regs);
-                regs = &hot_regs;
+        if (static_branch(&perf_swevent_enabled[event_id])) {
+                if (!regs) {
+                        perf_fetch_caller_regs(&hot_regs);
+                        regs = &hot_regs;
+                }
+                __perf_sw_event(event_id, nr, nmi, regs, addr);
         }
-        __perf_sw_event(event_id, nr, nmi, regs, addr);
 }
 
-extern atomic_t perf_sched_events;
+extern struct jump_label_key perf_sched_events;
 
 static inline void perf_event_task_sched_in(struct task_struct *task)
 {
-        COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task));
+        if (static_branch(&perf_sched_events))
+                __perf_event_task_sched_in(task);
 }
 
 static inline
@@ -1086,7 +1085,8 @@ void perf_event_task_sched_out(struct task_struct *task, struct task_struct *nex
 {
         perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
 
-        COND_STMT(&perf_sched_events, __perf_event_task_sched_out(task, next));
+        if (static_branch(&perf_sched_events))
+                __perf_event_task_sched_out(task, next);
 }
 
 extern void perf_event_mmap(struct vm_area_struct *vma);
diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h
index 97c84a58efb..d530a4460a0 100644
--- a/include/linux/tracepoint.h
+++ b/include/linux/tracepoint.h
@@ -29,7 +29,7 @@ struct tracepoint_func {
 
 struct tracepoint {
         const char *name;               /* Tracepoint name */
-        int state;                      /* State. */
+        struct jump_label_key key;
         void (*regfunc)(void);
         void (*unregfunc)(void);
         struct tracepoint_func __rcu *funcs;
@@ -146,9 +146,7 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin,
         extern struct tracepoint __tracepoint_##name;                   \
         static inline void trace_##name(proto)                          \
         {                                                               \
-                JUMP_LABEL(&__tracepoint_##name.state, do_trace);       \
-                return;                                                 \
-do_trace:                                                               \
+                if (static_branch(&__tracepoint_##name.key))            \
                         __DO_TRACE(&__tracepoint_##name,                \
                                 TP_PROTO(data_proto),                   \
                                 TP_ARGS(data_args),                     \
@@ -176,14 +174,14 @@ do_trace:								\
  * structures, so we create an array of pointers that will be used for iteration
  * on the tracepoints.
  */
-#define DEFINE_TRACE_FN(name, reg, unreg)                               \
-        static const char __tpstrtab_##name[]                           \
-        __attribute__((section("__tracepoints_strings"))) = #name;      \
-        struct tracepoint __tracepoint_##name                           \
-        __attribute__((section("__tracepoints"))) =                     \
-                { __tpstrtab_##name, 0, reg, unreg, NULL };             \
-        static struct tracepoint * const __tracepoint_ptr_##name __used \
-        __attribute__((section("__tracepoints_ptrs"))) =                \
+#define DEFINE_TRACE_FN(name, reg, unreg)                                \
+        static const char __tpstrtab_##name[]                            \
+        __attribute__((section("__tracepoints_strings"))) = #name;       \
+        struct tracepoint __tracepoint_##name                            \
+        __attribute__((section("__tracepoints"))) =                      \
+                { __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
+        static struct tracepoint * const __tracepoint_ptr_##name __used  \
+        __attribute__((section("__tracepoints_ptrs"))) =                 \
                 &__tracepoint_##name;
 
 #define DEFINE_TRACE(name)                                              \