locking/static_keys: Add a new static_key interface

There are various problems and short-comings with the current
static_key interface:

 - static_key_{true,false}() read like a branch depending on the key
   value, instead of the actual likely/unlikely branch depending on
   init value.

 - static_key_{true,false}() are, as stated above, tied to the
   static_key init values STATIC_KEY_INIT_{TRUE,FALSE}.

 - we're limited to the 2 (out of 4) possible options that compile to
   a default NOP because that's what our arch_static_branch() assembly
   emits.

So provide a new static_key interface:

  DEFINE_STATIC_KEY_TRUE(name);
  DEFINE_STATIC_KEY_FALSE(name);

Which define a key of different types with an initial true/false
value.

Then allow:

   static_branch_likely()
   static_branch_unlikely()

to take a key of either type and emit the right instruction for the
case.

This means adding a second arch_static_branch_jump() assembly helper
which emits a JMP per default.

In order to determine the right instruction for the right state,
encode the branch type in the LSB of jump_entry::key.

This is the final step in removing the naming confusion that has led to
a stream of avoidable bugs such as:

  a833581e372a ("x86, perf: Fix static_key bug in load_mm_cr4()")

... but it also allows new static key combinations that will give us
performance enhancements in the subsequent patches.

Tested-by: Rabin Vincent <rabin@rab.in> # arm
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Michael Ellerman <mpe@ellerman.id.au> # ppc
Acked-by: Heiko Carstens <heiko.carstens@de.ibm.com> # s390
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>

7 files changed, 129 insertions, 27 deletions

diff --git a/arch/arm/include/asm/jump_label.h b/arch/arm/include/asm/jump_label.h
index 5f337dc5c108..34f7b6980d21 100644
--- a/arch/arm/include/asm/jump_label.h
+++ b/arch/arm/include/asm/jump_label.h
@@ -4,23 +4,32 @@
 #ifndef __ASSEMBLY__
 
 #include <linux/types.h>
+#include <asm/unified.h>
 
 #define JUMP_LABEL_NOP_SIZE 4
 
-#ifdef CONFIG_THUMB2_KERNEL
-#define JUMP_LABEL_NOP  "nop.w"
-#else
-#define JUMP_LABEL_NOP  "nop"
-#endif
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
+{
+        asm_volatile_goto("1:\n\t"
+                 WASM(nop) "\n\t"
+                 ".pushsection __jump_table,  \"aw\"\n\t"
+                 ".word 1b, %l[l_yes], %c0\n\t"
+                 ".popsection\n\t"
+                 : :  "i" (&((char *)key)[branch]) :  : l_yes);
+
+        return false;
+l_yes:
+        return true;
+}
 
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
 {
         asm_volatile_goto("1:\n\t"
-                 JUMP_LABEL_NOP "\n\t"
+                 WASM(b) " %l[l_yes]\n\t"
                  ".pushsection __jump_table,  \"aw\"\n\t"
                  ".word 1b, %l[l_yes], %c0\n\t"
                  ".popsection\n\t"
-                 : :  "i" (key) :  : l_yes);
+                 : :  "i" (&((char *)key)[branch]) :  : l_yes);
 
         return false;
 l_yes:
diff --git a/arch/arm64/include/asm/jump_label.h b/arch/arm64/include/asm/jump_label.h
index c0e5165c2f76..1b5e0e843c3a 100644
--- a/arch/arm64/include/asm/jump_label.h
+++ b/arch/arm64/include/asm/jump_label.h
@@ -26,14 +26,28 @@
 
 #define JUMP_LABEL_NOP_SIZE             AARCH64_INSN_SIZE
 
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
         asm goto("1: nop\n\t"
                  ".pushsection __jump_table,  \"aw\"\n\t"
                  ".align 3\n\t"
                  ".quad 1b, %l[l_yes], %c0\n\t"
                  ".popsection\n\t"
-                 :  :  "i"(key) :  : l_yes);
+                 :  :  "i"(&((char *)key)[branch]) :  : l_yes);
+
+        return false;
+l_yes:
+        return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
+{
+        asm goto("1: b %l[l_yes]\n\t"
+                 ".pushsection __jump_table,  \"aw\"\n\t"
+                 ".align 3\n\t"
+                 ".quad 1b, %l[l_yes], %c0\n\t"
+                 ".popsection\n\t"
+                 :  :  "i"(&((char *)key)[branch]) :  : l_yes);
 
         return false;
 l_yes:
diff --git a/arch/mips/include/asm/jump_label.h b/arch/mips/include/asm/jump_label.h
index 608aa57799c8..e77672539e8e 100644
--- a/arch/mips/include/asm/jump_label.h
+++ b/arch/mips/include/asm/jump_label.h
@@ -26,14 +26,29 @@
 #define NOP_INSN "nop"
 #endif
 
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
         asm_volatile_goto("1:\t" NOP_INSN "\n\t"
                 "nop\n\t"
                 ".pushsection __jump_table,  \"aw\"\n\t"
                 WORD_INSN " 1b, %l[l_yes], %0\n\t"
                 ".popsection\n\t"
-                : :  "i" (key) : : l_yes);
+                : :  "i" (&((char *)key)[branch]) : : l_yes);
+
+        return false;
+l_yes:
+        return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
+{
+        asm_volatile_goto("1:\tj %l[l_yes]\n\t"
+                "nop\n\t"
+                ".pushsection __jump_table,  \"aw\"\n\t"
+                WORD_INSN " 1b, %l[l_yes], %0\n\t"
+                ".popsection\n\t"
+                : :  "i" (&((char *)key)[branch]) : : l_yes);
+
         return false;
 l_yes:
         return true;
diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h
index efbf9a322a23..47e155f15433 100644
--- a/arch/powerpc/include/asm/jump_label.h
+++ b/arch/powerpc/include/asm/jump_label.h
@@ -18,14 +18,29 @@
 #define JUMP_ENTRY_TYPE         stringify_in_c(FTR_ENTRY_LONG)
 #define JUMP_LABEL_NOP_SIZE     4
 
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
         asm_volatile_goto("1:\n\t"
                  "nop\n\t"
                  ".pushsection __jump_table,  \"aw\"\n\t"
                  JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
                  ".popsection \n\t"
-                 : :  "i" (key) : : l_yes);
+                 : :  "i" (&((char *)key)[branch]) : : l_yes);
+
+        return false;
+l_yes:
+        return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
+{
+        asm_volatile_goto("1:\n\t"
+                 "b %l[l_yes]\n\t"
+                 ".pushsection __jump_table,  \"aw\"\n\t"
+                 JUMP_ENTRY_TYPE "1b, %l[l_yes], %c0\n\t"
+                 ".popsection \n\t"
+                 : :  "i" (&((char *)key)[branch]) : : l_yes);
+
         return false;
 l_yes:
         return true;
diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h
index 69972b7957ee..7f9fd5e3f1bf 100644
--- a/arch/s390/include/asm/jump_label.h
+++ b/arch/s390/include/asm/jump_label.h
@@ -12,14 +12,29 @@
  * We use a brcl 0,2 instruction for jump labels at compile time so it
  * can be easily distinguished from a hotpatch generated instruction.
  */
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
         asm_volatile_goto("0:   brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
                 ".pushsection __jump_table, \"aw\"\n"
                 ".balign 8\n"
                 ".quad 0b, %l[label], %0\n"
                 ".popsection\n"
-                : : "X" (key) : : label);
+                : : "X" (&((char *)key)[branch]) : : label);
+
+        return false;
+label:
+        return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
+{
+        asm_volatile_goto("0:   brcl 15, %l[label]\n"
+                ".pushsection __jump_table, \"aw\"\n"
+                ".balign 8\n"
+                ".quad 0b, %l[label], %0\n"
+                ".popsection\n"
+                : : "X" (&((char *)key)[branch]) : : label);
+
         return false;
 label:
         return true;
diff --git a/arch/sparc/include/asm/jump_label.h b/arch/sparc/include/asm/jump_label.h
index cc9b04a2b11b..62d0354d1727 100644
--- a/arch/sparc/include/asm/jump_label.h
+++ b/arch/sparc/include/asm/jump_label.h
@@ -7,16 +7,33 @@
 
 #define JUMP_LABEL_NOP_SIZE 4
 
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
-                asm_volatile_goto("1:\n\t"
-                         "nop\n\t"
-                         "nop\n\t"
-                         ".pushsection __jump_table,  \"aw\"\n\t"
-                         ".align 4\n\t"
-                         ".word 1b, %l[l_yes], %c0\n\t"
-                         ".popsection \n\t"
-                         : :  "i" (key) : : l_yes);
+        asm_volatile_goto("1:\n\t"
+                 "nop\n\t"
+                 "nop\n\t"
+                 ".pushsection __jump_table,  \"aw\"\n\t"
+                 ".align 4\n\t"
+                 ".word 1b, %l[l_yes], %c0\n\t"
+                 ".popsection \n\t"
+                 : :  "i" (&((char *)key)[branch]) : : l_yes);
+
+        return false;
+l_yes:
+        return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
+{
+        asm_volatile_goto("1:\n\t"
+                 "b %l[l_yes]\n\t"
+                 "nop\n\t"
+                 ".pushsection __jump_table,  \"aw\"\n\t"
+                 ".align 4\n\t"
+                 ".word 1b, %l[l_yes], %c0\n\t"
+                 ".popsection \n\t"
+                 : :  "i" (&((char *)key)[branch]) : : l_yes);
+
         return false;
 l_yes:
         return true;
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index a4c1cf7e93f8..28d7a857f9d1 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -16,7 +16,7 @@
 # define STATIC_KEY_INIT_NOP GENERIC_NOP5_ATOMIC
 #endif
 
-static __always_inline bool arch_static_branch(struct static_key *key)
+static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
 {
         asm_volatile_goto("1:"
                 ".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
@@ -24,7 +24,24 @@ static __always_inline bool arch_static_branch(struct static_key *key)
                 _ASM_ALIGN "\n\t"
                 _ASM_PTR "1b, %l[l_yes], %c0 \n\t"
                 ".popsection \n\t"
-                : :  "i" (key) : : l_yes);
+                : :  "i" (&((char *)key)[branch]) : : l_yes);
+
+        return false;
+l_yes:
+        return true;
+}
+
+static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
+{
+        asm_volatile_goto("1:"
+                ".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
+                "2:\n\t"
+                ".pushsection __jump_table,  \"aw\" \n\t"
+                _ASM_ALIGN "\n\t"
+                _ASM_PTR "1b, %l[l_yes], %c0 \n\t"
+                ".popsection \n\t"
+                : :  "i" (&((char *)key)[branch]) : : l_yes);
+
         return false;
 l_yes:
         return true;