Merge branch 'x86/jumplabel' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 jumplabel changes from Peter Anvin:
 "One more x86 tree for this merge window.  This tree improves the
  handling of jump labels, so that most of the time we don't have to do
  a massive initial patching run.

  Furthermore, we will error out of the jump label is not what is
  expected, eg if it has been corrupted or tampered with"

* 'x86/jumplabel' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  x86/jump-label: Show where and what was wrong on errors
  x86/jump-label: Add safety checks to jump label conversions
  x86/jump-label: Do not bother updating nops if they are correct
  x86/jump-label: Use best default nops for inital jump label calls

1 files changed, 66 insertions, 4 deletions

diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index 460f5d9ceebb..ee11b7dfbfbb 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -24,18 +24,57 @@ union jump_code_union {
         } __attribute__((packed));
 };
 
+static void bug_at(unsigned char *ip, int line)
+{
+        /*
+         * The location is not an op that we were expecting.
+         * Something went wrong. Crash the box, as something could be
+         * corrupting the kernel.
+         */
+        pr_warning("Unexpected op at %pS [%p] (%02x %02x %02x %02x %02x) %s:%d\n",
+               ip, ip, ip[0], ip[1], ip[2], ip[3], ip[4], __FILE__, line);
+        BUG();
+}
+
 static void __jump_label_transform(struct jump_entry *entry,
                                    enum jump_label_type type,
-                                   void *(*poker)(void *, const void *, size_t))
+                                   void *(*poker)(void *, const void *, size_t),
+                                   int init)
 {
         union jump_code_union code;
+        const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
 
         if (type == JUMP_LABEL_ENABLE) {
+                /*
+                 * We are enabling this jump label. If it is not a nop
+                 * then something must have gone wrong.
+                 */
+                if (unlikely(memcmp((void *)entry->code, ideal_nop, 5) != 0))
+                        bug_at((void *)entry->code, __LINE__);
+
                 code.jump = 0xe9;
                 code.offset = entry->target -
                                 (entry->code + JUMP_LABEL_NOP_SIZE);
-        } else
+        } else {
+                /*
+                 * We are disabling this jump label. If it is not what
+                 * we think it is, then something must have gone wrong.
+                 * If this is the first initialization call, then we
+                 * are converting the default nop to the ideal nop.
+                 */
+                if (init) {
+                        const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
+                        if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0))
+                                bug_at((void *)entry->code, __LINE__);
+                } else {
+                        code.jump = 0xe9;
+                        code.offset = entry->target -
+                                (entry->code + JUMP_LABEL_NOP_SIZE);
+                        if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))
+                                bug_at((void *)entry->code, __LINE__);
+                }
                 memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
+        }
 
         /*
          * Make text_poke_bp() a default fallback poker.
@@ -57,15 +96,38 @@ void arch_jump_label_transform(struct jump_entry *entry,
 {
         get_online_cpus();
         mutex_lock(&text_mutex);
-        __jump_label_transform(entry, type, NULL);
+        __jump_label_transform(entry, type, NULL, 0);
         mutex_unlock(&text_mutex);
         put_online_cpus();
 }
 
+static enum {
+        JL_STATE_START,
+        JL_STATE_NO_UPDATE,
+        JL_STATE_UPDATE,
+} jlstate __initdata_or_module = JL_STATE_START;
+
 __init_or_module void arch_jump_label_transform_static(struct jump_entry *entry,
                                       enum jump_label_type type)
 {
-        __jump_label_transform(entry, type, text_poke_early);
+        /*
+         * This function is called at boot up and when modules are
+         * first loaded. Check if the default nop, the one that is
+         * inserted at compile time, is the ideal nop. If it is, then
+         * we do not need to update the nop, and we can leave it as is.
+         * If it is not, then we need to update the nop to the ideal nop.
+         */
+        if (jlstate == JL_STATE_START) {
+                const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
+                const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
+
+                if (memcmp(ideal_nop, default_nop, 5) != 0)
+                        jlstate = JL_STATE_UPDATE;
+                else
+                        jlstate = JL_STATE_NO_UPDATE;
+        }
+        if (jlstate == JL_STATE_UPDATE)
+                __jump_label_transform(entry, type, text_poke_early, 1);
 }
 
 #endif