1 files changed, 136 insertions, 22 deletions

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 703130f469ec..af397cc98d05 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -52,10 +52,25 @@ static int __init setup_noreplace_paravirt(char *str)
 __setup("noreplace-paravirt", setup_noreplace_paravirt);
 #endif
 
-#define DPRINTK(fmt, ...)                               \
-do {                                                    \
-        if (debug_alternative)                          \
-                printk(KERN_DEBUG fmt, ##__VA_ARGS__);  \
+#define DPRINTK(fmt, args...)                                           \
+do {                                                                    \
+        if (debug_alternative)                                          \
+                printk(KERN_DEBUG "%s: " fmt "\n", __func__, ##args);   \
+} while (0)
+
+#define DUMP_BYTES(buf, len, fmt, args...)                              \
+do {                                                                    \
+        if (unlikely(debug_alternative)) {                              \
+                int j;                                                  \
+                                                                        \
+                if (!(len))                                             \
+                        break;                                          \
+                                                                        \
+                printk(KERN_DEBUG fmt, ##args);                         \
+                for (j = 0; j < (len) - 1; j++)                         \
+                        printk(KERN_CONT "%02hhx ", buf[j]);            \
+                printk(KERN_CONT "%02hhx\n", buf[j]);                   \
+        }                                                               \
 } while (0)
 
 /*
@@ -243,12 +258,86 @@ extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
 extern s32 __smp_locks[], __smp_locks_end[];
 void *text_poke_early(void *addr, const void *opcode, size_t len);
 
-/* Replace instructions with better alternatives for this CPU type.
-   This runs before SMP is initialized to avoid SMP problems with
-   self modifying code. This implies that asymmetric systems where
-   APs have less capabilities than the boot processor are not handled.
-   Tough. Make sure you disable such features by hand. */
+/*
+ * Are we looking at a near JMP with a 1 or 4-byte displacement.
+ */
+static inline bool is_jmp(const u8 opcode)
+{
+        return opcode == 0xeb || opcode == 0xe9;
+}
+
+static void __init_or_module
+recompute_jump(struct alt_instr *a, u8 *orig_insn, u8 *repl_insn, u8 *insnbuf)
+{
+        u8 *next_rip, *tgt_rip;
+        s32 n_dspl, o_dspl;
+        int repl_len;
+
+        if (a->replacementlen != 5)
+                return;
+
+        o_dspl = *(s32 *)(insnbuf + 1);
+
+        /* next_rip of the replacement JMP */
+        next_rip = repl_insn + a->replacementlen;
+        /* target rip of the replacement JMP */
+        tgt_rip  = next_rip + o_dspl;
+        n_dspl = tgt_rip - orig_insn;
+
+        DPRINTK("target RIP: %p, new_displ: 0x%x", tgt_rip, n_dspl);
+
+        if (tgt_rip - orig_insn >= 0) {
+                if (n_dspl - 2 <= 127)
+                        goto two_byte_jmp;
+                else
+                        goto five_byte_jmp;
+        /* negative offset */
+        } else {
+                if (((n_dspl - 2) & 0xff) == (n_dspl - 2))
+                        goto two_byte_jmp;
+                else
+                        goto five_byte_jmp;
+        }
+
+two_byte_jmp:
+        n_dspl -= 2;
+
+        insnbuf[0] = 0xeb;
+        insnbuf[1] = (s8)n_dspl;
+        add_nops(insnbuf + 2, 3);
+
+        repl_len = 2;
+        goto done;
+
+five_byte_jmp:
+        n_dspl -= 5;
 
+        insnbuf[0] = 0xe9;
+        *(s32 *)&insnbuf[1] = n_dspl;
+
+        repl_len = 5;
+
+done:
+
+        DPRINTK("final displ: 0x%08x, JMP 0x%lx",
+                n_dspl, (unsigned long)orig_insn + n_dspl + repl_len);
+}
+
+static void __init_or_module optimize_nops(struct alt_instr *a, u8 *instr)
+{
+        add_nops(instr + (a->instrlen - a->padlen), a->padlen);
+
+        DUMP_BYTES(instr, a->instrlen, "%p: [%d:%d) optimized NOPs: ",
+                   instr, a->instrlen - a->padlen, a->padlen);
+}
+
+/*
+ * Replace instructions with better alternatives for this CPU type. This runs
+ * before SMP is initialized to avoid SMP problems with self modifying code.
+ * This implies that asymmetric systems where APs have less capabilities than
+ * the boot processor are not handled. Tough. Make sure you disable such
+ * features by hand.
+ */
 void __init_or_module apply_alternatives(struct alt_instr *start,
                                          struct alt_instr *end)
 {
@@ -256,10 +345,10 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
         u8 *instr, *replacement;
         u8 insnbuf[MAX_PATCH_LEN];
 
-        DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
+        DPRINTK("alt table %p -> %p", start, end);
         /*
          * The scan order should be from start to end. A later scanned
-         * alternative code can overwrite a previous scanned alternative code.
+         * alternative code can overwrite previously scanned alternative code.
          * Some kernel functions (e.g. memcpy, memset, etc) use this order to
          * patch code.
          *
@@ -267,29 +356,54 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
          * order.
          */
         for (a = start; a < end; a++) {
+                int insnbuf_sz = 0;
+
                 instr = (u8 *)&a->instr_offset + a->instr_offset;
                 replacement = (u8 *)&a->repl_offset + a->repl_offset;
-                BUG_ON(a->replacementlen > a->instrlen);
                 BUG_ON(a->instrlen > sizeof(insnbuf));
                 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
-                if (!boot_cpu_has(a->cpuid))
+                if (!boot_cpu_has(a->cpuid)) {
+                        if (a->padlen > 1)
+                                optimize_nops(a, instr);
+
                         continue;
+                }
+
+                DPRINTK("feat: %d*32+%d, old: (%p, len: %d), repl: (%p, len: %d)",
+                        a->cpuid >> 5,
+                        a->cpuid & 0x1f,
+                        instr, a->instrlen,
+                        replacement, a->replacementlen);
+
+                DUMP_BYTES(instr, a->instrlen, "%p: old_insn: ", instr);
+                DUMP_BYTES(replacement, a->replacementlen, "%p: rpl_insn: ", replacement);
 
                 memcpy(insnbuf, replacement, a->replacementlen);
+                insnbuf_sz = a->replacementlen;
 
                 /* 0xe8 is a relative jump; fix the offset. */
-                if (*insnbuf == 0xe8 && a->replacementlen == 5)
-                    *(s32 *)(insnbuf + 1) += replacement - instr;
+                if (*insnbuf == 0xe8 && a->replacementlen == 5) {
+                        *(s32 *)(insnbuf + 1) += replacement - instr;
+                        DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",
+                                *(s32 *)(insnbuf + 1),
+                                (unsigned long)instr + *(s32 *)(insnbuf + 1) + 5);
+                }
+
+                if (a->replacementlen && is_jmp(replacement[0]))
+                        recompute_jump(a, instr, replacement, insnbuf);
 
-                add_nops(insnbuf + a->replacementlen,
-                         a->instrlen - a->replacementlen);
+                if (a->instrlen > a->replacementlen) {
+                        add_nops(insnbuf + a->replacementlen,
+                                 a->instrlen - a->replacementlen);
+                        insnbuf_sz += a->instrlen - a->replacementlen;
+                }
+                DUMP_BYTES(insnbuf, insnbuf_sz, "%p: final_insn: ", instr);
 
-                text_poke_early(instr, insnbuf, a->instrlen);
+                text_poke_early(instr, insnbuf, insnbuf_sz);
         }
 }
 
 #ifdef CONFIG_SMP
-
 static void alternatives_smp_lock(const s32 *start, const s32 *end,
                                   u8 *text, u8 *text_end)
 {
@@ -371,8 +485,8 @@ void __init_or_module alternatives_smp_module_add(struct module *mod,
         smp->locks_end  = locks_end;
         smp->text       = text;
         smp->text_end   = text_end;
-        DPRINTK("%s: locks %p -> %p, text %p -> %p, name %s\n",
-                __func__, smp->locks, smp->locks_end,
+        DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",
+                smp->locks, smp->locks_end,
                 smp->text, smp->text_end, smp->name);
 
         list_add_tail(&smp->next, &smp_alt_modules);
@@ -440,7 +554,7 @@ int alternatives_text_reserved(void *start, void *end)
 
         return 0;
 }
-#endif
+#endif /* CONFIG_SMP */
 
 #ifdef CONFIG_PARAVIRT
 void __init_or_module apply_paravirt(struct paravirt_patch_site *start,