x86: kprobe-booster for x86-64

This patch adds kprobe-booster to kprobes_64.c.

- Changes are based on x86-32.
- Add REX prefix checking code.

Signed-off-by: Masami Hiramatsu <mhiramat@redhat.com>
Signed-off-by: Jim Keniston <jkenisto@us.ibm.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

1 files changed, 139 insertions, 3 deletions

diff --git a/arch/x86/kernel/kprobes_64.c b/arch/x86/kernel/kprobes_64.c
index f6837cd3bed5..bf0e18473677 100644
--- a/arch/x86/kernel/kprobes_64.c
+++ b/arch/x86/kernel/kprobes_64.c
@@ -55,6 +55,105 @@ struct kretprobe_blackpoint kretprobe_blacklist[] = {
 };
 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
 
+/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
+static __always_inline void set_jmp_op(void *from, void *to)
+{
+        struct __arch_jmp_op {
+                char op;
+                s32 raddr;
+        } __attribute__((packed)) * jop;
+        jop = (struct __arch_jmp_op *)from;
+        jop->raddr = (s32)((long)(to) - ((long)(from) + 5));
+        jop->op = RELATIVEJUMP_INSTRUCTION;
+}
+
+/*
+ * returns non-zero if opcode is boostable
+ * RIP relative instructions are adjusted at copying time
+ */
+static __always_inline int can_boost(kprobe_opcode_t *opcodes)
+{
+#define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
+        (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) |   \
+          (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) |   \
+          (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) |   \
+          (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf))    \
+         << (row % 64))
+        /*
+         * Undefined/reserved opcodes, conditional jump, Opcode Extension
+         * Groups, and some special opcodes can not boost.
+         */
+        static const unsigned long twobyte_is_boostable[256 / 64] = {
+                /*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f        */
+                /*      ----------------------------------------------        */
+                W(0x00, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0)|/* 00 */
+                W(0x10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)|/* 10 */
+                W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)|/* 20 */
+                W(0x30, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),/* 30 */
+                W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)|/* 40 */
+                W(0x50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)|/* 50 */
+                W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1)|/* 60 */
+                W(0x70, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1),/* 70 */
+                W(0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)|/* 80 */
+                W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1)|/* 90 */
+                W(0xa0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1)|/* a0 */
+                W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1),/* b0 */
+                W(0xc0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1)|/* c0 */
+                W(0xd0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1)|/* d0 */
+                W(0xe0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 1)|/* e0 */
+                W(0xf0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0) /* f0 */
+                /*      -----------------------------------------------       */
+                /*      0  1  2  3  4  5  6  7  8  9  a  b  c  d  e  f        */
+        };
+#undef W
+        kprobe_opcode_t opcode;
+        kprobe_opcode_t *orig_opcodes = opcodes;
+
+retry:
+        if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
+                return 0;
+        opcode = *(opcodes++);
+
+        /* 2nd-byte opcode */
+        if (opcode == 0x0f) {
+                if (opcodes - orig_opcodes > MAX_INSN_SIZE - 1)
+                        return 0;
+                return test_bit(*opcodes, twobyte_is_boostable);
+        }
+
+        switch (opcode & 0xf0) {
+        case 0x40:
+                goto retry; /* REX prefix is boostable */
+        case 0x60:
+                if (0x63 < opcode && opcode < 0x67)
+                        goto retry; /* prefixes */
+                /* can't boost Address-size override and bound */
+                return (opcode != 0x62 && opcode != 0x67);
+        case 0x70:
+                return 0; /* can't boost conditional jump */
+        case 0xc0:
+                /* can't boost software-interruptions */
+                return (0xc1 < opcode && opcode < 0xcc) || opcode == 0xcf;
+        case 0xd0:
+                /* can boost AA* and XLAT */
+                return (opcode == 0xd4 || opcode == 0xd5 || opcode == 0xd7);
+        case 0xe0:
+                /* can boost in/out and absolute jmps */
+                return ((opcode & 0x04) || opcode == 0xea);
+        case 0xf0:
+                if ((opcode & 0x0c) == 0 && opcode != 0xf1)
+                        goto retry; /* lock/rep(ne) prefix */
+                /* clear and set flags are boostable */
+                return (opcode == 0xf5 || (0xf7 < opcode && opcode < 0xfe));
+        default:
+                /* segment override prefixes are boostable */
+                if (opcode == 0x26 || opcode == 0x36 || opcode == 0x3e)
+                        goto retry; /* prefixes */
+                /* CS override prefix and call are not boostable */
+                return (opcode != 0x2e && opcode != 0x9a);
+        }
+}
+
 /*
  * returns non-zero if opcode modifies the interrupt flag.
  */
@@ -86,7 +185,7 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p)
 
 /*
  * Determine if the instruction uses the %rip-relative addressing mode.
- * If it does, return the address of the 32-bit displacement word.
+ * If it does, Return the address of the 32-bit displacement word.
  * If not, return null.
  */
 static s32 __kprobes *is_riprel(u8 *insn)
@@ -210,6 +309,11 @@ static void __kprobes arch_copy_kprobe(struct kprobe *p)
                 BUG_ON((s64) (s32) disp != disp); /* Sanity check.  */
                 *ripdisp = disp;
         }
+        if (can_boost(p->addr)) {
+                p->ainsn.boostable = 0;
+        } else {
+                p->ainsn.boostable = -1;
+        }
         p->opcode = *p->addr;
 }
 
@@ -226,7 +330,7 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p)
 void __kprobes arch_remove_kprobe(struct kprobe *p)
 {
         mutex_lock(&kprobe_mutex);
-        free_insn_slot(p->ainsn.insn, 0);
+        free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
         mutex_unlock(&kprobe_mutex);
 }
 
@@ -384,6 +488,15 @@ int __kprobes kprobe_handler(struct pt_regs *regs)
                 return 1;
 
 ss_probe:
+#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PM)
+        if (p->ainsn.boostable == 1 && !p->post_handler) {
+                /* Boost up -- we can execute copied instructions directly */
+                reset_current_kprobe();
+                regs->ip = (unsigned long)p->ainsn.insn;
+                preempt_enable_no_resched();
+                return 1;
+        }
+#endif
         prepare_singlestep(p, regs);
         kcb->kprobe_status = KPROBE_HIT_SS;
         return 1;
@@ -493,6 +606,11 @@ int __kprobes trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
  * 2) If the single-stepped instruction was a call, the return address
  * that is atop the stack is the address following the copied instruction.
  * We need to make it the address following the original instruction.
+ *
+ * If this is the first time we've single-stepped the instruction at
+ * this probepoint, and the instruction is boostable, boost it: add a
+ * jump instruction after the copied instruction, that jumps to the next
+ * instruction after the probepoint.
  */
 static void __kprobes resume_execution(struct kprobe *p,
                 struct pt_regs *regs, struct kprobe_ctlblk *kcb)
@@ -519,6 +637,7 @@ static void __kprobes resume_execution(struct kprobe *p,
         case 0xcf:
         case 0xea:      /* jmp absolute -- ip is correct */
                 /* ip is already adjusted, no more changes required */
+                p->ainsn.boostable = 1;
                 goto no_change;
         case 0xe8:      /* call relative - Fix return addr */
                 *tos = orig_rip + (*tos - copy_rip);
@@ -527,17 +646,34 @@ static void __kprobes resume_execution(struct kprobe *p,
                 if ((insn[1] & 0x30) == 0x10) {
                         /* call absolute, indirect */
                         /* Fix return addr; ip is correct. */
+                        /* not boostable */
                         *tos = orig_rip + (*tos - copy_rip);
                         goto no_change;
                 } else if (((insn[1] & 0x31) == 0x20) ||        /* jmp near, absolute indirect */
                            ((insn[1] & 0x31) == 0x21)) {        /* jmp far, absolute indirect */
-                        /* ip is correct. */
+                        /* ip is correct. And this is boostable */
+                        p->ainsn.boostable = 1;
                         goto no_change;
                 }
         default:
                 break;
         }
 
+        if (p->ainsn.boostable == 0) {
+                if ((regs->ip > copy_rip) &&
+                    (regs->ip - copy_rip) + 5 < MAX_INSN_SIZE) {
+                        /*
+                         * These instructions can be executed directly if it
+                         * jumps back to correct address.
+                         */
+                        set_jmp_op((void *)regs->ip,
+                                   (void *)orig_rip + (regs->ip - copy_rip));
+                        p->ainsn.boostable = 1;
+                } else {
+                        p->ainsn.boostable = -1;
+                }
+        }
+
         regs->ip = orig_rip + (regs->ip - copy_rip);
 
 no_change: