Merge branch 'perf-probes-for-linus-2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'perf-probes-for-linus-2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip:
  x86: Issue at least one memory barrier in stop_machine_text_poke()
  perf probe: Correct probe syntax on command line help
  perf probe: Add lazy line matching support
  perf probe: Show more lines after last line
  perf probe: Check function address range strictly in line finder
  perf probe: Use libdw callback routines
  perf probe: Use elfutils-libdw for analyzing debuginfo
  perf probe: Rename probe finder functions
  perf probe: Fix bugs in line range finder
  perf probe: Update perf probe document
  perf probe: Do not show --line option without dwarf support
  kprobes: Add documents of jump optimization
  kprobes/x86: Support kprobes jump optimization on x86
  x86: Add text_poke_smp for SMP cross modifying code
  kprobes/x86: Cleanup save/restore registers
  kprobes/x86: Boost probes when reentering
  kprobes: Jump optimization sysctl interface
  kprobes: Introduce kprobes jump optimization
  kprobes: Introduce generic insn_slot framework
  kprobes/x86: Cleanup RELATIVEJUMP_INSTRUCTION to RELATIVEJUMP_OPCODE

6 files changed, 611 insertions, 107 deletions

diff --git a/arch/Kconfig b/arch/Kconfig
index 215e46073c45..e5eb1337a537 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -41,6 +41,17 @@ config KPROBES
           for kernel debugging, non-intrusive instrumentation and testing.
           If in doubt, say "N".
 
+config OPTPROBES
+        bool "Kprobes jump optimization support (EXPERIMENTAL)"
+        default y
+        depends on KPROBES
+        depends on !PREEMPT
+        depends on HAVE_OPTPROBES
+        select KALLSYMS_ALL
+        help
+          This option will allow kprobes to optimize breakpoint to
+          a jump for reducing its overhead.
+
 config HAVE_EFFICIENT_UNALIGNED_ACCESS
         bool
         help
@@ -83,6 +94,8 @@ config HAVE_KPROBES
 config HAVE_KRETPROBES
         bool
 
+config HAVE_OPTPROBES
+        bool
 #
 # An arch should select this if it provides all these things:
 #
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 57ccdcec1469..f15f37bfbd62 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -31,6 +31,7 @@ config X86
         select ARCH_WANT_FRAME_POINTERS
         select HAVE_DMA_ATTRS
         select HAVE_KRETPROBES
+        select HAVE_OPTPROBES
         select HAVE_FTRACE_MCOUNT_RECORD
         select HAVE_DYNAMIC_FTRACE
         select HAVE_FUNCTION_TRACER
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index f1e253ceba4b..b09ec55650b3 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -165,10 +165,12 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
  * invalid instruction possible) or if the instructions are changed from a
  * consistent state to another consistent state atomically.
  * More care must be taken when modifying code in the SMP case because of
- * Intel's errata.
+ * Intel's errata. text_poke_smp() takes care that errata, but still
+ * doesn't support NMI/MCE handler code modifying.
  * On the local CPU you need to be protected again NMI or MCE handlers seeing an
  * inconsistent instruction while you patch.
  */
 extern void *text_poke(void *addr, const void *opcode, size_t len);
+extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
 
 #endif /* _ASM_X86_ALTERNATIVE_H */
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index 4fe681de1e76..4ffa345a8ccb 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -32,7 +32,10 @@ struct kprobe;
 
 typedef u8 kprobe_opcode_t;
 #define BREAKPOINT_INSTRUCTION  0xcc
-#define RELATIVEJUMP_INSTRUCTION 0xe9
+#define RELATIVEJUMP_OPCODE 0xe9
+#define RELATIVEJUMP_SIZE 5
+#define RELATIVECALL_OPCODE 0xe8
+#define RELATIVE_ADDR_SIZE 4
 #define MAX_INSN_SIZE 16
 #define MAX_STACK_SIZE 64
 #define MIN_STACK_SIZE(ADDR)                                           \
@@ -44,6 +47,17 @@ typedef u8 kprobe_opcode_t;
 
 #define flush_insn_slot(p)      do { } while (0)
 
+/* optinsn template addresses */
+extern kprobe_opcode_t optprobe_template_entry;
+extern kprobe_opcode_t optprobe_template_val;
+extern kprobe_opcode_t optprobe_template_call;
+extern kprobe_opcode_t optprobe_template_end;
+#define MAX_OPTIMIZED_LENGTH (MAX_INSN_SIZE + RELATIVE_ADDR_SIZE)
+#define MAX_OPTINSN_SIZE                                \
+        (((unsigned long)&optprobe_template_end -       \
+          (unsigned long)&optprobe_template_entry) +    \
+         MAX_OPTIMIZED_LENGTH + RELATIVEJUMP_SIZE)
+
 extern const int kretprobe_blacklist_size;
 
 void arch_remove_kprobe(struct kprobe *p);
@@ -64,6 +78,21 @@ struct arch_specific_insn {
         int boostable;
 };
 
+struct arch_optimized_insn {
+        /* copy of the original instructions */
+        kprobe_opcode_t copied_insn[RELATIVE_ADDR_SIZE];
+        /* detour code buffer */
+        kprobe_opcode_t *insn;
+        /* the size of instructions copied to detour code buffer */
+        size_t size;
+};
+
+/* Return true (!0) if optinsn is prepared for optimization. */
+static inline int arch_prepared_optinsn(struct arch_optimized_insn *optinsn)
+{
+        return optinsn->size;
+}
+
 struct prev_kprobe {
         struct kprobe *kp;
         unsigned long status;
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index e6ea0342c8f8..3a4bf35c179b 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -7,6 +7,7 @@
 #include <linux/mm.h>
 #include <linux/vmalloc.h>
 #include <linux/memory.h>
+#include <linux/stop_machine.h>
 #include <asm/alternative.h>
 #include <asm/sections.h>
 #include <asm/pgtable.h>
@@ -572,3 +573,62 @@ void *__kprobes text_poke(void *addr, const void *opcode, size_t len)
         local_irq_restore(flags);
         return addr;
 }
+
+/*
+ * Cross-modifying kernel text with stop_machine().
+ * This code originally comes from immediate value.
+ */
+static atomic_t stop_machine_first;
+static int wrote_text;
+
+struct text_poke_params {
+        void *addr;
+        const void *opcode;
+        size_t len;
+};
+
+static int __kprobes stop_machine_text_poke(void *data)
+{
+        struct text_poke_params *tpp = data;
+
+        if (atomic_dec_and_test(&stop_machine_first)) {
+                text_poke(tpp->addr, tpp->opcode, tpp->len);
+                smp_wmb();      /* Make sure other cpus see that this has run */
+                wrote_text = 1;
+        } else {
+                while (!wrote_text)
+                        cpu_relax();
+                smp_mb();       /* Load wrote_text before following execution */
+        }
+
+        flush_icache_range((unsigned long)tpp->addr,
+                           (unsigned long)tpp->addr + tpp->len);
+        return 0;
+}
+
+/**
+ * text_poke_smp - Update instructions on a live kernel on SMP
+ * @addr: address to modify
+ * @opcode: source of the copy
+ * @len: length to copy
+ *
+ * Modify multi-byte instruction by using stop_machine() on SMP. This allows
+ * user to poke/set multi-byte text on SMP. Only non-NMI/MCE code modifying
+ * should be allowed, since stop_machine() does _not_ protect code against
+ * NMI and MCE.
+ *
+ * Note: Must be called under get_online_cpus() and text_mutex.
+ */
+void *__kprobes text_poke_smp(void *addr, const void *opcode, size_t len)
+{
+        struct text_poke_params tpp;
+
+        tpp.addr = addr;
+        tpp.opcode = opcode;
+        tpp.len = len;
+        atomic_set(&stop_machine_first, 1);
+        wrote_text = 0;
+        stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
+        return addr;
+}
+
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index 5de9f4a9c3fd..b43bbaebe2c0 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -49,6 +49,7 @@
 #include <linux/module.h>
 #include <linux/kdebug.h>
 #include <linux/kallsyms.h>
+#include <linux/ftrace.h>
 
 #include <asm/cacheflush.h>
 #include <asm/desc.h>
@@ -106,16 +107,22 @@ struct kretprobe_blackpoint kretprobe_blacklist[] = {
 };
 const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
 
-/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
-static void __kprobes set_jmp_op(void *from, void *to)
+static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
 {
-        struct __arch_jmp_op {
-                char op;
+        struct __arch_relative_insn {
+                u8 op;
                 s32 raddr;
-        } __attribute__((packed)) * jop;
-        jop = (struct __arch_jmp_op *)from;
-        jop->raddr = (s32)((long)(to) - ((long)(from) + 5));
-        jop->op = RELATIVEJUMP_INSTRUCTION;
+        } __attribute__((packed)) *insn;
+
+        insn = (struct __arch_relative_insn *)from;
+        insn->raddr = (s32)((long)(to) - ((long)(from) + 5));
+        insn->op = op;
+}
+
+/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
+static void __kprobes synthesize_reljump(void *from, void *to)
+{
+        __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
 }
 
 /*
@@ -202,7 +209,7 @@ static int recover_probed_instruction(kprobe_opcode_t *buf, unsigned long addr)
         /*
          *  Basically, kp->ainsn.insn has an original instruction.
          *  However, RIP-relative instruction can not do single-stepping
-         *  at different place, fix_riprel() tweaks the displacement of
+         *  at different place, __copy_instruction() tweaks the displacement of
          *  that instruction. In that case, we can't recover the instruction
          *  from the kp->ainsn.insn.
          *
@@ -284,21 +291,37 @@ static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
 }
 
 /*
- * Adjust the displacement if the instruction uses the %rip-relative
- * addressing mode.
+ * Copy an instruction and adjust the displacement if the instruction
+ * uses the %rip-relative addressing mode.
  * If it does, Return the address of the 32-bit displacement word.
  * If not, return null.
  * Only applicable to 64-bit x86.
  */
-static void __kprobes fix_riprel(struct kprobe *p)
+static int __kprobes __copy_instruction(u8 *dest, u8 *src, int recover)
 {
-#ifdef CONFIG_X86_64
         struct insn insn;
-        kernel_insn_init(&insn, p->ainsn.insn);
+        int ret;
+        kprobe_opcode_t buf[MAX_INSN_SIZE];
 
+        kernel_insn_init(&insn, src);
+        if (recover) {
+                insn_get_opcode(&insn);
+                if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
+                        ret = recover_probed_instruction(buf,
+                                                         (unsigned long)src);
+                        if (ret)
+                                return 0;
+                        kernel_insn_init(&insn, buf);
+                }
+        }
+        insn_get_length(&insn);
+        memcpy(dest, insn.kaddr, insn.length);
+
+#ifdef CONFIG_X86_64
         if (insn_rip_relative(&insn)) {
                 s64 newdisp;
                 u8 *disp;
+                kernel_insn_init(&insn, dest);
                 insn_get_displacement(&insn);
                 /*
                  * The copied instruction uses the %rip-relative addressing
@@ -312,20 +335,23 @@ static void __kprobes fix_riprel(struct kprobe *p)
                  * extension of the original signed 32-bit displacement would
                  * have given.
                  */
-                newdisp = (u8 *) p->addr + (s64) insn.displacement.value -
-                          (u8 *) p->ainsn.insn;
+                newdisp = (u8 *) src + (s64) insn.displacement.value -
+                          (u8 *) dest;
                 BUG_ON((s64) (s32) newdisp != newdisp); /* Sanity check.  */
-                disp = (u8 *) p->ainsn.insn + insn_offset_displacement(&insn);
+                disp = (u8 *) dest + insn_offset_displacement(&insn);
                 *(s32 *) disp = (s32) newdisp;
         }
 #endif
+        return insn.length;
 }
 
 static void __kprobes arch_copy_kprobe(struct kprobe *p)
 {
-        memcpy(p->ainsn.insn, p->addr, MAX_INSN_SIZE * sizeof(kprobe_opcode_t));
-
-        fix_riprel(p);
+        /*
+         * Copy an instruction without recovering int3, because it will be
+         * put by another subsystem.
+         */
+        __copy_instruction(p->ainsn.insn, p->addr, 0);
 
         if (can_boost(p->addr))
                 p->ainsn.boostable = 0;
@@ -406,18 +432,6 @@ static void __kprobes restore_btf(void)
                 update_debugctlmsr(current->thread.debugctlmsr);
 }
 
-static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs)
-{
-        clear_btf();
-        regs->flags |= X86_EFLAGS_TF;
-        regs->flags &= ~X86_EFLAGS_IF;
-        /* single step inline if the instruction is an int3 */
-        if (p->opcode == BREAKPOINT_INSTRUCTION)
-                regs->ip = (unsigned long)p->addr;
-        else
-                regs->ip = (unsigned long)p->ainsn.insn;
-}
-
 void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
                                       struct pt_regs *regs)
 {
@@ -429,20 +443,50 @@ void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
         *sara = (unsigned long) &kretprobe_trampoline;
 }
 
+#ifdef CONFIG_OPTPROBES
+static int  __kprobes setup_detour_execution(struct kprobe *p,
+                                             struct pt_regs *regs,
+                                             int reenter);
+#else
+#define setup_detour_execution(p, regs, reenter) (0)
+#endif
+
 static void __kprobes setup_singlestep(struct kprobe *p, struct pt_regs *regs,
-                                       struct kprobe_ctlblk *kcb)
+                                       struct kprobe_ctlblk *kcb, int reenter)
 {
+        if (setup_detour_execution(p, regs, reenter))
+                return;
+
 #if !defined(CONFIG_PREEMPT)
         if (p->ainsn.boostable == 1 && !p->post_handler) {
                 /* Boost up -- we can execute copied instructions directly */
-                reset_current_kprobe();
+                if (!reenter)
+                        reset_current_kprobe();
+                /*
+                 * Reentering boosted probe doesn't reset current_kprobe,
+                 * nor set current_kprobe, because it doesn't use single
+                 * stepping.
+                 */
                 regs->ip = (unsigned long)p->ainsn.insn;
                 preempt_enable_no_resched();
                 return;
         }
 #endif
-        prepare_singlestep(p, regs);
-        kcb->kprobe_status = KPROBE_HIT_SS;
+        if (reenter) {
+                save_previous_kprobe(kcb);
+                set_current_kprobe(p, regs, kcb);
+                kcb->kprobe_status = KPROBE_REENTER;
+        } else
+                kcb->kprobe_status = KPROBE_HIT_SS;
+        /* Prepare real single stepping */
+        clear_btf();
+        regs->flags |= X86_EFLAGS_TF;
+        regs->flags &= ~X86_EFLAGS_IF;
+        /* single step inline if the instruction is an int3 */
+        if (p->opcode == BREAKPOINT_INSTRUCTION)
+                regs->ip = (unsigned long)p->addr;
+        else
+                regs->ip = (unsigned long)p->ainsn.insn;
 }
 
 /*
@@ -456,11 +500,8 @@ static int __kprobes reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
         switch (kcb->kprobe_status) {
         case KPROBE_HIT_SSDONE:
         case KPROBE_HIT_ACTIVE:
-                save_previous_kprobe(kcb);
-                set_current_kprobe(p, regs, kcb);
                 kprobes_inc_nmissed_count(p);
-                prepare_singlestep(p, regs);
-                kcb->kprobe_status = KPROBE_REENTER;
+                setup_singlestep(p, regs, kcb, 1);
                 break;
         case KPROBE_HIT_SS:
                 /* A probe has been hit in the codepath leading up to, or just
@@ -535,13 +576,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
                          * more here.
                          */
                         if (!p->pre_handler || !p->pre_handler(p, regs))
-                                setup_singlestep(p, regs, kcb);
+                                setup_singlestep(p, regs, kcb, 0);
                         return 1;
                 }
         } else if (kprobe_running()) {
                 p = __get_cpu_var(current_kprobe);
                 if (p->break_handler && p->break_handler(p, regs)) {
-                        setup_singlestep(p, regs, kcb);
+                        setup_singlestep(p, regs, kcb, 0);
                         return 1;
                 }
         } /* else: not a kprobe fault; let the kernel handle it */
@@ -550,6 +591,69 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
         return 0;
 }
 
+#ifdef CONFIG_X86_64
+#define SAVE_REGS_STRING                \
+        /* Skip cs, ip, orig_ax. */     \
+        "       subq $24, %rsp\n"       \
+        "       pushq %rdi\n"           \
+        "       pushq %rsi\n"           \
+        "       pushq %rdx\n"           \
+        "       pushq %rcx\n"           \
+        "       pushq %rax\n"           \
+        "       pushq %r8\n"            \
+        "       pushq %r9\n"            \
+        "       pushq %r10\n"           \
+        "       pushq %r11\n"           \
+        "       pushq %rbx\n"           \
+        "       pushq %rbp\n"           \
+        "       pushq %r12\n"           \
+        "       pushq %r13\n"           \
+        "       pushq %r14\n"           \
+        "       pushq %r15\n"
+#define RESTORE_REGS_STRING             \
+        "       popq %r15\n"            \
+        "       popq %r14\n"            \
+        "       popq %r13\n"            \
+        "       popq %r12\n"            \
+        "       popq %rbp\n"            \
+        "       popq %rbx\n"            \
+        "       popq %r11\n"            \
+        "       popq %r10\n"            \
+        "       popq %r9\n"             \
+        "       popq %r8\n"             \
+        "       popq %rax\n"            \
+        "       popq %rcx\n"            \
+        "       popq %rdx\n"            \
+        "       popq %rsi\n"            \
+        "       popq %rdi\n"            \
+        /* Skip orig_ax, ip, cs */      \
+        "       addq $24, %rsp\n"
+#else
+#define SAVE_REGS_STRING                \
+        /* Skip cs, ip, orig_ax and gs. */      \
+        "       subl $16, %esp\n"       \
+        "       pushl %fs\n"            \
+        "       pushl %ds\n"            \
+        "       pushl %es\n"            \
+        "       pushl %eax\n"           \
+        "       pushl %ebp\n"           \
+        "       pushl %edi\n"           \
+        "       pushl %esi\n"           \
+        "       pushl %edx\n"           \
+        "       pushl %ecx\n"           \
+        "       pushl %ebx\n"
+#define RESTORE_REGS_STRING             \
+        "       popl %ebx\n"            \
+        "       popl %ecx\n"            \
+        "       popl %edx\n"            \
+        "       popl %esi\n"            \
+        "       popl %edi\n"            \
+        "       popl %ebp\n"            \
+        "       popl %eax\n"            \
+        /* Skip ds, es, fs, gs, orig_ax, and ip. Note: don't pop cs here*/\
+        "       addl $24, %esp\n"
+#endif
+
 /*
  * When a retprobed function returns, this code saves registers and
  * calls trampoline_handler() runs, which calls the kretprobe's handler.
@@ -563,65 +667,16 @@ static void __used __kprobes kretprobe_trampoline_holder(void)
                         /* We don't bother saving the ss register */
                         "       pushq %rsp\n"
                         "       pushfq\n"
-                        /*
-                         * Skip cs, ip, orig_ax.
-                         * trampoline_handler() will plug in these values
-                         */
-                        "       subq $24, %rsp\n"
-                        "       pushq %rdi\n"
-                        "       pushq %rsi\n"
-                        "       pushq %rdx\n"
-                        "       pushq %rcx\n"
-                        "       pushq %rax\n"
-                        "       pushq %r8\n"
-                        "       pushq %r9\n"
-                        "       pushq %r10\n"
-                        "       pushq %r11\n"
-                        "       pushq %rbx\n"
-                        "       pushq %rbp\n"
-                        "       pushq %r12\n"
-                        "       pushq %r13\n"
-                        "       pushq %r14\n"
-                        "       pushq %r15\n"
+                        SAVE_REGS_STRING
                         "       movq %rsp, %rdi\n"
                         "       call trampoline_handler\n"
                         /* Replace saved sp with true return address. */
                         "       movq %rax, 152(%rsp)\n"
-                        "       popq %r15\n"
-                        "       popq %r14\n"
-                        "       popq %r13\n"
-                        "       popq %r12\n"
-                        "       popq %rbp\n"
-                        "       popq %rbx\n"
-                        "       popq %r11\n"
-                        "       popq %r10\n"
-                        "       popq %r9\n"
-                        "       popq %r8\n"
-                        "       popq %rax\n"
-                        "       popq %rcx\n"
-                        "       popq %rdx\n"
-                        "       popq %rsi\n"
-                        "       popq %rdi\n"
-                        /* Skip orig_ax, ip, cs */
-                        "       addq $24, %rsp\n"
+                        RESTORE_REGS_STRING
                         "       popfq\n"
 #else
                         "       pushf\n"
-                        /*
-                         * Skip cs, ip, orig_ax and gs.
-                         * trampoline_handler() will plug in these values
-                         */
-                        "       subl $16, %esp\n"
-                        "       pushl %fs\n"
-                        "       pushl %es\n"
-                        "       pushl %ds\n"
-                        "       pushl %eax\n"
-                        "       pushl %ebp\n"
-                        "       pushl %edi\n"
-                        "       pushl %esi\n"
-                        "       pushl %edx\n"
-                        "       pushl %ecx\n"
-                        "       pushl %ebx\n"
+                        SAVE_REGS_STRING
                         "       movl %esp, %eax\n"
                         "       call trampoline_handler\n"
                         /* Move flags to cs */
@@ -629,15 +684,7 @@ static void __used __kprobes kretprobe_trampoline_holder(void)
                         "       movl %edx, 52(%esp)\n"
                         /* Replace saved flags with true return address. */
                         "       movl %eax, 56(%esp)\n"
-                        "       popl %ebx\n"
-                        "       popl %ecx\n"
-                        "       popl %edx\n"
-                        "       popl %esi\n"
-                        "       popl %edi\n"
-                        "       popl %ebp\n"
-                        "       popl %eax\n"
-                        /* Skip ds, es, fs, gs, orig_ax and ip */
-                        "       addl $24, %esp\n"
+                        RESTORE_REGS_STRING
                         "       popf\n"
 #endif
                         "       ret\n");
@@ -805,8 +852,8 @@ static void __kprobes resume_execution(struct kprobe *p,
                          * These instructions can be executed directly if it
                          * jumps back to correct address.
                          */
-                        set_jmp_op((void *)regs->ip,
-                                   (void *)orig_ip + (regs->ip - copy_ip));
+                        synthesize_reljump((void *)regs->ip,
+                                (void *)orig_ip + (regs->ip - copy_ip));
                         p->ainsn.boostable = 1;
                 } else {
                         p->ainsn.boostable = -1;
@@ -1033,6 +1080,358 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
         return 0;
 }
 
+
+#ifdef CONFIG_OPTPROBES
+
+/* Insert a call instruction at address 'from', which calls address 'to'.*/
+static void __kprobes synthesize_relcall(void *from, void *to)
+{
+        __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
+}
+
+/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */
+static void __kprobes synthesize_set_arg1(kprobe_opcode_t *addr,
+                                          unsigned long val)
+{
+#ifdef CONFIG_X86_64
+        *addr++ = 0x48;
+        *addr++ = 0xbf;
+#else
+        *addr++ = 0xb8;
+#endif
+        *(unsigned long *)addr = val;
+}
+
+void __kprobes kprobes_optinsn_template_holder(void)
+{
+        asm volatile (
+                        ".global optprobe_template_entry\n"
+                        "optprobe_template_entry: \n"
+#ifdef CONFIG_X86_64
+                        /* We don't bother saving the ss register */
+                        "       pushq %rsp\n"
+                        "       pushfq\n"
+                        SAVE_REGS_STRING
+                        "       movq %rsp, %rsi\n"
+                        ".global optprobe_template_val\n"
+                        "optprobe_template_val: \n"
+                        ASM_NOP5
+                        ASM_NOP5
+                        ".global optprobe_template_call\n"
+                        "optprobe_template_call: \n"
+                        ASM_NOP5
+                        /* Move flags to rsp */
+                        "       movq 144(%rsp), %rdx\n"
+                        "       movq %rdx, 152(%rsp)\n"
+                        RESTORE_REGS_STRING
+                        /* Skip flags entry */
+                        "       addq $8, %rsp\n"
+                        "       popfq\n"
+#else /* CONFIG_X86_32 */
+                        "       pushf\n"
+                        SAVE_REGS_STRING
+                        "       movl %esp, %edx\n"
+                        ".global optprobe_template_val\n"
+                        "optprobe_template_val: \n"
+                        ASM_NOP5
+                        ".global optprobe_template_call\n"
+                        "optprobe_template_call: \n"
+                        ASM_NOP5
+                        RESTORE_REGS_STRING
+                        "       addl $4, %esp\n"        /* skip cs */
+                        "       popf\n"
+#endif
+                        ".global optprobe_template_end\n"
+                        "optprobe_template_end: \n");
+}
+
+#define TMPL_MOVE_IDX \
+        ((long)&optprobe_template_val - (long)&optprobe_template_entry)
+#define TMPL_CALL_IDX \
+        ((long)&optprobe_template_call - (long)&optprobe_template_entry)
+#define TMPL_END_IDX \
+        ((long)&optprobe_template_end - (long)&optprobe_template_entry)
+
+#define INT3_SIZE sizeof(kprobe_opcode_t)
+
+/* Optimized kprobe call back function: called from optinsn */
+static void __kprobes optimized_callback(struct optimized_kprobe *op,
+                                         struct pt_regs *regs)
+{
+        struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
+
+        preempt_disable();
+        if (kprobe_running()) {
+                kprobes_inc_nmissed_count(&op->kp);
+        } else {
+                /* Save skipped registers */
+#ifdef CONFIG_X86_64
+                regs->cs = __KERNEL_CS;
+#else
+                regs->cs = __KERNEL_CS | get_kernel_rpl();
+                regs->gs = 0;
+#endif
+                regs->ip = (unsigned long)op->kp.addr + INT3_SIZE;
+                regs->orig_ax = ~0UL;
+
+                __get_cpu_var(current_kprobe) = &op->kp;
+                kcb->kprobe_status = KPROBE_HIT_ACTIVE;
+                opt_pre_handler(&op->kp, regs);
+                __get_cpu_var(current_kprobe) = NULL;
+        }
+        preempt_enable_no_resched();
+}
+
+static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
+{
+        int len = 0, ret;
+
+        while (len < RELATIVEJUMP_SIZE) {
+                ret = __copy_instruction(dest + len, src + len, 1);
+                if (!ret || !can_boost(dest + len))
+                        return -EINVAL;
+                len += ret;
+        }
+        /* Check whether the address range is reserved */
+        if (ftrace_text_reserved(src, src + len - 1) ||
+            alternatives_text_reserved(src, src + len - 1))
+                return -EBUSY;
+
+        return len;
+}
+
+/* Check whether insn is indirect jump */
+static int __kprobes insn_is_indirect_jump(struct insn *insn)
+{
+        return ((insn->opcode.bytes[0] == 0xff &&
+                (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */
+                insn->opcode.bytes[0] == 0xea); /* Segment based jump */
+}
+
+/* Check whether insn jumps into specified address range */
+static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
+{
+        unsigned long target = 0;
+
+        switch (insn->opcode.bytes[0]) {
+        case 0xe0:      /* loopne */
+        case 0xe1:      /* loope */
+        case 0xe2:      /* loop */
+        case 0xe3:      /* jcxz */
+        case 0xe9:      /* near relative jump */
+        case 0xeb:      /* short relative jump */
+                break;
+        case 0x0f:
+                if ((insn->opcode.bytes[1] & 0xf0) == 0x80) /* jcc near */
+                        break;
+                return 0;
+        default:
+                if ((insn->opcode.bytes[0] & 0xf0) == 0x70) /* jcc short */
+                        break;
+                return 0;
+        }
+        target = (unsigned long)insn->next_byte + insn->immediate.value;
+
+        return (start <= target && target <= start + len);
+}
+
+/* Decode whole function to ensure any instructions don't jump into target */
+static int __kprobes can_optimize(unsigned long paddr)
+{
+        int ret;
+        unsigned long addr, size = 0, offset = 0;
+        struct insn insn;
+        kprobe_opcode_t buf[MAX_INSN_SIZE];
+        /* Dummy buffers for lookup_symbol_attrs */
+        static char __dummy_buf[KSYM_NAME_LEN];
+
+        /* Lookup symbol including addr */
+        if (!kallsyms_lookup(paddr, &size, &offset, NULL, __dummy_buf))
+                return 0;
+
+        /* Check there is enough space for a relative jump. */
+        if (size - offset < RELATIVEJUMP_SIZE)
+                return 0;
+
+        /* Decode instructions */
+        addr = paddr - offset;
+        while (addr < paddr - offset + size) { /* Decode until function end */
+                if (search_exception_tables(addr))
+                        /*
+                         * Since some fixup code will jumps into this function,
+                         * we can't optimize kprobe in this function.
+                         */
+                        return 0;
+                kernel_insn_init(&insn, (void *)addr);
+                insn_get_opcode(&insn);
+                if (insn.opcode.bytes[0] == BREAKPOINT_INSTRUCTION) {
+                        ret = recover_probed_instruction(buf, addr);
+                        if (ret)
+                                return 0;
+                        kernel_insn_init(&insn, buf);
+                }
+                insn_get_length(&insn);
+                /* Recover address */
+                insn.kaddr = (void *)addr;
+                insn.next_byte = (void *)(addr + insn.length);
+                /* Check any instructions don't jump into target */
+                if (insn_is_indirect_jump(&insn) ||
+                    insn_jump_into_range(&insn, paddr + INT3_SIZE,
+                                         RELATIVE_ADDR_SIZE))
+                        return 0;
+                addr += insn.length;
+        }
+
+        return 1;
+}
+
+/* Check optimized_kprobe can actually be optimized. */
+int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op)
+{
+        int i;
+        struct kprobe *p;
+
+        for (i = 1; i < op->optinsn.size; i++) {
+                p = get_kprobe(op->kp.addr + i);
+                if (p && !kprobe_disabled(p))
+                        return -EEXIST;
+        }
+
+        return 0;
+}
+
+/* Check the addr is within the optimized instructions. */
+int __kprobes arch_within_optimized_kprobe(struct optimized_kprobe *op,
+                                           unsigned long addr)
+{
+        return ((unsigned long)op->kp.addr <= addr &&
+                (unsigned long)op->kp.addr + op->optinsn.size > addr);
+}
+
+/* Free optimized instruction slot */
+static __kprobes
+void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
+{
+        if (op->optinsn.insn) {
+                free_optinsn_slot(op->optinsn.insn, dirty);
+                op->optinsn.insn = NULL;
+                op->optinsn.size = 0;
+        }
+}
+
+void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op)
+{
+        __arch_remove_optimized_kprobe(op, 1);
+}
+
+/*
+ * Copy replacing target instructions
+ * Target instructions MUST be relocatable (checked inside)
+ */
+int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op)
+{
+        u8 *buf;
+        int ret;
+        long rel;
+
+        if (!can_optimize((unsigned long)op->kp.addr))
+                return -EILSEQ;
+
+        op->optinsn.insn = get_optinsn_slot();
+        if (!op->optinsn.insn)
+                return -ENOMEM;
+
+        /*
+         * Verify if the address gap is in 2GB range, because this uses
+         * a relative jump.
+         */
+        rel = (long)op->optinsn.insn - (long)op->kp.addr + RELATIVEJUMP_SIZE;
+        if (abs(rel) > 0x7fffffff)
+                return -ERANGE;
+
+        buf = (u8 *)op->optinsn.insn;
+
+        /* Copy instructions into the out-of-line buffer */
+        ret = copy_optimized_instructions(buf + TMPL_END_IDX, op->kp.addr);
+        if (ret < 0) {
+                __arch_remove_optimized_kprobe(op, 0);
+                return ret;
+        }
+        op->optinsn.size = ret;
+
+        /* Copy arch-dep-instance from template */
+        memcpy(buf, &optprobe_template_entry, TMPL_END_IDX);
+
+        /* Set probe information */
+        synthesize_set_arg1(buf + TMPL_MOVE_IDX, (unsigned long)op);
+
+        /* Set probe function call */
+        synthesize_relcall(buf + TMPL_CALL_IDX, optimized_callback);
+
+        /* Set returning jmp instruction at the tail of out-of-line buffer */
+        synthesize_reljump(buf + TMPL_END_IDX + op->optinsn.size,
+                           (u8 *)op->kp.addr + op->optinsn.size);
+
+        flush_icache_range((unsigned long) buf,
+                           (unsigned long) buf + TMPL_END_IDX +
+                           op->optinsn.size + RELATIVEJUMP_SIZE);
+        return 0;
+}
+
+/* Replace a breakpoint (int3) with a relative jump.  */
+int __kprobes arch_optimize_kprobe(struct optimized_kprobe *op)
+{
+        unsigned char jmp_code[RELATIVEJUMP_SIZE];
+        s32 rel = (s32)((long)op->optinsn.insn -
+                        ((long)op->kp.addr + RELATIVEJUMP_SIZE));
+
+        /* Backup instructions which will be replaced by jump address */
+        memcpy(op->optinsn.copied_insn, op->kp.addr + INT3_SIZE,
+               RELATIVE_ADDR_SIZE);
+
+        jmp_code[0] = RELATIVEJUMP_OPCODE;
+        *(s32 *)(&jmp_code[1]) = rel;
+
+        /*
+         * text_poke_smp doesn't support NMI/MCE code modifying.
+         * However, since kprobes itself also doesn't support NMI/MCE
+         * code probing, it's not a problem.
+         */
+        text_poke_smp(op->kp.addr, jmp_code, RELATIVEJUMP_SIZE);
+        return 0;
+}
+
+/* Replace a relative jump with a breakpoint (int3).  */
+void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op)
+{
+        u8 buf[RELATIVEJUMP_SIZE];
+
+        /* Set int3 to first byte for kprobes */
+        buf[0] = BREAKPOINT_INSTRUCTION;
+        memcpy(buf + 1, op->optinsn.copied_insn, RELATIVE_ADDR_SIZE);
+        text_poke_smp(op->kp.addr, buf, RELATIVEJUMP_SIZE);
+}
+
+static int  __kprobes setup_detour_execution(struct kprobe *p,
+                                             struct pt_regs *regs,
+                                             int reenter)
+{
+        struct optimized_kprobe *op;
+
+        if (p->flags & KPROBE_FLAG_OPTIMIZED) {
+                /* This kprobe is really able to run optimized path. */
+                op = container_of(p, struct optimized_kprobe, kp);
+                /* Detour through copied instructions */
+                regs->ip = (unsigned long)op->optinsn.insn + TMPL_END_IDX;
+                if (!reenter)
+                        reset_current_kprobe();
+                preempt_enable_no_resched();
+                return 1;
+        }
+        return 0;
+}
+#endif
+
 int __init arch_init_kprobes(void)
 {
         return 0;