1 files changed, 119 insertions, 59 deletions

diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c
index 014d4729b134..1e4c710dfb92 100644
--- a/arch/s390/kernel/kprobes.c
+++ b/arch/s390/kernel/kprobes.c
@@ -29,6 +29,7 @@
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/hardirq.h>
+#include <linux/ftrace.h>
 #include <asm/cacheflush.h>
 #include <asm/sections.h>
 #include <asm/dis.h>
@@ -58,12 +59,23 @@ struct kprobe_insn_cache kprobe_dmainsn_slots = {
         .insn_size = MAX_INSN_SIZE,
 };
 
-static void __kprobes copy_instruction(struct kprobe *p)
+static void copy_instruction(struct kprobe *p)
 {
+        unsigned long ip = (unsigned long) p->addr;
         s64 disp, new_disp;
         u64 addr, new_addr;
 
-        memcpy(p->ainsn.insn, p->addr, insn_length(p->opcode >> 8));
+        if (ftrace_location(ip) == ip) {
+                /*
+                 * If kprobes patches the instruction that is morphed by
+                 * ftrace make sure that kprobes always sees the branch
+                 * "jg .+24" that skips the mcount block
+                 */
+                ftrace_generate_nop_insn((struct ftrace_insn *)p->ainsn.insn);
+                p->ainsn.is_ftrace_insn = 1;
+        } else
+                memcpy(p->ainsn.insn, p->addr, insn_length(*p->addr >> 8));
+        p->opcode = p->ainsn.insn[0];
         if (!probe_is_insn_relative_long(p->ainsn.insn))
                 return;
         /*
@@ -79,25 +91,14 @@ static void __kprobes copy_instruction(struct kprobe *p)
         new_disp = ((addr + (disp * 2)) - new_addr) / 2;
         *(s32 *)&p->ainsn.insn[1] = new_disp;
 }
+NOKPROBE_SYMBOL(copy_instruction);
 
 static inline int is_kernel_addr(void *addr)
 {
         return addr < (void *)_end;
 }
 
-static inline int is_module_addr(void *addr)
-{
-#ifdef CONFIG_64BIT
-        BUILD_BUG_ON(MODULES_LEN > (1UL << 31));
-        if (addr < (void *)MODULES_VADDR)
-                return 0;
-        if (addr > (void *)MODULES_END)
-                return 0;
-#endif
-        return 1;
-}
-
-static int __kprobes s390_get_insn_slot(struct kprobe *p)
+static int s390_get_insn_slot(struct kprobe *p)
 {
         /*
          * Get an insn slot that is within the same 2GB area like the original
@@ -111,8 +112,9 @@ static int __kprobes s390_get_insn_slot(struct kprobe *p)
                 p->ainsn.insn = get_insn_slot();
         return p->ainsn.insn ? 0 : -ENOMEM;
 }
+NOKPROBE_SYMBOL(s390_get_insn_slot);
 
-static void __kprobes s390_free_insn_slot(struct kprobe *p)
+static void s390_free_insn_slot(struct kprobe *p)
 {
         if (!p->ainsn.insn)
                 return;
@@ -122,8 +124,9 @@ static void __kprobes s390_free_insn_slot(struct kprobe *p)
                 free_insn_slot(p->ainsn.insn, 0);
         p->ainsn.insn = NULL;
 }
+NOKPROBE_SYMBOL(s390_free_insn_slot);
 
-int __kprobes arch_prepare_kprobe(struct kprobe *p)
+int arch_prepare_kprobe(struct kprobe *p)
 {
         if ((unsigned long) p->addr & 0x01)
                 return -EINVAL;
@@ -132,54 +135,79 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p)
                 return -EINVAL;
         if (s390_get_insn_slot(p))
                 return -ENOMEM;
-        p->opcode = *p->addr;
         copy_instruction(p);
         return 0;
 }
+NOKPROBE_SYMBOL(arch_prepare_kprobe);
 
-struct ins_replace_args {
-        kprobe_opcode_t *ptr;
-        kprobe_opcode_t opcode;
+int arch_check_ftrace_location(struct kprobe *p)
+{
+        return 0;
+}
+
+struct swap_insn_args {
+        struct kprobe *p;
+        unsigned int arm_kprobe : 1;
 };
 
-static int __kprobes swap_instruction(void *aref)
+static int swap_instruction(void *data)
 {
         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
         unsigned long status = kcb->kprobe_status;
-        struct ins_replace_args *args = aref;
-
+        struct swap_insn_args *args = data;
+        struct ftrace_insn new_insn, *insn;
+        struct kprobe *p = args->p;
+        size_t len;
+
+        new_insn.opc = args->arm_kprobe ? BREAKPOINT_INSTRUCTION : p->opcode;
+        len = sizeof(new_insn.opc);
+        if (!p->ainsn.is_ftrace_insn)
+                goto skip_ftrace;
+        len = sizeof(new_insn);
+        insn = (struct ftrace_insn *) p->addr;
+        if (args->arm_kprobe) {
+                if (is_ftrace_nop(insn))
+                        new_insn.disp = KPROBE_ON_FTRACE_NOP;
+                else
+                        new_insn.disp = KPROBE_ON_FTRACE_CALL;
+        } else {
+                ftrace_generate_call_insn(&new_insn, (unsigned long)p->addr);
+                if (insn->disp == KPROBE_ON_FTRACE_NOP)
+                        ftrace_generate_nop_insn(&new_insn);
+        }
+skip_ftrace:
         kcb->kprobe_status = KPROBE_SWAP_INST;
-        probe_kernel_write(args->ptr, &args->opcode, sizeof(args->opcode));
+        probe_kernel_write(p->addr, &new_insn, len);
         kcb->kprobe_status = status;
         return 0;
 }
+NOKPROBE_SYMBOL(swap_instruction);
 
-void __kprobes arch_arm_kprobe(struct kprobe *p)
+void arch_arm_kprobe(struct kprobe *p)
 {
-        struct ins_replace_args args;
+        struct swap_insn_args args = {.p = p, .arm_kprobe = 1};
 
-        args.ptr = p->addr;
-        args.opcode = BREAKPOINT_INSTRUCTION;
         stop_machine(swap_instruction, &args, NULL);
 }
+NOKPROBE_SYMBOL(arch_arm_kprobe);
 
-void __kprobes arch_disarm_kprobe(struct kprobe *p)
+void arch_disarm_kprobe(struct kprobe *p)
 {
-        struct ins_replace_args args;
+        struct swap_insn_args args = {.p = p, .arm_kprobe = 0};
 
-        args.ptr = p->addr;
-        args.opcode = p->opcode;
         stop_machine(swap_instruction, &args, NULL);
 }
+NOKPROBE_SYMBOL(arch_disarm_kprobe);
 
-void __kprobes arch_remove_kprobe(struct kprobe *p)
+void arch_remove_kprobe(struct kprobe *p)
 {
         s390_free_insn_slot(p);
 }
+NOKPROBE_SYMBOL(arch_remove_kprobe);
 
-static void __kprobes enable_singlestep(struct kprobe_ctlblk *kcb,
-                                        struct pt_regs *regs,
-                                        unsigned long ip)
+static void enable_singlestep(struct kprobe_ctlblk *kcb,
+                              struct pt_regs *regs,
+                              unsigned long ip)
 {
         struct per_regs per_kprobe;
 
@@ -199,10 +227,11 @@ static void __kprobes enable_singlestep(struct kprobe_ctlblk *kcb,
         regs->psw.mask &= ~(PSW_MASK_IO | PSW_MASK_EXT);
         regs->psw.addr = ip | PSW_ADDR_AMODE;
 }
+NOKPROBE_SYMBOL(enable_singlestep);
 
-static void __kprobes disable_singlestep(struct kprobe_ctlblk *kcb,
-                                         struct pt_regs *regs,
-                                         unsigned long ip)
+static void disable_singlestep(struct kprobe_ctlblk *kcb,
+                               struct pt_regs *regs,
+                               unsigned long ip)
 {
         /* Restore control regs and psw mask, set new psw address */
         __ctl_load(kcb->kprobe_saved_ctl, 9, 11);
@@ -210,41 +239,43 @@ static void __kprobes disable_singlestep(struct kprobe_ctlblk *kcb,
         regs->psw.mask |= kcb->kprobe_saved_imask;
         regs->psw.addr = ip | PSW_ADDR_AMODE;
 }
+NOKPROBE_SYMBOL(disable_singlestep);
 
 /*
  * Activate a kprobe by storing its pointer to current_kprobe. The
  * previous kprobe is stored in kcb->prev_kprobe. A stack of up to
  * two kprobes can be active, see KPROBE_REENTER.
  */
-static void __kprobes push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
+static void push_kprobe(struct kprobe_ctlblk *kcb, struct kprobe *p)
 {
         kcb->prev_kprobe.kp = __this_cpu_read(current_kprobe);
         kcb->prev_kprobe.status = kcb->kprobe_status;
         __this_cpu_write(current_kprobe, p);
 }
+NOKPROBE_SYMBOL(push_kprobe);
 
 /*
  * Deactivate a kprobe by backing up to the previous state. If the
  * current state is KPROBE_REENTER prev_kprobe.kp will be non-NULL,
  * for any other state prev_kprobe.kp will be NULL.
  */
-static void __kprobes pop_kprobe(struct kprobe_ctlblk *kcb)
+static void pop_kprobe(struct kprobe_ctlblk *kcb)
 {
         __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
         kcb->kprobe_status = kcb->prev_kprobe.status;
 }
+NOKPROBE_SYMBOL(pop_kprobe);
 
-void __kprobes arch_prepare_kretprobe(struct kretprobe_instance *ri,
-                                        struct pt_regs *regs)
+void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
 {
         ri->ret_addr = (kprobe_opcode_t *) regs->gprs[14];
 
         /* Replace the return addr with trampoline addr */
         regs->gprs[14] = (unsigned long) &kretprobe_trampoline;
 }
+NOKPROBE_SYMBOL(arch_prepare_kretprobe);
 
-static void __kprobes kprobe_reenter_check(struct kprobe_ctlblk *kcb,
-                                           struct kprobe *p)
+static void kprobe_reenter_check(struct kprobe_ctlblk *kcb, struct kprobe *p)
 {
         switch (kcb->kprobe_status) {
         case KPROBE_HIT_SSDONE:
@@ -264,8 +295,9 @@ static void __kprobes kprobe_reenter_check(struct kprobe_ctlblk *kcb,
                 BUG();
         }
 }
+NOKPROBE_SYMBOL(kprobe_reenter_check);
 
-static int __kprobes kprobe_handler(struct pt_regs *regs)
+static int kprobe_handler(struct pt_regs *regs)
 {
         struct kprobe_ctlblk *kcb;
         struct kprobe *p;
@@ -339,6 +371,7 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
         preempt_enable_no_resched();
         return 0;
 }
+NOKPROBE_SYMBOL(kprobe_handler);
 
 /*
  * Function return probe trampoline:
@@ -355,8 +388,7 @@ static void __used kretprobe_trampoline_holder(void)
 /*
  * Called when the probe at kretprobe trampoline is hit
  */
-static int __kprobes trampoline_probe_handler(struct kprobe *p,
-                                              struct pt_regs *regs)
+static int trampoline_probe_handler(struct kprobe *p, struct pt_regs *regs)
 {
         struct kretprobe_instance *ri;
         struct hlist_head *head, empty_rp;
@@ -444,6 +476,7 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p,
          */
         return 1;
 }
+NOKPROBE_SYMBOL(trampoline_probe_handler);
 
 /*
  * Called after single-stepping.  p->addr is the address of the
@@ -453,12 +486,30 @@ static int __kprobes trampoline_probe_handler(struct kprobe *p,
  * single-stepped a copy of the instruction.  The address of this
  * copy is p->ainsn.insn.
  */
-static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
+static void resume_execution(struct kprobe *p, struct pt_regs *regs)
 {
         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
         unsigned long ip = regs->psw.addr & PSW_ADDR_INSN;
         int fixup = probe_get_fixup_type(p->ainsn.insn);
 
+        /* Check if the kprobes location is an enabled ftrace caller */
+        if (p->ainsn.is_ftrace_insn) {
+                struct ftrace_insn *insn = (struct ftrace_insn *) p->addr;
+                struct ftrace_insn call_insn;
+
+                ftrace_generate_call_insn(&call_insn, (unsigned long) p->addr);
+                /*
+                 * A kprobe on an enabled ftrace call site actually single
+                 * stepped an unconditional branch (ftrace nop equivalent).
+                 * Now we need to fixup things and pretend that a brasl r0,...
+                 * was executed instead.
+                 */
+                if (insn->disp == KPROBE_ON_FTRACE_CALL) {
+                        ip += call_insn.disp * 2 - MCOUNT_INSN_SIZE;
+                        regs->gprs[0] = (unsigned long)p->addr + sizeof(*insn);
+                }
+        }
+
         if (fixup & FIXUP_PSW_NORMAL)
                 ip += (unsigned long) p->addr - (unsigned long) p->ainsn.insn;
 
@@ -476,8 +527,9 @@ static void __kprobes resume_execution(struct kprobe *p, struct pt_regs *regs)
 
         disable_singlestep(kcb, regs, ip);
 }
+NOKPROBE_SYMBOL(resume_execution);
 
-static int __kprobes post_kprobe_handler(struct pt_regs *regs)
+static int post_kprobe_handler(struct pt_regs *regs)
 {
         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
         struct kprobe *p = kprobe_running();
@@ -504,8 +556,9 @@ static int __kprobes post_kprobe_handler(struct pt_regs *regs)
 
         return 1;
 }
+NOKPROBE_SYMBOL(post_kprobe_handler);
 
-static int __kprobes kprobe_trap_handler(struct pt_regs *regs, int trapnr)
+static int kprobe_trap_handler(struct pt_regs *regs, int trapnr)
 {
         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
         struct kprobe *p = kprobe_running();
@@ -567,8 +620,9 @@ static int __kprobes kprobe_trap_handler(struct pt_regs *regs, int trapnr)
         }
         return 0;
 }
+NOKPROBE_SYMBOL(kprobe_trap_handler);
 
-int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
+int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
 {
         int ret;
 
@@ -579,12 +633,13 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
                 local_irq_restore(regs->psw.mask & ~PSW_MASK_PER);
         return ret;
 }
+NOKPROBE_SYMBOL(kprobe_fault_handler);
 
 /*
  * Wrapper routine to for handling exceptions.
  */
-int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
-                                       unsigned long val, void *data)
+int kprobe_exceptions_notify(struct notifier_block *self,
+                             unsigned long val, void *data)
 {
         struct die_args *args = (struct die_args *) data;
         struct pt_regs *regs = args->regs;
@@ -616,8 +671,9 @@ int __kprobes kprobe_exceptions_notify(struct notifier_block *self,
 
         return ret;
 }
+NOKPROBE_SYMBOL(kprobe_exceptions_notify);
 
-int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
+int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
 {
         struct jprobe *jp = container_of(p, struct jprobe, kp);
         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
@@ -635,13 +691,15 @@ int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
         memcpy(kcb->jprobes_stack, (void *) stack, MIN_STACK_SIZE(stack));
         return 1;
 }
+NOKPROBE_SYMBOL(setjmp_pre_handler);
 
-void __kprobes jprobe_return(void)
+void jprobe_return(void)
 {
         asm volatile(".word 0x0002");
 }
+NOKPROBE_SYMBOL(jprobe_return);
 
-int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
+int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
 {
         struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
         unsigned long stack;
@@ -655,6 +713,7 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
         preempt_enable_no_resched();
         return 1;
 }
+NOKPROBE_SYMBOL(longjmp_break_handler);
 
 static struct kprobe trampoline = {
         .addr = (kprobe_opcode_t *) &kretprobe_trampoline,
@@ -666,7 +725,8 @@ int __init arch_init_kprobes(void)
         return register_kprobe(&trampoline);
 }
 
-int __kprobes arch_trampoline_kprobe(struct kprobe *p)
+int arch_trampoline_kprobe(struct kprobe *p)
 {
         return p->addr == (kprobe_opcode_t *) &kretprobe_trampoline;
 }
+NOKPROBE_SYMBOL(arch_trampoline_kprobe);