aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-06-12 22:18:49 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2014-06-12 22:18:49 -0400
commit3737a12761636ebde0f09ef49daebb8eed18cc8a (patch)
tree965057f4bccd97049f8c0140f8670c5d4278ca3e /arch/x86/kernel
parentc29deef32e3699e40da3e9e82267610de04e6b54 (diff)
parent82b897782d10fcc4930c9d4a15b175348fdd2871 (diff)
Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull more perf updates from Ingo Molnar: "A second round of perf updates: - wide reaching kprobes sanitization and robustization, with the hope of fixing all 'probe this function crashes the kernel' bugs, by Masami Hiramatsu. - uprobes updates from Oleg Nesterov: tmpfs support, corner case fixes and robustization work. - perf tooling updates and fixes from Jiri Olsa, Namhyung Ki, Arnaldo et al: * Add support to accumulate hist periods (Namhyung Kim) * various fixes, refactorings and enhancements" * 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (101 commits) perf: Differentiate exec() and non-exec() comm events perf: Fix perf_event_comm() vs. exec() assumption uprobes/x86: Rename arch_uprobe->def to ->defparam, minor comment updates perf/documentation: Add description for conditional branch filter perf/x86: Add conditional branch filtering support perf/tool: Add conditional branch filter 'cond' to perf record perf: Add new conditional branch filter 'PERF_SAMPLE_BRANCH_COND' uprobes: Teach copy_insn() to support tmpfs uprobes: Shift ->readpage check from __copy_insn() to uprobe_register() perf/x86: Use common PMU interrupt disabled code perf/ARM: Use common PMU interrupt disabled code perf: Disable sampled events if no PMU interrupt perf: Fix use after free in perf_remove_from_context() perf tools: Fix 'make help' message error perf record: Fix poll return value propagation perf tools: Move elide bool into perf_hpp_fmt struct perf tools: Remove elide setup for SORT_MODE__MEMORY mode perf tools: Fix "==" into "=" in ui_browser__warning assignment perf tools: Allow overriding sysfs and proc finding with env var perf tools: Consider header files outside perf directory in tags target ...
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r--arch/x86/kernel/alternative.c3
-rw-r--r--arch/x86/kernel/apic/hw_nmi.c3
-rw-r--r--arch/x86/kernel/cpu/common.c4
-rw-r--r--arch/x86/kernel/cpu/perf_event.c21
-rw-r--r--arch/x86/kernel/cpu/perf_event_amd_ibs.c3
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_lbr.c5
-rw-r--r--arch/x86/kernel/dumpstack.c9
-rw-r--r--arch/x86/kernel/entry_32.S33
-rw-r--r--arch/x86/kernel/entry_64.S21
-rw-r--r--arch/x86/kernel/hw_breakpoint.c5
-rw-r--r--arch/x86/kernel/kprobes/core.c128
-rw-r--r--arch/x86/kernel/kprobes/ftrace.c17
-rw-r--r--arch/x86/kernel/kprobes/opt.c32
-rw-r--r--arch/x86/kernel/kvm.c4
-rw-r--r--arch/x86/kernel/nmi.c18
-rw-r--r--arch/x86/kernel/paravirt.c6
-rw-r--r--arch/x86/kernel/process_64.c7
-rw-r--r--arch/x86/kernel/traps.c145
-rw-r--r--arch/x86/kernel/uprobes.c505
19 files changed, 521 insertions, 448 deletions
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index df94598ad05a..703130f469ec 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -5,7 +5,6 @@
5#include <linux/mutex.h> 5#include <linux/mutex.h>
6#include <linux/list.h> 6#include <linux/list.h>
7#include <linux/stringify.h> 7#include <linux/stringify.h>
8#include <linux/kprobes.h>
9#include <linux/mm.h> 8#include <linux/mm.h>
10#include <linux/vmalloc.h> 9#include <linux/vmalloc.h>
11#include <linux/memory.h> 10#include <linux/memory.h>
@@ -551,7 +550,7 @@ void *__init_or_module text_poke_early(void *addr, const void *opcode,
551 * 550 *
552 * Note: Must be called under text_mutex. 551 * Note: Must be called under text_mutex.
553 */ 552 */
554void *__kprobes text_poke(void *addr, const void *opcode, size_t len) 553void *text_poke(void *addr, const void *opcode, size_t len)
555{ 554{
556 unsigned long flags; 555 unsigned long flags;
557 char *vaddr; 556 char *vaddr;
diff --git a/arch/x86/kernel/apic/hw_nmi.c b/arch/x86/kernel/apic/hw_nmi.c
index eab67047dec3..c3fcb5de5083 100644
--- a/arch/x86/kernel/apic/hw_nmi.c
+++ b/arch/x86/kernel/apic/hw_nmi.c
@@ -60,7 +60,7 @@ void arch_trigger_all_cpu_backtrace(void)
60 smp_mb__after_atomic(); 60 smp_mb__after_atomic();
61} 61}
62 62
63static int __kprobes 63static int
64arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs) 64arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs)
65{ 65{
66 int cpu; 66 int cpu;
@@ -80,6 +80,7 @@ arch_trigger_all_cpu_backtrace_handler(unsigned int cmd, struct pt_regs *regs)
80 80
81 return NMI_DONE; 81 return NMI_DONE;
82} 82}
83NOKPROBE_SYMBOL(arch_trigger_all_cpu_backtrace_handler);
83 84
84static int __init register_trigger_all_cpu_backtrace(void) 85static int __init register_trigger_all_cpu_backtrace(void)
85{ 86{
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 2cbbf88d8f2c..ef1b93f18ed1 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -8,6 +8,7 @@
8#include <linux/delay.h> 8#include <linux/delay.h>
9#include <linux/sched.h> 9#include <linux/sched.h>
10#include <linux/init.h> 10#include <linux/init.h>
11#include <linux/kprobes.h>
11#include <linux/kgdb.h> 12#include <linux/kgdb.h>
12#include <linux/smp.h> 13#include <linux/smp.h>
13#include <linux/io.h> 14#include <linux/io.h>
@@ -1193,6 +1194,7 @@ int is_debug_stack(unsigned long addr)
1193 (addr <= __get_cpu_var(debug_stack_addr) && 1194 (addr <= __get_cpu_var(debug_stack_addr) &&
1194 addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ)); 1195 addr > (__get_cpu_var(debug_stack_addr) - DEBUG_STKSZ));
1195} 1196}
1197NOKPROBE_SYMBOL(is_debug_stack);
1196 1198
1197DEFINE_PER_CPU(u32, debug_idt_ctr); 1199DEFINE_PER_CPU(u32, debug_idt_ctr);
1198 1200
@@ -1201,6 +1203,7 @@ void debug_stack_set_zero(void)
1201 this_cpu_inc(debug_idt_ctr); 1203 this_cpu_inc(debug_idt_ctr);
1202 load_current_idt(); 1204 load_current_idt();
1203} 1205}
1206NOKPROBE_SYMBOL(debug_stack_set_zero);
1204 1207
1205void debug_stack_reset(void) 1208void debug_stack_reset(void)
1206{ 1209{
@@ -1209,6 +1212,7 @@ void debug_stack_reset(void)
1209 if (this_cpu_dec_return(debug_idt_ctr) == 0) 1212 if (this_cpu_dec_return(debug_idt_ctr) == 0)
1210 load_current_idt(); 1213 load_current_idt();
1211} 1214}
1215NOKPROBE_SYMBOL(debug_stack_reset);
1212 1216
1213#else /* CONFIG_X86_64 */ 1217#else /* CONFIG_X86_64 */
1214 1218
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index 89f3b7c1af20..2bdfbff8a4f6 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -303,15 +303,6 @@ int x86_setup_perfctr(struct perf_event *event)
303 hwc->sample_period = x86_pmu.max_period; 303 hwc->sample_period = x86_pmu.max_period;
304 hwc->last_period = hwc->sample_period; 304 hwc->last_period = hwc->sample_period;
305 local64_set(&hwc->period_left, hwc->sample_period); 305 local64_set(&hwc->period_left, hwc->sample_period);
306 } else {
307 /*
308 * If we have a PMU initialized but no APIC
309 * interrupts, we cannot sample hardware
310 * events (user-space has to fall back and
311 * sample via a hrtimer based software event):
312 */
313 if (!x86_pmu.apic)
314 return -EOPNOTSUPP;
315 } 306 }
316 307
317 if (attr->type == PERF_TYPE_RAW) 308 if (attr->type == PERF_TYPE_RAW)
@@ -1293,7 +1284,7 @@ void perf_events_lapic_init(void)
1293 apic_write(APIC_LVTPC, APIC_DM_NMI); 1284 apic_write(APIC_LVTPC, APIC_DM_NMI);
1294} 1285}
1295 1286
1296static int __kprobes 1287static int
1297perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs) 1288perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1298{ 1289{
1299 u64 start_clock; 1290 u64 start_clock;
@@ -1311,6 +1302,7 @@ perf_event_nmi_handler(unsigned int cmd, struct pt_regs *regs)
1311 1302
1312 return ret; 1303 return ret;
1313} 1304}
1305NOKPROBE_SYMBOL(perf_event_nmi_handler);
1314 1306
1315struct event_constraint emptyconstraint; 1307struct event_constraint emptyconstraint;
1316struct event_constraint unconstrained; 1308struct event_constraint unconstrained;
@@ -1366,6 +1358,15 @@ static void __init pmu_check_apic(void)
1366 x86_pmu.apic = 0; 1358 x86_pmu.apic = 0;
1367 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); 1359 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n");
1368 pr_info("no hardware sampling interrupt available.\n"); 1360 pr_info("no hardware sampling interrupt available.\n");
1361
1362 /*
1363 * If we have a PMU initialized but no APIC
1364 * interrupts, we cannot sample hardware
1365 * events (user-space has to fall back and
1366 * sample via a hrtimer based software event):
1367 */
1368 pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1369
1369} 1370}
1370 1371
1371static struct attribute_group x86_pmu_format_group = { 1372static struct attribute_group x86_pmu_format_group = {
diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
index 4c36bbe3173a..cbb1be3ed9e4 100644
--- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c
+++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
@@ -593,7 +593,7 @@ out:
593 return 1; 593 return 1;
594} 594}
595 595
596static int __kprobes 596static int
597perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs) 597perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
598{ 598{
599 int handled = 0; 599 int handled = 0;
@@ -606,6 +606,7 @@ perf_ibs_nmi_handler(unsigned int cmd, struct pt_regs *regs)
606 606
607 return handled; 607 return handled;
608} 608}
609NOKPROBE_SYMBOL(perf_ibs_nmi_handler);
609 610
610static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name) 611static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
611{ 612{
diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index d82d155aca8c..9dd2459a4c73 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -384,6 +384,9 @@ static void intel_pmu_setup_sw_lbr_filter(struct perf_event *event)
384 if (br_type & PERF_SAMPLE_BRANCH_NO_TX) 384 if (br_type & PERF_SAMPLE_BRANCH_NO_TX)
385 mask |= X86_BR_NO_TX; 385 mask |= X86_BR_NO_TX;
386 386
387 if (br_type & PERF_SAMPLE_BRANCH_COND)
388 mask |= X86_BR_JCC;
389
387 /* 390 /*
388 * stash actual user request into reg, it may 391 * stash actual user request into reg, it may
389 * be used by fixup code for some CPU 392 * be used by fixup code for some CPU
@@ -678,6 +681,7 @@ static const int nhm_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
678 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL 681 * NHM/WSM erratum: must include IND_JMP to capture IND_CALL
679 */ 682 */
680 [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP, 683 [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL | LBR_IND_JMP,
684 [PERF_SAMPLE_BRANCH_COND] = LBR_JCC,
681}; 685};
682 686
683static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = { 687static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
@@ -689,6 +693,7 @@ static const int snb_lbr_sel_map[PERF_SAMPLE_BRANCH_MAX] = {
689 [PERF_SAMPLE_BRANCH_ANY_CALL] = LBR_REL_CALL | LBR_IND_CALL 693 [PERF_SAMPLE_BRANCH_ANY_CALL] = LBR_REL_CALL | LBR_IND_CALL
690 | LBR_FAR, 694 | LBR_FAR,
691 [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL, 695 [PERF_SAMPLE_BRANCH_IND_CALL] = LBR_IND_CALL,
696 [PERF_SAMPLE_BRANCH_COND] = LBR_JCC,
692}; 697};
693 698
694/* core */ 699/* core */
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c
index d9c12d3022a7..b74ebc7c4402 100644
--- a/arch/x86/kernel/dumpstack.c
+++ b/arch/x86/kernel/dumpstack.c
@@ -200,7 +200,7 @@ static arch_spinlock_t die_lock = __ARCH_SPIN_LOCK_UNLOCKED;
200static int die_owner = -1; 200static int die_owner = -1;
201static unsigned int die_nest_count; 201static unsigned int die_nest_count;
202 202
203unsigned __kprobes long oops_begin(void) 203unsigned long oops_begin(void)
204{ 204{
205 int cpu; 205 int cpu;
206 unsigned long flags; 206 unsigned long flags;
@@ -223,8 +223,9 @@ unsigned __kprobes long oops_begin(void)
223 return flags; 223 return flags;
224} 224}
225EXPORT_SYMBOL_GPL(oops_begin); 225EXPORT_SYMBOL_GPL(oops_begin);
226NOKPROBE_SYMBOL(oops_begin);
226 227
227void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr) 228void oops_end(unsigned long flags, struct pt_regs *regs, int signr)
228{ 229{
229 if (regs && kexec_should_crash(current)) 230 if (regs && kexec_should_crash(current))
230 crash_kexec(regs); 231 crash_kexec(regs);
@@ -247,8 +248,9 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr)
247 panic("Fatal exception"); 248 panic("Fatal exception");
248 do_exit(signr); 249 do_exit(signr);
249} 250}
251NOKPROBE_SYMBOL(oops_end);
250 252
251int __kprobes __die(const char *str, struct pt_regs *regs, long err) 253int __die(const char *str, struct pt_regs *regs, long err)
252{ 254{
253#ifdef CONFIG_X86_32 255#ifdef CONFIG_X86_32
254 unsigned short ss; 256 unsigned short ss;
@@ -291,6 +293,7 @@ int __kprobes __die(const char *str, struct pt_regs *regs, long err)
291#endif 293#endif
292 return 0; 294 return 0;
293} 295}
296NOKPROBE_SYMBOL(__die);
294 297
295/* 298/*
296 * This is gone through when something in the kernel has done something bad 299 * This is gone through when something in the kernel has done something bad
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 98313ffaae6a..f0da82b8e634 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -315,10 +315,6 @@ ENTRY(ret_from_kernel_thread)
315ENDPROC(ret_from_kernel_thread) 315ENDPROC(ret_from_kernel_thread)
316 316
317/* 317/*
318 * Interrupt exit functions should be protected against kprobes
319 */
320 .pushsection .kprobes.text, "ax"
321/*
322 * Return to user mode is not as complex as all this looks, 318 * Return to user mode is not as complex as all this looks,
323 * but we want the default path for a system call return to 319 * but we want the default path for a system call return to
324 * go as quickly as possible which is why some of this is 320 * go as quickly as possible which is why some of this is
@@ -372,10 +368,6 @@ need_resched:
372END(resume_kernel) 368END(resume_kernel)
373#endif 369#endif
374 CFI_ENDPROC 370 CFI_ENDPROC
375/*
376 * End of kprobes section
377 */
378 .popsection
379 371
380/* SYSENTER_RETURN points to after the "sysenter" instruction in 372/* SYSENTER_RETURN points to after the "sysenter" instruction in
381 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */ 373 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
@@ -495,10 +487,6 @@ sysexit_audit:
495 PTGS_TO_GS_EX 487 PTGS_TO_GS_EX
496ENDPROC(ia32_sysenter_target) 488ENDPROC(ia32_sysenter_target)
497 489
498/*
499 * syscall stub including irq exit should be protected against kprobes
500 */
501 .pushsection .kprobes.text, "ax"
502 # system call handler stub 490 # system call handler stub
503ENTRY(system_call) 491ENTRY(system_call)
504 RING0_INT_FRAME # can't unwind into user space anyway 492 RING0_INT_FRAME # can't unwind into user space anyway
@@ -690,10 +678,6 @@ syscall_badsys:
690 jmp resume_userspace 678 jmp resume_userspace
691END(syscall_badsys) 679END(syscall_badsys)
692 CFI_ENDPROC 680 CFI_ENDPROC
693/*
694 * End of kprobes section
695 */
696 .popsection
697 681
698.macro FIXUP_ESPFIX_STACK 682.macro FIXUP_ESPFIX_STACK
699/* 683/*
@@ -784,10 +768,6 @@ common_interrupt:
784ENDPROC(common_interrupt) 768ENDPROC(common_interrupt)
785 CFI_ENDPROC 769 CFI_ENDPROC
786 770
787/*
788 * Irq entries should be protected against kprobes
789 */
790 .pushsection .kprobes.text, "ax"
791#define BUILD_INTERRUPT3(name, nr, fn) \ 771#define BUILD_INTERRUPT3(name, nr, fn) \
792ENTRY(name) \ 772ENTRY(name) \
793 RING0_INT_FRAME; \ 773 RING0_INT_FRAME; \
@@ -964,10 +944,6 @@ ENTRY(spurious_interrupt_bug)
964 jmp error_code 944 jmp error_code
965 CFI_ENDPROC 945 CFI_ENDPROC
966END(spurious_interrupt_bug) 946END(spurious_interrupt_bug)
967/*
968 * End of kprobes section
969 */
970 .popsection
971 947
972#ifdef CONFIG_XEN 948#ifdef CONFIG_XEN
973/* Xen doesn't set %esp to be precisely what the normal sysenter 949/* Xen doesn't set %esp to be precisely what the normal sysenter
@@ -1242,11 +1218,6 @@ return_to_handler:
1242 jmp *%ecx 1218 jmp *%ecx
1243#endif 1219#endif
1244 1220
1245/*
1246 * Some functions should be protected against kprobes
1247 */
1248 .pushsection .kprobes.text, "ax"
1249
1250#ifdef CONFIG_TRACING 1221#ifdef CONFIG_TRACING
1251ENTRY(trace_page_fault) 1222ENTRY(trace_page_fault)
1252 RING0_EC_FRAME 1223 RING0_EC_FRAME
@@ -1460,7 +1431,3 @@ ENTRY(async_page_fault)
1460END(async_page_fault) 1431END(async_page_fault)
1461#endif 1432#endif
1462 1433
1463/*
1464 * End of kprobes section
1465 */
1466 .popsection
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 48a2644a082a..b25ca969edd2 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -284,8 +284,6 @@ ENDPROC(native_usergs_sysret64)
284 TRACE_IRQS_OFF 284 TRACE_IRQS_OFF
285 .endm 285 .endm
286 286
287/* save complete stack frame */
288 .pushsection .kprobes.text, "ax"
289ENTRY(save_paranoid) 287ENTRY(save_paranoid)
290 XCPT_FRAME 1 RDI+8 288 XCPT_FRAME 1 RDI+8
291 cld 289 cld
@@ -314,7 +312,6 @@ ENTRY(save_paranoid)
3141: ret 3121: ret
315 CFI_ENDPROC 313 CFI_ENDPROC
316END(save_paranoid) 314END(save_paranoid)
317 .popsection
318 315
319/* 316/*
320 * A newly forked process directly context switches into this address. 317 * A newly forked process directly context switches into this address.
@@ -772,10 +769,6 @@ END(interrupt)
772 call \func 769 call \func
773 .endm 770 .endm
774 771
775/*
776 * Interrupt entry/exit should be protected against kprobes
777 */
778 .pushsection .kprobes.text, "ax"
779 /* 772 /*
780 * The interrupt stubs push (~vector+0x80) onto the stack and 773 * The interrupt stubs push (~vector+0x80) onto the stack and
781 * then jump to common_interrupt. 774 * then jump to common_interrupt.
@@ -983,11 +976,6 @@ END(__do_double_fault)
983#endif 976#endif
984 977
985/* 978/*
986 * End of kprobes section
987 */
988 .popsection
989
990/*
991 * APIC interrupts. 979 * APIC interrupts.
992 */ 980 */
993.macro apicinterrupt3 num sym do_sym 981.macro apicinterrupt3 num sym do_sym
@@ -1321,11 +1309,6 @@ apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \
1321 hyperv_callback_vector hyperv_vector_handler 1309 hyperv_callback_vector hyperv_vector_handler
1322#endif /* CONFIG_HYPERV */ 1310#endif /* CONFIG_HYPERV */
1323 1311
1324/*
1325 * Some functions should be protected against kprobes
1326 */
1327 .pushsection .kprobes.text, "ax"
1328
1329idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK 1312idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
1330idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK 1313idtentry int3 do_int3 has_error_code=0 paranoid=1 shift_ist=DEBUG_STACK
1331idtentry stack_segment do_stack_segment has_error_code=1 paranoid=1 1314idtentry stack_segment do_stack_segment has_error_code=1 paranoid=1
@@ -1742,7 +1725,3 @@ ENTRY(ignore_sysret)
1742 CFI_ENDPROC 1725 CFI_ENDPROC
1743END(ignore_sysret) 1726END(ignore_sysret)
1744 1727
1745/*
1746 * End of kprobes section
1747 */
1748 .popsection
diff --git a/arch/x86/kernel/hw_breakpoint.c b/arch/x86/kernel/hw_breakpoint.c
index a67b47c31314..5f9cf20cdb68 100644
--- a/arch/x86/kernel/hw_breakpoint.c
+++ b/arch/x86/kernel/hw_breakpoint.c
@@ -32,7 +32,6 @@
32#include <linux/irqflags.h> 32#include <linux/irqflags.h>
33#include <linux/notifier.h> 33#include <linux/notifier.h>
34#include <linux/kallsyms.h> 34#include <linux/kallsyms.h>
35#include <linux/kprobes.h>
36#include <linux/percpu.h> 35#include <linux/percpu.h>
37#include <linux/kdebug.h> 36#include <linux/kdebug.h>
38#include <linux/kernel.h> 37#include <linux/kernel.h>
@@ -424,7 +423,7 @@ EXPORT_SYMBOL_GPL(hw_breakpoint_restore);
424 * NOTIFY_STOP returned for all other cases 423 * NOTIFY_STOP returned for all other cases
425 * 424 *
426 */ 425 */
427static int __kprobes hw_breakpoint_handler(struct die_args *args) 426static int hw_breakpoint_handler(struct die_args *args)
428{ 427{
429 int i, cpu, rc = NOTIFY_STOP; 428 int i, cpu, rc = NOTIFY_STOP;
430 struct perf_event *bp; 429 struct perf_event *bp;
@@ -511,7 +510,7 @@ static int __kprobes hw_breakpoint_handler(struct die_args *args)
511/* 510/*
512 * Handle debug exception notifications. 511 * Handle debug exception notifications.
513 */ 512 */
514int __kprobes hw_breakpoint_exceptions_notify( 513int hw_breakpoint_exceptions_notify(
515 struct notifier_block *unused, unsigned long val, void *data) 514 struct notifier_block *unused, unsigned long val, void *data)
516{ 515{
517 if (val != DIE_DEBUG) 516 if (val != DIE_DEBUG)
diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c
index 61b17dc2c277..7596df664901 100644
--- a/arch/x86/kernel/kprobes/core.c
+++ b/arch/x86/kernel/kprobes/core.c
@@ -112,7 +112,8 @@ struct kretprobe_blackpoint kretprobe_blacklist[] = {
112 112
113const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist); 113const int kretprobe_blacklist_size = ARRAY_SIZE(kretprobe_blacklist);
114 114
115static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op) 115static nokprobe_inline void
116__synthesize_relative_insn(void *from, void *to, u8 op)
116{ 117{
117 struct __arch_relative_insn { 118 struct __arch_relative_insn {
118 u8 op; 119 u8 op;
@@ -125,21 +126,23 @@ static void __kprobes __synthesize_relative_insn(void *from, void *to, u8 op)
125} 126}
126 127
127/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/ 128/* Insert a jump instruction at address 'from', which jumps to address 'to'.*/
128void __kprobes synthesize_reljump(void *from, void *to) 129void synthesize_reljump(void *from, void *to)
129{ 130{
130 __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE); 131 __synthesize_relative_insn(from, to, RELATIVEJUMP_OPCODE);
131} 132}
133NOKPROBE_SYMBOL(synthesize_reljump);
132 134
133/* Insert a call instruction at address 'from', which calls address 'to'.*/ 135/* Insert a call instruction at address 'from', which calls address 'to'.*/
134void __kprobes synthesize_relcall(void *from, void *to) 136void synthesize_relcall(void *from, void *to)
135{ 137{
136 __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE); 138 __synthesize_relative_insn(from, to, RELATIVECALL_OPCODE);
137} 139}
140NOKPROBE_SYMBOL(synthesize_relcall);
138 141
139/* 142/*
140 * Skip the prefixes of the instruction. 143 * Skip the prefixes of the instruction.
141 */ 144 */
142static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn) 145static kprobe_opcode_t *skip_prefixes(kprobe_opcode_t *insn)
143{ 146{
144 insn_attr_t attr; 147 insn_attr_t attr;
145 148
@@ -154,12 +157,13 @@ static kprobe_opcode_t *__kprobes skip_prefixes(kprobe_opcode_t *insn)
154#endif 157#endif
155 return insn; 158 return insn;
156} 159}
160NOKPROBE_SYMBOL(skip_prefixes);
157 161
158/* 162/*
159 * Returns non-zero if opcode is boostable. 163 * Returns non-zero if opcode is boostable.
160 * RIP relative instructions are adjusted at copying time in 64 bits mode 164 * RIP relative instructions are adjusted at copying time in 64 bits mode
161 */ 165 */
162int __kprobes can_boost(kprobe_opcode_t *opcodes) 166int can_boost(kprobe_opcode_t *opcodes)
163{ 167{
164 kprobe_opcode_t opcode; 168 kprobe_opcode_t opcode;
165 kprobe_opcode_t *orig_opcodes = opcodes; 169 kprobe_opcode_t *orig_opcodes = opcodes;
@@ -260,7 +264,7 @@ unsigned long recover_probed_instruction(kprobe_opcode_t *buf, unsigned long add
260} 264}
261 265
262/* Check if paddr is at an instruction boundary */ 266/* Check if paddr is at an instruction boundary */
263static int __kprobes can_probe(unsigned long paddr) 267static int can_probe(unsigned long paddr)
264{ 268{
265 unsigned long addr, __addr, offset = 0; 269 unsigned long addr, __addr, offset = 0;
266 struct insn insn; 270 struct insn insn;
@@ -299,7 +303,7 @@ static int __kprobes can_probe(unsigned long paddr)
299/* 303/*
300 * Returns non-zero if opcode modifies the interrupt flag. 304 * Returns non-zero if opcode modifies the interrupt flag.
301 */ 305 */
302static int __kprobes is_IF_modifier(kprobe_opcode_t *insn) 306static int is_IF_modifier(kprobe_opcode_t *insn)
303{ 307{
304 /* Skip prefixes */ 308 /* Skip prefixes */
305 insn = skip_prefixes(insn); 309 insn = skip_prefixes(insn);
@@ -322,7 +326,7 @@ static int __kprobes is_IF_modifier(kprobe_opcode_t *insn)
322 * If not, return null. 326 * If not, return null.
323 * Only applicable to 64-bit x86. 327 * Only applicable to 64-bit x86.
324 */ 328 */
325int __kprobes __copy_instruction(u8 *dest, u8 *src) 329int __copy_instruction(u8 *dest, u8 *src)
326{ 330{
327 struct insn insn; 331 struct insn insn;
328 kprobe_opcode_t buf[MAX_INSN_SIZE]; 332 kprobe_opcode_t buf[MAX_INSN_SIZE];
@@ -365,7 +369,7 @@ int __kprobes __copy_instruction(u8 *dest, u8 *src)
365 return insn.length; 369 return insn.length;
366} 370}
367 371
368static int __kprobes arch_copy_kprobe(struct kprobe *p) 372static int arch_copy_kprobe(struct kprobe *p)
369{ 373{
370 int ret; 374 int ret;
371 375
@@ -392,7 +396,7 @@ static int __kprobes arch_copy_kprobe(struct kprobe *p)
392 return 0; 396 return 0;
393} 397}
394 398
395int __kprobes arch_prepare_kprobe(struct kprobe *p) 399int arch_prepare_kprobe(struct kprobe *p)
396{ 400{
397 if (alternatives_text_reserved(p->addr, p->addr)) 401 if (alternatives_text_reserved(p->addr, p->addr))
398 return -EINVAL; 402 return -EINVAL;
@@ -407,17 +411,17 @@ int __kprobes arch_prepare_kprobe(struct kprobe *p)
407 return arch_copy_kprobe(p); 411 return arch_copy_kprobe(p);
408} 412}
409 413
410void __kprobes arch_arm_kprobe(struct kprobe *p) 414void arch_arm_kprobe(struct kprobe *p)
411{ 415{
412 text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1); 416 text_poke(p->addr, ((unsigned char []){BREAKPOINT_INSTRUCTION}), 1);
413} 417}
414 418
415void __kprobes arch_disarm_kprobe(struct kprobe *p) 419void arch_disarm_kprobe(struct kprobe *p)
416{ 420{
417 text_poke(p->addr, &p->opcode, 1); 421 text_poke(p->addr, &p->opcode, 1);
418} 422}
419 423
420void __kprobes arch_remove_kprobe(struct kprobe *p) 424void arch_remove_kprobe(struct kprobe *p)
421{ 425{
422 if (p->ainsn.insn) { 426 if (p->ainsn.insn) {
423 free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1)); 427 free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1));
@@ -425,7 +429,8 @@ void __kprobes arch_remove_kprobe(struct kprobe *p)
425 } 429 }
426} 430}
427 431
428static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) 432static nokprobe_inline void
433save_previous_kprobe(struct kprobe_ctlblk *kcb)
429{ 434{
430 kcb->prev_kprobe.kp = kprobe_running(); 435 kcb->prev_kprobe.kp = kprobe_running();
431 kcb->prev_kprobe.status = kcb->kprobe_status; 436 kcb->prev_kprobe.status = kcb->kprobe_status;
@@ -433,7 +438,8 @@ static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb)
433 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags; 438 kcb->prev_kprobe.saved_flags = kcb->kprobe_saved_flags;
434} 439}
435 440
436static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb) 441static nokprobe_inline void
442restore_previous_kprobe(struct kprobe_ctlblk *kcb)
437{ 443{
438 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp); 444 __this_cpu_write(current_kprobe, kcb->prev_kprobe.kp);
439 kcb->kprobe_status = kcb->prev_kprobe.status; 445 kcb->kprobe_status = kcb->prev_kprobe.status;
@@ -441,8 +447,9 @@ static void __kprobes restore_previous_kprobe(struct kprobe_ctlblk *kcb)
441 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags; 447 kcb->kprobe_saved_flags = kcb->prev_kprobe.saved_flags;
442} 448}
443 449
444static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs, 450static nokprobe_inline void
445 struct kprobe_ctlblk *kcb) 451set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
452 struct kprobe_ctlblk *kcb)
446{ 453{
447 __this_cpu_write(current_kprobe, p); 454 __this_cpu_write(current_kprobe, p);
448 kcb->kprobe_saved_flags = kcb->kprobe_old_flags 455 kcb->kprobe_saved_flags = kcb->kprobe_old_flags
@@ -451,7 +458,7 @@ static void __kprobes set_current_kprobe(struct kprobe *p, struct pt_regs *regs,
451 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF; 458 kcb->kprobe_saved_flags &= ~X86_EFLAGS_IF;
452} 459}
453 460
454static void __kprobes clear_btf(void) 461static nokprobe_inline void clear_btf(void)
455{ 462{
456 if (test_thread_flag(TIF_BLOCKSTEP)) { 463 if (test_thread_flag(TIF_BLOCKSTEP)) {
457 unsigned long debugctl = get_debugctlmsr(); 464 unsigned long debugctl = get_debugctlmsr();
@@ -461,7 +468,7 @@ static void __kprobes clear_btf(void)
461 } 468 }
462} 469}
463 470
464static void __kprobes restore_btf(void) 471static nokprobe_inline void restore_btf(void)
465{ 472{
466 if (test_thread_flag(TIF_BLOCKSTEP)) { 473 if (test_thread_flag(TIF_BLOCKSTEP)) {
467 unsigned long debugctl = get_debugctlmsr(); 474 unsigned long debugctl = get_debugctlmsr();
@@ -471,8 +478,7 @@ static void __kprobes restore_btf(void)
471 } 478 }
472} 479}
473 480
474void __kprobes 481void arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
475arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
476{ 482{
477 unsigned long *sara = stack_addr(regs); 483 unsigned long *sara = stack_addr(regs);
478 484
@@ -481,9 +487,10 @@ arch_prepare_kretprobe(struct kretprobe_instance *ri, struct pt_regs *regs)
481 /* Replace the return addr with trampoline addr */ 487 /* Replace the return addr with trampoline addr */
482 *sara = (unsigned long) &kretprobe_trampoline; 488 *sara = (unsigned long) &kretprobe_trampoline;
483} 489}
490NOKPROBE_SYMBOL(arch_prepare_kretprobe);
484 491
485static void __kprobes 492static void setup_singlestep(struct kprobe *p, struct pt_regs *regs,
486setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb, int reenter) 493 struct kprobe_ctlblk *kcb, int reenter)
487{ 494{
488 if (setup_detour_execution(p, regs, reenter)) 495 if (setup_detour_execution(p, regs, reenter))
489 return; 496 return;
@@ -519,22 +526,24 @@ setup_singlestep(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *k
519 else 526 else
520 regs->ip = (unsigned long)p->ainsn.insn; 527 regs->ip = (unsigned long)p->ainsn.insn;
521} 528}
529NOKPROBE_SYMBOL(setup_singlestep);
522 530
523/* 531/*
524 * We have reentered the kprobe_handler(), since another probe was hit while 532 * We have reentered the kprobe_handler(), since another probe was hit while
525 * within the handler. We save the original kprobes variables and just single 533 * within the handler. We save the original kprobes variables and just single
526 * step on the instruction of the new probe without calling any user handlers. 534 * step on the instruction of the new probe without calling any user handlers.
527 */ 535 */
528static int __kprobes 536static int reenter_kprobe(struct kprobe *p, struct pt_regs *regs,
529reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) 537 struct kprobe_ctlblk *kcb)
530{ 538{
531 switch (kcb->kprobe_status) { 539 switch (kcb->kprobe_status) {
532 case KPROBE_HIT_SSDONE: 540 case KPROBE_HIT_SSDONE:
533 case KPROBE_HIT_ACTIVE: 541 case KPROBE_HIT_ACTIVE:
542 case KPROBE_HIT_SS:
534 kprobes_inc_nmissed_count(p); 543 kprobes_inc_nmissed_count(p);
535 setup_singlestep(p, regs, kcb, 1); 544 setup_singlestep(p, regs, kcb, 1);
536 break; 545 break;
537 case KPROBE_HIT_SS: 546 case KPROBE_REENTER:
538 /* A probe has been hit in the codepath leading up to, or just 547 /* A probe has been hit in the codepath leading up to, or just
539 * after, single-stepping of a probed instruction. This entire 548 * after, single-stepping of a probed instruction. This entire
540 * codepath should strictly reside in .kprobes.text section. 549 * codepath should strictly reside in .kprobes.text section.
@@ -553,12 +562,13 @@ reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb
553 562
554 return 1; 563 return 1;
555} 564}
565NOKPROBE_SYMBOL(reenter_kprobe);
556 566
557/* 567/*
558 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 568 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
559 * remain disabled throughout this function. 569 * remain disabled throughout this function.
560 */ 570 */
561static int __kprobes kprobe_handler(struct pt_regs *regs) 571int kprobe_int3_handler(struct pt_regs *regs)
562{ 572{
563 kprobe_opcode_t *addr; 573 kprobe_opcode_t *addr;
564 struct kprobe *p; 574 struct kprobe *p;
@@ -621,12 +631,13 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
621 preempt_enable_no_resched(); 631 preempt_enable_no_resched();
622 return 0; 632 return 0;
623} 633}
634NOKPROBE_SYMBOL(kprobe_int3_handler);
624 635
625/* 636/*
626 * When a retprobed function returns, this code saves registers and 637 * When a retprobed function returns, this code saves registers and
627 * calls trampoline_handler() runs, which calls the kretprobe's handler. 638 * calls trampoline_handler() runs, which calls the kretprobe's handler.
628 */ 639 */
629static void __used __kprobes kretprobe_trampoline_holder(void) 640static void __used kretprobe_trampoline_holder(void)
630{ 641{
631 asm volatile ( 642 asm volatile (
632 ".global kretprobe_trampoline\n" 643 ".global kretprobe_trampoline\n"
@@ -657,11 +668,13 @@ static void __used __kprobes kretprobe_trampoline_holder(void)
657#endif 668#endif
658 " ret\n"); 669 " ret\n");
659} 670}
671NOKPROBE_SYMBOL(kretprobe_trampoline_holder);
672NOKPROBE_SYMBOL(kretprobe_trampoline);
660 673
661/* 674/*
662 * Called from kretprobe_trampoline 675 * Called from kretprobe_trampoline
663 */ 676 */
664__visible __used __kprobes void *trampoline_handler(struct pt_regs *regs) 677__visible __used void *trampoline_handler(struct pt_regs *regs)
665{ 678{
666 struct kretprobe_instance *ri = NULL; 679 struct kretprobe_instance *ri = NULL;
667 struct hlist_head *head, empty_rp; 680 struct hlist_head *head, empty_rp;
@@ -747,6 +760,7 @@ __visible __used __kprobes void *trampoline_handler(struct pt_regs *regs)
747 } 760 }
748 return (void *)orig_ret_address; 761 return (void *)orig_ret_address;
749} 762}
763NOKPROBE_SYMBOL(trampoline_handler);
750 764
751/* 765/*
752 * Called after single-stepping. p->addr is the address of the 766 * Called after single-stepping. p->addr is the address of the
@@ -775,8 +789,8 @@ __visible __used __kprobes void *trampoline_handler(struct pt_regs *regs)
775 * jump instruction after the copied instruction, that jumps to the next 789 * jump instruction after the copied instruction, that jumps to the next
776 * instruction after the probepoint. 790 * instruction after the probepoint.
777 */ 791 */
778static void __kprobes 792static void resume_execution(struct kprobe *p, struct pt_regs *regs,
779resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb) 793 struct kprobe_ctlblk *kcb)
780{ 794{
781 unsigned long *tos = stack_addr(regs); 795 unsigned long *tos = stack_addr(regs);
782 unsigned long copy_ip = (unsigned long)p->ainsn.insn; 796 unsigned long copy_ip = (unsigned long)p->ainsn.insn;
@@ -851,12 +865,13 @@ resume_execution(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *k
851no_change: 865no_change:
852 restore_btf(); 866 restore_btf();
853} 867}
868NOKPROBE_SYMBOL(resume_execution);
854 869
855/* 870/*
856 * Interrupts are disabled on entry as trap1 is an interrupt gate and they 871 * Interrupts are disabled on entry as trap1 is an interrupt gate and they
857 * remain disabled throughout this function. 872 * remain disabled throughout this function.
858 */ 873 */
859static int __kprobes post_kprobe_handler(struct pt_regs *regs) 874int kprobe_debug_handler(struct pt_regs *regs)
860{ 875{
861 struct kprobe *cur = kprobe_running(); 876 struct kprobe *cur = kprobe_running();
862 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 877 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
@@ -891,8 +906,9 @@ out:
891 906
892 return 1; 907 return 1;
893} 908}
909NOKPROBE_SYMBOL(kprobe_debug_handler);
894 910
895int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr) 911int kprobe_fault_handler(struct pt_regs *regs, int trapnr)
896{ 912{
897 struct kprobe *cur = kprobe_running(); 913 struct kprobe *cur = kprobe_running();
898 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 914 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
@@ -949,12 +965,13 @@ int __kprobes kprobe_fault_handler(struct pt_regs *regs, int trapnr)
949 965
950 return 0; 966 return 0;
951} 967}
968NOKPROBE_SYMBOL(kprobe_fault_handler);
952 969
953/* 970/*
954 * Wrapper routine for handling exceptions. 971 * Wrapper routine for handling exceptions.
955 */ 972 */
956int __kprobes 973int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val,
957kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data) 974 void *data)
958{ 975{
959 struct die_args *args = data; 976 struct die_args *args = data;
960 int ret = NOTIFY_DONE; 977 int ret = NOTIFY_DONE;
@@ -962,22 +979,7 @@ kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *d
962 if (args->regs && user_mode_vm(args->regs)) 979 if (args->regs && user_mode_vm(args->regs))
963 return ret; 980 return ret;
964 981
965 switch (val) { 982 if (val == DIE_GPF) {
966 case DIE_INT3:
967 if (kprobe_handler(args->regs))
968 ret = NOTIFY_STOP;
969 break;
970 case DIE_DEBUG:
971 if (post_kprobe_handler(args->regs)) {
972 /*
973 * Reset the BS bit in dr6 (pointed by args->err) to
974 * denote completion of processing
975 */
976 (*(unsigned long *)ERR_PTR(args->err)) &= ~DR_STEP;
977 ret = NOTIFY_STOP;
978 }
979 break;
980 case DIE_GPF:
981 /* 983 /*
982 * To be potentially processing a kprobe fault and to 984 * To be potentially processing a kprobe fault and to
983 * trust the result from kprobe_running(), we have 985 * trust the result from kprobe_running(), we have
@@ -986,14 +988,12 @@ kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *d
986 if (!preemptible() && kprobe_running() && 988 if (!preemptible() && kprobe_running() &&
987 kprobe_fault_handler(args->regs, args->trapnr)) 989 kprobe_fault_handler(args->regs, args->trapnr))
988 ret = NOTIFY_STOP; 990 ret = NOTIFY_STOP;
989 break;
990 default:
991 break;
992 } 991 }
993 return ret; 992 return ret;
994} 993}
994NOKPROBE_SYMBOL(kprobe_exceptions_notify);
995 995
996int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs) 996int setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
997{ 997{
998 struct jprobe *jp = container_of(p, struct jprobe, kp); 998 struct jprobe *jp = container_of(p, struct jprobe, kp);
999 unsigned long addr; 999 unsigned long addr;
@@ -1017,8 +1017,9 @@ int __kprobes setjmp_pre_handler(struct kprobe *p, struct pt_regs *regs)
1017 regs->ip = (unsigned long)(jp->entry); 1017 regs->ip = (unsigned long)(jp->entry);
1018 return 1; 1018 return 1;
1019} 1019}
1020NOKPROBE_SYMBOL(setjmp_pre_handler);
1020 1021
1021void __kprobes jprobe_return(void) 1022void jprobe_return(void)
1022{ 1023{
1023 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1024 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1024 1025
@@ -1034,8 +1035,10 @@ void __kprobes jprobe_return(void)
1034 " nop \n"::"b" 1035 " nop \n"::"b"
1035 (kcb->jprobe_saved_sp):"memory"); 1036 (kcb->jprobe_saved_sp):"memory");
1036} 1037}
1038NOKPROBE_SYMBOL(jprobe_return);
1039NOKPROBE_SYMBOL(jprobe_return_end);
1037 1040
1038int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs) 1041int longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1039{ 1042{
1040 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 1043 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
1041 u8 *addr = (u8 *) (regs->ip - 1); 1044 u8 *addr = (u8 *) (regs->ip - 1);
@@ -1063,13 +1066,22 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1063 } 1066 }
1064 return 0; 1067 return 0;
1065} 1068}
1069NOKPROBE_SYMBOL(longjmp_break_handler);
1070
1071bool arch_within_kprobe_blacklist(unsigned long addr)
1072{
1073 return (addr >= (unsigned long)__kprobes_text_start &&
1074 addr < (unsigned long)__kprobes_text_end) ||
1075 (addr >= (unsigned long)__entry_text_start &&
1076 addr < (unsigned long)__entry_text_end);
1077}
1066 1078
1067int __init arch_init_kprobes(void) 1079int __init arch_init_kprobes(void)
1068{ 1080{
1069 return 0; 1081 return 0;
1070} 1082}
1071 1083
1072int __kprobes arch_trampoline_kprobe(struct kprobe *p) 1084int arch_trampoline_kprobe(struct kprobe *p)
1073{ 1085{
1074 return 0; 1086 return 0;
1075} 1087}
diff --git a/arch/x86/kernel/kprobes/ftrace.c b/arch/x86/kernel/kprobes/ftrace.c
index 23ef5c556f06..717b02a22e67 100644
--- a/arch/x86/kernel/kprobes/ftrace.c
+++ b/arch/x86/kernel/kprobes/ftrace.c
@@ -25,8 +25,9 @@
25 25
26#include "common.h" 26#include "common.h"
27 27
28static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs, 28static nokprobe_inline
29 struct kprobe_ctlblk *kcb) 29int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
30 struct kprobe_ctlblk *kcb)
30{ 31{
31 /* 32 /*
32 * Emulate singlestep (and also recover regs->ip) 33 * Emulate singlestep (and also recover regs->ip)
@@ -41,18 +42,19 @@ static int __skip_singlestep(struct kprobe *p, struct pt_regs *regs,
41 return 1; 42 return 1;
42} 43}
43 44
44int __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs, 45int skip_singlestep(struct kprobe *p, struct pt_regs *regs,
45 struct kprobe_ctlblk *kcb) 46 struct kprobe_ctlblk *kcb)
46{ 47{
47 if (kprobe_ftrace(p)) 48 if (kprobe_ftrace(p))
48 return __skip_singlestep(p, regs, kcb); 49 return __skip_singlestep(p, regs, kcb);
49 else 50 else
50 return 0; 51 return 0;
51} 52}
53NOKPROBE_SYMBOL(skip_singlestep);
52 54
53/* Ftrace callback handler for kprobes */ 55/* Ftrace callback handler for kprobes */
54void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip, 56void kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
55 struct ftrace_ops *ops, struct pt_regs *regs) 57 struct ftrace_ops *ops, struct pt_regs *regs)
56{ 58{
57 struct kprobe *p; 59 struct kprobe *p;
58 struct kprobe_ctlblk *kcb; 60 struct kprobe_ctlblk *kcb;
@@ -84,8 +86,9 @@ void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
84end: 86end:
85 local_irq_restore(flags); 87 local_irq_restore(flags);
86} 88}
89NOKPROBE_SYMBOL(kprobe_ftrace_handler);
87 90
88int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p) 91int arch_prepare_kprobe_ftrace(struct kprobe *p)
89{ 92{
90 p->ainsn.insn = NULL; 93 p->ainsn.insn = NULL;
91 p->ainsn.boostable = -1; 94 p->ainsn.boostable = -1;
diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c
index 898160b42e43..f304773285ae 100644
--- a/arch/x86/kernel/kprobes/opt.c
+++ b/arch/x86/kernel/kprobes/opt.c
@@ -77,7 +77,7 @@ found:
77} 77}
78 78
79/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */ 79/* Insert a move instruction which sets a pointer to eax/rdi (1st arg). */
80static void __kprobes synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val) 80static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
81{ 81{
82#ifdef CONFIG_X86_64 82#ifdef CONFIG_X86_64
83 *addr++ = 0x48; 83 *addr++ = 0x48;
@@ -138,7 +138,8 @@ asm (
138#define INT3_SIZE sizeof(kprobe_opcode_t) 138#define INT3_SIZE sizeof(kprobe_opcode_t)
139 139
140/* Optimized kprobe call back function: called from optinsn */ 140/* Optimized kprobe call back function: called from optinsn */
141static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs) 141static void
142optimized_callback(struct optimized_kprobe *op, struct pt_regs *regs)
142{ 143{
143 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk(); 144 struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
144 unsigned long flags; 145 unsigned long flags;
@@ -168,8 +169,9 @@ static void __kprobes optimized_callback(struct optimized_kprobe *op, struct pt_
168 } 169 }
169 local_irq_restore(flags); 170 local_irq_restore(flags);
170} 171}
172NOKPROBE_SYMBOL(optimized_callback);
171 173
172static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src) 174static int copy_optimized_instructions(u8 *dest, u8 *src)
173{ 175{
174 int len = 0, ret; 176 int len = 0, ret;
175 177
@@ -189,7 +191,7 @@ static int __kprobes copy_optimized_instructions(u8 *dest, u8 *src)
189} 191}
190 192
191/* Check whether insn is indirect jump */ 193/* Check whether insn is indirect jump */
192static int __kprobes insn_is_indirect_jump(struct insn *insn) 194static int insn_is_indirect_jump(struct insn *insn)
193{ 195{
194 return ((insn->opcode.bytes[0] == 0xff && 196 return ((insn->opcode.bytes[0] == 0xff &&
195 (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */ 197 (X86_MODRM_REG(insn->modrm.value) & 6) == 4) || /* Jump */
@@ -224,7 +226,7 @@ static int insn_jump_into_range(struct insn *insn, unsigned long start, int len)
224} 226}
225 227
226/* Decode whole function to ensure any instructions don't jump into target */ 228/* Decode whole function to ensure any instructions don't jump into target */
227static int __kprobes can_optimize(unsigned long paddr) 229static int can_optimize(unsigned long paddr)
228{ 230{
229 unsigned long addr, size = 0, offset = 0; 231 unsigned long addr, size = 0, offset = 0;
230 struct insn insn; 232 struct insn insn;
@@ -275,7 +277,7 @@ static int __kprobes can_optimize(unsigned long paddr)
275} 277}
276 278
277/* Check optimized_kprobe can actually be optimized. */ 279/* Check optimized_kprobe can actually be optimized. */
278int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op) 280int arch_check_optimized_kprobe(struct optimized_kprobe *op)
279{ 281{
280 int i; 282 int i;
281 struct kprobe *p; 283 struct kprobe *p;
@@ -290,15 +292,15 @@ int __kprobes arch_check_optimized_kprobe(struct optimized_kprobe *op)
290} 292}
291 293
292/* Check the addr is within the optimized instructions. */ 294/* Check the addr is within the optimized instructions. */
293int __kprobes 295int arch_within_optimized_kprobe(struct optimized_kprobe *op,
294arch_within_optimized_kprobe(struct optimized_kprobe *op, unsigned long addr) 296 unsigned long addr)
295{ 297{
296 return ((unsigned long)op->kp.addr <= addr && 298 return ((unsigned long)op->kp.addr <= addr &&
297 (unsigned long)op->kp.addr + op->optinsn.size > addr); 299 (unsigned long)op->kp.addr + op->optinsn.size > addr);
298} 300}
299 301
300/* Free optimized instruction slot */ 302/* Free optimized instruction slot */
301static __kprobes 303static
302void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty) 304void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
303{ 305{
304 if (op->optinsn.insn) { 306 if (op->optinsn.insn) {
@@ -308,7 +310,7 @@ void __arch_remove_optimized_kprobe(struct optimized_kprobe *op, int dirty)
308 } 310 }
309} 311}
310 312
311void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op) 313void arch_remove_optimized_kprobe(struct optimized_kprobe *op)
312{ 314{
313 __arch_remove_optimized_kprobe(op, 1); 315 __arch_remove_optimized_kprobe(op, 1);
314} 316}
@@ -318,7 +320,7 @@ void __kprobes arch_remove_optimized_kprobe(struct optimized_kprobe *op)
318 * Target instructions MUST be relocatable (checked inside) 320 * Target instructions MUST be relocatable (checked inside)
319 * This is called when new aggr(opt)probe is allocated or reused. 321 * This is called when new aggr(opt)probe is allocated or reused.
320 */ 322 */
321int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op) 323int arch_prepare_optimized_kprobe(struct optimized_kprobe *op)
322{ 324{
323 u8 *buf; 325 u8 *buf;
324 int ret; 326 int ret;
@@ -372,7 +374,7 @@ int __kprobes arch_prepare_optimized_kprobe(struct optimized_kprobe *op)
372 * Replace breakpoints (int3) with relative jumps. 374 * Replace breakpoints (int3) with relative jumps.
373 * Caller must call with locking kprobe_mutex and text_mutex. 375 * Caller must call with locking kprobe_mutex and text_mutex.
374 */ 376 */
375void __kprobes arch_optimize_kprobes(struct list_head *oplist) 377void arch_optimize_kprobes(struct list_head *oplist)
376{ 378{
377 struct optimized_kprobe *op, *tmp; 379 struct optimized_kprobe *op, *tmp;
378 u8 insn_buf[RELATIVEJUMP_SIZE]; 380 u8 insn_buf[RELATIVEJUMP_SIZE];
@@ -398,7 +400,7 @@ void __kprobes arch_optimize_kprobes(struct list_head *oplist)
398} 400}
399 401
400/* Replace a relative jump with a breakpoint (int3). */ 402/* Replace a relative jump with a breakpoint (int3). */
401void __kprobes arch_unoptimize_kprobe(struct optimized_kprobe *op) 403void arch_unoptimize_kprobe(struct optimized_kprobe *op)
402{ 404{
403 u8 insn_buf[RELATIVEJUMP_SIZE]; 405 u8 insn_buf[RELATIVEJUMP_SIZE];
404 406
@@ -424,8 +426,7 @@ extern void arch_unoptimize_kprobes(struct list_head *oplist,
424 } 426 }
425} 427}
426 428
427int __kprobes 429int setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
428setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
429{ 430{
430 struct optimized_kprobe *op; 431 struct optimized_kprobe *op;
431 432
@@ -441,3 +442,4 @@ setup_detour_execution(struct kprobe *p, struct pt_regs *regs, int reenter)
441 } 442 }
442 return 0; 443 return 0;
443} 444}
445NOKPROBE_SYMBOL(setup_detour_execution);
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 7e97371387fd..3dd8e2c4d74a 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -251,8 +251,9 @@ u32 kvm_read_and_reset_pf_reason(void)
251 return reason; 251 return reason;
252} 252}
253EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason); 253EXPORT_SYMBOL_GPL(kvm_read_and_reset_pf_reason);
254NOKPROBE_SYMBOL(kvm_read_and_reset_pf_reason);
254 255
255dotraplinkage void __kprobes 256dotraplinkage void
256do_async_page_fault(struct pt_regs *regs, unsigned long error_code) 257do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
257{ 258{
258 enum ctx_state prev_state; 259 enum ctx_state prev_state;
@@ -276,6 +277,7 @@ do_async_page_fault(struct pt_regs *regs, unsigned long error_code)
276 break; 277 break;
277 } 278 }
278} 279}
280NOKPROBE_SYMBOL(do_async_page_fault);
279 281
280static void __init paravirt_ops_setup(void) 282static void __init paravirt_ops_setup(void)
281{ 283{
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index b4872b999a71..c3e985d1751c 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -110,7 +110,7 @@ static void nmi_max_handler(struct irq_work *w)
110 a->handler, whole_msecs, decimal_msecs); 110 a->handler, whole_msecs, decimal_msecs);
111} 111}
112 112
113static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b) 113static int nmi_handle(unsigned int type, struct pt_regs *regs, bool b2b)
114{ 114{
115 struct nmi_desc *desc = nmi_to_desc(type); 115 struct nmi_desc *desc = nmi_to_desc(type);
116 struct nmiaction *a; 116 struct nmiaction *a;
@@ -146,6 +146,7 @@ static int __kprobes nmi_handle(unsigned int type, struct pt_regs *regs, bool b2
146 /* return total number of NMI events handled */ 146 /* return total number of NMI events handled */
147 return handled; 147 return handled;
148} 148}
149NOKPROBE_SYMBOL(nmi_handle);
149 150
150int __register_nmi_handler(unsigned int type, struct nmiaction *action) 151int __register_nmi_handler(unsigned int type, struct nmiaction *action)
151{ 152{
@@ -208,7 +209,7 @@ void unregister_nmi_handler(unsigned int type, const char *name)
208} 209}
209EXPORT_SYMBOL_GPL(unregister_nmi_handler); 210EXPORT_SYMBOL_GPL(unregister_nmi_handler);
210 211
211static __kprobes void 212static void
212pci_serr_error(unsigned char reason, struct pt_regs *regs) 213pci_serr_error(unsigned char reason, struct pt_regs *regs)
213{ 214{
214 /* check to see if anyone registered against these types of errors */ 215 /* check to see if anyone registered against these types of errors */
@@ -238,8 +239,9 @@ pci_serr_error(unsigned char reason, struct pt_regs *regs)
238 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR; 239 reason = (reason & NMI_REASON_CLEAR_MASK) | NMI_REASON_CLEAR_SERR;
239 outb(reason, NMI_REASON_PORT); 240 outb(reason, NMI_REASON_PORT);
240} 241}
242NOKPROBE_SYMBOL(pci_serr_error);
241 243
242static __kprobes void 244static void
243io_check_error(unsigned char reason, struct pt_regs *regs) 245io_check_error(unsigned char reason, struct pt_regs *regs)
244{ 246{
245 unsigned long i; 247 unsigned long i;
@@ -269,8 +271,9 @@ io_check_error(unsigned char reason, struct pt_regs *regs)
269 reason &= ~NMI_REASON_CLEAR_IOCHK; 271 reason &= ~NMI_REASON_CLEAR_IOCHK;
270 outb(reason, NMI_REASON_PORT); 272 outb(reason, NMI_REASON_PORT);
271} 273}
274NOKPROBE_SYMBOL(io_check_error);
272 275
273static __kprobes void 276static void
274unknown_nmi_error(unsigned char reason, struct pt_regs *regs) 277unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
275{ 278{
276 int handled; 279 int handled;
@@ -298,11 +301,12 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
298 301
299 pr_emerg("Dazed and confused, but trying to continue\n"); 302 pr_emerg("Dazed and confused, but trying to continue\n");
300} 303}
304NOKPROBE_SYMBOL(unknown_nmi_error);
301 305
302static DEFINE_PER_CPU(bool, swallow_nmi); 306static DEFINE_PER_CPU(bool, swallow_nmi);
303static DEFINE_PER_CPU(unsigned long, last_nmi_rip); 307static DEFINE_PER_CPU(unsigned long, last_nmi_rip);
304 308
305static __kprobes void default_do_nmi(struct pt_regs *regs) 309static void default_do_nmi(struct pt_regs *regs)
306{ 310{
307 unsigned char reason = 0; 311 unsigned char reason = 0;
308 int handled; 312 int handled;
@@ -401,6 +405,7 @@ static __kprobes void default_do_nmi(struct pt_regs *regs)
401 else 405 else
402 unknown_nmi_error(reason, regs); 406 unknown_nmi_error(reason, regs);
403} 407}
408NOKPROBE_SYMBOL(default_do_nmi);
404 409
405/* 410/*
406 * NMIs can hit breakpoints which will cause it to lose its 411 * NMIs can hit breakpoints which will cause it to lose its
@@ -520,7 +525,7 @@ static inline void nmi_nesting_postprocess(void)
520} 525}
521#endif 526#endif
522 527
523dotraplinkage notrace __kprobes void 528dotraplinkage notrace void
524do_nmi(struct pt_regs *regs, long error_code) 529do_nmi(struct pt_regs *regs, long error_code)
525{ 530{
526 nmi_nesting_preprocess(regs); 531 nmi_nesting_preprocess(regs);
@@ -537,6 +542,7 @@ do_nmi(struct pt_regs *regs, long error_code)
537 /* On i386, may loop back to preprocess */ 542 /* On i386, may loop back to preprocess */
538 nmi_nesting_postprocess(); 543 nmi_nesting_postprocess();
539} 544}
545NOKPROBE_SYMBOL(do_nmi);
540 546
541void stop_nmi(void) 547void stop_nmi(void)
542{ 548{
diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
index 1b10af835c31..548d25f00c90 100644
--- a/arch/x86/kernel/paravirt.c
+++ b/arch/x86/kernel/paravirt.c
@@ -23,6 +23,7 @@
23#include <linux/efi.h> 23#include <linux/efi.h>
24#include <linux/bcd.h> 24#include <linux/bcd.h>
25#include <linux/highmem.h> 25#include <linux/highmem.h>
26#include <linux/kprobes.h>
26 27
27#include <asm/bug.h> 28#include <asm/bug.h>
28#include <asm/paravirt.h> 29#include <asm/paravirt.h>
@@ -389,6 +390,11 @@ __visible struct pv_cpu_ops pv_cpu_ops = {
389 .end_context_switch = paravirt_nop, 390 .end_context_switch = paravirt_nop,
390}; 391};
391 392
393/* At this point, native_get/set_debugreg has real function entries */
394NOKPROBE_SYMBOL(native_get_debugreg);
395NOKPROBE_SYMBOL(native_set_debugreg);
396NOKPROBE_SYMBOL(native_load_idt);
397
392struct pv_apic_ops pv_apic_ops = { 398struct pv_apic_ops pv_apic_ops = {
393#ifdef CONFIG_X86_LOCAL_APIC 399#ifdef CONFIG_X86_LOCAL_APIC
394 .startup_ipi_hook = paravirt_nop, 400 .startup_ipi_hook = paravirt_nop,
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 898d077617a9..ca5b02d405c3 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -413,12 +413,11 @@ void set_personality_ia32(bool x32)
413 set_thread_flag(TIF_ADDR32); 413 set_thread_flag(TIF_ADDR32);
414 414
415 /* Mark the associated mm as containing 32-bit tasks. */ 415 /* Mark the associated mm as containing 32-bit tasks. */
416 if (current->mm)
417 current->mm->context.ia32_compat = 1;
418
419 if (x32) { 416 if (x32) {
420 clear_thread_flag(TIF_IA32); 417 clear_thread_flag(TIF_IA32);
421 set_thread_flag(TIF_X32); 418 set_thread_flag(TIF_X32);
419 if (current->mm)
420 current->mm->context.ia32_compat = TIF_X32;
422 current->personality &= ~READ_IMPLIES_EXEC; 421 current->personality &= ~READ_IMPLIES_EXEC;
423 /* is_compat_task() uses the presence of the x32 422 /* is_compat_task() uses the presence of the x32
424 syscall bit flag to determine compat status */ 423 syscall bit flag to determine compat status */
@@ -426,6 +425,8 @@ void set_personality_ia32(bool x32)
426 } else { 425 } else {
427 set_thread_flag(TIF_IA32); 426 set_thread_flag(TIF_IA32);
428 clear_thread_flag(TIF_X32); 427 clear_thread_flag(TIF_X32);
428 if (current->mm)
429 current->mm->context.ia32_compat = TIF_IA32;
429 current->personality |= force_personality32; 430 current->personality |= force_personality32;
430 /* Prepare the first "return" to user space */ 431 /* Prepare the first "return" to user space */
431 current_thread_info()->status |= TS_COMPAT; 432 current_thread_info()->status |= TS_COMPAT;
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index f73b5d435bdc..c6eb418c5627 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -23,6 +23,7 @@
23#include <linux/kernel.h> 23#include <linux/kernel.h>
24#include <linux/module.h> 24#include <linux/module.h>
25#include <linux/ptrace.h> 25#include <linux/ptrace.h>
26#include <linux/uprobes.h>
26#include <linux/string.h> 27#include <linux/string.h>
27#include <linux/delay.h> 28#include <linux/delay.h>
28#include <linux/errno.h> 29#include <linux/errno.h>
@@ -106,7 +107,7 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
106 preempt_count_dec(); 107 preempt_count_dec();
107} 108}
108 109
109static int __kprobes 110static nokprobe_inline int
110do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str, 111do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
111 struct pt_regs *regs, long error_code) 112 struct pt_regs *regs, long error_code)
112{ 113{
@@ -136,7 +137,38 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
136 return -1; 137 return -1;
137} 138}
138 139
139static void __kprobes 140static siginfo_t *fill_trap_info(struct pt_regs *regs, int signr, int trapnr,
141 siginfo_t *info)
142{
143 unsigned long siaddr;
144 int sicode;
145
146 switch (trapnr) {
147 default:
148 return SEND_SIG_PRIV;
149
150 case X86_TRAP_DE:
151 sicode = FPE_INTDIV;
152 siaddr = uprobe_get_trap_addr(regs);
153 break;
154 case X86_TRAP_UD:
155 sicode = ILL_ILLOPN;
156 siaddr = uprobe_get_trap_addr(regs);
157 break;
158 case X86_TRAP_AC:
159 sicode = BUS_ADRALN;
160 siaddr = 0;
161 break;
162 }
163
164 info->si_signo = signr;
165 info->si_errno = 0;
166 info->si_code = sicode;
167 info->si_addr = (void __user *)siaddr;
168 return info;
169}
170
171static void
140do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, 172do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
141 long error_code, siginfo_t *info) 173 long error_code, siginfo_t *info)
142{ 174{
@@ -168,60 +200,43 @@ do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
168 } 200 }
169#endif 201#endif
170 202
171 if (info) 203 force_sig_info(signr, info ?: SEND_SIG_PRIV, tsk);
172 force_sig_info(signr, info, tsk);
173 else
174 force_sig(signr, tsk);
175} 204}
205NOKPROBE_SYMBOL(do_trap);
176 206
177#define DO_ERROR(trapnr, signr, str, name) \ 207static void do_error_trap(struct pt_regs *regs, long error_code, char *str,
178dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ 208 unsigned long trapnr, int signr)
179{ \ 209{
180 enum ctx_state prev_state; \ 210 enum ctx_state prev_state = exception_enter();
181 \ 211 siginfo_t info;
182 prev_state = exception_enter(); \ 212
183 if (notify_die(DIE_TRAP, str, regs, error_code, \ 213 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) !=
184 trapnr, signr) == NOTIFY_STOP) { \ 214 NOTIFY_STOP) {
185 exception_exit(prev_state); \ 215 conditional_sti(regs);
186 return; \ 216 do_trap(trapnr, signr, str, regs, error_code,
187 } \ 217 fill_trap_info(regs, signr, trapnr, &info));
188 conditional_sti(regs); \ 218 }
189 do_trap(trapnr, signr, str, regs, error_code, NULL); \ 219
190 exception_exit(prev_state); \ 220 exception_exit(prev_state);
191} 221}
192 222
193#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \ 223#define DO_ERROR(trapnr, signr, str, name) \
194dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ 224dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
195{ \ 225{ \
196 siginfo_t info; \ 226 do_error_trap(regs, error_code, str, trapnr, signr); \
197 enum ctx_state prev_state; \
198 \
199 info.si_signo = signr; \
200 info.si_errno = 0; \
201 info.si_code = sicode; \
202 info.si_addr = (void __user *)siaddr; \
203 prev_state = exception_enter(); \
204 if (notify_die(DIE_TRAP, str, regs, error_code, \
205 trapnr, signr) == NOTIFY_STOP) { \
206 exception_exit(prev_state); \
207 return; \
208 } \
209 conditional_sti(regs); \
210 do_trap(trapnr, signr, str, regs, error_code, &info); \
211 exception_exit(prev_state); \
212} 227}
213 228
214DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip ) 229DO_ERROR(X86_TRAP_DE, SIGFPE, "divide error", divide_error)
215DO_ERROR (X86_TRAP_OF, SIGSEGV, "overflow", overflow ) 230DO_ERROR(X86_TRAP_OF, SIGSEGV, "overflow", overflow)
216DO_ERROR (X86_TRAP_BR, SIGSEGV, "bounds", bounds ) 231DO_ERROR(X86_TRAP_BR, SIGSEGV, "bounds", bounds)
217DO_ERROR_INFO(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip ) 232DO_ERROR(X86_TRAP_UD, SIGILL, "invalid opcode", invalid_op)
218DO_ERROR (X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun ) 233DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, "coprocessor segment overrun",coprocessor_segment_overrun)
219DO_ERROR (X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS ) 234DO_ERROR(X86_TRAP_TS, SIGSEGV, "invalid TSS", invalid_TSS)
220DO_ERROR (X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present ) 235DO_ERROR(X86_TRAP_NP, SIGBUS, "segment not present", segment_not_present)
221#ifdef CONFIG_X86_32 236#ifdef CONFIG_X86_32
222DO_ERROR (X86_TRAP_SS, SIGBUS, "stack segment", stack_segment ) 237DO_ERROR(X86_TRAP_SS, SIGBUS, "stack segment", stack_segment)
223#endif 238#endif
224DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0 ) 239DO_ERROR(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check)
225 240
226#ifdef CONFIG_X86_64 241#ifdef CONFIG_X86_64
227/* Runs on IST stack */ 242/* Runs on IST stack */
@@ -263,7 +278,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
263} 278}
264#endif 279#endif
265 280
266dotraplinkage void __kprobes 281dotraplinkage void
267do_general_protection(struct pt_regs *regs, long error_code) 282do_general_protection(struct pt_regs *regs, long error_code)
268{ 283{
269 struct task_struct *tsk; 284 struct task_struct *tsk;
@@ -305,13 +320,14 @@ do_general_protection(struct pt_regs *regs, long error_code)
305 pr_cont("\n"); 320 pr_cont("\n");
306 } 321 }
307 322
308 force_sig(SIGSEGV, tsk); 323 force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
309exit: 324exit:
310 exception_exit(prev_state); 325 exception_exit(prev_state);
311} 326}
327NOKPROBE_SYMBOL(do_general_protection);
312 328
313/* May run on IST stack. */ 329/* May run on IST stack. */
314dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_code) 330dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code)
315{ 331{
316 enum ctx_state prev_state; 332 enum ctx_state prev_state;
317 333
@@ -327,13 +343,18 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
327 if (poke_int3_handler(regs)) 343 if (poke_int3_handler(regs))
328 return; 344 return;
329 345
330 prev_state = exception_enter();
331#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP 346#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
332 if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, 347 if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
333 SIGTRAP) == NOTIFY_STOP) 348 SIGTRAP) == NOTIFY_STOP)
334 goto exit; 349 goto exit;
335#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ 350#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
336 351
352#ifdef CONFIG_KPROBES
353 if (kprobe_int3_handler(regs))
354 return;
355#endif
356 prev_state = exception_enter();
357
337 if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, 358 if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
338 SIGTRAP) == NOTIFY_STOP) 359 SIGTRAP) == NOTIFY_STOP)
339 goto exit; 360 goto exit;
@@ -350,6 +371,7 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
350exit: 371exit:
351 exception_exit(prev_state); 372 exception_exit(prev_state);
352} 373}
374NOKPROBE_SYMBOL(do_int3);
353 375
354#ifdef CONFIG_X86_64 376#ifdef CONFIG_X86_64
355/* 377/*
@@ -357,7 +379,7 @@ exit:
357 * for scheduling or signal handling. The actual stack switch is done in 379 * for scheduling or signal handling. The actual stack switch is done in
358 * entry.S 380 * entry.S
359 */ 381 */
360asmlinkage __visible __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs) 382asmlinkage __visible struct pt_regs *sync_regs(struct pt_regs *eregs)
361{ 383{
362 struct pt_regs *regs = eregs; 384 struct pt_regs *regs = eregs;
363 /* Did already sync */ 385 /* Did already sync */
@@ -376,6 +398,7 @@ asmlinkage __visible __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
376 *regs = *eregs; 398 *regs = *eregs;
377 return regs; 399 return regs;
378} 400}
401NOKPROBE_SYMBOL(sync_regs);
379#endif 402#endif
380 403
381/* 404/*
@@ -402,7 +425,7 @@ asmlinkage __visible __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs)
402 * 425 *
403 * May run on IST stack. 426 * May run on IST stack.
404 */ 427 */
405dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) 428dotraplinkage void do_debug(struct pt_regs *regs, long error_code)
406{ 429{
407 struct task_struct *tsk = current; 430 struct task_struct *tsk = current;
408 enum ctx_state prev_state; 431 enum ctx_state prev_state;
@@ -410,8 +433,6 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
410 unsigned long dr6; 433 unsigned long dr6;
411 int si_code; 434 int si_code;
412 435
413 prev_state = exception_enter();
414
415 get_debugreg(dr6, 6); 436 get_debugreg(dr6, 6);
416 437
417 /* Filter out all the reserved bits which are preset to 1 */ 438 /* Filter out all the reserved bits which are preset to 1 */
@@ -440,6 +461,12 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
440 /* Store the virtualized DR6 value */ 461 /* Store the virtualized DR6 value */
441 tsk->thread.debugreg6 = dr6; 462 tsk->thread.debugreg6 = dr6;
442 463
464#ifdef CONFIG_KPROBES
465 if (kprobe_debug_handler(regs))
466 goto exit;
467#endif
468 prev_state = exception_enter();
469
443 if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code, 470 if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code,
444 SIGTRAP) == NOTIFY_STOP) 471 SIGTRAP) == NOTIFY_STOP)
445 goto exit; 472 goto exit;
@@ -482,13 +509,14 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
482exit: 509exit:
483 exception_exit(prev_state); 510 exception_exit(prev_state);
484} 511}
512NOKPROBE_SYMBOL(do_debug);
485 513
486/* 514/*
487 * Note that we play around with the 'TS' bit in an attempt to get 515 * Note that we play around with the 'TS' bit in an attempt to get
488 * the correct behaviour even in the presence of the asynchronous 516 * the correct behaviour even in the presence of the asynchronous
489 * IRQ13 behaviour 517 * IRQ13 behaviour
490 */ 518 */
491void math_error(struct pt_regs *regs, int error_code, int trapnr) 519static void math_error(struct pt_regs *regs, int error_code, int trapnr)
492{ 520{
493 struct task_struct *task = current; 521 struct task_struct *task = current;
494 siginfo_t info; 522 siginfo_t info;
@@ -518,7 +546,7 @@ void math_error(struct pt_regs *regs, int error_code, int trapnr)
518 task->thread.error_code = error_code; 546 task->thread.error_code = error_code;
519 info.si_signo = SIGFPE; 547 info.si_signo = SIGFPE;
520 info.si_errno = 0; 548 info.si_errno = 0;
521 info.si_addr = (void __user *)regs->ip; 549 info.si_addr = (void __user *)uprobe_get_trap_addr(regs);
522 if (trapnr == X86_TRAP_MF) { 550 if (trapnr == X86_TRAP_MF) {
523 unsigned short cwd, swd; 551 unsigned short cwd, swd;
524 /* 552 /*
@@ -645,7 +673,7 @@ void math_state_restore(void)
645 */ 673 */
646 if (unlikely(restore_fpu_checking(tsk))) { 674 if (unlikely(restore_fpu_checking(tsk))) {
647 drop_init_fpu(tsk); 675 drop_init_fpu(tsk);
648 force_sig(SIGSEGV, tsk); 676 force_sig_info(SIGSEGV, SEND_SIG_PRIV, tsk);
649 return; 677 return;
650 } 678 }
651 679
@@ -653,7 +681,7 @@ void math_state_restore(void)
653} 681}
654EXPORT_SYMBOL_GPL(math_state_restore); 682EXPORT_SYMBOL_GPL(math_state_restore);
655 683
656dotraplinkage void __kprobes 684dotraplinkage void
657do_device_not_available(struct pt_regs *regs, long error_code) 685do_device_not_available(struct pt_regs *regs, long error_code)
658{ 686{
659 enum ctx_state prev_state; 687 enum ctx_state prev_state;
@@ -679,6 +707,7 @@ do_device_not_available(struct pt_regs *regs, long error_code)
679#endif 707#endif
680 exception_exit(prev_state); 708 exception_exit(prev_state);
681} 709}
710NOKPROBE_SYMBOL(do_device_not_available);
682 711
683#ifdef CONFIG_X86_32 712#ifdef CONFIG_X86_32
684dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) 713dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index ace22916ade3..5d1cbfe4ae58 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -32,20 +32,20 @@
32 32
33/* Post-execution fixups. */ 33/* Post-execution fixups. */
34 34
35/* No fixup needed */
36#define UPROBE_FIX_NONE 0x0
37
38/* Adjust IP back to vicinity of actual insn */ 35/* Adjust IP back to vicinity of actual insn */
39#define UPROBE_FIX_IP 0x1 36#define UPROBE_FIX_IP 0x01
40 37
41/* Adjust the return address of a call insn */ 38/* Adjust the return address of a call insn */
42#define UPROBE_FIX_CALL 0x2 39#define UPROBE_FIX_CALL 0x02
43 40
44/* Instruction will modify TF, don't change it */ 41/* Instruction will modify TF, don't change it */
45#define UPROBE_FIX_SETF 0x4 42#define UPROBE_FIX_SETF 0x04
46 43
47#define UPROBE_FIX_RIP_AX 0x8000 44#define UPROBE_FIX_RIP_SI 0x08
48#define UPROBE_FIX_RIP_CX 0x4000 45#define UPROBE_FIX_RIP_DI 0x10
46#define UPROBE_FIX_RIP_BX 0x20
47#define UPROBE_FIX_RIP_MASK \
48 (UPROBE_FIX_RIP_SI | UPROBE_FIX_RIP_DI | UPROBE_FIX_RIP_BX)
49 49
50#define UPROBE_TRAP_NR UINT_MAX 50#define UPROBE_TRAP_NR UINT_MAX
51 51
@@ -67,6 +67,7 @@
67 * to keep gcc from statically optimizing it out, as variable_test_bit makes 67 * to keep gcc from statically optimizing it out, as variable_test_bit makes
68 * some versions of gcc to think only *(unsigned long*) is used. 68 * some versions of gcc to think only *(unsigned long*) is used.
69 */ 69 */
70#if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
70static volatile u32 good_insns_32[256 / 32] = { 71static volatile u32 good_insns_32[256 / 32] = {
71 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 72 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
72 /* ---------------------------------------------- */ 73 /* ---------------------------------------------- */
@@ -89,33 +90,12 @@ static volatile u32 good_insns_32[256 / 32] = {
89 /* ---------------------------------------------- */ 90 /* ---------------------------------------------- */
90 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 91 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
91}; 92};
93#else
94#define good_insns_32 NULL
95#endif
92 96
93/* Using this for both 64-bit and 32-bit apps */
94static volatile u32 good_2byte_insns[256 / 32] = {
95 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
96 /* ---------------------------------------------- */
97 W(0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1) | /* 00 */
98 W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1) , /* 10 */
99 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
100 W(0x30, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
101 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
102 W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
103 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 60 */
104 W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
105 W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
106 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
107 W(0xa0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1) | /* a0 */
108 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
109 W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
110 W(0xd0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
111 W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* e0 */
112 W(0xf0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) /* f0 */
113 /* ---------------------------------------------- */
114 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
115};
116
117#ifdef CONFIG_X86_64
118/* Good-instruction tables for 64-bit apps */ 97/* Good-instruction tables for 64-bit apps */
98#if defined(CONFIG_X86_64)
119static volatile u32 good_insns_64[256 / 32] = { 99static volatile u32 good_insns_64[256 / 32] = {
120 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 100 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
121 /* ---------------------------------------------- */ 101 /* ---------------------------------------------- */
@@ -138,7 +118,33 @@ static volatile u32 good_insns_64[256 / 32] = {
138 /* ---------------------------------------------- */ 118 /* ---------------------------------------------- */
139 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */ 119 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
140}; 120};
121#else
122#define good_insns_64 NULL
141#endif 123#endif
124
125/* Using this for both 64-bit and 32-bit apps */
126static volatile u32 good_2byte_insns[256 / 32] = {
127 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
128 /* ---------------------------------------------- */
129 W(0x00, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1) | /* 00 */
130 W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1) , /* 10 */
131 W(0x20, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
132 W(0x30, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) , /* 30 */
133 W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
134 W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
135 W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 60 */
136 W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1) , /* 70 */
137 W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
138 W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
139 W(0xa0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1) | /* a0 */
140 W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
141 W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
142 W(0xd0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
143 W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* e0 */
144 W(0xf0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0) /* f0 */
145 /* ---------------------------------------------- */
146 /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
147};
142#undef W 148#undef W
143 149
144/* 150/*
@@ -209,16 +215,25 @@ static bool is_prefix_bad(struct insn *insn)
209 return false; 215 return false;
210} 216}
211 217
212static int validate_insn_32bits(struct arch_uprobe *auprobe, struct insn *insn) 218static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
213{ 219{
214 insn_init(insn, auprobe->insn, false); 220 u32 volatile *good_insns;
221
222 insn_init(insn, auprobe->insn, x86_64);
223 /* has the side-effect of processing the entire instruction */
224 insn_get_length(insn);
225 if (WARN_ON_ONCE(!insn_complete(insn)))
226 return -ENOEXEC;
215 227
216 /* Skip good instruction prefixes; reject "bad" ones. */
217 insn_get_opcode(insn);
218 if (is_prefix_bad(insn)) 228 if (is_prefix_bad(insn))
219 return -ENOTSUPP; 229 return -ENOTSUPP;
220 230
221 if (test_bit(OPCODE1(insn), (unsigned long *)good_insns_32)) 231 if (x86_64)
232 good_insns = good_insns_64;
233 else
234 good_insns = good_insns_32;
235
236 if (test_bit(OPCODE1(insn), (unsigned long *)good_insns))
222 return 0; 237 return 0;
223 238
224 if (insn->opcode.nbytes == 2) { 239 if (insn->opcode.nbytes == 2) {
@@ -230,14 +245,18 @@ static int validate_insn_32bits(struct arch_uprobe *auprobe, struct insn *insn)
230} 245}
231 246
232#ifdef CONFIG_X86_64 247#ifdef CONFIG_X86_64
248static inline bool is_64bit_mm(struct mm_struct *mm)
249{
250 return !config_enabled(CONFIG_IA32_EMULATION) ||
251 !(mm->context.ia32_compat == TIF_IA32);
252}
233/* 253/*
234 * If arch_uprobe->insn doesn't use rip-relative addressing, return 254 * If arch_uprobe->insn doesn't use rip-relative addressing, return
235 * immediately. Otherwise, rewrite the instruction so that it accesses 255 * immediately. Otherwise, rewrite the instruction so that it accesses
236 * its memory operand indirectly through a scratch register. Set 256 * its memory operand indirectly through a scratch register. Set
237 * arch_uprobe->fixups and arch_uprobe->rip_rela_target_address 257 * defparam->fixups accordingly. (The contents of the scratch register
238 * accordingly. (The contents of the scratch register will be saved 258 * will be saved before we single-step the modified instruction,
239 * before we single-step the modified instruction, and restored 259 * and restored afterward).
240 * afterward.)
241 * 260 *
242 * We do this because a rip-relative instruction can access only a 261 * We do this because a rip-relative instruction can access only a
243 * relatively small area (+/- 2 GB from the instruction), and the XOL 262 * relatively small area (+/- 2 GB from the instruction), and the XOL
@@ -248,164 +267,192 @@ static int validate_insn_32bits(struct arch_uprobe *auprobe, struct insn *insn)
248 * 267 *
249 * Some useful facts about rip-relative instructions: 268 * Some useful facts about rip-relative instructions:
250 * 269 *
251 * - There's always a modrm byte. 270 * - There's always a modrm byte with bit layout "00 reg 101".
252 * - There's never a SIB byte. 271 * - There's never a SIB byte.
253 * - The displacement is always 4 bytes. 272 * - The displacement is always 4 bytes.
273 * - REX.B=1 bit in REX prefix, which normally extends r/m field,
274 * has no effect on rip-relative mode. It doesn't make modrm byte
275 * with r/m=101 refer to register 1101 = R13.
254 */ 276 */
255static void 277static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
256handle_riprel_insn(struct arch_uprobe *auprobe, struct insn *insn)
257{ 278{
258 u8 *cursor; 279 u8 *cursor;
259 u8 reg; 280 u8 reg;
281 u8 reg2;
260 282
261 if (!insn_rip_relative(insn)) 283 if (!insn_rip_relative(insn))
262 return; 284 return;
263 285
264 /* 286 /*
265 * insn_rip_relative() would have decoded rex_prefix, modrm. 287 * insn_rip_relative() would have decoded rex_prefix, vex_prefix, modrm.
266 * Clear REX.b bit (extension of MODRM.rm field): 288 * Clear REX.b bit (extension of MODRM.rm field):
267 * we want to encode rax/rcx, not r8/r9. 289 * we want to encode low numbered reg, not r8+.
268 */ 290 */
269 if (insn->rex_prefix.nbytes) { 291 if (insn->rex_prefix.nbytes) {
270 cursor = auprobe->insn + insn_offset_rex_prefix(insn); 292 cursor = auprobe->insn + insn_offset_rex_prefix(insn);
271 *cursor &= 0xfe; /* Clearing REX.B bit */ 293 /* REX byte has 0100wrxb layout, clearing REX.b bit */
294 *cursor &= 0xfe;
295 }
296 /*
297 * Similar treatment for VEX3 prefix.
298 * TODO: add XOP/EVEX treatment when insn decoder supports them
299 */
300 if (insn->vex_prefix.nbytes == 3) {
301 /*
302 * vex2: c5 rvvvvLpp (has no b bit)
303 * vex3/xop: c4/8f rxbmmmmm wvvvvLpp
304 * evex: 62 rxbR00mm wvvvv1pp zllBVaaa
305 * (evex will need setting of both b and x since
306 * in non-sib encoding evex.x is 4th bit of MODRM.rm)
307 * Setting VEX3.b (setting because it has inverted meaning):
308 */
309 cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;
310 *cursor |= 0x20;
272 } 311 }
273 312
274 /* 313 /*
314 * Convert from rip-relative addressing to register-relative addressing
315 * via a scratch register.
316 *
317 * This is tricky since there are insns with modrm byte
318 * which also use registers not encoded in modrm byte:
319 * [i]div/[i]mul: implicitly use dx:ax
320 * shift ops: implicitly use cx
321 * cmpxchg: implicitly uses ax
322 * cmpxchg8/16b: implicitly uses dx:ax and bx:cx
323 * Encoding: 0f c7/1 modrm
324 * The code below thinks that reg=1 (cx), chooses si as scratch.
325 * mulx: implicitly uses dx: mulx r/m,r1,r2 does r1:r2 = dx * r/m.
326 * First appeared in Haswell (BMI2 insn). It is vex-encoded.
327 * Example where none of bx,cx,dx can be used as scratch reg:
328 * c4 e2 63 f6 0d disp32 mulx disp32(%rip),%ebx,%ecx
329 * [v]pcmpistri: implicitly uses cx, xmm0
330 * [v]pcmpistrm: implicitly uses xmm0
331 * [v]pcmpestri: implicitly uses ax, dx, cx, xmm0
332 * [v]pcmpestrm: implicitly uses ax, dx, xmm0
333 * Evil SSE4.2 string comparison ops from hell.
334 * maskmovq/[v]maskmovdqu: implicitly uses (ds:rdi) as destination.
335 * Encoding: 0f f7 modrm, 66 0f f7 modrm, vex-encoded: c5 f9 f7 modrm.
336 * Store op1, byte-masked by op2 msb's in each byte, to (ds:rdi).
337 * AMD says it has no 3-operand form (vex.vvvv must be 1111)
338 * and that it can have only register operands, not mem
339 * (its modrm byte must have mode=11).
340 * If these restrictions will ever be lifted,
341 * we'll need code to prevent selection of di as scratch reg!
342 *
343 * Summary: I don't know any insns with modrm byte which
344 * use SI register implicitly. DI register is used only
345 * by one insn (maskmovq) and BX register is used
346 * only by one too (cmpxchg8b).
347 * BP is stack-segment based (may be a problem?).
348 * AX, DX, CX are off-limits (many implicit users).
349 * SP is unusable (it's stack pointer - think about "pop mem";
350 * also, rsp+disp32 needs sib encoding -> insn length change).
351 */
352
353 reg = MODRM_REG(insn); /* Fetch modrm.reg */
354 reg2 = 0xff; /* Fetch vex.vvvv */
355 if (insn->vex_prefix.nbytes == 2)
356 reg2 = insn->vex_prefix.bytes[1];
357 else if (insn->vex_prefix.nbytes == 3)
358 reg2 = insn->vex_prefix.bytes[2];
359 /*
360 * TODO: add XOP, EXEV vvvv reading.
361 *
362 * vex.vvvv field is in bits 6-3, bits are inverted.
363 * But in 32-bit mode, high-order bit may be ignored.
364 * Therefore, let's consider only 3 low-order bits.
365 */
366 reg2 = ((reg2 >> 3) & 0x7) ^ 0x7;
367 /*
368 * Register numbering is ax,cx,dx,bx, sp,bp,si,di, r8..r15.
369 *
370 * Choose scratch reg. Order is important: must not select bx
371 * if we can use si (cmpxchg8b case!)
372 */
373 if (reg != 6 && reg2 != 6) {
374 reg2 = 6;
375 auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;
376 } else if (reg != 7 && reg2 != 7) {
377 reg2 = 7;
378 auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;
379 /* TODO (paranoia): force maskmovq to not use di */
380 } else {
381 reg2 = 3;
382 auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;
383 }
384 /*
275 * Point cursor at the modrm byte. The next 4 bytes are the 385 * Point cursor at the modrm byte. The next 4 bytes are the
276 * displacement. Beyond the displacement, for some instructions, 386 * displacement. Beyond the displacement, for some instructions,
277 * is the immediate operand. 387 * is the immediate operand.
278 */ 388 */
279 cursor = auprobe->insn + insn_offset_modrm(insn); 389 cursor = auprobe->insn + insn_offset_modrm(insn);
280 insn_get_length(insn);
281
282 /* 390 /*
283 * Convert from rip-relative addressing to indirect addressing 391 * Change modrm from "00 reg 101" to "10 reg reg2". Example:
284 * via a scratch register. Change the r/m field from 0x5 (%rip) 392 * 89 05 disp32 mov %eax,disp32(%rip) becomes
285 * to 0x0 (%rax) or 0x1 (%rcx), and squeeze out the offset field. 393 * 89 86 disp32 mov %eax,disp32(%rsi)
286 */ 394 */
287 reg = MODRM_REG(insn); 395 *cursor = 0x80 | (reg << 3) | reg2;
288 if (reg == 0) { 396}
289 /*
290 * The register operand (if any) is either the A register
291 * (%rax, %eax, etc.) or (if the 0x4 bit is set in the
292 * REX prefix) %r8. In any case, we know the C register
293 * is NOT the register operand, so we use %rcx (register
294 * #1) for the scratch register.
295 */
296 auprobe->fixups = UPROBE_FIX_RIP_CX;
297 /* Change modrm from 00 000 101 to 00 000 001. */
298 *cursor = 0x1;
299 } else {
300 /* Use %rax (register #0) for the scratch register. */
301 auprobe->fixups = UPROBE_FIX_RIP_AX;
302 /* Change modrm from 00 xxx 101 to 00 xxx 000 */
303 *cursor = (reg << 3);
304 }
305
306 /* Target address = address of next instruction + (signed) offset */
307 auprobe->rip_rela_target_address = (long)insn->length + insn->displacement.value;
308 397
309 /* Displacement field is gone; slide immediate field (if any) over. */ 398static inline unsigned long *
310 if (insn->immediate.nbytes) { 399scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)
311 cursor++; 400{
312 memmove(cursor, cursor + insn->displacement.nbytes, insn->immediate.nbytes); 401 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)
313 } 402 return &regs->si;
403 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)
404 return &regs->di;
405 return &regs->bx;
314} 406}
315 407
316/* 408/*
317 * If we're emulating a rip-relative instruction, save the contents 409 * If we're emulating a rip-relative instruction, save the contents
318 * of the scratch register and store the target address in that register. 410 * of the scratch register and store the target address in that register.
319 */ 411 */
320static void 412static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
321pre_xol_rip_insn(struct arch_uprobe *auprobe, struct pt_regs *regs,
322 struct arch_uprobe_task *autask)
323{
324 if (auprobe->fixups & UPROBE_FIX_RIP_AX) {
325 autask->saved_scratch_register = regs->ax;
326 regs->ax = current->utask->vaddr;
327 regs->ax += auprobe->rip_rela_target_address;
328 } else if (auprobe->fixups & UPROBE_FIX_RIP_CX) {
329 autask->saved_scratch_register = regs->cx;
330 regs->cx = current->utask->vaddr;
331 regs->cx += auprobe->rip_rela_target_address;
332 }
333}
334
335static void
336handle_riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs, long *correction)
337{ 413{
338 if (auprobe->fixups & (UPROBE_FIX_RIP_AX | UPROBE_FIX_RIP_CX)) { 414 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
339 struct arch_uprobe_task *autask; 415 struct uprobe_task *utask = current->utask;
340 416 unsigned long *sr = scratch_reg(auprobe, regs);
341 autask = &current->utask->autask;
342 if (auprobe->fixups & UPROBE_FIX_RIP_AX)
343 regs->ax = autask->saved_scratch_register;
344 else
345 regs->cx = autask->saved_scratch_register;
346 417
347 /* 418 utask->autask.saved_scratch_register = *sr;
348 * The original instruction includes a displacement, and so 419 *sr = utask->vaddr + auprobe->defparam.ilen;
349 * is 4 bytes longer than what we've just single-stepped.
350 * Caller may need to apply other fixups to handle stuff
351 * like "jmpq *...(%rip)" and "callq *...(%rip)".
352 */
353 if (correction)
354 *correction += 4;
355 } 420 }
356} 421}
357 422
358static int validate_insn_64bits(struct arch_uprobe *auprobe, struct insn *insn) 423static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
359{ 424{
360 insn_init(insn, auprobe->insn, true); 425 if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
361 426 struct uprobe_task *utask = current->utask;
362 /* Skip good instruction prefixes; reject "bad" ones. */ 427 unsigned long *sr = scratch_reg(auprobe, regs);
363 insn_get_opcode(insn);
364 if (is_prefix_bad(insn))
365 return -ENOTSUPP;
366 428
367 if (test_bit(OPCODE1(insn), (unsigned long *)good_insns_64)) 429 *sr = utask->autask.saved_scratch_register;
368 return 0;
369
370 if (insn->opcode.nbytes == 2) {
371 if (test_bit(OPCODE2(insn), (unsigned long *)good_2byte_insns))
372 return 0;
373 } 430 }
374 return -ENOTSUPP;
375} 431}
376 432#else /* 32-bit: */
377static int validate_insn_bits(struct arch_uprobe *auprobe, struct mm_struct *mm, struct insn *insn) 433static inline bool is_64bit_mm(struct mm_struct *mm)
378{ 434{
379 if (mm->context.ia32_compat) 435 return false;
380 return validate_insn_32bits(auprobe, insn);
381 return validate_insn_64bits(auprobe, insn);
382} 436}
383#else /* 32-bit: */
384/* 437/*
385 * No RIP-relative addressing on 32-bit 438 * No RIP-relative addressing on 32-bit
386 */ 439 */
387static void handle_riprel_insn(struct arch_uprobe *auprobe, struct insn *insn) 440static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
388{ 441{
389} 442}
390static void pre_xol_rip_insn(struct arch_uprobe *auprobe, struct pt_regs *regs, 443static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
391 struct arch_uprobe_task *autask)
392{ 444{
393} 445}
394static void handle_riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs, 446static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
395 long *correction)
396{ 447{
397} 448}
398
399static int validate_insn_bits(struct arch_uprobe *auprobe, struct mm_struct *mm, struct insn *insn)
400{
401 return validate_insn_32bits(auprobe, insn);
402}
403#endif /* CONFIG_X86_64 */ 449#endif /* CONFIG_X86_64 */
404 450
405struct uprobe_xol_ops { 451struct uprobe_xol_ops {
406 bool (*emulate)(struct arch_uprobe *, struct pt_regs *); 452 bool (*emulate)(struct arch_uprobe *, struct pt_regs *);
407 int (*pre_xol)(struct arch_uprobe *, struct pt_regs *); 453 int (*pre_xol)(struct arch_uprobe *, struct pt_regs *);
408 int (*post_xol)(struct arch_uprobe *, struct pt_regs *); 454 int (*post_xol)(struct arch_uprobe *, struct pt_regs *);
455 void (*abort)(struct arch_uprobe *, struct pt_regs *);
409}; 456};
410 457
411static inline int sizeof_long(void) 458static inline int sizeof_long(void)
@@ -415,50 +462,67 @@ static inline int sizeof_long(void)
415 462
416static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs) 463static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
417{ 464{
418 pre_xol_rip_insn(auprobe, regs, &current->utask->autask); 465 riprel_pre_xol(auprobe, regs);
419 return 0; 466 return 0;
420} 467}
421 468
422/* 469static int push_ret_address(struct pt_regs *regs, unsigned long ip)
423 * Adjust the return address pushed by a call insn executed out of line.
424 */
425static int adjust_ret_addr(unsigned long sp, long correction)
426{ 470{
427 int rasize = sizeof_long(); 471 unsigned long new_sp = regs->sp - sizeof_long();
428 long ra;
429
430 if (copy_from_user(&ra, (void __user *)sp, rasize))
431 return -EFAULT;
432 472
433 ra += correction; 473 if (copy_to_user((void __user *)new_sp, &ip, sizeof_long()))
434 if (copy_to_user((void __user *)sp, &ra, rasize))
435 return -EFAULT; 474 return -EFAULT;
436 475
476 regs->sp = new_sp;
437 return 0; 477 return 0;
438} 478}
439 479
480/*
481 * We have to fix things up as follows:
482 *
483 * Typically, the new ip is relative to the copied instruction. We need
484 * to make it relative to the original instruction (FIX_IP). Exceptions
485 * are return instructions and absolute or indirect jump or call instructions.
486 *
487 * If the single-stepped instruction was a call, the return address that
488 * is atop the stack is the address following the copied instruction. We
489 * need to make it the address following the original instruction (FIX_CALL).
490 *
491 * If the original instruction was a rip-relative instruction such as
492 * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
493 * instruction using a scratch register -- e.g., "movl %edx,0xnnnn(%rsi)".
494 * We need to restore the contents of the scratch register
495 * (FIX_RIP_reg).
496 */
440static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs) 497static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
441{ 498{
442 struct uprobe_task *utask = current->utask; 499 struct uprobe_task *utask = current->utask;
443 long correction = (long)(utask->vaddr - utask->xol_vaddr);
444 500
445 handle_riprel_post_xol(auprobe, regs, &correction); 501 riprel_post_xol(auprobe, regs);
446 if (auprobe->fixups & UPROBE_FIX_IP) 502 if (auprobe->defparam.fixups & UPROBE_FIX_IP) {
503 long correction = utask->vaddr - utask->xol_vaddr;
447 regs->ip += correction; 504 regs->ip += correction;
448 505 } else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {
449 if (auprobe->fixups & UPROBE_FIX_CALL) { 506 regs->sp += sizeof_long(); /* Pop incorrect return address */
450 if (adjust_ret_addr(regs->sp, correction)) { 507 if (push_ret_address(regs, utask->vaddr + auprobe->defparam.ilen))
451 regs->sp += sizeof_long();
452 return -ERESTART; 508 return -ERESTART;
453 }
454 } 509 }
510 /* popf; tell the caller to not touch TF */
511 if (auprobe->defparam.fixups & UPROBE_FIX_SETF)
512 utask->autask.saved_tf = true;
455 513
456 return 0; 514 return 0;
457} 515}
458 516
517static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
518{
519 riprel_post_xol(auprobe, regs);
520}
521
459static struct uprobe_xol_ops default_xol_ops = { 522static struct uprobe_xol_ops default_xol_ops = {
460 .pre_xol = default_pre_xol_op, 523 .pre_xol = default_pre_xol_op,
461 .post_xol = default_post_xol_op, 524 .post_xol = default_post_xol_op,
525 .abort = default_abort_op,
462}; 526};
463 527
464static bool branch_is_call(struct arch_uprobe *auprobe) 528static bool branch_is_call(struct arch_uprobe *auprobe)
@@ -520,7 +584,6 @@ static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
520 unsigned long offs = (long)auprobe->branch.offs; 584 unsigned long offs = (long)auprobe->branch.offs;
521 585
522 if (branch_is_call(auprobe)) { 586 if (branch_is_call(auprobe)) {
523 unsigned long new_sp = regs->sp - sizeof_long();
524 /* 587 /*
525 * If it fails we execute this (mangled, see the comment in 588 * If it fails we execute this (mangled, see the comment in
526 * branch_clear_offset) insn out-of-line. In the likely case 589 * branch_clear_offset) insn out-of-line. In the likely case
@@ -530,9 +593,8 @@ static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
530 * 593 *
531 * But there is corner case, see the comment in ->post_xol(). 594 * But there is corner case, see the comment in ->post_xol().
532 */ 595 */
533 if (copy_to_user((void __user *)new_sp, &new_ip, sizeof_long())) 596 if (push_ret_address(regs, new_ip))
534 return false; 597 return false;
535 regs->sp = new_sp;
536 } else if (!check_jmp_cond(auprobe, regs)) { 598 } else if (!check_jmp_cond(auprobe, regs)) {
537 offs = 0; 599 offs = 0;
538 } 600 }
@@ -583,11 +645,7 @@ static struct uprobe_xol_ops branch_xol_ops = {
583static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) 645static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
584{ 646{
585 u8 opc1 = OPCODE1(insn); 647 u8 opc1 = OPCODE1(insn);
586 648 int i;
587 /* has the side-effect of processing the entire instruction */
588 insn_get_length(insn);
589 if (WARN_ON_ONCE(!insn_complete(insn)))
590 return -ENOEXEC;
591 649
592 switch (opc1) { 650 switch (opc1) {
593 case 0xeb: /* jmp 8 */ 651 case 0xeb: /* jmp 8 */
@@ -612,6 +670,16 @@ static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
612 return -ENOSYS; 670 return -ENOSYS;
613 } 671 }
614 672
673 /*
674 * 16-bit overrides such as CALLW (66 e8 nn nn) are not supported.
675 * Intel and AMD behavior differ in 64-bit mode: Intel ignores 66 prefix.
676 * No one uses these insns, reject any branch insns with such prefix.
677 */
678 for (i = 0; i < insn->prefixes.nbytes; i++) {
679 if (insn->prefixes.bytes[i] == 0x66)
680 return -ENOTSUPP;
681 }
682
615 auprobe->branch.opc1 = opc1; 683 auprobe->branch.opc1 = opc1;
616 auprobe->branch.ilen = insn->length; 684 auprobe->branch.ilen = insn->length;
617 auprobe->branch.offs = insn->immediate.value; 685 auprobe->branch.offs = insn->immediate.value;
@@ -630,10 +698,10 @@ static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
630int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) 698int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)
631{ 699{
632 struct insn insn; 700 struct insn insn;
633 bool fix_ip = true, fix_call = false; 701 u8 fix_ip_or_call = UPROBE_FIX_IP;
634 int ret; 702 int ret;
635 703
636 ret = validate_insn_bits(auprobe, mm, &insn); 704 ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
637 if (ret) 705 if (ret)
638 return ret; 706 return ret;
639 707
@@ -642,44 +710,39 @@ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm,
642 return ret; 710 return ret;
643 711
644 /* 712 /*
645 * Figure out which fixups arch_uprobe_post_xol() will need to perform, 713 * Figure out which fixups default_post_xol_op() will need to perform,
646 * and annotate arch_uprobe->fixups accordingly. To start with, ->fixups 714 * and annotate defparam->fixups accordingly.
647 * is either zero or it reflects rip-related fixups.
648 */ 715 */
649 switch (OPCODE1(&insn)) { 716 switch (OPCODE1(&insn)) {
650 case 0x9d: /* popf */ 717 case 0x9d: /* popf */
651 auprobe->fixups |= UPROBE_FIX_SETF; 718 auprobe->defparam.fixups |= UPROBE_FIX_SETF;
652 break; 719 break;
653 case 0xc3: /* ret or lret -- ip is correct */ 720 case 0xc3: /* ret or lret -- ip is correct */
654 case 0xcb: 721 case 0xcb:
655 case 0xc2: 722 case 0xc2:
656 case 0xca: 723 case 0xca:
657 fix_ip = false; 724 case 0xea: /* jmp absolute -- ip is correct */
725 fix_ip_or_call = 0;
658 break; 726 break;
659 case 0x9a: /* call absolute - Fix return addr, not ip */ 727 case 0x9a: /* call absolute - Fix return addr, not ip */
660 fix_call = true; 728 fix_ip_or_call = UPROBE_FIX_CALL;
661 fix_ip = false;
662 break;
663 case 0xea: /* jmp absolute -- ip is correct */
664 fix_ip = false;
665 break; 729 break;
666 case 0xff: 730 case 0xff:
667 insn_get_modrm(&insn);
668 switch (MODRM_REG(&insn)) { 731 switch (MODRM_REG(&insn)) {
669 case 2: case 3: /* call or lcall, indirect */ 732 case 2: case 3: /* call or lcall, indirect */
670 fix_call = true; 733 fix_ip_or_call = UPROBE_FIX_CALL;
734 break;
671 case 4: case 5: /* jmp or ljmp, indirect */ 735 case 4: case 5: /* jmp or ljmp, indirect */
672 fix_ip = false; 736 fix_ip_or_call = 0;
737 break;
673 } 738 }
674 /* fall through */ 739 /* fall through */
675 default: 740 default:
676 handle_riprel_insn(auprobe, &insn); 741 riprel_analyze(auprobe, &insn);
677 } 742 }
678 743
679 if (fix_ip) 744 auprobe->defparam.ilen = insn.length;
680 auprobe->fixups |= UPROBE_FIX_IP; 745 auprobe->defparam.fixups |= fix_ip_or_call;
681 if (fix_call)
682 auprobe->fixups |= UPROBE_FIX_CALL;
683 746
684 auprobe->ops = &default_xol_ops; 747 auprobe->ops = &default_xol_ops;
685 return 0; 748 return 0;
@@ -694,6 +757,12 @@ int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
694{ 757{
695 struct uprobe_task *utask = current->utask; 758 struct uprobe_task *utask = current->utask;
696 759
760 if (auprobe->ops->pre_xol) {
761 int err = auprobe->ops->pre_xol(auprobe, regs);
762 if (err)
763 return err;
764 }
765
697 regs->ip = utask->xol_vaddr; 766 regs->ip = utask->xol_vaddr;
698 utask->autask.saved_trap_nr = current->thread.trap_nr; 767 utask->autask.saved_trap_nr = current->thread.trap_nr;
699 current->thread.trap_nr = UPROBE_TRAP_NR; 768 current->thread.trap_nr = UPROBE_TRAP_NR;
@@ -703,8 +772,6 @@ int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
703 if (test_tsk_thread_flag(current, TIF_BLOCKSTEP)) 772 if (test_tsk_thread_flag(current, TIF_BLOCKSTEP))
704 set_task_blockstep(current, false); 773 set_task_blockstep(current, false);
705 774
706 if (auprobe->ops->pre_xol)
707 return auprobe->ops->pre_xol(auprobe, regs);
708 return 0; 775 return 0;
709} 776}
710 777
@@ -732,56 +799,42 @@ bool arch_uprobe_xol_was_trapped(struct task_struct *t)
732 * single-step, we single-stepped a copy of the instruction. 799 * single-step, we single-stepped a copy of the instruction.
733 * 800 *
734 * This function prepares to resume execution after the single-step. 801 * This function prepares to resume execution after the single-step.
735 * We have to fix things up as follows:
736 *
737 * Typically, the new ip is relative to the copied instruction. We need
738 * to make it relative to the original instruction (FIX_IP). Exceptions
739 * are return instructions and absolute or indirect jump or call instructions.
740 *
741 * If the single-stepped instruction was a call, the return address that
742 * is atop the stack is the address following the copied instruction. We
743 * need to make it the address following the original instruction (FIX_CALL).
744 *
745 * If the original instruction was a rip-relative instruction such as
746 * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
747 * instruction using a scratch register -- e.g., "movl %edx,(%rax)".
748 * We need to restore the contents of the scratch register and adjust
749 * the ip, keeping in mind that the instruction we executed is 4 bytes
750 * shorter than the original instruction (since we squeezed out the offset
751 * field). (FIX_RIP_AX or FIX_RIP_CX)
752 */ 802 */
753int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) 803int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
754{ 804{
755 struct uprobe_task *utask = current->utask; 805 struct uprobe_task *utask = current->utask;
806 bool send_sigtrap = utask->autask.saved_tf;
807 int err = 0;
756 808
757 WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR); 809 WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
810 current->thread.trap_nr = utask->autask.saved_trap_nr;
758 811
759 if (auprobe->ops->post_xol) { 812 if (auprobe->ops->post_xol) {
760 int err = auprobe->ops->post_xol(auprobe, regs); 813 err = auprobe->ops->post_xol(auprobe, regs);
761 if (err) { 814 if (err) {
762 arch_uprobe_abort_xol(auprobe, regs);
763 /* 815 /*
764 * Restart the probed insn. ->post_xol() must ensure 816 * Restore ->ip for restart or post mortem analysis.
765 * this is really possible if it returns -ERESTART. 817 * ->post_xol() must not return -ERESTART unless this
818 * is really possible.
766 */ 819 */
820 regs->ip = utask->vaddr;
767 if (err == -ERESTART) 821 if (err == -ERESTART)
768 return 0; 822 err = 0;
769 return err; 823 send_sigtrap = false;
770 } 824 }
771 } 825 }
772
773 current->thread.trap_nr = utask->autask.saved_trap_nr;
774 /* 826 /*
775 * arch_uprobe_pre_xol() doesn't save the state of TIF_BLOCKSTEP 827 * arch_uprobe_pre_xol() doesn't save the state of TIF_BLOCKSTEP
776 * so we can get an extra SIGTRAP if we do not clear TF. We need 828 * so we can get an extra SIGTRAP if we do not clear TF. We need
777 * to examine the opcode to make it right. 829 * to examine the opcode to make it right.
778 */ 830 */
779 if (utask->autask.saved_tf) 831 if (send_sigtrap)
780 send_sig(SIGTRAP, current, 0); 832 send_sig(SIGTRAP, current, 0);
781 else if (!(auprobe->fixups & UPROBE_FIX_SETF)) 833
834 if (!utask->autask.saved_tf)
782 regs->flags &= ~X86_EFLAGS_TF; 835 regs->flags &= ~X86_EFLAGS_TF;
783 836
784 return 0; 837 return err;
785} 838}
786 839
787/* callback routine for handling exceptions. */ 840/* callback routine for handling exceptions. */
@@ -815,18 +868,18 @@ int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val,
815 868
816/* 869/*
817 * This function gets called when XOL instruction either gets trapped or 870 * This function gets called when XOL instruction either gets trapped or
818 * the thread has a fatal signal, or if arch_uprobe_post_xol() failed. 871 * the thread has a fatal signal. Reset the instruction pointer to its
819 * Reset the instruction pointer to its probed address for the potential 872 * probed address for the potential restart or for post mortem analysis.
820 * restart or for post mortem analysis.
821 */ 873 */
822void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs) 874void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
823{ 875{
824 struct uprobe_task *utask = current->utask; 876 struct uprobe_task *utask = current->utask;
825 877
826 current->thread.trap_nr = utask->autask.saved_trap_nr; 878 if (auprobe->ops->abort)
827 handle_riprel_post_xol(auprobe, regs, NULL); 879 auprobe->ops->abort(auprobe, regs);
828 instruction_pointer_set(regs, utask->vaddr);
829 880
881 current->thread.trap_nr = utask->autask.saved_trap_nr;
882 regs->ip = utask->vaddr;
830 /* clear TF if it was set by us in arch_uprobe_pre_xol() */ 883 /* clear TF if it was set by us in arch_uprobe_pre_xol() */
831 if (!utask->autask.saved_tf) 884 if (!utask->autask.saved_tf)
832 regs->flags &= ~X86_EFLAGS_TF; 885 regs->flags &= ~X86_EFLAGS_TF;
generated by cgit v1.2.2 (git 2.25.0) at 2025-01-28 12:36:36 -0500