x86: Save rbp in pt_regs on irq entry

From the x86_64 low level interrupt handlers, the frame pointer is
saved right after the partial pt_regs frame.

rbp is not supposed to be part of the irq partial saved registers,
but it only requires to extend the pt_regs frame by 8 bytes to
do so, plus a tiny stack offset fixup on irq exit.

This changes a bit the semantics or get_irq_entry() that is supposed
to provide only the value of caller saved registers and the cpu
saved frame. However it's a win for unwinders that can walk through
stack frames on top of get_irq_regs() snapshots.

A noticeable impact is that it makes perf events cpu-clock and
task-clock events based callchains working on x86_64.

Let's then save rbp into the irq pt_regs.

As a result with:

	perf record -e cpu-clock perf bench sched messaging
	perf report --stdio

Before:
    20.94%             perf  [kernel.kallsyms]        [k] lock_acquire
                       |
                       --- lock_acquire
                          |
                          |--44.01%-- __write_nocancel
                          |
                          |--43.18%-- __read
                          |
                          |--6.08%-- fork
                          |          create_worker
                          |
                          |--0.88%-- _dl_fixup
                          |
                          |--0.65%-- do_lookup_x
                          |
                          |--0.53%-- __GI___libc_read
                           --4.67%-- [...]

After:
    19.23%         perf  [kernel.kallsyms]    [k] __lock_acquire
                   |
                   --- __lock_acquire
                      |
                      |--97.74%-- lock_acquire
                      |          |
                      |          |--21.82%-- _raw_spin_lock
                      |          |          |
                      |          |          |--37.26%-- unix_stream_recvmsg
                      |          |          |          sock_aio_read
                      |          |          |          do_sync_read
                      |          |          |          vfs_read
                      |          |          |          sys_read
                      |          |          |          system_call
                      |          |          |          __read
                      |          |          |
                      |          |          |--24.09%-- unix_stream_sendmsg
                      |          |          |          sock_aio_write
                      |          |          |          do_sync_write
                      |          |          |          vfs_write
                      |          |          |          sys_write
                      |          |          |          system_call
                      |          |          |          __write_nocancel

v2: Fix cfi annotations.

Reported-by: Soeren Sandmann Pedersen <sandmann@redhat.com>
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: H. Peter Anvin <hpa@zytor.com
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Stephane Eranian <eranian@google.com>
Cc: Jan Beulich <JBeulich@novell.com>

1 files changed, 23 insertions, 13 deletions

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index e3ba417e8697..d3b895f375d3 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -299,17 +299,21 @@ ENDPROC(native_usergs_sysret64)
 ENTRY(save_args)
         XCPT_FRAME
         cld
-        movq_cfi rdi, RDI+16-ARGOFFSET
-        movq_cfi rsi, RSI+16-ARGOFFSET
-        movq_cfi rdx, RDX+16-ARGOFFSET
-        movq_cfi rcx, RCX+16-ARGOFFSET
-        movq_cfi rax, RAX+16-ARGOFFSET
-        movq_cfi  r8,  R8+16-ARGOFFSET
-        movq_cfi  r9,  R9+16-ARGOFFSET
-        movq_cfi r10, R10+16-ARGOFFSET
-        movq_cfi r11, R11+16-ARGOFFSET
-
-        leaq -ARGOFFSET+16(%rsp),%rdi   /* arg1 for handler */
+        /*
+         * start from rbp in pt_regs and jump over
+         * return address.
+         */
+        movq_cfi rdi, RDI+8-RBP
+        movq_cfi rsi, RSI+8-RBP
+        movq_cfi rdx, RDX+8-RBP
+        movq_cfi rcx, RCX+8-RBP
+        movq_cfi rax, RAX+8-RBP
+        movq_cfi  r8,  R8+8-RBP
+        movq_cfi  r9,  R9+8-RBP
+        movq_cfi r10, R10+8-RBP
+        movq_cfi r11, R11+8-RBP
+
+        leaq -RBP+8(%rsp),%rdi  /* arg1 for handler */
         movq_cfi rbp, 8         /* push %rbp */
         leaq 8(%rsp), %rbp              /* mov %rsp, %ebp */
         testl $3, CS(%rdi)
@@ -782,8 +786,9 @@ END(interrupt)
 
 /* 0(%rsp): ~(interrupt number) */
         .macro interrupt func
-        subq $ORIG_RAX-ARGOFFSET+8, %rsp
-        CFI_ADJUST_CFA_OFFSET ORIG_RAX-ARGOFFSET+8
+        /* reserve pt_regs for scratch regs and rbp */
+        subq $ORIG_RAX-RBP, %rsp
+        CFI_ADJUST_CFA_OFFSET ORIG_RAX-RBP
         call save_args
         PARTIAL_FRAME 0
         call \func
@@ -808,9 +813,14 @@ ret_from_intr:
         TRACE_IRQS_OFF
         decl PER_CPU_VAR(irq_count)
         leaveq
+
         CFI_RESTORE             rbp
         CFI_DEF_CFA_REGISTER    rsp
         CFI_ADJUST_CFA_OFFSET   -8
+
+        /* we did not save rbx, restore only from ARGOFFSET */
+        addq $8, %rsp
+        CFI_ADJUST_CFA_OFFSET   -8
 exit_intr:
         GET_THREAD_INFO(%rcx)
         testl $3,CS-ARGOFFSET(%rsp)