aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--arch/x86_64/ia32/ia32entry.S51
-rw-r--r--arch/x86_64/kernel/entry.S207
-rw-r--r--include/asm-x86_64/calling.h23
-rw-r--r--include/asm-x86_64/dwarf2.h8
4 files changed, 220 insertions, 69 deletions
diff --git a/arch/x86_64/ia32/ia32entry.S b/arch/x86_64/ia32/ia32entry.S
index 5244f803203d..e0eb0c712fe9 100644
--- a/arch/x86_64/ia32/ia32entry.S
+++ b/arch/x86_64/ia32/ia32entry.S
@@ -55,20 +55,34 @@
55 * with the int 0x80 path. 55 * with the int 0x80 path.
56 */ 56 */
57ENTRY(ia32_sysenter_target) 57ENTRY(ia32_sysenter_target)
58 CFI_STARTPROC 58 CFI_STARTPROC simple
59 CFI_DEF_CFA rsp,0
60 CFI_REGISTER rsp,rbp
59 swapgs 61 swapgs
60 movq %gs:pda_kernelstack, %rsp 62 movq %gs:pda_kernelstack, %rsp
61 addq $(PDA_STACKOFFSET),%rsp 63 addq $(PDA_STACKOFFSET),%rsp
62 sti 64 sti
63 movl %ebp,%ebp /* zero extension */ 65 movl %ebp,%ebp /* zero extension */
64 pushq $__USER32_DS 66 pushq $__USER32_DS
67 CFI_ADJUST_CFA_OFFSET 8
68 /*CFI_REL_OFFSET ss,0*/
65 pushq %rbp 69 pushq %rbp
70 CFI_ADJUST_CFA_OFFSET 8
71 CFI_REL_OFFSET rsp,0
66 pushfq 72 pushfq
73 CFI_ADJUST_CFA_OFFSET 8
74 /*CFI_REL_OFFSET rflags,0*/
67 movl $VSYSCALL32_SYSEXIT, %r10d 75 movl $VSYSCALL32_SYSEXIT, %r10d
76 CFI_REGISTER rip,r10
68 pushq $__USER32_CS 77 pushq $__USER32_CS
78 CFI_ADJUST_CFA_OFFSET 8
79 /*CFI_REL_OFFSET cs,0*/
69 movl %eax, %eax 80 movl %eax, %eax
70 pushq %r10 81 pushq %r10
82 CFI_ADJUST_CFA_OFFSET 8
83 CFI_REL_OFFSET rip,0
71 pushq %rax 84 pushq %rax
85 CFI_ADJUST_CFA_OFFSET 8
72 cld 86 cld
73 SAVE_ARGS 0,0,1 87 SAVE_ARGS 0,0,1
74 /* no need to do an access_ok check here because rbp has been 88 /* no need to do an access_ok check here because rbp has been
@@ -79,6 +93,7 @@ ENTRY(ia32_sysenter_target)
79 .previous 93 .previous
80 GET_THREAD_INFO(%r10) 94 GET_THREAD_INFO(%r10)
81 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) 95 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
96 CFI_REMEMBER_STATE
82 jnz sysenter_tracesys 97 jnz sysenter_tracesys
83sysenter_do_call: 98sysenter_do_call:
84 cmpl $(IA32_NR_syscalls),%eax 99 cmpl $(IA32_NR_syscalls),%eax
@@ -94,14 +109,20 @@ sysenter_do_call:
94 andl $~0x200,EFLAGS-R11(%rsp) 109 andl $~0x200,EFLAGS-R11(%rsp)
95 RESTORE_ARGS 1,24,1,1,1,1 110 RESTORE_ARGS 1,24,1,1,1,1
96 popfq 111 popfq
112 CFI_ADJUST_CFA_OFFSET -8
113 /*CFI_RESTORE rflags*/
97 popq %rcx /* User %esp */ 114 popq %rcx /* User %esp */
115 CFI_ADJUST_CFA_OFFSET -8
116 CFI_REGISTER rsp,rcx
98 movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */ 117 movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */
118 CFI_REGISTER rip,rdx
99 swapgs 119 swapgs
100 sti /* sti only takes effect after the next instruction */ 120 sti /* sti only takes effect after the next instruction */
101 /* sysexit */ 121 /* sysexit */
102 .byte 0xf, 0x35 122 .byte 0xf, 0x35
103 123
104sysenter_tracesys: 124sysenter_tracesys:
125 CFI_RESTORE_STATE
105 SAVE_REST 126 SAVE_REST
106 CLEAR_RREGS 127 CLEAR_RREGS
107 movq $-ENOSYS,RAX(%rsp) /* really needed? */ 128 movq $-ENOSYS,RAX(%rsp) /* really needed? */
@@ -140,21 +161,28 @@ sysenter_tracesys:
140 * with the int 0x80 path. 161 * with the int 0x80 path.
141 */ 162 */
142ENTRY(ia32_cstar_target) 163ENTRY(ia32_cstar_target)
143 CFI_STARTPROC 164 CFI_STARTPROC simple
165 CFI_DEF_CFA rsp,0
166 CFI_REGISTER rip,rcx
167 /*CFI_REGISTER rflags,r11*/
144 swapgs 168 swapgs
145 movl %esp,%r8d 169 movl %esp,%r8d
170 CFI_REGISTER rsp,r8
146 movq %gs:pda_kernelstack,%rsp 171 movq %gs:pda_kernelstack,%rsp
147 sti 172 sti
148 SAVE_ARGS 8,1,1 173 SAVE_ARGS 8,1,1
149 movl %eax,%eax /* zero extension */ 174 movl %eax,%eax /* zero extension */
150 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 175 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
151 movq %rcx,RIP-ARGOFFSET(%rsp) 176 movq %rcx,RIP-ARGOFFSET(%rsp)
177 CFI_REL_OFFSET rip,RIP-ARGOFFSET
152 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */ 178 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
153 movl %ebp,%ecx 179 movl %ebp,%ecx
154 movq $__USER32_CS,CS-ARGOFFSET(%rsp) 180 movq $__USER32_CS,CS-ARGOFFSET(%rsp)
155 movq $__USER32_DS,SS-ARGOFFSET(%rsp) 181 movq $__USER32_DS,SS-ARGOFFSET(%rsp)
156 movq %r11,EFLAGS-ARGOFFSET(%rsp) 182 movq %r11,EFLAGS-ARGOFFSET(%rsp)
183 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
157 movq %r8,RSP-ARGOFFSET(%rsp) 184 movq %r8,RSP-ARGOFFSET(%rsp)
185 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
158 /* no need to do an access_ok check here because r8 has been 186 /* no need to do an access_ok check here because r8 has been
159 32bit zero extended */ 187 32bit zero extended */
160 /* hardware stack frame is complete now */ 188 /* hardware stack frame is complete now */
@@ -164,6 +192,7 @@ ENTRY(ia32_cstar_target)
164 .previous 192 .previous
165 GET_THREAD_INFO(%r10) 193 GET_THREAD_INFO(%r10)
166 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) 194 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
195 CFI_REMEMBER_STATE
167 jnz cstar_tracesys 196 jnz cstar_tracesys
168cstar_do_call: 197cstar_do_call:
169 cmpl $IA32_NR_syscalls,%eax 198 cmpl $IA32_NR_syscalls,%eax
@@ -177,12 +206,16 @@ cstar_do_call:
177 jnz int_ret_from_sys_call 206 jnz int_ret_from_sys_call
178 RESTORE_ARGS 1,-ARG_SKIP,1,1,1 207 RESTORE_ARGS 1,-ARG_SKIP,1,1,1
179 movl RIP-ARGOFFSET(%rsp),%ecx 208 movl RIP-ARGOFFSET(%rsp),%ecx
209 CFI_REGISTER rip,rcx
180 movl EFLAGS-ARGOFFSET(%rsp),%r11d 210 movl EFLAGS-ARGOFFSET(%rsp),%r11d
211 /*CFI_REGISTER rflags,r11*/
181 movl RSP-ARGOFFSET(%rsp),%esp 212 movl RSP-ARGOFFSET(%rsp),%esp
213 CFI_RESTORE rsp
182 swapgs 214 swapgs
183 sysretl 215 sysretl
184 216
185cstar_tracesys: 217cstar_tracesys:
218 CFI_RESTORE_STATE
186 SAVE_REST 219 SAVE_REST
187 CLEAR_RREGS 220 CLEAR_RREGS
188 movq $-ENOSYS,RAX(%rsp) /* really needed? */ 221 movq $-ENOSYS,RAX(%rsp) /* really needed? */
@@ -226,11 +259,18 @@ ia32_badarg:
226 */ 259 */
227 260
228ENTRY(ia32_syscall) 261ENTRY(ia32_syscall)
229 CFI_STARTPROC 262 CFI_STARTPROC simple
263 CFI_DEF_CFA rsp,SS+8-RIP
264 /*CFI_REL_OFFSET ss,SS-RIP*/
265 CFI_REL_OFFSET rsp,RSP-RIP
266 /*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
267 /*CFI_REL_OFFSET cs,CS-RIP*/
268 CFI_REL_OFFSET rip,RIP-RIP
230 swapgs 269 swapgs
231 sti 270 sti
232 movl %eax,%eax 271 movl %eax,%eax
233 pushq %rax 272 pushq %rax
273 CFI_ADJUST_CFA_OFFSET 8
234 cld 274 cld
235 /* note the registers are not zero extended to the sf. 275 /* note the registers are not zero extended to the sf.
236 this could be a problem. */ 276 this could be a problem. */
@@ -278,6 +318,8 @@ quiet_ni_syscall:
278 jmp ia32_ptregs_common 318 jmp ia32_ptregs_common
279 .endm 319 .endm
280 320
321 CFI_STARTPROC
322
281 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi 323 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
282 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi 324 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
283 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx 325 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx
@@ -290,8 +332,9 @@ quiet_ni_syscall:
290 PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx 332 PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx
291 333
292ENTRY(ia32_ptregs_common) 334ENTRY(ia32_ptregs_common)
293 CFI_STARTPROC
294 popq %r11 335 popq %r11
336 CFI_ADJUST_CFA_OFFSET -8
337 CFI_REGISTER rip, r11
295 SAVE_REST 338 SAVE_REST
296 call *%rax 339 call *%rax
297 RESTORE_REST 340 RESTORE_REST
diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 873c39d8f818..7937971d1853 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -79,16 +79,19 @@
79 xorl %eax, %eax 79 xorl %eax, %eax
80 pushq %rax /* ss */ 80 pushq %rax /* ss */
81 CFI_ADJUST_CFA_OFFSET 8 81 CFI_ADJUST_CFA_OFFSET 8
82 /*CFI_REL_OFFSET ss,0*/
82 pushq %rax /* rsp */ 83 pushq %rax /* rsp */
83 CFI_ADJUST_CFA_OFFSET 8 84 CFI_ADJUST_CFA_OFFSET 8
84 CFI_OFFSET rip,0 85 CFI_REL_OFFSET rsp,0
85 pushq $(1<<9) /* eflags - interrupts on */ 86 pushq $(1<<9) /* eflags - interrupts on */
86 CFI_ADJUST_CFA_OFFSET 8 87 CFI_ADJUST_CFA_OFFSET 8
88 /*CFI_REL_OFFSET rflags,0*/
87 pushq $__KERNEL_CS /* cs */ 89 pushq $__KERNEL_CS /* cs */
88 CFI_ADJUST_CFA_OFFSET 8 90 CFI_ADJUST_CFA_OFFSET 8
91 /*CFI_REL_OFFSET cs,0*/
89 pushq \child_rip /* rip */ 92 pushq \child_rip /* rip */
90 CFI_ADJUST_CFA_OFFSET 8 93 CFI_ADJUST_CFA_OFFSET 8
91 CFI_OFFSET rip,0 94 CFI_REL_OFFSET rip,0
92 pushq %rax /* orig rax */ 95 pushq %rax /* orig rax */
93 CFI_ADJUST_CFA_OFFSET 8 96 CFI_ADJUST_CFA_OFFSET 8
94 .endm 97 .endm
@@ -98,32 +101,39 @@
98 CFI_ADJUST_CFA_OFFSET -(6*8) 101 CFI_ADJUST_CFA_OFFSET -(6*8)
99 .endm 102 .endm
100 103
101 .macro CFI_DEFAULT_STACK 104 .macro CFI_DEFAULT_STACK start=1
102 CFI_ADJUST_CFA_OFFSET (SS) 105 .if \start
103 CFI_OFFSET r15,R15-SS 106 CFI_STARTPROC simple
104 CFI_OFFSET r14,R14-SS 107 CFI_DEF_CFA rsp,SS+8
105 CFI_OFFSET r13,R13-SS 108 .else
106 CFI_OFFSET r12,R12-SS 109 CFI_DEF_CFA_OFFSET SS+8
107 CFI_OFFSET rbp,RBP-SS 110 .endif
108 CFI_OFFSET rbx,RBX-SS 111 CFI_REL_OFFSET r15,R15
109 CFI_OFFSET r11,R11-SS 112 CFI_REL_OFFSET r14,R14
110 CFI_OFFSET r10,R10-SS 113 CFI_REL_OFFSET r13,R13
111 CFI_OFFSET r9,R9-SS 114 CFI_REL_OFFSET r12,R12
112 CFI_OFFSET r8,R8-SS 115 CFI_REL_OFFSET rbp,RBP
113 CFI_OFFSET rax,RAX-SS 116 CFI_REL_OFFSET rbx,RBX
114 CFI_OFFSET rcx,RCX-SS 117 CFI_REL_OFFSET r11,R11
115 CFI_OFFSET rdx,RDX-SS 118 CFI_REL_OFFSET r10,R10
116 CFI_OFFSET rsi,RSI-SS 119 CFI_REL_OFFSET r9,R9
117 CFI_OFFSET rdi,RDI-SS 120 CFI_REL_OFFSET r8,R8
118 CFI_OFFSET rsp,RSP-SS 121 CFI_REL_OFFSET rax,RAX
119 CFI_OFFSET rip,RIP-SS 122 CFI_REL_OFFSET rcx,RCX
123 CFI_REL_OFFSET rdx,RDX
124 CFI_REL_OFFSET rsi,RSI
125 CFI_REL_OFFSET rdi,RDI
126 CFI_REL_OFFSET rip,RIP
127 /*CFI_REL_OFFSET cs,CS*/
128 /*CFI_REL_OFFSET rflags,EFLAGS*/
129 CFI_REL_OFFSET rsp,RSP
130 /*CFI_REL_OFFSET ss,SS*/
120 .endm 131 .endm
121/* 132/*
122 * A newly forked process directly context switches into this. 133 * A newly forked process directly context switches into this.
123 */ 134 */
124/* rdi: prev */ 135/* rdi: prev */
125ENTRY(ret_from_fork) 136ENTRY(ret_from_fork)
126 CFI_STARTPROC
127 CFI_DEFAULT_STACK 137 CFI_DEFAULT_STACK
128 call schedule_tail 138 call schedule_tail
129 GET_THREAD_INFO(%rcx) 139 GET_THREAD_INFO(%rcx)
@@ -172,16 +182,21 @@ rff_trace:
172 */ 182 */
173 183
174ENTRY(system_call) 184ENTRY(system_call)
175 CFI_STARTPROC 185 CFI_STARTPROC simple
186 CFI_DEF_CFA rsp,0
187 CFI_REGISTER rip,rcx
188 /*CFI_REGISTER rflags,r11*/
176 swapgs 189 swapgs
177 movq %rsp,%gs:pda_oldrsp 190 movq %rsp,%gs:pda_oldrsp
178 movq %gs:pda_kernelstack,%rsp 191 movq %gs:pda_kernelstack,%rsp
179 sti 192 sti
180 SAVE_ARGS 8,1 193 SAVE_ARGS 8,1
181 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 194 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
182 movq %rcx,RIP-ARGOFFSET(%rsp) 195 movq %rcx,RIP-ARGOFFSET(%rsp)
196 CFI_REL_OFFSET rip,RIP-ARGOFFSET
183 GET_THREAD_INFO(%rcx) 197 GET_THREAD_INFO(%rcx)
184 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) 198 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
199 CFI_REMEMBER_STATE
185 jnz tracesys 200 jnz tracesys
186 cmpq $__NR_syscall_max,%rax 201 cmpq $__NR_syscall_max,%rax
187 ja badsys 202 ja badsys
@@ -201,9 +216,12 @@ sysret_check:
201 cli 216 cli
202 movl threadinfo_flags(%rcx),%edx 217 movl threadinfo_flags(%rcx),%edx
203 andl %edi,%edx 218 andl %edi,%edx
219 CFI_REMEMBER_STATE
204 jnz sysret_careful 220 jnz sysret_careful
205 movq RIP-ARGOFFSET(%rsp),%rcx 221 movq RIP-ARGOFFSET(%rsp),%rcx
222 CFI_REGISTER rip,rcx
206 RESTORE_ARGS 0,-ARG_SKIP,1 223 RESTORE_ARGS 0,-ARG_SKIP,1
224 /*CFI_REGISTER rflags,r11*/
207 movq %gs:pda_oldrsp,%rsp 225 movq %gs:pda_oldrsp,%rsp
208 swapgs 226 swapgs
209 sysretq 227 sysretq
@@ -211,12 +229,15 @@ sysret_check:
211 /* Handle reschedules */ 229 /* Handle reschedules */
212 /* edx: work, edi: workmask */ 230 /* edx: work, edi: workmask */
213sysret_careful: 231sysret_careful:
232 CFI_RESTORE_STATE
214 bt $TIF_NEED_RESCHED,%edx 233 bt $TIF_NEED_RESCHED,%edx
215 jnc sysret_signal 234 jnc sysret_signal
216 sti 235 sti
217 pushq %rdi 236 pushq %rdi
237 CFI_ADJUST_CFA_OFFSET 8
218 call schedule 238 call schedule
219 popq %rdi 239 popq %rdi
240 CFI_ADJUST_CFA_OFFSET -8
220 jmp sysret_check 241 jmp sysret_check
221 242
222 /* Handle a signal */ 243 /* Handle a signal */
@@ -234,8 +255,13 @@ sysret_signal:
2341: movl $_TIF_NEED_RESCHED,%edi 2551: movl $_TIF_NEED_RESCHED,%edi
235 jmp sysret_check 256 jmp sysret_check
236 257
258badsys:
259 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
260 jmp ret_from_sys_call
261
237 /* Do syscall tracing */ 262 /* Do syscall tracing */
238tracesys: 263tracesys:
264 CFI_RESTORE_STATE
239 SAVE_REST 265 SAVE_REST
240 movq $-ENOSYS,RAX(%rsp) 266 movq $-ENOSYS,RAX(%rsp)
241 FIXUP_TOP_OF_STACK %rdi 267 FIXUP_TOP_OF_STACK %rdi
@@ -254,16 +280,29 @@ tracesys:
254 RESTORE_TOP_OF_STACK %rbx 280 RESTORE_TOP_OF_STACK %rbx
255 RESTORE_REST 281 RESTORE_REST
256 jmp ret_from_sys_call 282 jmp ret_from_sys_call
283 CFI_ENDPROC
257 284
258badsys:
259 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
260 jmp ret_from_sys_call
261
262/* 285/*
263 * Syscall return path ending with IRET. 286 * Syscall return path ending with IRET.
264 * Has correct top of stack, but partial stack frame. 287 * Has correct top of stack, but partial stack frame.
265 */ 288 */
266ENTRY(int_ret_from_sys_call) 289ENTRY(int_ret_from_sys_call)
290 CFI_STARTPROC simple
291 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
292 /*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
293 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
294 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
295 /*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
296 CFI_REL_OFFSET rip,RIP-ARGOFFSET
297 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
298 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
299 CFI_REL_OFFSET rax,RAX-ARGOFFSET
300 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
301 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
302 CFI_REL_OFFSET r8,R8-ARGOFFSET
303 CFI_REL_OFFSET r9,R9-ARGOFFSET
304 CFI_REL_OFFSET r10,R10-ARGOFFSET
305 CFI_REL_OFFSET r11,R11-ARGOFFSET
267 cli 306 cli
268 testl $3,CS-ARGOFFSET(%rsp) 307 testl $3,CS-ARGOFFSET(%rsp)
269 je retint_restore_args 308 je retint_restore_args
@@ -284,8 +323,10 @@ int_careful:
284 jnc int_very_careful 323 jnc int_very_careful
285 sti 324 sti
286 pushq %rdi 325 pushq %rdi
326 CFI_ADJUST_CFA_OFFSET 8
287 call schedule 327 call schedule
288 popq %rdi 328 popq %rdi
329 CFI_ADJUST_CFA_OFFSET -8
289 cli 330 cli
290 jmp int_with_check 331 jmp int_with_check
291 332
@@ -297,9 +338,11 @@ int_very_careful:
297 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx 338 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
298 jz int_signal 339 jz int_signal
299 pushq %rdi 340 pushq %rdi
341 CFI_ADJUST_CFA_OFFSET 8
300 leaq 8(%rsp),%rdi # &ptregs -> arg1 342 leaq 8(%rsp),%rdi # &ptregs -> arg1
301 call syscall_trace_leave 343 call syscall_trace_leave
302 popq %rdi 344 popq %rdi
345 CFI_ADJUST_CFA_OFFSET -8
303 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi 346 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
304 cli 347 cli
305 jmp int_restore_rest 348 jmp int_restore_rest
@@ -329,6 +372,8 @@ int_restore_rest:
329 jmp ptregscall_common 372 jmp ptregscall_common
330 .endm 373 .endm
331 374
375 CFI_STARTPROC
376
332 PTREGSCALL stub_clone, sys_clone, %r8 377 PTREGSCALL stub_clone, sys_clone, %r8
333 PTREGSCALL stub_fork, sys_fork, %rdi 378 PTREGSCALL stub_fork, sys_fork, %rdi
334 PTREGSCALL stub_vfork, sys_vfork, %rdi 379 PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -337,40 +382,49 @@ int_restore_rest:
337 PTREGSCALL stub_iopl, sys_iopl, %rsi 382 PTREGSCALL stub_iopl, sys_iopl, %rsi
338 383
339ENTRY(ptregscall_common) 384ENTRY(ptregscall_common)
340 CFI_STARTPROC
341 popq %r11 385 popq %r11
342 CFI_ADJUST_CFA_OFFSET -8 386 CFI_ADJUST_CFA_OFFSET -8
387 CFI_REGISTER rip, r11
343 SAVE_REST 388 SAVE_REST
344 movq %r11, %r15 389 movq %r11, %r15
390 CFI_REGISTER rip, r15
345 FIXUP_TOP_OF_STACK %r11 391 FIXUP_TOP_OF_STACK %r11
346 call *%rax 392 call *%rax
347 RESTORE_TOP_OF_STACK %r11 393 RESTORE_TOP_OF_STACK %r11
348 movq %r15, %r11 394 movq %r15, %r11
395 CFI_REGISTER rip, r11
349 RESTORE_REST 396 RESTORE_REST
350 pushq %r11 397 pushq %r11
351 CFI_ADJUST_CFA_OFFSET 8 398 CFI_ADJUST_CFA_OFFSET 8
399 CFI_REL_OFFSET rip, 0
352 ret 400 ret
353 CFI_ENDPROC 401 CFI_ENDPROC
354 402
355ENTRY(stub_execve) 403ENTRY(stub_execve)
356 CFI_STARTPROC 404 CFI_STARTPROC
357 popq %r11 405 popq %r11
358 CFI_ADJUST_CFA_OFFSET -8 406 CFI_ADJUST_CFA_OFFSET -8
407 CFI_REGISTER rip, r11
359 SAVE_REST 408 SAVE_REST
360 movq %r11, %r15 409 movq %r11, %r15
410 CFI_REGISTER rip, r15
361 FIXUP_TOP_OF_STACK %r11 411 FIXUP_TOP_OF_STACK %r11
362 call sys_execve 412 call sys_execve
363 GET_THREAD_INFO(%rcx) 413 GET_THREAD_INFO(%rcx)
364 bt $TIF_IA32,threadinfo_flags(%rcx) 414 bt $TIF_IA32,threadinfo_flags(%rcx)
415 CFI_REMEMBER_STATE
365 jc exec_32bit 416 jc exec_32bit
366 RESTORE_TOP_OF_STACK %r11 417 RESTORE_TOP_OF_STACK %r11
367 movq %r15, %r11 418 movq %r15, %r11
419 CFI_REGISTER rip, r11
368 RESTORE_REST 420 RESTORE_REST
369 push %r11 421 pushq %r11
422 CFI_ADJUST_CFA_OFFSET 8
423 CFI_REL_OFFSET rip, 0
370 ret 424 ret
371 425
372exec_32bit: 426exec_32bit:
373 CFI_ADJUST_CFA_OFFSET REST_SKIP 427 CFI_RESTORE_STATE
374 movq %rax,RAX(%rsp) 428 movq %rax,RAX(%rsp)
375 RESTORE_REST 429 RESTORE_REST
376 jmp int_ret_from_sys_call 430 jmp int_ret_from_sys_call
@@ -382,7 +436,8 @@ exec_32bit:
382 */ 436 */
383ENTRY(stub_rt_sigreturn) 437ENTRY(stub_rt_sigreturn)
384 CFI_STARTPROC 438 CFI_STARTPROC
385 addq $8, %rsp 439 addq $8, %rsp
440 CFI_ADJUST_CFA_OFFSET -8
386 SAVE_REST 441 SAVE_REST
387 movq %rsp,%rdi 442 movq %rsp,%rdi
388 FIXUP_TOP_OF_STACK %r11 443 FIXUP_TOP_OF_STACK %r11
@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
392 jmp int_ret_from_sys_call 447 jmp int_ret_from_sys_call
393 CFI_ENDPROC 448 CFI_ENDPROC
394 449
450/*
451 * initial frame state for interrupts and exceptions
452 */
453 .macro _frame ref
454 CFI_STARTPROC simple
455 CFI_DEF_CFA rsp,SS+8-\ref
456 /*CFI_REL_OFFSET ss,SS-\ref*/
457 CFI_REL_OFFSET rsp,RSP-\ref
458 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
459 /*CFI_REL_OFFSET cs,CS-\ref*/
460 CFI_REL_OFFSET rip,RIP-\ref
461 .endm
462
463/* initial frame state for interrupts (and exceptions without error code) */
464#define INTR_FRAME _frame RIP
465/* initial frame state for exceptions with error code (and interrupts with
466 vector already pushed) */
467#define XCPT_FRAME _frame ORIG_RAX
468
395/* 469/*
396 * Interrupt entry/exit. 470 * Interrupt entry/exit.
397 * 471 *
@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
402 476
403/* 0(%rsp): interrupt number */ 477/* 0(%rsp): interrupt number */
404 .macro interrupt func 478 .macro interrupt func
405 CFI_STARTPROC simple
406 CFI_DEF_CFA rsp,(SS-RDI)
407 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
408 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
409 cld 479 cld
410#ifdef CONFIG_DEBUG_INFO 480#ifdef CONFIG_DEBUG_INFO
411 SAVE_ALL 481 SAVE_ALL
@@ -425,23 +495,27 @@ ENTRY(stub_rt_sigreturn)
425 swapgs 495 swapgs
4261: incl %gs:pda_irqcount # RED-PEN should check preempt count 4961: incl %gs:pda_irqcount # RED-PEN should check preempt count
427 movq %gs:pda_irqstackptr,%rax 497 movq %gs:pda_irqstackptr,%rax
428 cmoveq %rax,%rsp 498 cmoveq %rax,%rsp /*todo This needs CFI annotation! */
429 pushq %rdi # save old stack 499 pushq %rdi # save old stack
500 CFI_ADJUST_CFA_OFFSET 8
430 call \func 501 call \func
431 .endm 502 .endm
432 503
433ENTRY(common_interrupt) 504ENTRY(common_interrupt)
505 XCPT_FRAME
434 interrupt do_IRQ 506 interrupt do_IRQ
435 /* 0(%rsp): oldrsp-ARGOFFSET */ 507 /* 0(%rsp): oldrsp-ARGOFFSET */
436ret_from_intr: 508ret_from_intr:
437 popq %rdi 509 popq %rdi
510 CFI_ADJUST_CFA_OFFSET -8
438 cli 511 cli
439 decl %gs:pda_irqcount 512 decl %gs:pda_irqcount
440#ifdef CONFIG_DEBUG_INFO 513#ifdef CONFIG_DEBUG_INFO
441 movq RBP(%rdi),%rbp 514 movq RBP(%rdi),%rbp
515 CFI_DEF_CFA_REGISTER rsp
442#endif 516#endif
443 leaq ARGOFFSET(%rdi),%rsp 517 leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
444exit_intr: 518exit_intr:
445 GET_THREAD_INFO(%rcx) 519 GET_THREAD_INFO(%rcx)
446 testl $3,CS-ARGOFFSET(%rsp) 520 testl $3,CS-ARGOFFSET(%rsp)
447 je retint_kernel 521 je retint_kernel
@@ -453,9 +527,10 @@ exit_intr:
453 */ 527 */
454retint_with_reschedule: 528retint_with_reschedule:
455 movl $_TIF_WORK_MASK,%edi 529 movl $_TIF_WORK_MASK,%edi
456retint_check: 530retint_check:
457 movl threadinfo_flags(%rcx),%edx 531 movl threadinfo_flags(%rcx),%edx
458 andl %edi,%edx 532 andl %edi,%edx
533 CFI_REMEMBER_STATE
459 jnz retint_careful 534 jnz retint_careful
460retint_swapgs: 535retint_swapgs:
461 swapgs 536 swapgs
@@ -476,14 +551,17 @@ bad_iret:
476 jmp do_exit 551 jmp do_exit
477 .previous 552 .previous
478 553
479 /* edi: workmask, edx: work */ 554 /* edi: workmask, edx: work */
480retint_careful: 555retint_careful:
556 CFI_RESTORE_STATE
481 bt $TIF_NEED_RESCHED,%edx 557 bt $TIF_NEED_RESCHED,%edx
482 jnc retint_signal 558 jnc retint_signal
483 sti 559 sti
484 pushq %rdi 560 pushq %rdi
561 CFI_ADJUST_CFA_OFFSET 8
485 call schedule 562 call schedule
486 popq %rdi 563 popq %rdi
564 CFI_ADJUST_CFA_OFFSET -8
487 GET_THREAD_INFO(%rcx) 565 GET_THREAD_INFO(%rcx)
488 cli 566 cli
489 jmp retint_check 567 jmp retint_check
@@ -523,7 +601,9 @@ retint_kernel:
523 * APIC interrupts. 601 * APIC interrupts.
524 */ 602 */
525 .macro apicinterrupt num,func 603 .macro apicinterrupt num,func
604 INTR_FRAME
526 pushq $\num-256 605 pushq $\num-256
606 CFI_ADJUST_CFA_OFFSET 8
527 interrupt \func 607 interrupt \func
528 jmp ret_from_intr 608 jmp ret_from_intr
529 CFI_ENDPROC 609 CFI_ENDPROC
@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
569 * Exception entry points. 649 * Exception entry points.
570 */ 650 */
571 .macro zeroentry sym 651 .macro zeroentry sym
652 INTR_FRAME
572 pushq $0 /* push error code/oldrax */ 653 pushq $0 /* push error code/oldrax */
654 CFI_ADJUST_CFA_OFFSET 8
573 pushq %rax /* push real oldrax to the rdi slot */ 655 pushq %rax /* push real oldrax to the rdi slot */
656 CFI_ADJUST_CFA_OFFSET 8
574 leaq \sym(%rip),%rax 657 leaq \sym(%rip),%rax
575 jmp error_entry 658 jmp error_entry
659 CFI_ENDPROC
576 .endm 660 .endm
577 661
578 .macro errorentry sym 662 .macro errorentry sym
663 XCPT_FRAME
579 pushq %rax 664 pushq %rax
665 CFI_ADJUST_CFA_OFFSET 8
580 leaq \sym(%rip),%rax 666 leaq \sym(%rip),%rax
581 jmp error_entry 667 jmp error_entry
668 CFI_ENDPROC
582 .endm 669 .endm
583 670
584 /* error code is on the stack already */ 671 /* error code is on the stack already */
@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
605 * and the exception handler in %rax. 692 * and the exception handler in %rax.
606 */ 693 */
607ENTRY(error_entry) 694ENTRY(error_entry)
608 CFI_STARTPROC simple 695 _frame RDI
609 CFI_DEF_CFA rsp,(SS-RDI)
610 CFI_REL_OFFSET rsp,(RSP-RDI)
611 CFI_REL_OFFSET rip,(RIP-RDI)
612 /* rdi slot contains rax, oldrax contains error code */ 696 /* rdi slot contains rax, oldrax contains error code */
613 cld 697 cld
614 subq $14*8,%rsp 698 subq $14*8,%rsp
@@ -690,7 +774,9 @@ error_kernelspace:
690 /* Reload gs selector with exception handling */ 774 /* Reload gs selector with exception handling */
691 /* edi: new selector */ 775 /* edi: new selector */
692ENTRY(load_gs_index) 776ENTRY(load_gs_index)
777 CFI_STARTPROC
693 pushf 778 pushf
779 CFI_ADJUST_CFA_OFFSET 8
694 cli 780 cli
695 swapgs 781 swapgs
696gs_change: 782gs_change:
@@ -698,7 +784,9 @@ gs_change:
6982: mfence /* workaround */ 7842: mfence /* workaround */
699 swapgs 785 swapgs
700 popf 786 popf
787 CFI_ADJUST_CFA_OFFSET -8
701 ret 788 ret
789 CFI_ENDPROC
702 790
703 .section __ex_table,"a" 791 .section __ex_table,"a"
704 .align 8 792 .align 8
@@ -810,7 +898,7 @@ ENTRY(device_not_available)
810 898
811 /* runs on exception stack */ 899 /* runs on exception stack */
812KPROBE_ENTRY(debug) 900KPROBE_ENTRY(debug)
813 CFI_STARTPROC 901 INTR_FRAME
814 pushq $0 902 pushq $0
815 CFI_ADJUST_CFA_OFFSET 8 903 CFI_ADJUST_CFA_OFFSET 8
816 paranoidentry do_debug 904 paranoidentry do_debug
@@ -820,9 +908,9 @@ KPROBE_ENTRY(debug)
820 908
821 /* runs on exception stack */ 909 /* runs on exception stack */
822ENTRY(nmi) 910ENTRY(nmi)
823 CFI_STARTPROC 911 INTR_FRAME
824 pushq $-1 912 pushq $-1
825 CFI_ADJUST_CFA_OFFSET 8 913 CFI_ADJUST_CFA_OFFSET 8
826 paranoidentry do_nmi 914 paranoidentry do_nmi
827 /* 915 /*
828 * "Paranoid" exit path from exception stack. 916 * "Paranoid" exit path from exception stack.
@@ -888,7 +976,7 @@ ENTRY(reserved)
888 976
889 /* runs on exception stack */ 977 /* runs on exception stack */
890ENTRY(double_fault) 978ENTRY(double_fault)
891 CFI_STARTPROC 979 XCPT_FRAME
892 paranoidentry do_double_fault 980 paranoidentry do_double_fault
893 jmp paranoid_exit 981 jmp paranoid_exit
894 CFI_ENDPROC 982 CFI_ENDPROC
@@ -901,7 +989,7 @@ ENTRY(segment_not_present)
901 989
902 /* runs on exception stack */ 990 /* runs on exception stack */
903ENTRY(stack_segment) 991ENTRY(stack_segment)
904 CFI_STARTPROC 992 XCPT_FRAME
905 paranoidentry do_stack_segment 993 paranoidentry do_stack_segment
906 jmp paranoid_exit 994 jmp paranoid_exit
907 CFI_ENDPROC 995 CFI_ENDPROC
@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
922#ifdef CONFIG_X86_MCE 1010#ifdef CONFIG_X86_MCE
923 /* runs on exception stack */ 1011 /* runs on exception stack */
924ENTRY(machine_check) 1012ENTRY(machine_check)
925 CFI_STARTPROC 1013 INTR_FRAME
926 pushq $0 1014 pushq $0
927 CFI_ADJUST_CFA_OFFSET 8 1015 CFI_ADJUST_CFA_OFFSET 8
928 paranoidentry do_machine_check 1016 paranoidentry do_machine_check
@@ -934,14 +1022,19 @@ ENTRY(call_debug)
934 zeroentry do_call_debug 1022 zeroentry do_call_debug
935 1023
936ENTRY(call_softirq) 1024ENTRY(call_softirq)
1025 CFI_STARTPROC
937 movq %gs:pda_irqstackptr,%rax 1026 movq %gs:pda_irqstackptr,%rax
938 pushq %r15 1027 pushq %r15
1028 CFI_ADJUST_CFA_OFFSET 8
939 movq %rsp,%r15 1029 movq %rsp,%r15
1030 CFI_DEF_CFA_REGISTER r15
940 incl %gs:pda_irqcount 1031 incl %gs:pda_irqcount
941 cmove %rax,%rsp 1032 cmove %rax,%rsp
942 call __do_softirq 1033 call __do_softirq
943 movq %r15,%rsp 1034 movq %r15,%rsp
1035 CFI_DEF_CFA_REGISTER rsp
944 decl %gs:pda_irqcount 1036 decl %gs:pda_irqcount
945 popq %r15 1037 popq %r15
1038 CFI_ADJUST_CFA_OFFSET -8
946 ret 1039 ret
947 1040 CFI_ENDPROC
diff --git a/include/asm-x86_64/calling.h b/include/asm-x86_64/calling.h
index 0bc12655fa5b..fc2c5a6c262a 100644
--- a/include/asm-x86_64/calling.h
+++ b/include/asm-x86_64/calling.h
@@ -65,27 +65,36 @@
65 .if \skipr11 65 .if \skipr11
66 .else 66 .else
67 movq (%rsp),%r11 67 movq (%rsp),%r11
68 CFI_RESTORE r11
68 .endif 69 .endif
69 .if \skipr8910 70 .if \skipr8910
70 .else 71 .else
71 movq 1*8(%rsp),%r10 72 movq 1*8(%rsp),%r10
73 CFI_RESTORE r10
72 movq 2*8(%rsp),%r9 74 movq 2*8(%rsp),%r9
75 CFI_RESTORE r9
73 movq 3*8(%rsp),%r8 76 movq 3*8(%rsp),%r8
77 CFI_RESTORE r8
74 .endif 78 .endif
75 .if \skiprax 79 .if \skiprax
76 .else 80 .else
77 movq 4*8(%rsp),%rax 81 movq 4*8(%rsp),%rax
82 CFI_RESTORE rax
78 .endif 83 .endif
79 .if \skiprcx 84 .if \skiprcx
80 .else 85 .else
81 movq 5*8(%rsp),%rcx 86 movq 5*8(%rsp),%rcx
87 CFI_RESTORE rcx
82 .endif 88 .endif
83 .if \skiprdx 89 .if \skiprdx
84 .else 90 .else
85 movq 6*8(%rsp),%rdx 91 movq 6*8(%rsp),%rdx
92 CFI_RESTORE rdx
86 .endif 93 .endif
87 movq 7*8(%rsp),%rsi 94 movq 7*8(%rsp),%rsi
95 CFI_RESTORE rsi
88 movq 8*8(%rsp),%rdi 96 movq 8*8(%rsp),%rdi
97 CFI_RESTORE rdi
89 .if ARG_SKIP+\addskip > 0 98 .if ARG_SKIP+\addskip > 0
90 addq $ARG_SKIP+\addskip,%rsp 99 addq $ARG_SKIP+\addskip,%rsp
91 CFI_ADJUST_CFA_OFFSET -(ARG_SKIP+\addskip) 100 CFI_ADJUST_CFA_OFFSET -(ARG_SKIP+\addskip)
@@ -124,11 +133,17 @@
124 133
125 .macro RESTORE_REST 134 .macro RESTORE_REST
126 movq (%rsp),%r15 135 movq (%rsp),%r15
136 CFI_RESTORE r15
127 movq 1*8(%rsp),%r14 137 movq 1*8(%rsp),%r14
138 CFI_RESTORE r14
128 movq 2*8(%rsp),%r13 139 movq 2*8(%rsp),%r13
140 CFI_RESTORE r13
129 movq 3*8(%rsp),%r12 141 movq 3*8(%rsp),%r12
142 CFI_RESTORE r12
130 movq 4*8(%rsp),%rbp 143 movq 4*8(%rsp),%rbp
144 CFI_RESTORE rbp
131 movq 5*8(%rsp),%rbx 145 movq 5*8(%rsp),%rbx
146 CFI_RESTORE rbx
132 addq $REST_SKIP,%rsp 147 addq $REST_SKIP,%rsp
133 CFI_ADJUST_CFA_OFFSET -(REST_SKIP) 148 CFI_ADJUST_CFA_OFFSET -(REST_SKIP)
134 .endm 149 .endm
@@ -146,11 +161,3 @@
146 .macro icebp 161 .macro icebp
147 .byte 0xf1 162 .byte 0xf1
148 .endm 163 .endm
149
150#ifdef CONFIG_FRAME_POINTER
151#define ENTER enter
152#define LEAVE leave
153#else
154#define ENTER
155#define LEAVE
156#endif
diff --git a/include/asm-x86_64/dwarf2.h b/include/asm-x86_64/dwarf2.h
index afd4212e860b..582757fc0365 100644
--- a/include/asm-x86_64/dwarf2.h
+++ b/include/asm-x86_64/dwarf2.h
@@ -24,6 +24,10 @@
24#define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset 24#define CFI_ADJUST_CFA_OFFSET .cfi_adjust_cfa_offset
25#define CFI_OFFSET .cfi_offset 25#define CFI_OFFSET .cfi_offset
26#define CFI_REL_OFFSET .cfi_rel_offset 26#define CFI_REL_OFFSET .cfi_rel_offset
27#define CFI_REGISTER .cfi_register
28#define CFI_RESTORE .cfi_restore
29#define CFI_REMEMBER_STATE .cfi_remember_state
30#define CFI_RESTORE_STATE .cfi_restore_state
27 31
28#else 32#else
29 33
@@ -36,6 +40,10 @@
36#define CFI_ADJUST_CFA_OFFSET # 40#define CFI_ADJUST_CFA_OFFSET #
37#define CFI_OFFSET # 41#define CFI_OFFSET #
38#define CFI_REL_OFFSET # 42#define CFI_REL_OFFSET #
43#define CFI_REGISTER #
44#define CFI_RESTORE #
45#define CFI_REMEMBER_STATE #
46#define CFI_RESTORE_STATE #
39 47
40#endif 48#endif
41 49
generated by cgit v1.2.2 (git 2.25.0) at 2025-12-21 05:58:26 -0500