aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86_64
diff options
context:
space:
mode:
authorJan Beulich <jbeulich@novell.com>2005-09-12 12:49:24 -0400
committerLinus Torvalds <torvalds@g5.osdl.org>2005-09-12 13:50:56 -0400
commit7effaa882af523085f7acadc5871b75a7e506baf (patch)
tree420890a24a9904bdaaec20dd9909d2f6f1e0d2f6 /arch/x86_64
parentb3ab8382245541ea030faaa1645f66258fde452d (diff)
[PATCH] x86-64: Fix CFI information
Being the foundation for reliable stack unwinding, this fixes CFI unwind annotations in many low-level x86_64 routines, plus a config option (available to all architectures, and also present in the previously sent patch adding such annotations to i386 code) to enable them separatly rather than only along with adding full debug information. Signed-off-by: Jan Beulich <jbeulich@novell.com> Signed-off-by: Andi Kleen <ak@suse.de> Signed-off-by: Linus Torvalds <torvalds@osdl.org>
Diffstat (limited to 'arch/x86_64')
-rw-r--r--arch/x86_64/ia32/ia32entry.S51
-rw-r--r--arch/x86_64/kernel/entry.S207
2 files changed, 197 insertions, 61 deletions
diff --git a/arch/x86_64/ia32/ia32entry.S b/arch/x86_64/ia32/ia32entry.S
index 5244f803203d..e0eb0c712fe9 100644
--- a/arch/x86_64/ia32/ia32entry.S
+++ b/arch/x86_64/ia32/ia32entry.S
@@ -55,20 +55,34 @@
55 * with the int 0x80 path. 55 * with the int 0x80 path.
56 */ 56 */
57ENTRY(ia32_sysenter_target) 57ENTRY(ia32_sysenter_target)
58 CFI_STARTPROC 58 CFI_STARTPROC simple
59 CFI_DEF_CFA rsp,0
60 CFI_REGISTER rsp,rbp
59 swapgs 61 swapgs
60 movq %gs:pda_kernelstack, %rsp 62 movq %gs:pda_kernelstack, %rsp
61 addq $(PDA_STACKOFFSET),%rsp 63 addq $(PDA_STACKOFFSET),%rsp
62 sti 64 sti
63 movl %ebp,%ebp /* zero extension */ 65 movl %ebp,%ebp /* zero extension */
64 pushq $__USER32_DS 66 pushq $__USER32_DS
67 CFI_ADJUST_CFA_OFFSET 8
68 /*CFI_REL_OFFSET ss,0*/
65 pushq %rbp 69 pushq %rbp
70 CFI_ADJUST_CFA_OFFSET 8
71 CFI_REL_OFFSET rsp,0
66 pushfq 72 pushfq
73 CFI_ADJUST_CFA_OFFSET 8
74 /*CFI_REL_OFFSET rflags,0*/
67 movl $VSYSCALL32_SYSEXIT, %r10d 75 movl $VSYSCALL32_SYSEXIT, %r10d
76 CFI_REGISTER rip,r10
68 pushq $__USER32_CS 77 pushq $__USER32_CS
78 CFI_ADJUST_CFA_OFFSET 8
79 /*CFI_REL_OFFSET cs,0*/
69 movl %eax, %eax 80 movl %eax, %eax
70 pushq %r10 81 pushq %r10
82 CFI_ADJUST_CFA_OFFSET 8
83 CFI_REL_OFFSET rip,0
71 pushq %rax 84 pushq %rax
85 CFI_ADJUST_CFA_OFFSET 8
72 cld 86 cld
73 SAVE_ARGS 0,0,1 87 SAVE_ARGS 0,0,1
74 /* no need to do an access_ok check here because rbp has been 88 /* no need to do an access_ok check here because rbp has been
@@ -79,6 +93,7 @@ ENTRY(ia32_sysenter_target)
79 .previous 93 .previous
80 GET_THREAD_INFO(%r10) 94 GET_THREAD_INFO(%r10)
81 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) 95 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
96 CFI_REMEMBER_STATE
82 jnz sysenter_tracesys 97 jnz sysenter_tracesys
83sysenter_do_call: 98sysenter_do_call:
84 cmpl $(IA32_NR_syscalls),%eax 99 cmpl $(IA32_NR_syscalls),%eax
@@ -94,14 +109,20 @@ sysenter_do_call:
94 andl $~0x200,EFLAGS-R11(%rsp) 109 andl $~0x200,EFLAGS-R11(%rsp)
95 RESTORE_ARGS 1,24,1,1,1,1 110 RESTORE_ARGS 1,24,1,1,1,1
96 popfq 111 popfq
112 CFI_ADJUST_CFA_OFFSET -8
113 /*CFI_RESTORE rflags*/
97 popq %rcx /* User %esp */ 114 popq %rcx /* User %esp */
115 CFI_ADJUST_CFA_OFFSET -8
116 CFI_REGISTER rsp,rcx
98 movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */ 117 movl $VSYSCALL32_SYSEXIT,%edx /* User %eip */
118 CFI_REGISTER rip,rdx
99 swapgs 119 swapgs
100 sti /* sti only takes effect after the next instruction */ 120 sti /* sti only takes effect after the next instruction */
101 /* sysexit */ 121 /* sysexit */
102 .byte 0xf, 0x35 122 .byte 0xf, 0x35
103 123
104sysenter_tracesys: 124sysenter_tracesys:
125 CFI_RESTORE_STATE
105 SAVE_REST 126 SAVE_REST
106 CLEAR_RREGS 127 CLEAR_RREGS
107 movq $-ENOSYS,RAX(%rsp) /* really needed? */ 128 movq $-ENOSYS,RAX(%rsp) /* really needed? */
@@ -140,21 +161,28 @@ sysenter_tracesys:
140 * with the int 0x80 path. 161 * with the int 0x80 path.
141 */ 162 */
142ENTRY(ia32_cstar_target) 163ENTRY(ia32_cstar_target)
143 CFI_STARTPROC 164 CFI_STARTPROC simple
165 CFI_DEF_CFA rsp,0
166 CFI_REGISTER rip,rcx
167 /*CFI_REGISTER rflags,r11*/
144 swapgs 168 swapgs
145 movl %esp,%r8d 169 movl %esp,%r8d
170 CFI_REGISTER rsp,r8
146 movq %gs:pda_kernelstack,%rsp 171 movq %gs:pda_kernelstack,%rsp
147 sti 172 sti
148 SAVE_ARGS 8,1,1 173 SAVE_ARGS 8,1,1
149 movl %eax,%eax /* zero extension */ 174 movl %eax,%eax /* zero extension */
150 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 175 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
151 movq %rcx,RIP-ARGOFFSET(%rsp) 176 movq %rcx,RIP-ARGOFFSET(%rsp)
177 CFI_REL_OFFSET rip,RIP-ARGOFFSET
152 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */ 178 movq %rbp,RCX-ARGOFFSET(%rsp) /* this lies slightly to ptrace */
153 movl %ebp,%ecx 179 movl %ebp,%ecx
154 movq $__USER32_CS,CS-ARGOFFSET(%rsp) 180 movq $__USER32_CS,CS-ARGOFFSET(%rsp)
155 movq $__USER32_DS,SS-ARGOFFSET(%rsp) 181 movq $__USER32_DS,SS-ARGOFFSET(%rsp)
156 movq %r11,EFLAGS-ARGOFFSET(%rsp) 182 movq %r11,EFLAGS-ARGOFFSET(%rsp)
183 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
157 movq %r8,RSP-ARGOFFSET(%rsp) 184 movq %r8,RSP-ARGOFFSET(%rsp)
185 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
158 /* no need to do an access_ok check here because r8 has been 186 /* no need to do an access_ok check here because r8 has been
159 32bit zero extended */ 187 32bit zero extended */
160 /* hardware stack frame is complete now */ 188 /* hardware stack frame is complete now */
@@ -164,6 +192,7 @@ ENTRY(ia32_cstar_target)
164 .previous 192 .previous
165 GET_THREAD_INFO(%r10) 193 GET_THREAD_INFO(%r10)
166 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10) 194 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%r10)
195 CFI_REMEMBER_STATE
167 jnz cstar_tracesys 196 jnz cstar_tracesys
168cstar_do_call: 197cstar_do_call:
169 cmpl $IA32_NR_syscalls,%eax 198 cmpl $IA32_NR_syscalls,%eax
@@ -177,12 +206,16 @@ cstar_do_call:
177 jnz int_ret_from_sys_call 206 jnz int_ret_from_sys_call
178 RESTORE_ARGS 1,-ARG_SKIP,1,1,1 207 RESTORE_ARGS 1,-ARG_SKIP,1,1,1
179 movl RIP-ARGOFFSET(%rsp),%ecx 208 movl RIP-ARGOFFSET(%rsp),%ecx
209 CFI_REGISTER rip,rcx
180 movl EFLAGS-ARGOFFSET(%rsp),%r11d 210 movl EFLAGS-ARGOFFSET(%rsp),%r11d
211 /*CFI_REGISTER rflags,r11*/
181 movl RSP-ARGOFFSET(%rsp),%esp 212 movl RSP-ARGOFFSET(%rsp),%esp
213 CFI_RESTORE rsp
182 swapgs 214 swapgs
183 sysretl 215 sysretl
184 216
185cstar_tracesys: 217cstar_tracesys:
218 CFI_RESTORE_STATE
186 SAVE_REST 219 SAVE_REST
187 CLEAR_RREGS 220 CLEAR_RREGS
188 movq $-ENOSYS,RAX(%rsp) /* really needed? */ 221 movq $-ENOSYS,RAX(%rsp) /* really needed? */
@@ -226,11 +259,18 @@ ia32_badarg:
226 */ 259 */
227 260
228ENTRY(ia32_syscall) 261ENTRY(ia32_syscall)
229 CFI_STARTPROC 262 CFI_STARTPROC simple
263 CFI_DEF_CFA rsp,SS+8-RIP
264 /*CFI_REL_OFFSET ss,SS-RIP*/
265 CFI_REL_OFFSET rsp,RSP-RIP
266 /*CFI_REL_OFFSET rflags,EFLAGS-RIP*/
267 /*CFI_REL_OFFSET cs,CS-RIP*/
268 CFI_REL_OFFSET rip,RIP-RIP
230 swapgs 269 swapgs
231 sti 270 sti
232 movl %eax,%eax 271 movl %eax,%eax
233 pushq %rax 272 pushq %rax
273 CFI_ADJUST_CFA_OFFSET 8
234 cld 274 cld
235 /* note the registers are not zero extended to the sf. 275 /* note the registers are not zero extended to the sf.
236 this could be a problem. */ 276 this could be a problem. */
@@ -278,6 +318,8 @@ quiet_ni_syscall:
278 jmp ia32_ptregs_common 318 jmp ia32_ptregs_common
279 .endm 319 .endm
280 320
321 CFI_STARTPROC
322
281 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi 323 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
282 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi 324 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
283 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx 325 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx
@@ -290,8 +332,9 @@ quiet_ni_syscall:
290 PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx 332 PTREGSCALL stub32_rt_sigsuspend, sys_rt_sigsuspend, %rdx
291 333
292ENTRY(ia32_ptregs_common) 334ENTRY(ia32_ptregs_common)
293 CFI_STARTPROC
294 popq %r11 335 popq %r11
336 CFI_ADJUST_CFA_OFFSET -8
337 CFI_REGISTER rip, r11
295 SAVE_REST 338 SAVE_REST
296 call *%rax 339 call *%rax
297 RESTORE_REST 340 RESTORE_REST
diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 873c39d8f818..7937971d1853 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -79,16 +79,19 @@
79 xorl %eax, %eax 79 xorl %eax, %eax
80 pushq %rax /* ss */ 80 pushq %rax /* ss */
81 CFI_ADJUST_CFA_OFFSET 8 81 CFI_ADJUST_CFA_OFFSET 8
82 /*CFI_REL_OFFSET ss,0*/
82 pushq %rax /* rsp */ 83 pushq %rax /* rsp */
83 CFI_ADJUST_CFA_OFFSET 8 84 CFI_ADJUST_CFA_OFFSET 8
84 CFI_OFFSET rip,0 85 CFI_REL_OFFSET rsp,0
85 pushq $(1<<9) /* eflags - interrupts on */ 86 pushq $(1<<9) /* eflags - interrupts on */
86 CFI_ADJUST_CFA_OFFSET 8 87 CFI_ADJUST_CFA_OFFSET 8
88 /*CFI_REL_OFFSET rflags,0*/
87 pushq $__KERNEL_CS /* cs */ 89 pushq $__KERNEL_CS /* cs */
88 CFI_ADJUST_CFA_OFFSET 8 90 CFI_ADJUST_CFA_OFFSET 8
91 /*CFI_REL_OFFSET cs,0*/
89 pushq \child_rip /* rip */ 92 pushq \child_rip /* rip */
90 CFI_ADJUST_CFA_OFFSET 8 93 CFI_ADJUST_CFA_OFFSET 8
91 CFI_OFFSET rip,0 94 CFI_REL_OFFSET rip,0
92 pushq %rax /* orig rax */ 95 pushq %rax /* orig rax */
93 CFI_ADJUST_CFA_OFFSET 8 96 CFI_ADJUST_CFA_OFFSET 8
94 .endm 97 .endm
@@ -98,32 +101,39 @@
98 CFI_ADJUST_CFA_OFFSET -(6*8) 101 CFI_ADJUST_CFA_OFFSET -(6*8)
99 .endm 102 .endm
100 103
101 .macro CFI_DEFAULT_STACK 104 .macro CFI_DEFAULT_STACK start=1
102 CFI_ADJUST_CFA_OFFSET (SS) 105 .if \start
103 CFI_OFFSET r15,R15-SS 106 CFI_STARTPROC simple
104 CFI_OFFSET r14,R14-SS 107 CFI_DEF_CFA rsp,SS+8
105 CFI_OFFSET r13,R13-SS 108 .else
106 CFI_OFFSET r12,R12-SS 109 CFI_DEF_CFA_OFFSET SS+8
107 CFI_OFFSET rbp,RBP-SS 110 .endif
108 CFI_OFFSET rbx,RBX-SS 111 CFI_REL_OFFSET r15,R15
109 CFI_OFFSET r11,R11-SS 112 CFI_REL_OFFSET r14,R14
110 CFI_OFFSET r10,R10-SS 113 CFI_REL_OFFSET r13,R13
111 CFI_OFFSET r9,R9-SS 114 CFI_REL_OFFSET r12,R12
112 CFI_OFFSET r8,R8-SS 115 CFI_REL_OFFSET rbp,RBP
113 CFI_OFFSET rax,RAX-SS 116 CFI_REL_OFFSET rbx,RBX
114 CFI_OFFSET rcx,RCX-SS 117 CFI_REL_OFFSET r11,R11
115 CFI_OFFSET rdx,RDX-SS 118 CFI_REL_OFFSET r10,R10
116 CFI_OFFSET rsi,RSI-SS 119 CFI_REL_OFFSET r9,R9
117 CFI_OFFSET rdi,RDI-SS 120 CFI_REL_OFFSET r8,R8
118 CFI_OFFSET rsp,RSP-SS 121 CFI_REL_OFFSET rax,RAX
119 CFI_OFFSET rip,RIP-SS 122 CFI_REL_OFFSET rcx,RCX
123 CFI_REL_OFFSET rdx,RDX
124 CFI_REL_OFFSET rsi,RSI
125 CFI_REL_OFFSET rdi,RDI
126 CFI_REL_OFFSET rip,RIP
127 /*CFI_REL_OFFSET cs,CS*/
128 /*CFI_REL_OFFSET rflags,EFLAGS*/
129 CFI_REL_OFFSET rsp,RSP
130 /*CFI_REL_OFFSET ss,SS*/
120 .endm 131 .endm
121/* 132/*
122 * A newly forked process directly context switches into this. 133 * A newly forked process directly context switches into this.
123 */ 134 */
124/* rdi: prev */ 135/* rdi: prev */
125ENTRY(ret_from_fork) 136ENTRY(ret_from_fork)
126 CFI_STARTPROC
127 CFI_DEFAULT_STACK 137 CFI_DEFAULT_STACK
128 call schedule_tail 138 call schedule_tail
129 GET_THREAD_INFO(%rcx) 139 GET_THREAD_INFO(%rcx)
@@ -172,16 +182,21 @@ rff_trace:
172 */ 182 */
173 183
174ENTRY(system_call) 184ENTRY(system_call)
175 CFI_STARTPROC 185 CFI_STARTPROC simple
186 CFI_DEF_CFA rsp,0
187 CFI_REGISTER rip,rcx
188 /*CFI_REGISTER rflags,r11*/
176 swapgs 189 swapgs
177 movq %rsp,%gs:pda_oldrsp 190 movq %rsp,%gs:pda_oldrsp
178 movq %gs:pda_kernelstack,%rsp 191 movq %gs:pda_kernelstack,%rsp
179 sti 192 sti
180 SAVE_ARGS 8,1 193 SAVE_ARGS 8,1
181 movq %rax,ORIG_RAX-ARGOFFSET(%rsp) 194 movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
182 movq %rcx,RIP-ARGOFFSET(%rsp) 195 movq %rcx,RIP-ARGOFFSET(%rsp)
196 CFI_REL_OFFSET rip,RIP-ARGOFFSET
183 GET_THREAD_INFO(%rcx) 197 GET_THREAD_INFO(%rcx)
184 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx) 198 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),threadinfo_flags(%rcx)
199 CFI_REMEMBER_STATE
185 jnz tracesys 200 jnz tracesys
186 cmpq $__NR_syscall_max,%rax 201 cmpq $__NR_syscall_max,%rax
187 ja badsys 202 ja badsys
@@ -201,9 +216,12 @@ sysret_check:
201 cli 216 cli
202 movl threadinfo_flags(%rcx),%edx 217 movl threadinfo_flags(%rcx),%edx
203 andl %edi,%edx 218 andl %edi,%edx
219 CFI_REMEMBER_STATE
204 jnz sysret_careful 220 jnz sysret_careful
205 movq RIP-ARGOFFSET(%rsp),%rcx 221 movq RIP-ARGOFFSET(%rsp),%rcx
222 CFI_REGISTER rip,rcx
206 RESTORE_ARGS 0,-ARG_SKIP,1 223 RESTORE_ARGS 0,-ARG_SKIP,1
224 /*CFI_REGISTER rflags,r11*/
207 movq %gs:pda_oldrsp,%rsp 225 movq %gs:pda_oldrsp,%rsp
208 swapgs 226 swapgs
209 sysretq 227 sysretq
@@ -211,12 +229,15 @@ sysret_check:
211 /* Handle reschedules */ 229 /* Handle reschedules */
212 /* edx: work, edi: workmask */ 230 /* edx: work, edi: workmask */
213sysret_careful: 231sysret_careful:
232 CFI_RESTORE_STATE
214 bt $TIF_NEED_RESCHED,%edx 233 bt $TIF_NEED_RESCHED,%edx
215 jnc sysret_signal 234 jnc sysret_signal
216 sti 235 sti
217 pushq %rdi 236 pushq %rdi
237 CFI_ADJUST_CFA_OFFSET 8
218 call schedule 238 call schedule
219 popq %rdi 239 popq %rdi
240 CFI_ADJUST_CFA_OFFSET -8
220 jmp sysret_check 241 jmp sysret_check
221 242
222 /* Handle a signal */ 243 /* Handle a signal */
@@ -234,8 +255,13 @@ sysret_signal:
2341: movl $_TIF_NEED_RESCHED,%edi 2551: movl $_TIF_NEED_RESCHED,%edi
235 jmp sysret_check 256 jmp sysret_check
236 257
258badsys:
259 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
260 jmp ret_from_sys_call
261
237 /* Do syscall tracing */ 262 /* Do syscall tracing */
238tracesys: 263tracesys:
264 CFI_RESTORE_STATE
239 SAVE_REST 265 SAVE_REST
240 movq $-ENOSYS,RAX(%rsp) 266 movq $-ENOSYS,RAX(%rsp)
241 FIXUP_TOP_OF_STACK %rdi 267 FIXUP_TOP_OF_STACK %rdi
@@ -254,16 +280,29 @@ tracesys:
254 RESTORE_TOP_OF_STACK %rbx 280 RESTORE_TOP_OF_STACK %rbx
255 RESTORE_REST 281 RESTORE_REST
256 jmp ret_from_sys_call 282 jmp ret_from_sys_call
283 CFI_ENDPROC
257 284
258badsys:
259 movq $-ENOSYS,RAX-ARGOFFSET(%rsp)
260 jmp ret_from_sys_call
261
262/* 285/*
263 * Syscall return path ending with IRET. 286 * Syscall return path ending with IRET.
264 * Has correct top of stack, but partial stack frame. 287 * Has correct top of stack, but partial stack frame.
265 */ 288 */
266ENTRY(int_ret_from_sys_call) 289ENTRY(int_ret_from_sys_call)
290 CFI_STARTPROC simple
291 CFI_DEF_CFA rsp,SS+8-ARGOFFSET
292 /*CFI_REL_OFFSET ss,SS-ARGOFFSET*/
293 CFI_REL_OFFSET rsp,RSP-ARGOFFSET
294 /*CFI_REL_OFFSET rflags,EFLAGS-ARGOFFSET*/
295 /*CFI_REL_OFFSET cs,CS-ARGOFFSET*/
296 CFI_REL_OFFSET rip,RIP-ARGOFFSET
297 CFI_REL_OFFSET rdx,RDX-ARGOFFSET
298 CFI_REL_OFFSET rcx,RCX-ARGOFFSET
299 CFI_REL_OFFSET rax,RAX-ARGOFFSET
300 CFI_REL_OFFSET rdi,RDI-ARGOFFSET
301 CFI_REL_OFFSET rsi,RSI-ARGOFFSET
302 CFI_REL_OFFSET r8,R8-ARGOFFSET
303 CFI_REL_OFFSET r9,R9-ARGOFFSET
304 CFI_REL_OFFSET r10,R10-ARGOFFSET
305 CFI_REL_OFFSET r11,R11-ARGOFFSET
267 cli 306 cli
268 testl $3,CS-ARGOFFSET(%rsp) 307 testl $3,CS-ARGOFFSET(%rsp)
269 je retint_restore_args 308 je retint_restore_args
@@ -284,8 +323,10 @@ int_careful:
284 jnc int_very_careful 323 jnc int_very_careful
285 sti 324 sti
286 pushq %rdi 325 pushq %rdi
326 CFI_ADJUST_CFA_OFFSET 8
287 call schedule 327 call schedule
288 popq %rdi 328 popq %rdi
329 CFI_ADJUST_CFA_OFFSET -8
289 cli 330 cli
290 jmp int_with_check 331 jmp int_with_check
291 332
@@ -297,9 +338,11 @@ int_very_careful:
297 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx 338 testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edx
298 jz int_signal 339 jz int_signal
299 pushq %rdi 340 pushq %rdi
341 CFI_ADJUST_CFA_OFFSET 8
300 leaq 8(%rsp),%rdi # &ptregs -> arg1 342 leaq 8(%rsp),%rdi # &ptregs -> arg1
301 call syscall_trace_leave 343 call syscall_trace_leave
302 popq %rdi 344 popq %rdi
345 CFI_ADJUST_CFA_OFFSET -8
303 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi 346 andl $~(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP),%edi
304 cli 347 cli
305 jmp int_restore_rest 348 jmp int_restore_rest
@@ -329,6 +372,8 @@ int_restore_rest:
329 jmp ptregscall_common 372 jmp ptregscall_common
330 .endm 373 .endm
331 374
375 CFI_STARTPROC
376
332 PTREGSCALL stub_clone, sys_clone, %r8 377 PTREGSCALL stub_clone, sys_clone, %r8
333 PTREGSCALL stub_fork, sys_fork, %rdi 378 PTREGSCALL stub_fork, sys_fork, %rdi
334 PTREGSCALL stub_vfork, sys_vfork, %rdi 379 PTREGSCALL stub_vfork, sys_vfork, %rdi
@@ -337,40 +382,49 @@ int_restore_rest:
337 PTREGSCALL stub_iopl, sys_iopl, %rsi 382 PTREGSCALL stub_iopl, sys_iopl, %rsi
338 383
339ENTRY(ptregscall_common) 384ENTRY(ptregscall_common)
340 CFI_STARTPROC
341 popq %r11 385 popq %r11
342 CFI_ADJUST_CFA_OFFSET -8 386 CFI_ADJUST_CFA_OFFSET -8
387 CFI_REGISTER rip, r11
343 SAVE_REST 388 SAVE_REST
344 movq %r11, %r15 389 movq %r11, %r15
390 CFI_REGISTER rip, r15
345 FIXUP_TOP_OF_STACK %r11 391 FIXUP_TOP_OF_STACK %r11
346 call *%rax 392 call *%rax
347 RESTORE_TOP_OF_STACK %r11 393 RESTORE_TOP_OF_STACK %r11
348 movq %r15, %r11 394 movq %r15, %r11
395 CFI_REGISTER rip, r11
349 RESTORE_REST 396 RESTORE_REST
350 pushq %r11 397 pushq %r11
351 CFI_ADJUST_CFA_OFFSET 8 398 CFI_ADJUST_CFA_OFFSET 8
399 CFI_REL_OFFSET rip, 0
352 ret 400 ret
353 CFI_ENDPROC 401 CFI_ENDPROC
354 402
355ENTRY(stub_execve) 403ENTRY(stub_execve)
356 CFI_STARTPROC 404 CFI_STARTPROC
357 popq %r11 405 popq %r11
358 CFI_ADJUST_CFA_OFFSET -8 406 CFI_ADJUST_CFA_OFFSET -8
407 CFI_REGISTER rip, r11
359 SAVE_REST 408 SAVE_REST
360 movq %r11, %r15 409 movq %r11, %r15
410 CFI_REGISTER rip, r15
361 FIXUP_TOP_OF_STACK %r11 411 FIXUP_TOP_OF_STACK %r11
362 call sys_execve 412 call sys_execve
363 GET_THREAD_INFO(%rcx) 413 GET_THREAD_INFO(%rcx)
364 bt $TIF_IA32,threadinfo_flags(%rcx) 414 bt $TIF_IA32,threadinfo_flags(%rcx)
415 CFI_REMEMBER_STATE
365 jc exec_32bit 416 jc exec_32bit
366 RESTORE_TOP_OF_STACK %r11 417 RESTORE_TOP_OF_STACK %r11
367 movq %r15, %r11 418 movq %r15, %r11
419 CFI_REGISTER rip, r11
368 RESTORE_REST 420 RESTORE_REST
369 push %r11 421 pushq %r11
422 CFI_ADJUST_CFA_OFFSET 8
423 CFI_REL_OFFSET rip, 0
370 ret 424 ret
371 425
372exec_32bit: 426exec_32bit:
373 CFI_ADJUST_CFA_OFFSET REST_SKIP 427 CFI_RESTORE_STATE
374 movq %rax,RAX(%rsp) 428 movq %rax,RAX(%rsp)
375 RESTORE_REST 429 RESTORE_REST
376 jmp int_ret_from_sys_call 430 jmp int_ret_from_sys_call
@@ -382,7 +436,8 @@ exec_32bit:
382 */ 436 */
383ENTRY(stub_rt_sigreturn) 437ENTRY(stub_rt_sigreturn)
384 CFI_STARTPROC 438 CFI_STARTPROC
385 addq $8, %rsp 439 addq $8, %rsp
440 CFI_ADJUST_CFA_OFFSET -8
386 SAVE_REST 441 SAVE_REST
387 movq %rsp,%rdi 442 movq %rsp,%rdi
388 FIXUP_TOP_OF_STACK %r11 443 FIXUP_TOP_OF_STACK %r11
@@ -392,6 +447,25 @@ ENTRY(stub_rt_sigreturn)
392 jmp int_ret_from_sys_call 447 jmp int_ret_from_sys_call
393 CFI_ENDPROC 448 CFI_ENDPROC
394 449
450/*
451 * initial frame state for interrupts and exceptions
452 */
453 .macro _frame ref
454 CFI_STARTPROC simple
455 CFI_DEF_CFA rsp,SS+8-\ref
456 /*CFI_REL_OFFSET ss,SS-\ref*/
457 CFI_REL_OFFSET rsp,RSP-\ref
458 /*CFI_REL_OFFSET rflags,EFLAGS-\ref*/
459 /*CFI_REL_OFFSET cs,CS-\ref*/
460 CFI_REL_OFFSET rip,RIP-\ref
461 .endm
462
463/* initial frame state for interrupts (and exceptions without error code) */
464#define INTR_FRAME _frame RIP
465/* initial frame state for exceptions with error code (and interrupts with
466 vector already pushed) */
467#define XCPT_FRAME _frame ORIG_RAX
468
395/* 469/*
396 * Interrupt entry/exit. 470 * Interrupt entry/exit.
397 * 471 *
@@ -402,10 +476,6 @@ ENTRY(stub_rt_sigreturn)
402 476
403/* 0(%rsp): interrupt number */ 477/* 0(%rsp): interrupt number */
404 .macro interrupt func 478 .macro interrupt func
405 CFI_STARTPROC simple
406 CFI_DEF_CFA rsp,(SS-RDI)
407 CFI_REL_OFFSET rsp,(RSP-ORIG_RAX)
408 CFI_REL_OFFSET rip,(RIP-ORIG_RAX)
409 cld 479 cld
410#ifdef CONFIG_DEBUG_INFO 480#ifdef CONFIG_DEBUG_INFO
411 SAVE_ALL 481 SAVE_ALL
@@ -425,23 +495,27 @@ ENTRY(stub_rt_sigreturn)
425 swapgs 495 swapgs
4261: incl %gs:pda_irqcount # RED-PEN should check preempt count 4961: incl %gs:pda_irqcount # RED-PEN should check preempt count
427 movq %gs:pda_irqstackptr,%rax 497 movq %gs:pda_irqstackptr,%rax
428 cmoveq %rax,%rsp 498 cmoveq %rax,%rsp /*todo This needs CFI annotation! */
429 pushq %rdi # save old stack 499 pushq %rdi # save old stack
500 CFI_ADJUST_CFA_OFFSET 8
430 call \func 501 call \func
431 .endm 502 .endm
432 503
433ENTRY(common_interrupt) 504ENTRY(common_interrupt)
505 XCPT_FRAME
434 interrupt do_IRQ 506 interrupt do_IRQ
435 /* 0(%rsp): oldrsp-ARGOFFSET */ 507 /* 0(%rsp): oldrsp-ARGOFFSET */
436ret_from_intr: 508ret_from_intr:
437 popq %rdi 509 popq %rdi
510 CFI_ADJUST_CFA_OFFSET -8
438 cli 511 cli
439 decl %gs:pda_irqcount 512 decl %gs:pda_irqcount
440#ifdef CONFIG_DEBUG_INFO 513#ifdef CONFIG_DEBUG_INFO
441 movq RBP(%rdi),%rbp 514 movq RBP(%rdi),%rbp
515 CFI_DEF_CFA_REGISTER rsp
442#endif 516#endif
443 leaq ARGOFFSET(%rdi),%rsp 517 leaq ARGOFFSET(%rdi),%rsp /*todo This needs CFI annotation! */
444exit_intr: 518exit_intr:
445 GET_THREAD_INFO(%rcx) 519 GET_THREAD_INFO(%rcx)
446 testl $3,CS-ARGOFFSET(%rsp) 520 testl $3,CS-ARGOFFSET(%rsp)
447 je retint_kernel 521 je retint_kernel
@@ -453,9 +527,10 @@ exit_intr:
453 */ 527 */
454retint_with_reschedule: 528retint_with_reschedule:
455 movl $_TIF_WORK_MASK,%edi 529 movl $_TIF_WORK_MASK,%edi
456retint_check: 530retint_check:
457 movl threadinfo_flags(%rcx),%edx 531 movl threadinfo_flags(%rcx),%edx
458 andl %edi,%edx 532 andl %edi,%edx
533 CFI_REMEMBER_STATE
459 jnz retint_careful 534 jnz retint_careful
460retint_swapgs: 535retint_swapgs:
461 swapgs 536 swapgs
@@ -476,14 +551,17 @@ bad_iret:
476 jmp do_exit 551 jmp do_exit
477 .previous 552 .previous
478 553
479 /* edi: workmask, edx: work */ 554 /* edi: workmask, edx: work */
480retint_careful: 555retint_careful:
556 CFI_RESTORE_STATE
481 bt $TIF_NEED_RESCHED,%edx 557 bt $TIF_NEED_RESCHED,%edx
482 jnc retint_signal 558 jnc retint_signal
483 sti 559 sti
484 pushq %rdi 560 pushq %rdi
561 CFI_ADJUST_CFA_OFFSET 8
485 call schedule 562 call schedule
486 popq %rdi 563 popq %rdi
564 CFI_ADJUST_CFA_OFFSET -8
487 GET_THREAD_INFO(%rcx) 565 GET_THREAD_INFO(%rcx)
488 cli 566 cli
489 jmp retint_check 567 jmp retint_check
@@ -523,7 +601,9 @@ retint_kernel:
523 * APIC interrupts. 601 * APIC interrupts.
524 */ 602 */
525 .macro apicinterrupt num,func 603 .macro apicinterrupt num,func
604 INTR_FRAME
526 pushq $\num-256 605 pushq $\num-256
606 CFI_ADJUST_CFA_OFFSET 8
527 interrupt \func 607 interrupt \func
528 jmp ret_from_intr 608 jmp ret_from_intr
529 CFI_ENDPROC 609 CFI_ENDPROC
@@ -569,16 +649,23 @@ ENTRY(spurious_interrupt)
569 * Exception entry points. 649 * Exception entry points.
570 */ 650 */
571 .macro zeroentry sym 651 .macro zeroentry sym
652 INTR_FRAME
572 pushq $0 /* push error code/oldrax */ 653 pushq $0 /* push error code/oldrax */
654 CFI_ADJUST_CFA_OFFSET 8
573 pushq %rax /* push real oldrax to the rdi slot */ 655 pushq %rax /* push real oldrax to the rdi slot */
656 CFI_ADJUST_CFA_OFFSET 8
574 leaq \sym(%rip),%rax 657 leaq \sym(%rip),%rax
575 jmp error_entry 658 jmp error_entry
659 CFI_ENDPROC
576 .endm 660 .endm
577 661
578 .macro errorentry sym 662 .macro errorentry sym
663 XCPT_FRAME
579 pushq %rax 664 pushq %rax
665 CFI_ADJUST_CFA_OFFSET 8
580 leaq \sym(%rip),%rax 666 leaq \sym(%rip),%rax
581 jmp error_entry 667 jmp error_entry
668 CFI_ENDPROC
582 .endm 669 .endm
583 670
584 /* error code is on the stack already */ 671 /* error code is on the stack already */
@@ -605,10 +692,7 @@ ENTRY(spurious_interrupt)
605 * and the exception handler in %rax. 692 * and the exception handler in %rax.
606 */ 693 */
607ENTRY(error_entry) 694ENTRY(error_entry)
608 CFI_STARTPROC simple 695 _frame RDI
609 CFI_DEF_CFA rsp,(SS-RDI)
610 CFI_REL_OFFSET rsp,(RSP-RDI)
611 CFI_REL_OFFSET rip,(RIP-RDI)
612 /* rdi slot contains rax, oldrax contains error code */ 696 /* rdi slot contains rax, oldrax contains error code */
613 cld 697 cld
614 subq $14*8,%rsp 698 subq $14*8,%rsp
@@ -690,7 +774,9 @@ error_kernelspace:
690 /* Reload gs selector with exception handling */ 774 /* Reload gs selector with exception handling */
691 /* edi: new selector */ 775 /* edi: new selector */
692ENTRY(load_gs_index) 776ENTRY(load_gs_index)
777 CFI_STARTPROC
693 pushf 778 pushf
779 CFI_ADJUST_CFA_OFFSET 8
694 cli 780 cli
695 swapgs 781 swapgs
696gs_change: 782gs_change:
@@ -698,7 +784,9 @@ gs_change:
6982: mfence /* workaround */ 7842: mfence /* workaround */
699 swapgs 785 swapgs
700 popf 786 popf
787 CFI_ADJUST_CFA_OFFSET -8
701 ret 788 ret
789 CFI_ENDPROC
702 790
703 .section __ex_table,"a" 791 .section __ex_table,"a"
704 .align 8 792 .align 8
@@ -810,7 +898,7 @@ ENTRY(device_not_available)
810 898
811 /* runs on exception stack */ 899 /* runs on exception stack */
812KPROBE_ENTRY(debug) 900KPROBE_ENTRY(debug)
813 CFI_STARTPROC 901 INTR_FRAME
814 pushq $0 902 pushq $0
815 CFI_ADJUST_CFA_OFFSET 8 903 CFI_ADJUST_CFA_OFFSET 8
816 paranoidentry do_debug 904 paranoidentry do_debug
@@ -820,9 +908,9 @@ KPROBE_ENTRY(debug)
820 908
821 /* runs on exception stack */ 909 /* runs on exception stack */
822ENTRY(nmi) 910ENTRY(nmi)
823 CFI_STARTPROC 911 INTR_FRAME
824 pushq $-1 912 pushq $-1
825 CFI_ADJUST_CFA_OFFSET 8 913 CFI_ADJUST_CFA_OFFSET 8
826 paranoidentry do_nmi 914 paranoidentry do_nmi
827 /* 915 /*
828 * "Paranoid" exit path from exception stack. 916 * "Paranoid" exit path from exception stack.
@@ -888,7 +976,7 @@ ENTRY(reserved)
888 976
889 /* runs on exception stack */ 977 /* runs on exception stack */
890ENTRY(double_fault) 978ENTRY(double_fault)
891 CFI_STARTPROC 979 XCPT_FRAME
892 paranoidentry do_double_fault 980 paranoidentry do_double_fault
893 jmp paranoid_exit 981 jmp paranoid_exit
894 CFI_ENDPROC 982 CFI_ENDPROC
@@ -901,7 +989,7 @@ ENTRY(segment_not_present)
901 989
902 /* runs on exception stack */ 990 /* runs on exception stack */
903ENTRY(stack_segment) 991ENTRY(stack_segment)
904 CFI_STARTPROC 992 XCPT_FRAME
905 paranoidentry do_stack_segment 993 paranoidentry do_stack_segment
906 jmp paranoid_exit 994 jmp paranoid_exit
907 CFI_ENDPROC 995 CFI_ENDPROC
@@ -922,7 +1010,7 @@ ENTRY(spurious_interrupt_bug)
922#ifdef CONFIG_X86_MCE 1010#ifdef CONFIG_X86_MCE
923 /* runs on exception stack */ 1011 /* runs on exception stack */
924ENTRY(machine_check) 1012ENTRY(machine_check)
925 CFI_STARTPROC 1013 INTR_FRAME
926 pushq $0 1014 pushq $0
927 CFI_ADJUST_CFA_OFFSET 8 1015 CFI_ADJUST_CFA_OFFSET 8
928 paranoidentry do_machine_check 1016 paranoidentry do_machine_check
@@ -934,14 +1022,19 @@ ENTRY(call_debug)
934 zeroentry do_call_debug 1022 zeroentry do_call_debug
935 1023
936ENTRY(call_softirq) 1024ENTRY(call_softirq)
1025 CFI_STARTPROC
937 movq %gs:pda_irqstackptr,%rax 1026 movq %gs:pda_irqstackptr,%rax
938 pushq %r15 1027 pushq %r15
1028 CFI_ADJUST_CFA_OFFSET 8
939 movq %rsp,%r15 1029 movq %rsp,%r15
1030 CFI_DEF_CFA_REGISTER r15
940 incl %gs:pda_irqcount 1031 incl %gs:pda_irqcount
941 cmove %rax,%rsp 1032 cmove %rax,%rsp
942 call __do_softirq 1033 call __do_softirq
943 movq %r15,%rsp 1034 movq %r15,%rsp
1035 CFI_DEF_CFA_REGISTER rsp
944 decl %gs:pda_irqcount 1036 decl %gs:pda_irqcount
945 popq %r15 1037 popq %r15
1038 CFI_ADJUST_CFA_OFFSET -8
946 ret 1039 ret
947 1040 CFI_ENDPROC
generated by cgit v1.2.2 (git 2.25.0) at 2024-09-21 22:32:52 -0400