diff options
Diffstat (limited to 'arch/x86/kernel/entry_64.S')
-rw-r--r-- | arch/x86/kernel/entry_64.S | 34 |
1 files changed, 17 insertions, 17 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 4833f3a19650..c52b60919163 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S | |||
@@ -210,7 +210,7 @@ ENTRY(native_usergs_sysret64) | |||
210 | 210 | ||
211 | /* %rsp:at FRAMEEND */ | 211 | /* %rsp:at FRAMEEND */ |
212 | .macro FIXUP_TOP_OF_STACK tmp offset=0 | 212 | .macro FIXUP_TOP_OF_STACK tmp offset=0 |
213 | movq %gs:pda_oldrsp,\tmp | 213 | movq PER_CPU_VAR(old_rsp),\tmp |
214 | movq \tmp,RSP+\offset(%rsp) | 214 | movq \tmp,RSP+\offset(%rsp) |
215 | movq $__USER_DS,SS+\offset(%rsp) | 215 | movq $__USER_DS,SS+\offset(%rsp) |
216 | movq $__USER_CS,CS+\offset(%rsp) | 216 | movq $__USER_CS,CS+\offset(%rsp) |
@@ -221,7 +221,7 @@ ENTRY(native_usergs_sysret64) | |||
221 | 221 | ||
222 | .macro RESTORE_TOP_OF_STACK tmp offset=0 | 222 | .macro RESTORE_TOP_OF_STACK tmp offset=0 |
223 | movq RSP+\offset(%rsp),\tmp | 223 | movq RSP+\offset(%rsp),\tmp |
224 | movq \tmp,%gs:pda_oldrsp | 224 | movq \tmp,PER_CPU_VAR(old_rsp) |
225 | movq EFLAGS+\offset(%rsp),\tmp | 225 | movq EFLAGS+\offset(%rsp),\tmp |
226 | movq \tmp,R11+\offset(%rsp) | 226 | movq \tmp,R11+\offset(%rsp) |
227 | .endm | 227 | .endm |
@@ -337,15 +337,15 @@ ENTRY(save_args) | |||
337 | je 1f | 337 | je 1f |
338 | SWAPGS | 338 | SWAPGS |
339 | /* | 339 | /* |
340 | * irqcount is used to check if a CPU is already on an interrupt stack | 340 | * irq_count is used to check if a CPU is already on an interrupt stack |
341 | * or not. While this is essentially redundant with preempt_count it is | 341 | * or not. While this is essentially redundant with preempt_count it is |
342 | * a little cheaper to use a separate counter in the PDA (short of | 342 | * a little cheaper to use a separate counter in the PDA (short of |
343 | * moving irq_enter into assembly, which would be too much work) | 343 | * moving irq_enter into assembly, which would be too much work) |
344 | */ | 344 | */ |
345 | 1: incl %gs:pda_irqcount | 345 | 1: incl PER_CPU_VAR(irq_count) |
346 | jne 2f | 346 | jne 2f |
347 | popq_cfi %rax /* move return address... */ | 347 | popq_cfi %rax /* move return address... */ |
348 | mov %gs:pda_irqstackptr,%rsp | 348 | mov PER_CPU_VAR(irq_stack_ptr),%rsp |
349 | EMPTY_FRAME 0 | 349 | EMPTY_FRAME 0 |
350 | pushq_cfi %rax /* ... to the new stack */ | 350 | pushq_cfi %rax /* ... to the new stack */ |
351 | /* | 351 | /* |
@@ -468,7 +468,7 @@ END(ret_from_fork) | |||
468 | ENTRY(system_call) | 468 | ENTRY(system_call) |
469 | CFI_STARTPROC simple | 469 | CFI_STARTPROC simple |
470 | CFI_SIGNAL_FRAME | 470 | CFI_SIGNAL_FRAME |
471 | CFI_DEF_CFA rsp,PDA_STACKOFFSET | 471 | CFI_DEF_CFA rsp,KERNEL_STACK_OFFSET |
472 | CFI_REGISTER rip,rcx | 472 | CFI_REGISTER rip,rcx |
473 | /*CFI_REGISTER rflags,r11*/ | 473 | /*CFI_REGISTER rflags,r11*/ |
474 | SWAPGS_UNSAFE_STACK | 474 | SWAPGS_UNSAFE_STACK |
@@ -479,8 +479,8 @@ ENTRY(system_call) | |||
479 | */ | 479 | */ |
480 | ENTRY(system_call_after_swapgs) | 480 | ENTRY(system_call_after_swapgs) |
481 | 481 | ||
482 | movq %rsp,%gs:pda_oldrsp | 482 | movq %rsp,PER_CPU_VAR(old_rsp) |
483 | movq %gs:pda_kernelstack,%rsp | 483 | movq PER_CPU_VAR(kernel_stack),%rsp |
484 | /* | 484 | /* |
485 | * No need to follow this irqs off/on section - it's straight | 485 | * No need to follow this irqs off/on section - it's straight |
486 | * and short: | 486 | * and short: |
@@ -523,7 +523,7 @@ sysret_check: | |||
523 | CFI_REGISTER rip,rcx | 523 | CFI_REGISTER rip,rcx |
524 | RESTORE_ARGS 0,-ARG_SKIP,1 | 524 | RESTORE_ARGS 0,-ARG_SKIP,1 |
525 | /*CFI_REGISTER rflags,r11*/ | 525 | /*CFI_REGISTER rflags,r11*/ |
526 | movq %gs:pda_oldrsp, %rsp | 526 | movq PER_CPU_VAR(old_rsp), %rsp |
527 | USERGS_SYSRET64 | 527 | USERGS_SYSRET64 |
528 | 528 | ||
529 | CFI_RESTORE_STATE | 529 | CFI_RESTORE_STATE |
@@ -833,11 +833,11 @@ common_interrupt: | |||
833 | XCPT_FRAME | 833 | XCPT_FRAME |
834 | addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ | 834 | addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ |
835 | interrupt do_IRQ | 835 | interrupt do_IRQ |
836 | /* 0(%rsp): oldrsp-ARGOFFSET */ | 836 | /* 0(%rsp): old_rsp-ARGOFFSET */ |
837 | ret_from_intr: | 837 | ret_from_intr: |
838 | DISABLE_INTERRUPTS(CLBR_NONE) | 838 | DISABLE_INTERRUPTS(CLBR_NONE) |
839 | TRACE_IRQS_OFF | 839 | TRACE_IRQS_OFF |
840 | decl %gs:pda_irqcount | 840 | decl PER_CPU_VAR(irq_count) |
841 | leaveq | 841 | leaveq |
842 | CFI_DEF_CFA_REGISTER rsp | 842 | CFI_DEF_CFA_REGISTER rsp |
843 | CFI_ADJUST_CFA_OFFSET -8 | 843 | CFI_ADJUST_CFA_OFFSET -8 |
@@ -1260,14 +1260,14 @@ ENTRY(call_softirq) | |||
1260 | CFI_REL_OFFSET rbp,0 | 1260 | CFI_REL_OFFSET rbp,0 |
1261 | mov %rsp,%rbp | 1261 | mov %rsp,%rbp |
1262 | CFI_DEF_CFA_REGISTER rbp | 1262 | CFI_DEF_CFA_REGISTER rbp |
1263 | incl %gs:pda_irqcount | 1263 | incl PER_CPU_VAR(irq_count) |
1264 | cmove %gs:pda_irqstackptr,%rsp | 1264 | cmove PER_CPU_VAR(irq_stack_ptr),%rsp |
1265 | push %rbp # backlink for old unwinder | 1265 | push %rbp # backlink for old unwinder |
1266 | call __do_softirq | 1266 | call __do_softirq |
1267 | leaveq | 1267 | leaveq |
1268 | CFI_DEF_CFA_REGISTER rsp | 1268 | CFI_DEF_CFA_REGISTER rsp |
1269 | CFI_ADJUST_CFA_OFFSET -8 | 1269 | CFI_ADJUST_CFA_OFFSET -8 |
1270 | decl %gs:pda_irqcount | 1270 | decl PER_CPU_VAR(irq_count) |
1271 | ret | 1271 | ret |
1272 | CFI_ENDPROC | 1272 | CFI_ENDPROC |
1273 | END(call_softirq) | 1273 | END(call_softirq) |
@@ -1297,15 +1297,15 @@ ENTRY(xen_do_hypervisor_callback) # do_hypervisor_callback(struct *pt_regs) | |||
1297 | movq %rdi, %rsp # we don't return, adjust the stack frame | 1297 | movq %rdi, %rsp # we don't return, adjust the stack frame |
1298 | CFI_ENDPROC | 1298 | CFI_ENDPROC |
1299 | DEFAULT_FRAME | 1299 | DEFAULT_FRAME |
1300 | 11: incl %gs:pda_irqcount | 1300 | 11: incl PER_CPU_VAR(irq_count) |
1301 | movq %rsp,%rbp | 1301 | movq %rsp,%rbp |
1302 | CFI_DEF_CFA_REGISTER rbp | 1302 | CFI_DEF_CFA_REGISTER rbp |
1303 | cmovzq %gs:pda_irqstackptr,%rsp | 1303 | cmovzq PER_CPU_VAR(irq_stack_ptr),%rsp |
1304 | pushq %rbp # backlink for old unwinder | 1304 | pushq %rbp # backlink for old unwinder |
1305 | call xen_evtchn_do_upcall | 1305 | call xen_evtchn_do_upcall |
1306 | popq %rsp | 1306 | popq %rsp |
1307 | CFI_DEF_CFA_REGISTER rsp | 1307 | CFI_DEF_CFA_REGISTER rsp |
1308 | decl %gs:pda_irqcount | 1308 | decl PER_CPU_VAR(irq_count) |
1309 | jmp error_exit | 1309 | jmp error_exit |
1310 | CFI_ENDPROC | 1310 | CFI_ENDPROC |
1311 | END(do_hypervisor_callback) | 1311 | END(do_hypervisor_callback) |