diff options
Diffstat (limited to 'arch/x86/kernel/process_64.c')
-rw-r--r-- | arch/x86/kernel/process_64.c | 230 |
1 files changed, 25 insertions, 205 deletions
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c index 85b4cb5c1980..abb7e6a7f0c6 100644 --- a/arch/x86/kernel/process_64.c +++ b/arch/x86/kernel/process_64.c | |||
@@ -16,6 +16,7 @@ | |||
16 | 16 | ||
17 | #include <stdarg.h> | 17 | #include <stdarg.h> |
18 | 18 | ||
19 | #include <linux/stackprotector.h> | ||
19 | #include <linux/cpu.h> | 20 | #include <linux/cpu.h> |
20 | #include <linux/errno.h> | 21 | #include <linux/errno.h> |
21 | #include <linux/sched.h> | 22 | #include <linux/sched.h> |
@@ -47,7 +48,6 @@ | |||
47 | #include <asm/processor.h> | 48 | #include <asm/processor.h> |
48 | #include <asm/i387.h> | 49 | #include <asm/i387.h> |
49 | #include <asm/mmu_context.h> | 50 | #include <asm/mmu_context.h> |
50 | #include <asm/pda.h> | ||
51 | #include <asm/prctl.h> | 51 | #include <asm/prctl.h> |
52 | #include <asm/desc.h> | 52 | #include <asm/desc.h> |
53 | #include <asm/proto.h> | 53 | #include <asm/proto.h> |
@@ -58,6 +58,12 @@ | |||
58 | 58 | ||
59 | asmlinkage extern void ret_from_fork(void); | 59 | asmlinkage extern void ret_from_fork(void); |
60 | 60 | ||
61 | DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; | ||
62 | EXPORT_PER_CPU_SYMBOL(current_task); | ||
63 | |||
64 | DEFINE_PER_CPU(unsigned long, old_rsp); | ||
65 | static DEFINE_PER_CPU(unsigned char, is_idle); | ||
66 | |||
61 | unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED; | 67 | unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED; |
62 | 68 | ||
63 | static ATOMIC_NOTIFIER_HEAD(idle_notifier); | 69 | static ATOMIC_NOTIFIER_HEAD(idle_notifier); |
@@ -76,13 +82,13 @@ EXPORT_SYMBOL_GPL(idle_notifier_unregister); | |||
76 | 82 | ||
77 | void enter_idle(void) | 83 | void enter_idle(void) |
78 | { | 84 | { |
79 | write_pda(isidle, 1); | 85 | percpu_write(is_idle, 1); |
80 | atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); | 86 | atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL); |
81 | } | 87 | } |
82 | 88 | ||
83 | static void __exit_idle(void) | 89 | static void __exit_idle(void) |
84 | { | 90 | { |
85 | if (test_and_clear_bit_pda(0, isidle) == 0) | 91 | if (x86_test_and_clear_bit_percpu(0, is_idle) == 0) |
86 | return; | 92 | return; |
87 | atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); | 93 | atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); |
88 | } | 94 | } |
@@ -112,6 +118,16 @@ static inline void play_dead(void) | |||
112 | void cpu_idle(void) | 118 | void cpu_idle(void) |
113 | { | 119 | { |
114 | current_thread_info()->status |= TS_POLLING; | 120 | current_thread_info()->status |= TS_POLLING; |
121 | |||
122 | /* | ||
123 | * If we're the non-boot CPU, nothing set the stack canary up | ||
124 | * for us. CPU0 already has it initialized but no harm in | ||
125 | * doing it again. This is a good place for updating it, as | ||
126 | * we wont ever return from this function (so the invalid | ||
127 | * canaries already on the stack wont ever trigger). | ||
128 | */ | ||
129 | boot_init_stack_canary(); | ||
130 | |||
115 | /* endless idle loop with no priority at all */ | 131 | /* endless idle loop with no priority at all */ |
116 | while (1) { | 132 | while (1) { |
117 | tick_nohz_stop_sched_tick(1); | 133 | tick_nohz_stop_sched_tick(1); |
@@ -221,61 +237,6 @@ void show_regs(struct pt_regs *regs) | |||
221 | show_trace(NULL, regs, (void *)(regs + 1), regs->bp); | 237 | show_trace(NULL, regs, (void *)(regs + 1), regs->bp); |
222 | } | 238 | } |
223 | 239 | ||
224 | /* | ||
225 | * Free current thread data structures etc.. | ||
226 | */ | ||
227 | void exit_thread(void) | ||
228 | { | ||
229 | struct task_struct *me = current; | ||
230 | struct thread_struct *t = &me->thread; | ||
231 | |||
232 | if (me->thread.io_bitmap_ptr) { | ||
233 | struct tss_struct *tss = &per_cpu(init_tss, get_cpu()); | ||
234 | |||
235 | kfree(t->io_bitmap_ptr); | ||
236 | t->io_bitmap_ptr = NULL; | ||
237 | clear_thread_flag(TIF_IO_BITMAP); | ||
238 | /* | ||
239 | * Careful, clear this in the TSS too: | ||
240 | */ | ||
241 | memset(tss->io_bitmap, 0xff, t->io_bitmap_max); | ||
242 | t->io_bitmap_max = 0; | ||
243 | put_cpu(); | ||
244 | } | ||
245 | |||
246 | ds_exit_thread(current); | ||
247 | } | ||
248 | |||
249 | void flush_thread(void) | ||
250 | { | ||
251 | struct task_struct *tsk = current; | ||
252 | |||
253 | if (test_tsk_thread_flag(tsk, TIF_ABI_PENDING)) { | ||
254 | clear_tsk_thread_flag(tsk, TIF_ABI_PENDING); | ||
255 | if (test_tsk_thread_flag(tsk, TIF_IA32)) { | ||
256 | clear_tsk_thread_flag(tsk, TIF_IA32); | ||
257 | } else { | ||
258 | set_tsk_thread_flag(tsk, TIF_IA32); | ||
259 | current_thread_info()->status |= TS_COMPAT; | ||
260 | } | ||
261 | } | ||
262 | clear_tsk_thread_flag(tsk, TIF_DEBUG); | ||
263 | |||
264 | tsk->thread.debugreg0 = 0; | ||
265 | tsk->thread.debugreg1 = 0; | ||
266 | tsk->thread.debugreg2 = 0; | ||
267 | tsk->thread.debugreg3 = 0; | ||
268 | tsk->thread.debugreg6 = 0; | ||
269 | tsk->thread.debugreg7 = 0; | ||
270 | memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); | ||
271 | /* | ||
272 | * Forget coprocessor state.. | ||
273 | */ | ||
274 | tsk->fpu_counter = 0; | ||
275 | clear_fpu(tsk); | ||
276 | clear_used_math(); | ||
277 | } | ||
278 | |||
279 | void release_thread(struct task_struct *dead_task) | 240 | void release_thread(struct task_struct *dead_task) |
280 | { | 241 | { |
281 | if (dead_task->mm) { | 242 | if (dead_task->mm) { |
@@ -397,7 +358,7 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) | |||
397 | load_gs_index(0); | 358 | load_gs_index(0); |
398 | regs->ip = new_ip; | 359 | regs->ip = new_ip; |
399 | regs->sp = new_sp; | 360 | regs->sp = new_sp; |
400 | write_pda(oldrsp, new_sp); | 361 | percpu_write(old_rsp, new_sp); |
401 | regs->cs = __USER_CS; | 362 | regs->cs = __USER_CS; |
402 | regs->ss = __USER_DS; | 363 | regs->ss = __USER_DS; |
403 | regs->flags = 0x200; | 364 | regs->flags = 0x200; |
@@ -409,118 +370,6 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp) | |||
409 | } | 370 | } |
410 | EXPORT_SYMBOL_GPL(start_thread); | 371 | EXPORT_SYMBOL_GPL(start_thread); |
411 | 372 | ||
412 | static void hard_disable_TSC(void) | ||
413 | { | ||
414 | write_cr4(read_cr4() | X86_CR4_TSD); | ||
415 | } | ||
416 | |||
417 | void disable_TSC(void) | ||
418 | { | ||
419 | preempt_disable(); | ||
420 | if (!test_and_set_thread_flag(TIF_NOTSC)) | ||
421 | /* | ||
422 | * Must flip the CPU state synchronously with | ||
423 | * TIF_NOTSC in the current running context. | ||
424 | */ | ||
425 | hard_disable_TSC(); | ||
426 | preempt_enable(); | ||
427 | } | ||
428 | |||
429 | static void hard_enable_TSC(void) | ||
430 | { | ||
431 | write_cr4(read_cr4() & ~X86_CR4_TSD); | ||
432 | } | ||
433 | |||
434 | static void enable_TSC(void) | ||
435 | { | ||
436 | preempt_disable(); | ||
437 | if (test_and_clear_thread_flag(TIF_NOTSC)) | ||
438 | /* | ||
439 | * Must flip the CPU state synchronously with | ||
440 | * TIF_NOTSC in the current running context. | ||
441 | */ | ||
442 | hard_enable_TSC(); | ||
443 | preempt_enable(); | ||
444 | } | ||
445 | |||
446 | int get_tsc_mode(unsigned long adr) | ||
447 | { | ||
448 | unsigned int val; | ||
449 | |||
450 | if (test_thread_flag(TIF_NOTSC)) | ||
451 | val = PR_TSC_SIGSEGV; | ||
452 | else | ||
453 | val = PR_TSC_ENABLE; | ||
454 | |||
455 | return put_user(val, (unsigned int __user *)adr); | ||
456 | } | ||
457 | |||
458 | int set_tsc_mode(unsigned int val) | ||
459 | { | ||
460 | if (val == PR_TSC_SIGSEGV) | ||
461 | disable_TSC(); | ||
462 | else if (val == PR_TSC_ENABLE) | ||
463 | enable_TSC(); | ||
464 | else | ||
465 | return -EINVAL; | ||
466 | |||
467 | return 0; | ||
468 | } | ||
469 | |||
470 | /* | ||
471 | * This special macro can be used to load a debugging register | ||
472 | */ | ||
473 | #define loaddebug(thread, r) set_debugreg(thread->debugreg ## r, r) | ||
474 | |||
475 | static inline void __switch_to_xtra(struct task_struct *prev_p, | ||
476 | struct task_struct *next_p, | ||
477 | struct tss_struct *tss) | ||
478 | { | ||
479 | struct thread_struct *prev, *next; | ||
480 | |||
481 | prev = &prev_p->thread, | ||
482 | next = &next_p->thread; | ||
483 | |||
484 | if (test_tsk_thread_flag(next_p, TIF_DS_AREA_MSR) || | ||
485 | test_tsk_thread_flag(prev_p, TIF_DS_AREA_MSR)) | ||
486 | ds_switch_to(prev_p, next_p); | ||
487 | else if (next->debugctlmsr != prev->debugctlmsr) | ||
488 | update_debugctlmsr(next->debugctlmsr); | ||
489 | |||
490 | if (test_tsk_thread_flag(next_p, TIF_DEBUG)) { | ||
491 | loaddebug(next, 0); | ||
492 | loaddebug(next, 1); | ||
493 | loaddebug(next, 2); | ||
494 | loaddebug(next, 3); | ||
495 | /* no 4 and 5 */ | ||
496 | loaddebug(next, 6); | ||
497 | loaddebug(next, 7); | ||
498 | } | ||
499 | |||
500 | if (test_tsk_thread_flag(prev_p, TIF_NOTSC) ^ | ||
501 | test_tsk_thread_flag(next_p, TIF_NOTSC)) { | ||
502 | /* prev and next are different */ | ||
503 | if (test_tsk_thread_flag(next_p, TIF_NOTSC)) | ||
504 | hard_disable_TSC(); | ||
505 | else | ||
506 | hard_enable_TSC(); | ||
507 | } | ||
508 | |||
509 | if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) { | ||
510 | /* | ||
511 | * Copy the relevant range of the IO bitmap. | ||
512 | * Normally this is 128 bytes or less: | ||
513 | */ | ||
514 | memcpy(tss->io_bitmap, next->io_bitmap_ptr, | ||
515 | max(prev->io_bitmap_max, next->io_bitmap_max)); | ||
516 | } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) { | ||
517 | /* | ||
518 | * Clear any possible leftover bits: | ||
519 | */ | ||
520 | memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); | ||
521 | } | ||
522 | } | ||
523 | |||
524 | /* | 373 | /* |
525 | * switch_to(x,y) should switch tasks from x to y. | 374 | * switch_to(x,y) should switch tasks from x to y. |
526 | * | 375 | * |
@@ -618,21 +467,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) | |||
618 | /* | 467 | /* |
619 | * Switch the PDA and FPU contexts. | 468 | * Switch the PDA and FPU contexts. |
620 | */ | 469 | */ |
621 | prev->usersp = read_pda(oldrsp); | 470 | prev->usersp = percpu_read(old_rsp); |
622 | write_pda(oldrsp, next->usersp); | 471 | percpu_write(old_rsp, next->usersp); |
623 | write_pda(pcurrent, next_p); | 472 | percpu_write(current_task, next_p); |
624 | 473 | ||
625 | write_pda(kernelstack, | 474 | percpu_write(kernel_stack, |
626 | (unsigned long)task_stack_page(next_p) + | 475 | (unsigned long)task_stack_page(next_p) + |
627 | THREAD_SIZE - PDA_STACKOFFSET); | 476 | THREAD_SIZE - KERNEL_STACK_OFFSET); |
628 | #ifdef CONFIG_CC_STACKPROTECTOR | ||
629 | write_pda(stack_canary, next_p->stack_canary); | ||
630 | /* | ||
631 | * Build time only check to make sure the stack_canary is at | ||
632 | * offset 40 in the pda; this is a gcc ABI requirement | ||
633 | */ | ||
634 | BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40); | ||
635 | #endif | ||
636 | 477 | ||
637 | /* | 478 | /* |
638 | * Now maybe reload the debug registers and handle I/O bitmaps | 479 | * Now maybe reload the debug registers and handle I/O bitmaps |
@@ -686,11 +527,6 @@ void set_personality_64bit(void) | |||
686 | current->personality &= ~READ_IMPLIES_EXEC; | 527 | current->personality &= ~READ_IMPLIES_EXEC; |
687 | } | 528 | } |
688 | 529 | ||
689 | asmlinkage long sys_fork(struct pt_regs *regs) | ||
690 | { | ||
691 | return do_fork(SIGCHLD, regs->sp, regs, 0, NULL, NULL); | ||
692 | } | ||
693 | |||
694 | asmlinkage long | 530 | asmlinkage long |
695 | sys_clone(unsigned long clone_flags, unsigned long newsp, | 531 | sys_clone(unsigned long clone_flags, unsigned long newsp, |
696 | void __user *parent_tid, void __user *child_tid, struct pt_regs *regs) | 532 | void __user *parent_tid, void __user *child_tid, struct pt_regs *regs) |
@@ -700,22 +536,6 @@ sys_clone(unsigned long clone_flags, unsigned long newsp, | |||
700 | return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid); | 536 | return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid); |
701 | } | 537 | } |
702 | 538 | ||
703 | /* | ||
704 | * This is trivial, and on the face of it looks like it | ||
705 | * could equally well be done in user mode. | ||
706 | * | ||
707 | * Not so, for quite unobvious reasons - register pressure. | ||
708 | * In user mode vfork() cannot have a stack frame, and if | ||
709 | * done by calling the "clone()" system call directly, you | ||
710 | * do not have enough call-clobbered registers to hold all | ||
711 | * the information you need. | ||
712 | */ | ||
713 | asmlinkage long sys_vfork(struct pt_regs *regs) | ||
714 | { | ||
715 | return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->sp, regs, 0, | ||
716 | NULL, NULL); | ||
717 | } | ||
718 | |||
719 | unsigned long get_wchan(struct task_struct *p) | 539 | unsigned long get_wchan(struct task_struct *p) |
720 | { | 540 | { |
721 | unsigned long stack; | 541 | unsigned long stack; |