aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/process_64.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel/process_64.c')
-rw-r--r--arch/x86/kernel/process_64.c199
1 files changed, 104 insertions, 95 deletions
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index e12e0e4dd256..cd8c0ed02b7e 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -37,11 +37,11 @@
37#include <linux/kdebug.h> 37#include <linux/kdebug.h>
38#include <linux/tick.h> 38#include <linux/tick.h>
39#include <linux/prctl.h> 39#include <linux/prctl.h>
40#include <linux/uaccess.h>
41#include <linux/io.h>
40 42
41#include <asm/uaccess.h>
42#include <asm/pgtable.h> 43#include <asm/pgtable.h>
43#include <asm/system.h> 44#include <asm/system.h>
44#include <asm/io.h>
45#include <asm/processor.h> 45#include <asm/processor.h>
46#include <asm/i387.h> 46#include <asm/i387.h>
47#include <asm/mmu_context.h> 47#include <asm/mmu_context.h>
@@ -51,6 +51,7 @@
51#include <asm/proto.h> 51#include <asm/proto.h>
52#include <asm/ia32.h> 52#include <asm/ia32.h>
53#include <asm/idle.h> 53#include <asm/idle.h>
54#include <asm/syscalls.h>
54 55
55asmlinkage extern void ret_from_fork(void); 56asmlinkage extern void ret_from_fork(void);
56 57
@@ -85,30 +86,12 @@ void exit_idle(void)
85 __exit_idle(); 86 __exit_idle();
86} 87}
87 88
88#ifdef CONFIG_HOTPLUG_CPU 89#ifndef CONFIG_SMP
89DECLARE_PER_CPU(int, cpu_state);
90
91#include <asm/nmi.h>
92/* We halt the CPU with physical CPU hotplug */
93static inline void play_dead(void)
94{
95 idle_task_exit();
96 c1e_remove_cpu(raw_smp_processor_id());
97
98 mb();
99 /* Ack it */
100 __get_cpu_var(cpu_state) = CPU_DEAD;
101
102 local_irq_disable();
103 /* mask all interrupts, flush any and all caches, and halt */
104 wbinvd_halt();
105}
106#else
107static inline void play_dead(void) 90static inline void play_dead(void)
108{ 91{
109 BUG(); 92 BUG();
110} 93}
111#endif /* CONFIG_HOTPLUG_CPU */ 94#endif
112 95
113/* 96/*
114 * The idle thread. There's no useful work to be 97 * The idle thread. There's no useful work to be
@@ -153,7 +136,7 @@ void cpu_idle(void)
153} 136}
154 137
155/* Prints also some state that isn't saved in the pt_regs */ 138/* Prints also some state that isn't saved in the pt_regs */
156void __show_regs(struct pt_regs * regs) 139void __show_regs(struct pt_regs *regs, int all)
157{ 140{
158 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs; 141 unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
159 unsigned long d0, d1, d2, d3, d6, d7; 142 unsigned long d0, d1, d2, d3, d6, d7;
@@ -162,60 +145,65 @@ void __show_regs(struct pt_regs * regs)
162 145
163 printk("\n"); 146 printk("\n");
164 print_modules(); 147 print_modules();
165 printk("Pid: %d, comm: %.20s %s %s %.*s\n", 148 printk(KERN_INFO "Pid: %d, comm: %.20s %s %s %.*s\n",
166 current->pid, current->comm, print_tainted(), 149 current->pid, current->comm, print_tainted(),
167 init_utsname()->release, 150 init_utsname()->release,
168 (int)strcspn(init_utsname()->version, " "), 151 (int)strcspn(init_utsname()->version, " "),
169 init_utsname()->version); 152 init_utsname()->version);
170 printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip); 153 printk(KERN_INFO "RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->ip);
171 printk_address(regs->ip, 1); 154 printk_address(regs->ip, 1);
172 printk("RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss, regs->sp, 155 printk(KERN_INFO "RSP: %04lx:%016lx EFLAGS: %08lx\n", regs->ss,
173 regs->flags); 156 regs->sp, regs->flags);
174 printk("RAX: %016lx RBX: %016lx RCX: %016lx\n", 157 printk(KERN_INFO "RAX: %016lx RBX: %016lx RCX: %016lx\n",
175 regs->ax, regs->bx, regs->cx); 158 regs->ax, regs->bx, regs->cx);
176 printk("RDX: %016lx RSI: %016lx RDI: %016lx\n", 159 printk(KERN_INFO "RDX: %016lx RSI: %016lx RDI: %016lx\n",
177 regs->dx, regs->si, regs->di); 160 regs->dx, regs->si, regs->di);
178 printk("RBP: %016lx R08: %016lx R09: %016lx\n", 161 printk(KERN_INFO "RBP: %016lx R08: %016lx R09: %016lx\n",
179 regs->bp, regs->r8, regs->r9); 162 regs->bp, regs->r8, regs->r9);
180 printk("R10: %016lx R11: %016lx R12: %016lx\n", 163 printk(KERN_INFO "R10: %016lx R11: %016lx R12: %016lx\n",
181 regs->r10, regs->r11, regs->r12); 164 regs->r10, regs->r11, regs->r12);
182 printk("R13: %016lx R14: %016lx R15: %016lx\n", 165 printk(KERN_INFO "R13: %016lx R14: %016lx R15: %016lx\n",
183 regs->r13, regs->r14, regs->r15); 166 regs->r13, regs->r14, regs->r15);
184 167
185 asm("movl %%ds,%0" : "=r" (ds)); 168 asm("movl %%ds,%0" : "=r" (ds));
186 asm("movl %%cs,%0" : "=r" (cs)); 169 asm("movl %%cs,%0" : "=r" (cs));
187 asm("movl %%es,%0" : "=r" (es)); 170 asm("movl %%es,%0" : "=r" (es));
188 asm("movl %%fs,%0" : "=r" (fsindex)); 171 asm("movl %%fs,%0" : "=r" (fsindex));
189 asm("movl %%gs,%0" : "=r" (gsindex)); 172 asm("movl %%gs,%0" : "=r" (gsindex));
190 173
191 rdmsrl(MSR_FS_BASE, fs); 174 rdmsrl(MSR_FS_BASE, fs);
192 rdmsrl(MSR_GS_BASE, gs); 175 rdmsrl(MSR_GS_BASE, gs);
193 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs); 176 rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
177
178 if (!all)
179 return;
194 180
195 cr0 = read_cr0(); 181 cr0 = read_cr0();
196 cr2 = read_cr2(); 182 cr2 = read_cr2();
197 cr3 = read_cr3(); 183 cr3 = read_cr3();
198 cr4 = read_cr4(); 184 cr4 = read_cr4();
199 185
200 printk("FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n", 186 printk(KERN_INFO "FS: %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
201 fs,fsindex,gs,gsindex,shadowgs); 187 fs, fsindex, gs, gsindex, shadowgs);
202 printk("CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0); 188 printk(KERN_INFO "CS: %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds,
203 printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4); 189 es, cr0);
190 printk(KERN_INFO "CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3,
191 cr4);
204 192
205 get_debugreg(d0, 0); 193 get_debugreg(d0, 0);
206 get_debugreg(d1, 1); 194 get_debugreg(d1, 1);
207 get_debugreg(d2, 2); 195 get_debugreg(d2, 2);
208 printk("DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2); 196 printk(KERN_INFO "DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
209 get_debugreg(d3, 3); 197 get_debugreg(d3, 3);
210 get_debugreg(d6, 6); 198 get_debugreg(d6, 6);
211 get_debugreg(d7, 7); 199 get_debugreg(d7, 7);
212 printk("DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7); 200 printk(KERN_INFO "DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
213} 201}
214 202
215void show_regs(struct pt_regs *regs) 203void show_regs(struct pt_regs *regs)
216{ 204{
217 printk("CPU %d:", smp_processor_id()); 205 printk(KERN_INFO "CPU %d:", smp_processor_id());
218 __show_regs(regs); 206 __show_regs(regs, 1);
219 show_trace(NULL, regs, (void *)(regs + 1), regs->bp); 207 show_trace(NULL, regs, (void *)(regs + 1), regs->bp);
220} 208}
221 209
@@ -240,6 +228,14 @@ void exit_thread(void)
240 t->io_bitmap_max = 0; 228 t->io_bitmap_max = 0;
241 put_cpu(); 229 put_cpu();
242 } 230 }
231#ifdef CONFIG_X86_DS
232 /* Free any DS contexts that have not been properly released. */
233 if (unlikely(t->ds_ctx)) {
234 /* we clear debugctl to make sure DS is not used. */
235 update_debugctlmsr(0);
236 ds_free(t->ds_ctx);
237 }
238#endif /* CONFIG_X86_DS */
243} 239}
244 240
245void flush_thread(void) 241void flush_thread(void)
@@ -315,10 +311,10 @@ void prepare_to_copy(struct task_struct *tsk)
315 311
316int copy_thread(int nr, unsigned long clone_flags, unsigned long sp, 312int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
317 unsigned long unused, 313 unsigned long unused,
318 struct task_struct * p, struct pt_regs * regs) 314 struct task_struct *p, struct pt_regs *regs)
319{ 315{
320 int err; 316 int err;
321 struct pt_regs * childregs; 317 struct pt_regs *childregs;
322 struct task_struct *me = current; 318 struct task_struct *me = current;
323 319
324 childregs = ((struct pt_regs *) 320 childregs = ((struct pt_regs *)
@@ -363,10 +359,10 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
363 if (test_thread_flag(TIF_IA32)) 359 if (test_thread_flag(TIF_IA32))
364 err = do_set_thread_area(p, -1, 360 err = do_set_thread_area(p, -1,
365 (struct user_desc __user *)childregs->si, 0); 361 (struct user_desc __user *)childregs->si, 0);
366 else 362 else
367#endif 363#endif
368 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8); 364 err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
369 if (err) 365 if (err)
370 goto out; 366 goto out;
371 } 367 }
372 err = 0; 368 err = 0;
@@ -473,13 +469,27 @@ static inline void __switch_to_xtra(struct task_struct *prev_p,
473 next = &next_p->thread; 469 next = &next_p->thread;
474 470
475 debugctl = prev->debugctlmsr; 471 debugctl = prev->debugctlmsr;
476 if (next->ds_area_msr != prev->ds_area_msr) { 472
477 /* we clear debugctl to make sure DS 473#ifdef CONFIG_X86_DS
478 * is not in use when we change it */ 474 {
479 debugctl = 0; 475 unsigned long ds_prev = 0, ds_next = 0;
480 update_debugctlmsr(0); 476
481 wrmsrl(MSR_IA32_DS_AREA, next->ds_area_msr); 477 if (prev->ds_ctx)
478 ds_prev = (unsigned long)prev->ds_ctx->ds;
479 if (next->ds_ctx)
480 ds_next = (unsigned long)next->ds_ctx->ds;
481
482 if (ds_next != ds_prev) {
483 /*
484 * We clear debugctl to make sure DS
485 * is not in use when we change it:
486 */
487 debugctl = 0;
488 update_debugctlmsr(0);
489 wrmsrl(MSR_IA32_DS_AREA, ds_next);
490 }
482 } 491 }
492#endif /* CONFIG_X86_DS */
483 493
484 if (next->debugctlmsr != debugctl) 494 if (next->debugctlmsr != debugctl)
485 update_debugctlmsr(next->debugctlmsr); 495 update_debugctlmsr(next->debugctlmsr);
@@ -517,13 +527,13 @@ static inline void __switch_to_xtra(struct task_struct *prev_p,
517 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max); 527 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
518 } 528 }
519 529
520#ifdef X86_BTS 530#ifdef CONFIG_X86_PTRACE_BTS
521 if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS)) 531 if (test_tsk_thread_flag(prev_p, TIF_BTS_TRACE_TS))
522 ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS); 532 ptrace_bts_take_timestamp(prev_p, BTS_TASK_DEPARTS);
523 533
524 if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS)) 534 if (test_tsk_thread_flag(next_p, TIF_BTS_TRACE_TS))
525 ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES); 535 ptrace_bts_take_timestamp(next_p, BTS_TASK_ARRIVES);
526#endif 536#endif /* CONFIG_X86_PTRACE_BTS */
527} 537}
528 538
529/* 539/*
@@ -545,7 +555,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
545 unsigned fsindex, gsindex; 555 unsigned fsindex, gsindex;
546 556
547 /* we're going to use this soon, after a few expensive things */ 557 /* we're going to use this soon, after a few expensive things */
548 if (next_p->fpu_counter>5) 558 if (next_p->fpu_counter > 5)
549 prefetch(next->xstate); 559 prefetch(next->xstate);
550 560
551 /* 561 /*
@@ -553,13 +563,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
553 */ 563 */
554 load_sp0(tss, next); 564 load_sp0(tss, next);
555 565
556 /* 566 /*
557 * Switch DS and ES. 567 * Switch DS and ES.
558 * This won't pick up thread selector changes, but I guess that is ok. 568 * This won't pick up thread selector changes, but I guess that is ok.
559 */ 569 */
560 savesegment(es, prev->es); 570 savesegment(es, prev->es);
561 if (unlikely(next->es | prev->es)) 571 if (unlikely(next->es | prev->es))
562 loadsegment(es, next->es); 572 loadsegment(es, next->es);
563 573
564 savesegment(ds, prev->ds); 574 savesegment(ds, prev->ds);
565 if (unlikely(next->ds | prev->ds)) 575 if (unlikely(next->ds | prev->ds))
@@ -585,7 +595,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
585 */ 595 */
586 arch_leave_lazy_cpu_mode(); 596 arch_leave_lazy_cpu_mode();
587 597
588 /* 598 /*
589 * Switch FS and GS. 599 * Switch FS and GS.
590 * 600 *
591 * Segment register != 0 always requires a reload. Also 601 * Segment register != 0 always requires a reload. Also
@@ -594,13 +604,13 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
594 */ 604 */
595 if (unlikely(fsindex | next->fsindex | prev->fs)) { 605 if (unlikely(fsindex | next->fsindex | prev->fs)) {
596 loadsegment(fs, next->fsindex); 606 loadsegment(fs, next->fsindex);
597 /* 607 /*
598 * Check if the user used a selector != 0; if yes 608 * Check if the user used a selector != 0; if yes
599 * clear 64bit base, since overloaded base is always 609 * clear 64bit base, since overloaded base is always
600 * mapped to the Null selector 610 * mapped to the Null selector
601 */ 611 */
602 if (fsindex) 612 if (fsindex)
603 prev->fs = 0; 613 prev->fs = 0;
604 } 614 }
605 /* when next process has a 64bit base use it */ 615 /* when next process has a 64bit base use it */
606 if (next->fs) 616 if (next->fs)
@@ -610,7 +620,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
610 if (unlikely(gsindex | next->gsindex | prev->gs)) { 620 if (unlikely(gsindex | next->gsindex | prev->gs)) {
611 load_gs_index(next->gsindex); 621 load_gs_index(next->gsindex);
612 if (gsindex) 622 if (gsindex)
613 prev->gs = 0; 623 prev->gs = 0;
614 } 624 }
615 if (next->gs) 625 if (next->gs)
616 wrmsrl(MSR_KERNEL_GS_BASE, next->gs); 626 wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
@@ -619,12 +629,12 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
619 /* Must be after DS reload */ 629 /* Must be after DS reload */
620 unlazy_fpu(prev_p); 630 unlazy_fpu(prev_p);
621 631
622 /* 632 /*
623 * Switch the PDA and FPU contexts. 633 * Switch the PDA and FPU contexts.
624 */ 634 */
625 prev->usersp = read_pda(oldrsp); 635 prev->usersp = read_pda(oldrsp);
626 write_pda(oldrsp, next->usersp); 636 write_pda(oldrsp, next->usersp);
627 write_pda(pcurrent, next_p); 637 write_pda(pcurrent, next_p);
628 638
629 write_pda(kernelstack, 639 write_pda(kernelstack,
630 (unsigned long)task_stack_page(next_p) + 640 (unsigned long)task_stack_page(next_p) +
@@ -665,7 +675,7 @@ long sys_execve(char __user *name, char __user * __user *argv,
665 char __user * __user *envp, struct pt_regs *regs) 675 char __user * __user *envp, struct pt_regs *regs)
666{ 676{
667 long error; 677 long error;
668 char * filename; 678 char *filename;
669 679
670 filename = getname(name); 680 filename = getname(name);
671 error = PTR_ERR(filename); 681 error = PTR_ERR(filename);
@@ -723,55 +733,55 @@ asmlinkage long sys_vfork(struct pt_regs *regs)
723unsigned long get_wchan(struct task_struct *p) 733unsigned long get_wchan(struct task_struct *p)
724{ 734{
725 unsigned long stack; 735 unsigned long stack;
726 u64 fp,ip; 736 u64 fp, ip;
727 int count = 0; 737 int count = 0;
728 738
729 if (!p || p == current || p->state==TASK_RUNNING) 739 if (!p || p == current || p->state == TASK_RUNNING)
730 return 0; 740 return 0;
731 stack = (unsigned long)task_stack_page(p); 741 stack = (unsigned long)task_stack_page(p);
732 if (p->thread.sp < stack || p->thread.sp > stack+THREAD_SIZE) 742 if (p->thread.sp < stack || p->thread.sp >= stack+THREAD_SIZE)
733 return 0; 743 return 0;
734 fp = *(u64 *)(p->thread.sp); 744 fp = *(u64 *)(p->thread.sp);
735 do { 745 do {
736 if (fp < (unsigned long)stack || 746 if (fp < (unsigned long)stack ||
737 fp > (unsigned long)stack+THREAD_SIZE) 747 fp >= (unsigned long)stack+THREAD_SIZE)
738 return 0; 748 return 0;
739 ip = *(u64 *)(fp+8); 749 ip = *(u64 *)(fp+8);
740 if (!in_sched_functions(ip)) 750 if (!in_sched_functions(ip))
741 return ip; 751 return ip;
742 fp = *(u64 *)fp; 752 fp = *(u64 *)fp;
743 } while (count++ < 16); 753 } while (count++ < 16);
744 return 0; 754 return 0;
745} 755}
746 756
747long do_arch_prctl(struct task_struct *task, int code, unsigned long addr) 757long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
748{ 758{
749 int ret = 0; 759 int ret = 0;
750 int doit = task == current; 760 int doit = task == current;
751 int cpu; 761 int cpu;
752 762
753 switch (code) { 763 switch (code) {
754 case ARCH_SET_GS: 764 case ARCH_SET_GS:
755 if (addr >= TASK_SIZE_OF(task)) 765 if (addr >= TASK_SIZE_OF(task))
756 return -EPERM; 766 return -EPERM;
757 cpu = get_cpu(); 767 cpu = get_cpu();
758 /* handle small bases via the GDT because that's faster to 768 /* handle small bases via the GDT because that's faster to
759 switch. */ 769 switch. */
760 if (addr <= 0xffffffff) { 770 if (addr <= 0xffffffff) {
761 set_32bit_tls(task, GS_TLS, addr); 771 set_32bit_tls(task, GS_TLS, addr);
762 if (doit) { 772 if (doit) {
763 load_TLS(&task->thread, cpu); 773 load_TLS(&task->thread, cpu);
764 load_gs_index(GS_TLS_SEL); 774 load_gs_index(GS_TLS_SEL);
765 } 775 }
766 task->thread.gsindex = GS_TLS_SEL; 776 task->thread.gsindex = GS_TLS_SEL;
767 task->thread.gs = 0; 777 task->thread.gs = 0;
768 } else { 778 } else {
769 task->thread.gsindex = 0; 779 task->thread.gsindex = 0;
770 task->thread.gs = addr; 780 task->thread.gs = addr;
771 if (doit) { 781 if (doit) {
772 load_gs_index(0); 782 load_gs_index(0);
773 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr); 783 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
774 } 784 }
775 } 785 }
776 put_cpu(); 786 put_cpu();
777 break; 787 break;
@@ -825,8 +835,7 @@ long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
825 rdmsrl(MSR_KERNEL_GS_BASE, base); 835 rdmsrl(MSR_KERNEL_GS_BASE, base);
826 else 836 else
827 base = task->thread.gs; 837 base = task->thread.gs;
828 } 838 } else
829 else
830 base = task->thread.gs; 839 base = task->thread.gs;
831 ret = put_user(base, (unsigned long __user *)addr); 840 ret = put_user(base, (unsigned long __user *)addr);
832 break; 841 break;