diff options
Diffstat (limited to 'arch/x86/kernel/i387.c')
-rw-r--r-- | arch/x86/kernel/i387.c | 168 |
1 files changed, 149 insertions, 19 deletions
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c index eb9ddd8efb8..1f20608d4ca 100644 --- a/arch/x86/kernel/i387.c +++ b/arch/x86/kernel/i387.c | |||
@@ -21,9 +21,12 @@ | |||
21 | # include <asm/sigcontext32.h> | 21 | # include <asm/sigcontext32.h> |
22 | # include <asm/user32.h> | 22 | # include <asm/user32.h> |
23 | #else | 23 | #else |
24 | # define save_i387_ia32 save_i387 | 24 | # define save_i387_xstate_ia32 save_i387_xstate |
25 | # define restore_i387_ia32 restore_i387 | 25 | # define restore_i387_xstate_ia32 restore_i387_xstate |
26 | # define _fpstate_ia32 _fpstate | 26 | # define _fpstate_ia32 _fpstate |
27 | # define _xstate_ia32 _xstate | ||
28 | # define sig_xstate_ia32_size sig_xstate_size | ||
29 | # define fx_sw_reserved_ia32 fx_sw_reserved | ||
27 | # define user_i387_ia32_struct user_i387_struct | 30 | # define user_i387_ia32_struct user_i387_struct |
28 | # define user32_fxsr_struct user_fxsr_struct | 31 | # define user32_fxsr_struct user_fxsr_struct |
29 | #endif | 32 | #endif |
@@ -36,6 +39,7 @@ | |||
36 | 39 | ||
37 | static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu; | 40 | static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu; |
38 | unsigned int xstate_size; | 41 | unsigned int xstate_size; |
42 | unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32); | ||
39 | static struct i387_fxsave_struct fx_scratch __cpuinitdata; | 43 | static struct i387_fxsave_struct fx_scratch __cpuinitdata; |
40 | 44 | ||
41 | void __cpuinit mxcsr_feature_mask_init(void) | 45 | void __cpuinit mxcsr_feature_mask_init(void) |
@@ -61,6 +65,11 @@ void __init init_thread_xstate(void) | |||
61 | return; | 65 | return; |
62 | } | 66 | } |
63 | 67 | ||
68 | if (cpu_has_xsave) { | ||
69 | xsave_cntxt_init(); | ||
70 | return; | ||
71 | } | ||
72 | |||
64 | if (cpu_has_fxsr) | 73 | if (cpu_has_fxsr) |
65 | xstate_size = sizeof(struct i387_fxsave_struct); | 74 | xstate_size = sizeof(struct i387_fxsave_struct); |
66 | #ifdef CONFIG_X86_32 | 75 | #ifdef CONFIG_X86_32 |
@@ -83,9 +92,19 @@ void __cpuinit fpu_init(void) | |||
83 | 92 | ||
84 | write_cr0(oldcr0 & ~(X86_CR0_TS|X86_CR0_EM)); /* clear TS and EM */ | 93 | write_cr0(oldcr0 & ~(X86_CR0_TS|X86_CR0_EM)); /* clear TS and EM */ |
85 | 94 | ||
95 | /* | ||
96 | * Boot processor to setup the FP and extended state context info. | ||
97 | */ | ||
98 | if (!smp_processor_id()) | ||
99 | init_thread_xstate(); | ||
100 | xsave_init(); | ||
101 | |||
86 | mxcsr_feature_mask_init(); | 102 | mxcsr_feature_mask_init(); |
87 | /* clean state in init */ | 103 | /* clean state in init */ |
88 | current_thread_info()->status = 0; | 104 | if (cpu_has_xsave) |
105 | current_thread_info()->status = TS_XSAVE; | ||
106 | else | ||
107 | current_thread_info()->status = 0; | ||
89 | clear_used_math(); | 108 | clear_used_math(); |
90 | } | 109 | } |
91 | #endif /* CONFIG_X86_64 */ | 110 | #endif /* CONFIG_X86_64 */ |
@@ -195,6 +214,13 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset, | |||
195 | */ | 214 | */ |
196 | target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask; | 215 | target->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask; |
197 | 216 | ||
217 | /* | ||
218 | * update the header bits in the xsave header, indicating the | ||
219 | * presence of FP and SSE state. | ||
220 | */ | ||
221 | if (cpu_has_xsave) | ||
222 | target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE; | ||
223 | |||
198 | return ret; | 224 | return ret; |
199 | } | 225 | } |
200 | 226 | ||
@@ -395,6 +421,12 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset, | |||
395 | if (!ret) | 421 | if (!ret) |
396 | convert_to_fxsr(target, &env); | 422 | convert_to_fxsr(target, &env); |
397 | 423 | ||
424 | /* | ||
425 | * update the header bit in the xsave header, indicating the | ||
426 | * presence of FP. | ||
427 | */ | ||
428 | if (cpu_has_xsave) | ||
429 | target->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FP; | ||
398 | return ret; | 430 | return ret; |
399 | } | 431 | } |
400 | 432 | ||
@@ -407,7 +439,6 @@ static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf) | |||
407 | struct task_struct *tsk = current; | 439 | struct task_struct *tsk = current; |
408 | struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave; | 440 | struct i387_fsave_struct *fp = &tsk->thread.xstate->fsave; |
409 | 441 | ||
410 | unlazy_fpu(tsk); | ||
411 | fp->status = fp->swd; | 442 | fp->status = fp->swd; |
412 | if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct))) | 443 | if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct))) |
413 | return -1; | 444 | return -1; |
@@ -421,8 +452,6 @@ static int save_i387_fxsave(struct _fpstate_ia32 __user *buf) | |||
421 | struct user_i387_ia32_struct env; | 452 | struct user_i387_ia32_struct env; |
422 | int err = 0; | 453 | int err = 0; |
423 | 454 | ||
424 | unlazy_fpu(tsk); | ||
425 | |||
426 | convert_from_fxsr(&env, tsk); | 455 | convert_from_fxsr(&env, tsk); |
427 | if (__copy_to_user(buf, &env, sizeof(env))) | 456 | if (__copy_to_user(buf, &env, sizeof(env))) |
428 | return -1; | 457 | return -1; |
@@ -432,16 +461,54 @@ static int save_i387_fxsave(struct _fpstate_ia32 __user *buf) | |||
432 | if (err) | 461 | if (err) |
433 | return -1; | 462 | return -1; |
434 | 463 | ||
435 | if (__copy_to_user(&buf->_fxsr_env[0], fx, | 464 | if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size)) |
436 | sizeof(struct i387_fxsave_struct))) | 465 | return -1; |
466 | return 1; | ||
467 | } | ||
468 | |||
469 | static int save_i387_xsave(void __user *buf) | ||
470 | { | ||
471 | struct task_struct *tsk = current; | ||
472 | struct _fpstate_ia32 __user *fx = buf; | ||
473 | int err = 0; | ||
474 | |||
475 | /* | ||
476 | * For legacy compatible, we always set FP/SSE bits in the bit | ||
477 | * vector while saving the state to the user context. | ||
478 | * This will enable us capturing any changes(during sigreturn) to | ||
479 | * the FP/SSE bits by the legacy applications which don't touch | ||
480 | * xstate_bv in the xsave header. | ||
481 | * | ||
482 | * xsave aware applications can change the xstate_bv in the xsave | ||
483 | * header as well as change any contents in the memory layout. | ||
484 | * xrestore as part of sigreturn will capture all the changes. | ||
485 | */ | ||
486 | tsk->thread.xstate->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE; | ||
487 | |||
488 | if (save_i387_fxsave(fx) < 0) | ||
489 | return -1; | ||
490 | |||
491 | err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32, | ||
492 | sizeof(struct _fpx_sw_bytes)); | ||
493 | err |= __put_user(FP_XSTATE_MAGIC2, | ||
494 | (__u32 __user *) (buf + sig_xstate_ia32_size | ||
495 | - FP_XSTATE_MAGIC2_SIZE)); | ||
496 | if (err) | ||
437 | return -1; | 497 | return -1; |
498 | |||
438 | return 1; | 499 | return 1; |
439 | } | 500 | } |
440 | 501 | ||
441 | int save_i387_ia32(struct _fpstate_ia32 __user *buf) | 502 | int save_i387_xstate_ia32(void __user *buf) |
442 | { | 503 | { |
504 | struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf; | ||
505 | struct task_struct *tsk = current; | ||
506 | |||
443 | if (!used_math()) | 507 | if (!used_math()) |
444 | return 0; | 508 | return 0; |
509 | |||
510 | if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size)) | ||
511 | return -EACCES; | ||
445 | /* | 512 | /* |
446 | * This will cause a "finit" to be triggered by the next | 513 | * This will cause a "finit" to be triggered by the next |
447 | * attempted FPU operation by the 'current' process. | 514 | * attempted FPU operation by the 'current' process. |
@@ -451,13 +518,17 @@ int save_i387_ia32(struct _fpstate_ia32 __user *buf) | |||
451 | if (!HAVE_HWFP) { | 518 | if (!HAVE_HWFP) { |
452 | return fpregs_soft_get(current, NULL, | 519 | return fpregs_soft_get(current, NULL, |
453 | 0, sizeof(struct user_i387_ia32_struct), | 520 | 0, sizeof(struct user_i387_ia32_struct), |
454 | NULL, buf) ? -1 : 1; | 521 | NULL, fp) ? -1 : 1; |
455 | } | 522 | } |
456 | 523 | ||
524 | unlazy_fpu(tsk); | ||
525 | |||
526 | if (cpu_has_xsave) | ||
527 | return save_i387_xsave(fp); | ||
457 | if (cpu_has_fxsr) | 528 | if (cpu_has_fxsr) |
458 | return save_i387_fxsave(buf); | 529 | return save_i387_fxsave(fp); |
459 | else | 530 | else |
460 | return save_i387_fsave(buf); | 531 | return save_i387_fsave(fp); |
461 | } | 532 | } |
462 | 533 | ||
463 | static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf) | 534 | static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf) |
@@ -468,14 +539,15 @@ static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf) | |||
468 | sizeof(struct i387_fsave_struct)); | 539 | sizeof(struct i387_fsave_struct)); |
469 | } | 540 | } |
470 | 541 | ||
471 | static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf) | 542 | static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf, |
543 | unsigned int size) | ||
472 | { | 544 | { |
473 | struct task_struct *tsk = current; | 545 | struct task_struct *tsk = current; |
474 | struct user_i387_ia32_struct env; | 546 | struct user_i387_ia32_struct env; |
475 | int err; | 547 | int err; |
476 | 548 | ||
477 | err = __copy_from_user(&tsk->thread.xstate->fxsave, &buf->_fxsr_env[0], | 549 | err = __copy_from_user(&tsk->thread.xstate->fxsave, &buf->_fxsr_env[0], |
478 | sizeof(struct i387_fxsave_struct)); | 550 | size); |
479 | /* mxcsr reserved bits must be masked to zero for security reasons */ | 551 | /* mxcsr reserved bits must be masked to zero for security reasons */ |
480 | tsk->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask; | 552 | tsk->thread.xstate->fxsave.mxcsr &= mxcsr_feature_mask; |
481 | if (err || __copy_from_user(&env, buf, sizeof(env))) | 553 | if (err || __copy_from_user(&env, buf, sizeof(env))) |
@@ -485,14 +557,69 @@ static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf) | |||
485 | return 0; | 557 | return 0; |
486 | } | 558 | } |
487 | 559 | ||
488 | int restore_i387_ia32(struct _fpstate_ia32 __user *buf) | 560 | static int restore_i387_xsave(void __user *buf) |
561 | { | ||
562 | struct _fpx_sw_bytes fx_sw_user; | ||
563 | struct _fpstate_ia32 __user *fx_user = | ||
564 | ((struct _fpstate_ia32 __user *) buf); | ||
565 | struct i387_fxsave_struct __user *fx = | ||
566 | (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0]; | ||
567 | struct xsave_hdr_struct *xsave_hdr = | ||
568 | ¤t->thread.xstate->xsave.xsave_hdr; | ||
569 | u64 mask; | ||
570 | int err; | ||
571 | |||
572 | if (check_for_xstate(fx, buf, &fx_sw_user)) | ||
573 | goto fx_only; | ||
574 | |||
575 | mask = fx_sw_user.xstate_bv; | ||
576 | |||
577 | err = restore_i387_fxsave(buf, fx_sw_user.xstate_size); | ||
578 | |||
579 | xsave_hdr->xstate_bv &= pcntxt_mask; | ||
580 | /* | ||
581 | * These bits must be zero. | ||
582 | */ | ||
583 | xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0; | ||
584 | |||
585 | /* | ||
586 | * Init the state that is not present in the memory layout | ||
587 | * and enabled by the OS. | ||
588 | */ | ||
589 | mask = ~(pcntxt_mask & ~mask); | ||
590 | xsave_hdr->xstate_bv &= mask; | ||
591 | |||
592 | return err; | ||
593 | fx_only: | ||
594 | /* | ||
595 | * Couldn't find the extended state information in the memory | ||
596 | * layout. Restore the FP/SSE and init the other extended state | ||
597 | * enabled by the OS. | ||
598 | */ | ||
599 | xsave_hdr->xstate_bv = XSTATE_FPSSE; | ||
600 | return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct)); | ||
601 | } | ||
602 | |||
603 | int restore_i387_xstate_ia32(void __user *buf) | ||
489 | { | 604 | { |
490 | int err; | 605 | int err; |
491 | struct task_struct *tsk = current; | 606 | struct task_struct *tsk = current; |
607 | struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf; | ||
492 | 608 | ||
493 | if (HAVE_HWFP) | 609 | if (HAVE_HWFP) |
494 | clear_fpu(tsk); | 610 | clear_fpu(tsk); |
495 | 611 | ||
612 | if (!buf) { | ||
613 | if (used_math()) { | ||
614 | clear_fpu(tsk); | ||
615 | clear_used_math(); | ||
616 | } | ||
617 | |||
618 | return 0; | ||
619 | } else | ||
620 | if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size)) | ||
621 | return -EACCES; | ||
622 | |||
496 | if (!used_math()) { | 623 | if (!used_math()) { |
497 | err = init_fpu(tsk); | 624 | err = init_fpu(tsk); |
498 | if (err) | 625 | if (err) |
@@ -500,14 +627,17 @@ int restore_i387_ia32(struct _fpstate_ia32 __user *buf) | |||
500 | } | 627 | } |
501 | 628 | ||
502 | if (HAVE_HWFP) { | 629 | if (HAVE_HWFP) { |
503 | if (cpu_has_fxsr) | 630 | if (cpu_has_xsave) |
504 | err = restore_i387_fxsave(buf); | 631 | err = restore_i387_xsave(buf); |
632 | else if (cpu_has_fxsr) | ||
633 | err = restore_i387_fxsave(fp, sizeof(struct | ||
634 | i387_fxsave_struct)); | ||
505 | else | 635 | else |
506 | err = restore_i387_fsave(buf); | 636 | err = restore_i387_fsave(fp); |
507 | } else { | 637 | } else { |
508 | err = fpregs_soft_set(current, NULL, | 638 | err = fpregs_soft_set(current, NULL, |
509 | 0, sizeof(struct user_i387_ia32_struct), | 639 | 0, sizeof(struct user_i387_ia32_struct), |
510 | NULL, buf) != 0; | 640 | NULL, fp) != 0; |
511 | } | 641 | } |
512 | set_used_math(); | 642 | set_used_math(); |
513 | 643 | ||