aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86
diff options
context:
space:
mode:
authorKonrad Rzeszutek Wilk <konrad.wilk@oracle.com>2012-10-19 15:19:19 -0400
committerKonrad Rzeszutek Wilk <konrad.wilk@oracle.com>2012-10-19 15:19:19 -0400
commite05dacd71db0a5da7c1a44bcaab2a8a240b9c233 (patch)
tree31382cf1c7d62c03126448affb2fc86e8c4aaa8b /arch/x86
parent3ab0b83bf6a1e834f4b884150d8012990c75d25d (diff)
parentddffeb8c4d0331609ef2581d84de4d763607bd37 (diff)
Merge commit 'v3.7-rc1' into stable/for-linus-3.7
* commit 'v3.7-rc1': (10892 commits) Linux 3.7-rc1 x86, boot: Explicitly include autoconf.h for hostprogs perf: Fix UAPI fallout ARM: config: make sure that platforms are ordered by option string ARM: config: sort select statements alphanumerically UAPI: (Scripted) Disintegrate include/linux/byteorder UAPI: (Scripted) Disintegrate include/linux UAPI: Unexport linux/blk_types.h UAPI: Unexport part of linux/ppp-comp.h perf: Handle new rbtree implementation procfs: don't need a PATH_MAX allocation to hold a string representation of an int vfs: embed struct filename inside of names_cache allocation if possible audit: make audit_inode take struct filename vfs: make path_openat take a struct filename pointer vfs: turn do_path_lookup into wrapper around struct filename variant audit: allow audit code to satisfy getname requests from its names_list vfs: define struct filename and have getname() return it btrfs: Fix compilation with user namespace support enabled userns: Fix posix_acl_file_xattr_userns gid conversion userns: Properly print bluetooth socket uids ...
Diffstat (limited to 'arch/x86')
-rw-r--r--arch/x86/Kconfig136
-rw-r--r--arch/x86/Kconfig.cpu5
-rw-r--r--arch/x86/Makefile2
-rw-r--r--arch/x86/boot/Makefile5
-rw-r--r--arch/x86/boot/compressed/Makefile3
-rw-r--r--arch/x86/boot/compressed/eboot.c34
-rw-r--r--arch/x86/boot/compressed/eboot.h4
-rw-r--r--arch/x86/boot/header.S4
-rw-r--r--arch/x86/boot/mkcpustr.c2
-rw-r--r--arch/x86/configs/i386_defconfig23
-rw-r--r--arch/x86/configs/x86_64_defconfig23
-rw-r--r--arch/x86/crypto/Makefile4
-rw-r--r--arch/x86/crypto/aes_glue.c1
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c258
-rw-r--r--arch/x86/crypto/blowfish_glue.c4
-rw-r--r--arch/x86/crypto/camellia_glue.c1382
-rw-r--r--arch/x86/crypto/cast5-avx-x86_64-asm_64.S376
-rw-r--r--arch/x86/crypto/cast5_avx_glue.c530
-rw-r--r--arch/x86/crypto/cast6-avx-x86_64-asm_64.S383
-rw-r--r--arch/x86/crypto/cast6_avx_glue.c648
-rw-r--r--arch/x86/crypto/ghash-clmulni-intel_glue.c2
-rw-r--r--arch/x86/crypto/glue_helper.c2
-rw-r--r--arch/x86/crypto/salsa20_glue.c1
-rw-r--r--arch/x86/crypto/serpent_avx_glue.c10
-rw-r--r--arch/x86/crypto/serpent_sse2_glue.c10
-rw-r--r--arch/x86/crypto/twofish-avx-x86_64-asm_64.S227
-rw-r--r--arch/x86/crypto/twofish_avx_glue.c10
-rw-r--r--arch/x86/crypto/twofish_glue.c1
-rw-r--r--arch/x86/crypto/twofish_glue_3way.c5
-rw-r--r--arch/x86/ia32/ia32_signal.c31
-rw-r--r--arch/x86/ia32/ia32entry.S8
-rw-r--r--arch/x86/ia32/sys_ia32.c17
-rw-r--r--arch/x86/include/asm/Kbuild2
-rw-r--r--arch/x86/include/asm/alternative-asm.h9
-rw-r--r--arch/x86/include/asm/alternative.h36
-rw-r--r--arch/x86/include/asm/apic.h2
-rw-r--r--arch/x86/include/asm/atomic.h28
-rw-r--r--arch/x86/include/asm/bitops.h14
-rw-r--r--arch/x86/include/asm/calling.h50
-rw-r--r--arch/x86/include/asm/checksum.h4
-rw-r--r--arch/x86/include/asm/cmpxchg.h4
-rw-r--r--arch/x86/include/asm/compat.h74
-rw-r--r--arch/x86/include/asm/cpufeature.h6
-rw-r--r--arch/x86/include/asm/fpu-internal.h422
-rw-r--r--arch/x86/include/asm/ftrace.h56
-rw-r--r--arch/x86/include/asm/futex.h19
-rw-r--r--arch/x86/include/asm/hardirq.h4
-rw-r--r--arch/x86/include/asm/hpet.h2
-rw-r--r--arch/x86/include/asm/hugetlb.h4
-rw-r--r--arch/x86/include/asm/i387.h29
-rw-r--r--arch/x86/include/asm/ia32.h67
-rw-r--r--arch/x86/include/asm/iommu_table.h6
-rw-r--r--arch/x86/include/asm/kprobes.h1
-rw-r--r--arch/x86/include/asm/kvm.h17
-rw-r--r--arch/x86/include/asm/kvm_emulate.h48
-rw-r--r--arch/x86/include/asm/kvm_host.h52
-rw-r--r--arch/x86/include/asm/kvm_para.h6
-rw-r--r--arch/x86/include/asm/mce.h13
-rw-r--r--arch/x86/include/asm/microcode.h10
-rw-r--r--arch/x86/include/asm/mmzone.h4
-rw-r--r--arch/x86/include/asm/msr-index.h8
-rw-r--r--arch/x86/include/asm/mutex.h4
-rw-r--r--arch/x86/include/asm/numa.h4
-rw-r--r--arch/x86/include/asm/pci.h2
-rw-r--r--arch/x86/include/asm/perf_event.h2
-rw-r--r--arch/x86/include/asm/perf_regs.h33
-rw-r--r--arch/x86/include/asm/pgtable.h15
-rw-r--r--arch/x86/include/asm/pgtable_32.h1
-rw-r--r--arch/x86/include/asm/pgtable_64.h1
-rw-r--r--arch/x86/include/asm/pgtable_types.h4
-rw-r--r--arch/x86/include/asm/posix_types.h10
-rw-r--r--arch/x86/include/asm/processor-flags.h1
-rw-r--r--arch/x86/include/asm/processor.h8
-rw-r--r--arch/x86/include/asm/rcu.h32
-rw-r--r--arch/x86/include/asm/seccomp.h4
-rw-r--r--arch/x86/include/asm/signal.h4
-rw-r--r--arch/x86/include/asm/smap.h91
-rw-r--r--arch/x86/include/asm/string.h4
-rw-r--r--arch/x86/include/asm/suspend.h4
-rw-r--r--arch/x86/include/asm/svm.h205
-rw-r--r--arch/x86/include/asm/sys_ia32.h4
-rw-r--r--arch/x86/include/asm/syscalls.h2
-rw-r--r--arch/x86/include/asm/thread_info.h12
-rw-r--r--arch/x86/include/asm/uaccess.h32
-rw-r--r--arch/x86/include/asm/uaccess_32.h3
-rw-r--r--arch/x86/include/asm/uaccess_64.h3
-rw-r--r--arch/x86/include/asm/unistd.h1
-rw-r--r--arch/x86/include/asm/uprobes.h3
-rw-r--r--arch/x86/include/asm/user.h4
-rw-r--r--arch/x86/include/asm/vdso.h3
-rw-r--r--arch/x86/include/asm/vgtod.h4
-rw-r--r--arch/x86/include/asm/vmx.h127
-rw-r--r--arch/x86/include/asm/xen/interface.h4
-rw-r--r--arch/x86/include/asm/xen/page.h3
-rw-r--r--arch/x86/include/asm/xor.h4
-rw-r--r--arch/x86/include/asm/xor_32.h58
-rw-r--r--arch/x86/include/asm/xor_64.h63
-rw-r--r--arch/x86/include/asm/xor_avx.h54
-rw-r--r--arch/x86/include/asm/xsave.h23
-rw-r--r--arch/x86/include/uapi/asm/Kbuild6
-rw-r--r--arch/x86/kernel/Makefile7
-rw-r--r--arch/x86/kernel/acpi/boot.c2
-rw-r--r--arch/x86/kernel/acpi/sleep.c15
-rw-r--r--arch/x86/kernel/alternative.c111
-rw-r--r--arch/x86/kernel/apic/apic.c2
-rw-r--r--arch/x86/kernel/apic/apic_numachip.c4
-rw-r--r--arch/x86/kernel/asm-offsets.c3
-rw-r--r--arch/x86/kernel/cpu/Makefile2
-rw-r--r--arch/x86/kernel/cpu/amd.c67
-rw-r--r--arch/x86/kernel/cpu/bugs.c7
-rw-r--r--arch/x86/kernel/cpu/common.c63
-rw-r--r--arch/x86/kernel/cpu/intel.c4
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce-inject.c8
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce-internal.h12
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce.c94
-rw-r--r--arch/x86/kernel/cpu/mcheck/mce_intel.c168
-rw-r--r--arch/x86/kernel/cpu/mkcapflags.pl5
-rw-r--r--arch/x86/kernel/cpu/perf_event.h4
-rw-r--r--arch/x86/kernel/cpu/perf_event_amd_ibs.c73
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c27
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_ds.c14
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_lbr.c3
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.c36
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.h6
-rw-r--r--arch/x86/kernel/cpu/perf_event_knc.c248
-rw-r--r--arch/x86/kernel/cpu/perfctr-watchdog.c4
-rw-r--r--arch/x86/kernel/cpu/proc.c5
-rw-r--r--arch/x86/kernel/cpuid.c5
-rw-r--r--arch/x86/kernel/devicetree.c51
-rw-r--r--arch/x86/kernel/entry_32.S146
-rw-r--r--arch/x86/kernel/entry_64.S232
-rw-r--r--arch/x86/kernel/ftrace.c73
-rw-r--r--arch/x86/kernel/head_32.S31
-rw-r--r--arch/x86/kernel/i387.c292
-rw-r--r--arch/x86/kernel/i8259.c2
-rw-r--r--arch/x86/kernel/irq.c4
-rw-r--r--arch/x86/kernel/kgdb.c2
-rw-r--r--arch/x86/kernel/kprobes.c67
-rw-r--r--arch/x86/kernel/kvm.c3
-rw-r--r--arch/x86/kernel/microcode_amd.c357
-rw-r--r--arch/x86/kernel/microcode_core.c70
-rw-r--r--arch/x86/kernel/microcode_intel.c3
-rw-r--r--arch/x86/kernel/msr.c5
-rw-r--r--arch/x86/kernel/perf_regs.c105
-rw-r--r--arch/x86/kernel/probe_roms.c2
-rw-r--r--arch/x86/kernel/process.c87
-rw-r--r--arch/x86/kernel/process_32.c37
-rw-r--r--arch/x86/kernel/process_64.c39
-rw-r--r--arch/x86/kernel/ptrace.c8
-rw-r--r--arch/x86/kernel/rtc.c2
-rw-r--r--arch/x86/kernel/setup.c5
-rw-r--r--arch/x86/kernel/signal.c233
-rw-r--r--arch/x86/kernel/smpboot.c20
-rw-r--r--arch/x86/kernel/step.c53
-rw-r--r--arch/x86/kernel/sys_i386_32.c40
-rw-r--r--arch/x86/kernel/traps.c174
-rw-r--r--arch/x86/kernel/uprobes.c52
-rw-r--r--arch/x86/kernel/vm86_32.c6
-rw-r--r--arch/x86/kernel/vsyscall_64.c49
-rw-r--r--arch/x86/kernel/x8664_ksyms_64.c6
-rw-r--r--arch/x86/kernel/xsave.c517
-rw-r--r--arch/x86/kvm/Kconfig2
-rw-r--r--arch/x86/kvm/Makefile2
-rw-r--r--arch/x86/kvm/cpuid.c14
-rw-r--r--arch/x86/kvm/emulate.c538
-rw-r--r--arch/x86/kvm/i8254.c64
-rw-r--r--arch/x86/kvm/i8254.h6
-rw-r--r--arch/x86/kvm/i8259.c70
-rw-r--r--arch/x86/kvm/irq.h2
-rw-r--r--arch/x86/kvm/kvm_timer.h18
-rw-r--r--arch/x86/kvm/lapic.c484
-rw-r--r--arch/x86/kvm/lapic.h61
-rw-r--r--arch/x86/kvm/mmu.c240
-rw-r--r--arch/x86/kvm/mmu.h25
-rw-r--r--arch/x86/kvm/mmu_audit.c8
-rw-r--r--arch/x86/kvm/paging_tmpl.h199
-rw-r--r--arch/x86/kvm/pmu.c2
-rw-r--r--arch/x86/kvm/svm.c82
-rw-r--r--arch/x86/kvm/timer.c47
-rw-r--r--arch/x86/kvm/trace.h89
-rw-r--r--arch/x86/kvm/vmx.c268
-rw-r--r--arch/x86/kvm/x86.c400
-rw-r--r--arch/x86/kvm/x86.h1
-rw-r--r--arch/x86/lguest/Kconfig1
-rw-r--r--arch/x86/lib/copy_user_64.S7
-rw-r--r--arch/x86/lib/copy_user_nocache_64.S3
-rw-r--r--arch/x86/lib/getuser.S10
-rw-r--r--arch/x86/lib/insn.c4
-rw-r--r--arch/x86/lib/putuser.S8
-rw-r--r--arch/x86/lib/usercopy_32.c13
-rw-r--r--arch/x86/lib/usercopy_64.c3
-rw-r--r--arch/x86/mm/fault.c32
-rw-r--r--arch/x86/mm/hugetlbpage.c3
-rw-r--r--arch/x86/mm/init.c2
-rw-r--r--arch/x86/mm/init_32.c2
-rw-r--r--arch/x86/mm/pat.c87
-rw-r--r--arch/x86/mm/pat_rbtree.c34
-rw-r--r--arch/x86/mm/tlb.c4
-rw-r--r--arch/x86/net/bpf_jit_comp.c34
-rw-r--r--arch/x86/pci/acpi.c3
-rw-r--r--arch/x86/pci/mmconfig-shared.c2
-rw-r--r--arch/x86/pci/visws.c5
-rw-r--r--arch/x86/platform/efi/Makefile1
-rw-r--r--arch/x86/platform/efi/efi-bgrt.c76
-rw-r--r--arch/x86/platform/efi/efi.c66
-rw-r--r--arch/x86/realmode/rm/wakeup.h2
-rw-r--r--arch/x86/realmode/rm/wakeup_asm.S29
-rw-r--r--arch/x86/syscalls/Makefile17
-rw-r--r--arch/x86/syscalls/syscall_32.tbl2
-rw-r--r--arch/x86/tools/Makefile2
-rw-r--r--arch/x86/um/Kconfig5
-rw-r--r--arch/x86/um/asm/checksum.h144
-rw-r--r--arch/x86/um/asm/checksum_32.h140
-rw-r--r--arch/x86/um/asm/checksum_64.h125
-rw-r--r--arch/x86/um/asm/elf.h2
-rw-r--r--arch/x86/um/asm/ptrace.h58
-rw-r--r--arch/x86/um/asm/ptrace_32.h28
-rw-r--r--arch/x86/um/asm/ptrace_64.h46
-rw-r--r--arch/x86/um/bugs_32.c6
-rw-r--r--arch/x86/um/bugs_64.c2
-rw-r--r--arch/x86/um/fault.c2
-rw-r--r--arch/x86/um/ldt.c10
-rw-r--r--arch/x86/um/mem_64.c6
-rw-r--r--arch/x86/um/os-Linux/registers.c4
-rw-r--r--arch/x86/um/os-Linux/task_size.c2
-rw-r--r--arch/x86/um/os-Linux/tls.c2
-rw-r--r--arch/x86/um/ptrace_32.c8
-rw-r--r--arch/x86/um/ptrace_user.c2
-rw-r--r--arch/x86/um/shared/sysdep/kernel-offsets.h3
-rw-r--r--arch/x86/um/shared/sysdep/ptrace.h2
-rw-r--r--arch/x86/um/shared/sysdep/stub.h4
-rw-r--r--arch/x86/um/shared/sysdep/syscalls.h2
-rw-r--r--arch/x86/um/shared/sysdep/syscalls_32.h4
-rw-r--r--arch/x86/um/signal.c10
-rw-r--r--arch/x86/um/stub_32.S2
-rw-r--r--arch/x86/um/stub_64.S2
-rw-r--r--arch/x86/um/stub_segv.c6
-rw-r--r--arch/x86/um/sys_call_table_32.c3
-rw-r--r--arch/x86/um/syscalls_32.c27
-rw-r--r--arch/x86/um/syscalls_64.c23
-rw-r--r--arch/x86/um/sysrq_32.c12
-rw-r--r--arch/x86/um/sysrq_64.c2
-rw-r--r--arch/x86/um/tls_32.c12
-rw-r--r--arch/x86/um/tls_64.c2
-rw-r--r--arch/x86/vdso/vclock_gettime.c22
-rw-r--r--arch/x86/xen/enlighten.c4
-rw-r--r--arch/x86/xen/mmu.c3
-rw-r--r--arch/x86/xen/p2m.c27
-rw-r--r--arch/x86/xen/setup.c4
-rw-r--r--arch/x86/xen/smp.c6
250 files changed, 8741 insertions, 4869 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 8ec3a1aa4abd..46c3bff3ced2 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -7,11 +7,14 @@ config 64BIT
7 Say no to build a 32-bit kernel - formerly known as i386 7 Say no to build a 32-bit kernel - formerly known as i386
8 8
9config X86_32 9config X86_32
10 def_bool !64BIT 10 def_bool y
11 depends on !64BIT
11 select CLKSRC_I8253 12 select CLKSRC_I8253
13 select HAVE_UID16
12 14
13config X86_64 15config X86_64
14 def_bool 64BIT 16 def_bool y
17 depends on 64BIT
15 select X86_DEV_DMA_OPS 18 select X86_DEV_DMA_OPS
16 19
17### Arch settings 20### Arch settings
@@ -36,6 +39,7 @@ config X86
36 select HAVE_KRETPROBES 39 select HAVE_KRETPROBES
37 select HAVE_OPTPROBES 40 select HAVE_OPTPROBES
38 select HAVE_FTRACE_MCOUNT_RECORD 41 select HAVE_FTRACE_MCOUNT_RECORD
42 select HAVE_FENTRY if X86_64
39 select HAVE_C_RECORDMCOUNT 43 select HAVE_C_RECORDMCOUNT
40 select HAVE_DYNAMIC_FTRACE 44 select HAVE_DYNAMIC_FTRACE
41 select HAVE_FUNCTION_TRACER 45 select HAVE_FUNCTION_TRACER
@@ -43,6 +47,7 @@ config X86
43 select HAVE_FUNCTION_GRAPH_FP_TEST 47 select HAVE_FUNCTION_GRAPH_FP_TEST
44 select HAVE_FUNCTION_TRACE_MCOUNT_TEST 48 select HAVE_FUNCTION_TRACE_MCOUNT_TEST
45 select HAVE_SYSCALL_TRACEPOINTS 49 select HAVE_SYSCALL_TRACEPOINTS
50 select SYSCTL_EXCEPTION_TRACE
46 select HAVE_KVM 51 select HAVE_KVM
47 select HAVE_ARCH_KGDB 52 select HAVE_ARCH_KGDB
48 select HAVE_ARCH_TRACEHOOK 53 select HAVE_ARCH_TRACEHOOK
@@ -60,6 +65,9 @@ config X86
60 select HAVE_MIXED_BREAKPOINTS_REGS 65 select HAVE_MIXED_BREAKPOINTS_REGS
61 select PERF_EVENTS 66 select PERF_EVENTS
62 select HAVE_PERF_EVENTS_NMI 67 select HAVE_PERF_EVENTS_NMI
68 select HAVE_PERF_REGS
69 select HAVE_PERF_USER_STACK_DUMP
70 select HAVE_DEBUG_KMEMLEAK
63 select ANON_INODES 71 select ANON_INODES
64 select HAVE_ALIGNED_STRUCT_PAGE if SLUB && !M386 72 select HAVE_ALIGNED_STRUCT_PAGE if SLUB && !M386
65 select HAVE_CMPXCHG_LOCAL if !M386 73 select HAVE_CMPXCHG_LOCAL if !M386
@@ -80,6 +88,7 @@ config X86
80 select IRQ_FORCED_THREADING 88 select IRQ_FORCED_THREADING
81 select USE_GENERIC_SMP_HELPERS if SMP 89 select USE_GENERIC_SMP_HELPERS if SMP
82 select HAVE_BPF_JIT if X86_64 90 select HAVE_BPF_JIT if X86_64
91 select HAVE_ARCH_TRANSPARENT_HUGEPAGE
83 select CLKEVT_I8253 92 select CLKEVT_I8253
84 select ARCH_HAVE_NMI_SAFE_CMPXCHG 93 select ARCH_HAVE_NMI_SAFE_CMPXCHG
85 select GENERIC_IOMAP 94 select GENERIC_IOMAP
@@ -97,9 +106,16 @@ config X86
97 select KTIME_SCALAR if X86_32 106 select KTIME_SCALAR if X86_32
98 select GENERIC_STRNCPY_FROM_USER 107 select GENERIC_STRNCPY_FROM_USER
99 select GENERIC_STRNLEN_USER 108 select GENERIC_STRNLEN_USER
109 select HAVE_RCU_USER_QS if X86_64
110 select HAVE_IRQ_TIME_ACCOUNTING
111 select GENERIC_KERNEL_THREAD
112 select GENERIC_KERNEL_EXECVE
113 select MODULES_USE_ELF_REL if X86_32
114 select MODULES_USE_ELF_RELA if X86_64
100 115
101config INSTRUCTION_DECODER 116config INSTRUCTION_DECODER
102 def_bool (KPROBES || PERF_EVENTS || UPROBES) 117 def_bool y
118 depends on KPROBES || PERF_EVENTS || UPROBES
103 119
104config OUTPUT_FORMAT 120config OUTPUT_FORMAT
105 string 121 string
@@ -127,13 +143,15 @@ config SBUS
127 bool 143 bool
128 144
129config NEED_DMA_MAP_STATE 145config NEED_DMA_MAP_STATE
130 def_bool (X86_64 || INTEL_IOMMU || DMA_API_DEBUG) 146 def_bool y
147 depends on X86_64 || INTEL_IOMMU || DMA_API_DEBUG
131 148
132config NEED_SG_DMA_LENGTH 149config NEED_SG_DMA_LENGTH
133 def_bool y 150 def_bool y
134 151
135config GENERIC_ISA_DMA 152config GENERIC_ISA_DMA
136 def_bool ISA_DMA_API 153 def_bool y
154 depends on ISA_DMA_API
137 155
138config GENERIC_BUG 156config GENERIC_BUG
139 def_bool y 157 def_bool y
@@ -150,13 +168,16 @@ config GENERIC_GPIO
150 bool 168 bool
151 169
152config ARCH_MAY_HAVE_PC_FDC 170config ARCH_MAY_HAVE_PC_FDC
153 def_bool ISA_DMA_API 171 def_bool y
172 depends on ISA_DMA_API
154 173
155config RWSEM_GENERIC_SPINLOCK 174config RWSEM_GENERIC_SPINLOCK
156 def_bool !X86_XADD 175 def_bool y
176 depends on !X86_XADD
157 177
158config RWSEM_XCHGADD_ALGORITHM 178config RWSEM_XCHGADD_ALGORITHM
159 def_bool X86_XADD 179 def_bool y
180 depends on X86_XADD
160 181
161config GENERIC_CALIBRATE_DELAY 182config GENERIC_CALIBRATE_DELAY
162 def_bool y 183 def_bool y
@@ -573,23 +594,18 @@ config PARAVIRT_TIME_ACCOUNTING
573 594
574source "arch/x86/xen/Kconfig" 595source "arch/x86/xen/Kconfig"
575 596
576config KVM_CLOCK
577 bool "KVM paravirtualized clock"
578 select PARAVIRT
579 select PARAVIRT_CLOCK
580 ---help---
581 Turning on this option will allow you to run a paravirtualized clock
582 when running over the KVM hypervisor. Instead of relying on a PIT
583 (or probably other) emulation by the underlying device model, the host
584 provides the guest with timing infrastructure such as time of day, and
585 system time
586
587config KVM_GUEST 597config KVM_GUEST
588 bool "KVM Guest support" 598 bool "KVM Guest support (including kvmclock)"
599 select PARAVIRT
589 select PARAVIRT 600 select PARAVIRT
601 select PARAVIRT_CLOCK
602 default y if PARAVIRT_GUEST
590 ---help--- 603 ---help---
591 This option enables various optimizations for running under the KVM 604 This option enables various optimizations for running under the KVM
592 hypervisor. 605 hypervisor. It includes a paravirtualized clock, so that instead
606 of relying on a PIT (or probably other) emulation by the
607 underlying device model, the host provides the guest with
608 timing infrastructure such as time of day, and system time
593 609
594source "arch/x86/lguest/Kconfig" 610source "arch/x86/lguest/Kconfig"
595 611
@@ -746,13 +762,14 @@ config SWIOTLB
746 def_bool y if X86_64 762 def_bool y if X86_64
747 ---help--- 763 ---help---
748 Support for software bounce buffers used on x86-64 systems 764 Support for software bounce buffers used on x86-64 systems
749 which don't have a hardware IOMMU (e.g. the current generation 765 which don't have a hardware IOMMU. Using this PCI devices
750 of Intel's x86-64 CPUs). Using this PCI devices which can only 766 which can only access 32-bits of memory can be used on systems
751 access 32-bits of memory can be used on systems with more than 767 with more than 3 GB of memory.
752 3 GB of memory. If unsure, say Y. 768 If unsure, say Y.
753 769
754config IOMMU_HELPER 770config IOMMU_HELPER
755 def_bool (CALGARY_IOMMU || GART_IOMMU || SWIOTLB || AMD_IOMMU) 771 def_bool y
772 depends on CALGARY_IOMMU || GART_IOMMU || SWIOTLB || AMD_IOMMU
756 773
757config MAXSMP 774config MAXSMP
758 bool "Enable Maximum number of SMP Processors and NUMA Nodes" 775 bool "Enable Maximum number of SMP Processors and NUMA Nodes"
@@ -796,17 +813,6 @@ config SCHED_MC
796 making when dealing with multi-core CPU chips at a cost of slightly 813 making when dealing with multi-core CPU chips at a cost of slightly
797 increased overhead in some places. If unsure say N here. 814 increased overhead in some places. If unsure say N here.
798 815
799config IRQ_TIME_ACCOUNTING
800 bool "Fine granularity task level IRQ time accounting"
801 default n
802 ---help---
803 Select this option to enable fine granularity task irq time
804 accounting. This is done by reading a timestamp on each
805 transitions between softirq and hardirq state, so there can be a
806 small performance impact.
807
808 If in doubt, say N here.
809
810source "kernel/Kconfig.preempt" 816source "kernel/Kconfig.preempt"
811 817
812config X86_UP_APIC 818config X86_UP_APIC
@@ -871,6 +877,7 @@ config X86_REROUTE_FOR_BROKEN_BOOT_IRQS
871 877
872config X86_MCE 878config X86_MCE
873 bool "Machine Check / overheating reporting" 879 bool "Machine Check / overheating reporting"
880 default y
874 ---help--- 881 ---help---
875 Machine Check support allows the processor to notify the 882 Machine Check support allows the processor to notify the
876 kernel if it detects a problem (e.g. overheating, data corruption). 883 kernel if it detects a problem (e.g. overheating, data corruption).
@@ -982,25 +989,25 @@ config X86_REBOOTFIXUPS
982 Say N otherwise. 989 Say N otherwise.
983 990
984config MICROCODE 991config MICROCODE
985 tristate "/dev/cpu/microcode - microcode support" 992 tristate "CPU microcode loading support"
986 select FW_LOADER 993 select FW_LOADER
987 ---help--- 994 ---help---
995
988 If you say Y here, you will be able to update the microcode on 996 If you say Y here, you will be able to update the microcode on
989 certain Intel and AMD processors. The Intel support is for the 997 certain Intel and AMD processors. The Intel support is for the
990 IA32 family, e.g. Pentium Pro, Pentium II, Pentium III, 998 IA32 family, e.g. Pentium Pro, Pentium II, Pentium III, Pentium 4,
991 Pentium 4, Xeon etc. The AMD support is for family 0x10 and 999 Xeon etc. The AMD support is for families 0x10 and later. You will
992 0x11 processors, e.g. Opteron, Phenom and Turion 64 Ultra. 1000 obviously need the actual microcode binary data itself which is not
993 You will obviously need the actual microcode binary data itself 1001 shipped with the Linux kernel.
994 which is not shipped with the Linux kernel.
995 1002
996 This option selects the general module only, you need to select 1003 This option selects the general module only, you need to select
997 at least one vendor specific module as well. 1004 at least one vendor specific module as well.
998 1005
999 To compile this driver as a module, choose M here: the 1006 To compile this driver as a module, choose M here: the module
1000 module will be called microcode. 1007 will be called microcode.
1001 1008
1002config MICROCODE_INTEL 1009config MICROCODE_INTEL
1003 bool "Intel microcode patch loading support" 1010 bool "Intel microcode loading support"
1004 depends on MICROCODE 1011 depends on MICROCODE
1005 default MICROCODE 1012 default MICROCODE
1006 select FW_LOADER 1013 select FW_LOADER
@@ -1013,7 +1020,7 @@ config MICROCODE_INTEL
1013 <http://www.urbanmyth.org/microcode/>. 1020 <http://www.urbanmyth.org/microcode/>.
1014 1021
1015config MICROCODE_AMD 1022config MICROCODE_AMD
1016 bool "AMD microcode patch loading support" 1023 bool "AMD microcode loading support"
1017 depends on MICROCODE 1024 depends on MICROCODE
1018 select FW_LOADER 1025 select FW_LOADER
1019 ---help--- 1026 ---help---
@@ -1159,10 +1166,12 @@ config X86_PAE
1159 consumes more pagetable space per process. 1166 consumes more pagetable space per process.
1160 1167
1161config ARCH_PHYS_ADDR_T_64BIT 1168config ARCH_PHYS_ADDR_T_64BIT
1162 def_bool X86_64 || X86_PAE 1169 def_bool y
1170 depends on X86_64 || X86_PAE
1163 1171
1164config ARCH_DMA_ADDR_T_64BIT 1172config ARCH_DMA_ADDR_T_64BIT
1165 def_bool X86_64 || HIGHMEM64G 1173 def_bool y
1174 depends on X86_64 || HIGHMEM64G
1166 1175
1167config DIRECT_GBPAGES 1176config DIRECT_GBPAGES
1168 bool "Enable 1GB pages for kernel pagetables" if EXPERT 1177 bool "Enable 1GB pages for kernel pagetables" if EXPERT
@@ -1285,8 +1294,8 @@ config ARCH_SELECT_MEMORY_MODEL
1285 depends on ARCH_SPARSEMEM_ENABLE 1294 depends on ARCH_SPARSEMEM_ENABLE
1286 1295
1287config ARCH_MEMORY_PROBE 1296config ARCH_MEMORY_PROBE
1288 def_bool X86_64 1297 def_bool y
1289 depends on MEMORY_HOTPLUG 1298 depends on X86_64 && MEMORY_HOTPLUG
1290 1299
1291config ARCH_PROC_KCORE_TEXT 1300config ARCH_PROC_KCORE_TEXT
1292 def_bool y 1301 def_bool y
@@ -1487,6 +1496,17 @@ config ARCH_RANDOM
1487 If supported, this is a high bandwidth, cryptographically 1496 If supported, this is a high bandwidth, cryptographically
1488 secure hardware random number generator. 1497 secure hardware random number generator.
1489 1498
1499config X86_SMAP
1500 def_bool y
1501 prompt "Supervisor Mode Access Prevention" if EXPERT
1502 ---help---
1503 Supervisor Mode Access Prevention (SMAP) is a security
1504 feature in newer Intel processors. There is a small
1505 performance cost if this enabled and turned on; there is
1506 also a small increase in the kernel size if this is enabled.
1507
1508 If unsure, say Y.
1509
1490config EFI 1510config EFI
1491 bool "EFI runtime service support" 1511 bool "EFI runtime service support"
1492 depends on ACPI 1512 depends on ACPI
@@ -1975,7 +1995,6 @@ config PCI_MMCONFIG
1975 1995
1976config PCI_CNB20LE_QUIRK 1996config PCI_CNB20LE_QUIRK
1977 bool "Read CNB20LE Host Bridge Windows" if EXPERT 1997 bool "Read CNB20LE Host Bridge Windows" if EXPERT
1978 default n
1979 depends on PCI && EXPERIMENTAL 1998 depends on PCI && EXPERIMENTAL
1980 help 1999 help
1981 Read the PCI windows out of the CNB20LE host bridge. This allows 2000 Read the PCI windows out of the CNB20LE host bridge. This allows
@@ -2157,6 +2176,7 @@ config IA32_EMULATION
2157 bool "IA32 Emulation" 2176 bool "IA32 Emulation"
2158 depends on X86_64 2177 depends on X86_64
2159 select COMPAT_BINFMT_ELF 2178 select COMPAT_BINFMT_ELF
2179 select HAVE_UID16
2160 ---help--- 2180 ---help---
2161 Include code to run legacy 32-bit programs under a 2181 Include code to run legacy 32-bit programs under a
2162 64-bit kernel. You should likely turn this on, unless you're 2182 64-bit kernel. You should likely turn this on, unless you're
@@ -2186,18 +2206,18 @@ config COMPAT
2186 depends on IA32_EMULATION || X86_X32 2206 depends on IA32_EMULATION || X86_X32
2187 select ARCH_WANT_OLD_COMPAT_IPC 2207 select ARCH_WANT_OLD_COMPAT_IPC
2188 2208
2209if COMPAT
2189config COMPAT_FOR_U64_ALIGNMENT 2210config COMPAT_FOR_U64_ALIGNMENT
2190 def_bool COMPAT 2211 def_bool y
2191 depends on X86_64
2192 2212
2193config SYSVIPC_COMPAT 2213config SYSVIPC_COMPAT
2194 def_bool y 2214 def_bool y
2195 depends on COMPAT && SYSVIPC 2215 depends on SYSVIPC
2196 2216
2197config KEYS_COMPAT 2217config KEYS_COMPAT
2198 bool 2218 def_bool y
2199 depends on COMPAT && KEYS 2219 depends on KEYS
2200 default y 2220endif
2201 2221
2202endmenu 2222endmenu
2203 2223
diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
index 706e12e9984b..f3b86d0df44e 100644
--- a/arch/x86/Kconfig.cpu
+++ b/arch/x86/Kconfig.cpu
@@ -306,7 +306,8 @@ config X86_INTERNODE_CACHE_SHIFT
306 default X86_L1_CACHE_SHIFT 306 default X86_L1_CACHE_SHIFT
307 307
308config X86_CMPXCHG 308config X86_CMPXCHG
309 def_bool X86_64 || (X86_32 && !M386) 309 def_bool y
310 depends on X86_64 || (X86_32 && !M386)
310 311
311config X86_L1_CACHE_SHIFT 312config X86_L1_CACHE_SHIFT
312 int 313 int
@@ -317,7 +318,7 @@ config X86_L1_CACHE_SHIFT
317 318
318config X86_XADD 319config X86_XADD
319 def_bool y 320 def_bool y
320 depends on X86_64 || !M386 321 depends on !M386
321 322
322config X86_PPRO_FENCE 323config X86_PPRO_FENCE
323 bool "PentiumPro memory ordering errata workaround" 324 bool "PentiumPro memory ordering errata workaround"
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
index 682e9c210baa..58790bd85c1d 100644
--- a/arch/x86/Makefile
+++ b/arch/x86/Makefile
@@ -92,7 +92,7 @@ endif
92ifdef CONFIG_X86_X32 92ifdef CONFIG_X86_X32
93 x32_ld_ok := $(call try-run,\ 93 x32_ld_ok := $(call try-run,\
94 /bin/echo -e '1: .quad 1b' | \ 94 /bin/echo -e '1: .quad 1b' | \
95 $(CC) $(KBUILD_AFLAGS) -c -xassembler -o "$$TMP" - && \ 95 $(CC) $(KBUILD_AFLAGS) -c -x assembler -o "$$TMP" - && \
96 $(OBJCOPY) -O elf32-x86-64 "$$TMP" "$$TMPO" && \ 96 $(OBJCOPY) -O elf32-x86-64 "$$TMP" "$$TMPO" && \
97 $(LD) -m elf32_x86_64 "$$TMPO" -o "$$TMP",y,n) 97 $(LD) -m elf32_x86_64 "$$TMPO" -o "$$TMP",y,n)
98 ifeq ($(x32_ld_ok),y) 98 ifeq ($(x32_ld_ok),y)
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
index f7535bedc33f..ccce0ed67dde 100644
--- a/arch/x86/boot/Makefile
+++ b/arch/x86/boot/Makefile
@@ -37,7 +37,8 @@ setup-y += video-bios.o
37targets += $(setup-y) 37targets += $(setup-y)
38hostprogs-y := mkcpustr tools/build 38hostprogs-y := mkcpustr tools/build
39 39
40HOST_EXTRACFLAGS += -I$(srctree)/tools/include $(LINUXINCLUDE) \ 40HOST_EXTRACFLAGS += -I$(srctree)/tools/include \
41 -include include/generated/autoconf.h \
41 -D__EXPORTED_HEADERS__ 42 -D__EXPORTED_HEADERS__
42 43
43$(obj)/cpu.o: $(obj)/cpustr.h 44$(obj)/cpu.o: $(obj)/cpustr.h
@@ -52,7 +53,7 @@ $(obj)/cpustr.h: $(obj)/mkcpustr FORCE
52 53
53# How to compile the 16-bit code. Note we always compile for -march=i386, 54# How to compile the 16-bit code. Note we always compile for -march=i386,
54# that way we can complain to the user if the CPU is insufficient. 55# that way we can complain to the user if the CPU is insufficient.
55KBUILD_CFLAGS := $(LINUXINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \ 56KBUILD_CFLAGS := $(USERINCLUDE) -g -Os -D_SETUP -D__KERNEL__ \
56 -DDISABLE_BRANCH_PROFILING \ 57 -DDISABLE_BRANCH_PROFILING \
57 -Wall -Wstrict-prototypes \ 58 -Wall -Wstrict-prototypes \
58 -march=i386 -mregparm=3 \ 59 -march=i386 -mregparm=3 \
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
index e398bb5d63bb..8a84501acb1b 100644
--- a/arch/x86/boot/compressed/Makefile
+++ b/arch/x86/boot/compressed/Makefile
@@ -28,6 +28,9 @@ VMLINUX_OBJS = $(obj)/vmlinux.lds $(obj)/head_$(BITS).o $(obj)/misc.o \
28 $(obj)/string.o $(obj)/cmdline.o $(obj)/early_serial_console.o \ 28 $(obj)/string.o $(obj)/cmdline.o $(obj)/early_serial_console.o \
29 $(obj)/piggy.o 29 $(obj)/piggy.o
30 30
31$(obj)/eboot.o: KBUILD_CFLAGS += -fshort-wchar -mno-red-zone
32$(obj)/efi_stub_$(BITS).o: KBUILD_CLFAGS += -fshort-wchar -mno-red-zone
33
31ifeq ($(CONFIG_EFI_STUB), y) 34ifeq ($(CONFIG_EFI_STUB), y)
32 VMLINUX_OBJS += $(obj)/eboot.o $(obj)/efi_stub_$(BITS).o 35 VMLINUX_OBJS += $(obj)/eboot.o $(obj)/efi_stub_$(BITS).o
33endif 36endif
diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c
index b3e0227df2c9..c760e073963e 100644
--- a/arch/x86/boot/compressed/eboot.c
+++ b/arch/x86/boot/compressed/eboot.c
@@ -276,8 +276,9 @@ static efi_status_t setup_gop(struct screen_info *si, efi_guid_t *proto,
276 nr_gops = size / sizeof(void *); 276 nr_gops = size / sizeof(void *);
277 for (i = 0; i < nr_gops; i++) { 277 for (i = 0; i < nr_gops; i++) {
278 struct efi_graphics_output_mode_info *info; 278 struct efi_graphics_output_mode_info *info;
279 efi_guid_t pciio_proto = EFI_PCI_IO_PROTOCOL_GUID; 279 efi_guid_t conout_proto = EFI_CONSOLE_OUT_DEVICE_GUID;
280 void *pciio; 280 bool conout_found = false;
281 void *dummy;
281 void *h = gop_handle[i]; 282 void *h = gop_handle[i];
282 283
283 status = efi_call_phys3(sys_table->boottime->handle_protocol, 284 status = efi_call_phys3(sys_table->boottime->handle_protocol,
@@ -285,19 +286,21 @@ static efi_status_t setup_gop(struct screen_info *si, efi_guid_t *proto,
285 if (status != EFI_SUCCESS) 286 if (status != EFI_SUCCESS)
286 continue; 287 continue;
287 288
288 efi_call_phys3(sys_table->boottime->handle_protocol, 289 status = efi_call_phys3(sys_table->boottime->handle_protocol,
289 h, &pciio_proto, &pciio); 290 h, &conout_proto, &dummy);
291
292 if (status == EFI_SUCCESS)
293 conout_found = true;
290 294
291 status = efi_call_phys4(gop->query_mode, gop, 295 status = efi_call_phys4(gop->query_mode, gop,
292 gop->mode->mode, &size, &info); 296 gop->mode->mode, &size, &info);
293 if (status == EFI_SUCCESS && (!first_gop || pciio)) { 297 if (status == EFI_SUCCESS && (!first_gop || conout_found)) {
294 /* 298 /*
295 * Apple provide GOPs that are not backed by 299 * Systems that use the UEFI Console Splitter may
296 * real hardware (they're used to handle 300 * provide multiple GOP devices, not all of which are
297 * multiple displays). The workaround is to 301 * backed by real hardware. The workaround is to search
298 * search for a GOP implementing the PCIIO 302 * for a GOP implementing the ConOut protocol, and if
299 * protocol, and if one isn't found, to just 303 * one isn't found, to just fall back to the first GOP.
300 * fallback to the first GOP.
301 */ 304 */
302 width = info->horizontal_resolution; 305 width = info->horizontal_resolution;
303 height = info->vertical_resolution; 306 height = info->vertical_resolution;
@@ -308,10 +311,10 @@ static efi_status_t setup_gop(struct screen_info *si, efi_guid_t *proto,
308 pixels_per_scan_line = info->pixels_per_scan_line; 311 pixels_per_scan_line = info->pixels_per_scan_line;
309 312
310 /* 313 /*
311 * Once we've found a GOP supporting PCIIO, 314 * Once we've found a GOP supporting ConOut,
312 * don't bother looking any further. 315 * don't bother looking any further.
313 */ 316 */
314 if (pciio) 317 if (conout_found)
315 break; 318 break;
316 319
317 first_gop = gop; 320 first_gop = gop;
@@ -328,7 +331,6 @@ static efi_status_t setup_gop(struct screen_info *si, efi_guid_t *proto,
328 si->lfb_width = width; 331 si->lfb_width = width;
329 si->lfb_height = height; 332 si->lfb_height = height;
330 si->lfb_base = fb_base; 333 si->lfb_base = fb_base;
331 si->lfb_size = fb_size;
332 si->pages = 1; 334 si->pages = 1;
333 335
334 if (pixel_format == PIXEL_RGB_RESERVED_8BIT_PER_COLOR) { 336 if (pixel_format == PIXEL_RGB_RESERVED_8BIT_PER_COLOR) {
@@ -376,6 +378,10 @@ static efi_status_t setup_gop(struct screen_info *si, efi_guid_t *proto,
376 si->rsvd_pos = 0; 378 si->rsvd_pos = 0;
377 } 379 }
378 380
381 si->lfb_size = si->lfb_linelength * si->lfb_height;
382
383 si->capabilities |= VIDEO_CAPABILITY_SKIP_QUIRKS;
384
379free_handle: 385free_handle:
380 efi_call_phys1(sys_table->boottime->free_pool, gop_handle); 386 efi_call_phys1(sys_table->boottime->free_pool, gop_handle);
381 return status; 387 return status;
diff --git a/arch/x86/boot/compressed/eboot.h b/arch/x86/boot/compressed/eboot.h
index 3b6e15627c55..e5b0a8f91c5f 100644
--- a/arch/x86/boot/compressed/eboot.h
+++ b/arch/x86/boot/compressed/eboot.h
@@ -14,6 +14,10 @@
14#define EFI_PAGE_SIZE (1UL << EFI_PAGE_SHIFT) 14#define EFI_PAGE_SIZE (1UL << EFI_PAGE_SHIFT)
15#define EFI_READ_CHUNK_SIZE (1024 * 1024) 15#define EFI_READ_CHUNK_SIZE (1024 * 1024)
16 16
17#define EFI_CONSOLE_OUT_DEVICE_GUID \
18 EFI_GUID(0xd3b36f2c, 0xd551, 0x11d4, 0x9a, 0x46, 0x0, 0x90, 0x27, \
19 0x3f, 0xc1, 0x4d)
20
17#define PIXEL_RGB_RESERVED_8BIT_PER_COLOR 0 21#define PIXEL_RGB_RESERVED_8BIT_PER_COLOR 0
18#define PIXEL_BGR_RESERVED_8BIT_PER_COLOR 1 22#define PIXEL_BGR_RESERVED_8BIT_PER_COLOR 1
19#define PIXEL_BIT_MASK 2 23#define PIXEL_BIT_MASK 2
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index b4e15dd6786a..2a017441b8b2 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -32,10 +32,6 @@ SYSSEG = 0x1000 /* historical load address >> 4 */
32#define SVGA_MODE ASK_VGA 32#define SVGA_MODE ASK_VGA
33#endif 33#endif
34 34
35#ifndef RAMDISK
36#define RAMDISK 0
37#endif
38
39#ifndef ROOT_RDONLY 35#ifndef ROOT_RDONLY
40#define ROOT_RDONLY 1 36#define ROOT_RDONLY 1
41#endif 37#endif
diff --git a/arch/x86/boot/mkcpustr.c b/arch/x86/boot/mkcpustr.c
index 919257f526f2..4579eff0ef4d 100644
--- a/arch/x86/boot/mkcpustr.c
+++ b/arch/x86/boot/mkcpustr.c
@@ -15,6 +15,8 @@
15 15
16#include <stdio.h> 16#include <stdio.h>
17 17
18#include "../include/asm/required-features.h"
19#include "../include/asm/cpufeature.h"
18#include "../kernel/cpu/capflags.c" 20#include "../kernel/cpu/capflags.c"
19 21
20int main(void) 22int main(void)
diff --git a/arch/x86/configs/i386_defconfig b/arch/x86/configs/i386_defconfig
index 119db67dcb03..5598547281a7 100644
--- a/arch/x86/configs/i386_defconfig
+++ b/arch/x86/configs/i386_defconfig
@@ -8,6 +8,8 @@ CONFIG_TASK_DELAY_ACCT=y
8CONFIG_TASK_XACCT=y 8CONFIG_TASK_XACCT=y
9CONFIG_TASK_IO_ACCOUNTING=y 9CONFIG_TASK_IO_ACCOUNTING=y
10CONFIG_AUDIT=y 10CONFIG_AUDIT=y
11CONFIG_NO_HZ=y
12CONFIG_HIGH_RES_TIMERS=y
11CONFIG_LOG_BUF_SHIFT=18 13CONFIG_LOG_BUF_SHIFT=18
12CONFIG_CGROUPS=y 14CONFIG_CGROUPS=y
13CONFIG_CGROUP_FREEZER=y 15CONFIG_CGROUP_FREEZER=y
@@ -34,8 +36,6 @@ CONFIG_SGI_PARTITION=y
34CONFIG_SUN_PARTITION=y 36CONFIG_SUN_PARTITION=y
35CONFIG_KARMA_PARTITION=y 37CONFIG_KARMA_PARTITION=y
36CONFIG_EFI_PARTITION=y 38CONFIG_EFI_PARTITION=y
37CONFIG_NO_HZ=y
38CONFIG_HIGH_RES_TIMERS=y
39CONFIG_SMP=y 39CONFIG_SMP=y
40CONFIG_X86_GENERIC=y 40CONFIG_X86_GENERIC=y
41CONFIG_HPET_TIMER=y 41CONFIG_HPET_TIMER=y
@@ -144,8 +144,6 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
144CONFIG_DEBUG_DEVRES=y 144CONFIG_DEBUG_DEVRES=y
145CONFIG_CONNECTOR=y 145CONFIG_CONNECTOR=y
146CONFIG_BLK_DEV_LOOP=y 146CONFIG_BLK_DEV_LOOP=y
147CONFIG_BLK_DEV_RAM=y
148CONFIG_BLK_DEV_RAM_SIZE=16384
149CONFIG_BLK_DEV_SD=y 147CONFIG_BLK_DEV_SD=y
150CONFIG_BLK_DEV_SR=y 148CONFIG_BLK_DEV_SR=y
151CONFIG_BLK_DEV_SR_VENDOR=y 149CONFIG_BLK_DEV_SR_VENDOR=y
@@ -231,8 +229,6 @@ CONFIG_SND_HRTIMER=y
231CONFIG_SND_HDA_INTEL=y 229CONFIG_SND_HDA_INTEL=y
232CONFIG_SND_HDA_HWDEP=y 230CONFIG_SND_HDA_HWDEP=y
233CONFIG_HIDRAW=y 231CONFIG_HIDRAW=y
234CONFIG_HID_PID=y
235CONFIG_USB_HIDDEV=y
236CONFIG_HID_GYRATION=y 232CONFIG_HID_GYRATION=y
237CONFIG_LOGITECH_FF=y 233CONFIG_LOGITECH_FF=y
238CONFIG_HID_NTRIG=y 234CONFIG_HID_NTRIG=y
@@ -243,11 +239,11 @@ CONFIG_HID_SAMSUNG=y
243CONFIG_HID_SONY=y 239CONFIG_HID_SONY=y
244CONFIG_HID_SUNPLUS=y 240CONFIG_HID_SUNPLUS=y
245CONFIG_HID_TOPSEED=y 241CONFIG_HID_TOPSEED=y
242CONFIG_HID_PID=y
243CONFIG_USB_HIDDEV=y
246CONFIG_USB=y 244CONFIG_USB=y
247CONFIG_USB_DEBUG=y 245CONFIG_USB_DEBUG=y
248CONFIG_USB_ANNOUNCE_NEW_DEVICES=y 246CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
249CONFIG_USB_DEVICEFS=y
250# CONFIG_USB_DEVICE_CLASS is not set
251CONFIG_USB_MON=y 247CONFIG_USB_MON=y
252CONFIG_USB_EHCI_HCD=y 248CONFIG_USB_EHCI_HCD=y
253# CONFIG_USB_EHCI_TT_NEWSCHED is not set 249# CONFIG_USB_EHCI_TT_NEWSCHED is not set
@@ -262,10 +258,9 @@ CONFIG_RTC_CLASS=y
262CONFIG_DMADEVICES=y 258CONFIG_DMADEVICES=y
263CONFIG_EEEPC_LAPTOP=y 259CONFIG_EEEPC_LAPTOP=y
264CONFIG_EFI_VARS=y 260CONFIG_EFI_VARS=y
265CONFIG_EXT3_FS=y 261CONFIG_EXT4_FS=y
266# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set 262CONFIG_EXT4_FS_POSIX_ACL=y
267CONFIG_EXT3_FS_POSIX_ACL=y 263CONFIG_EXT4_FS_SECURITY=y
268CONFIG_EXT3_FS_SECURITY=y
269CONFIG_QUOTA=y 264CONFIG_QUOTA=y
270CONFIG_QUOTA_NETLINK_INTERFACE=y 265CONFIG_QUOTA_NETLINK_INTERFACE=y
271# CONFIG_PRINT_QUOTA_WARNING is not set 266# CONFIG_PRINT_QUOTA_WARNING is not set
@@ -280,7 +275,6 @@ CONFIG_PROC_KCORE=y
280CONFIG_TMPFS_POSIX_ACL=y 275CONFIG_TMPFS_POSIX_ACL=y
281CONFIG_HUGETLBFS=y 276CONFIG_HUGETLBFS=y
282CONFIG_NFS_FS=y 277CONFIG_NFS_FS=y
283CONFIG_NFS_V3=y
284CONFIG_NFS_V3_ACL=y 278CONFIG_NFS_V3_ACL=y
285CONFIG_NFS_V4=y 279CONFIG_NFS_V4=y
286CONFIG_ROOT_NFS=y 280CONFIG_ROOT_NFS=y
@@ -299,13 +293,11 @@ CONFIG_DEBUG_KERNEL=y
299CONFIG_SCHEDSTATS=y 293CONFIG_SCHEDSTATS=y
300CONFIG_TIMER_STATS=y 294CONFIG_TIMER_STATS=y
301CONFIG_DEBUG_STACK_USAGE=y 295CONFIG_DEBUG_STACK_USAGE=y
302CONFIG_SYSCTL_SYSCALL_CHECK=y
303CONFIG_BLK_DEV_IO_TRACE=y 296CONFIG_BLK_DEV_IO_TRACE=y
304CONFIG_PROVIDE_OHCI1394_DMA_INIT=y 297CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
305CONFIG_EARLY_PRINTK_DBGP=y 298CONFIG_EARLY_PRINTK_DBGP=y
306CONFIG_DEBUG_STACKOVERFLOW=y 299CONFIG_DEBUG_STACKOVERFLOW=y
307# CONFIG_DEBUG_RODATA_TEST is not set 300# CONFIG_DEBUG_RODATA_TEST is not set
308CONFIG_DEBUG_NX_TEST=m
309CONFIG_DEBUG_BOOT_PARAMS=y 301CONFIG_DEBUG_BOOT_PARAMS=y
310CONFIG_OPTIMIZE_INLINING=y 302CONFIG_OPTIMIZE_INLINING=y
311CONFIG_KEYS_DEBUG_PROC_KEYS=y 303CONFIG_KEYS_DEBUG_PROC_KEYS=y
@@ -316,4 +308,3 @@ CONFIG_SECURITY_SELINUX_BOOTPARAM=y
316CONFIG_SECURITY_SELINUX_DISABLE=y 308CONFIG_SECURITY_SELINUX_DISABLE=y
317CONFIG_CRYPTO_AES_586=y 309CONFIG_CRYPTO_AES_586=y
318# CONFIG_CRYPTO_ANSI_CPRNG is not set 310# CONFIG_CRYPTO_ANSI_CPRNG is not set
319CONFIG_CRC_T10DIF=y
diff --git a/arch/x86/configs/x86_64_defconfig b/arch/x86/configs/x86_64_defconfig
index 76eb2903809f..671524d0f6c0 100644
--- a/arch/x86/configs/x86_64_defconfig
+++ b/arch/x86/configs/x86_64_defconfig
@@ -8,6 +8,8 @@ CONFIG_TASK_DELAY_ACCT=y
8CONFIG_TASK_XACCT=y 8CONFIG_TASK_XACCT=y
9CONFIG_TASK_IO_ACCOUNTING=y 9CONFIG_TASK_IO_ACCOUNTING=y
10CONFIG_AUDIT=y 10CONFIG_AUDIT=y
11CONFIG_NO_HZ=y
12CONFIG_HIGH_RES_TIMERS=y
11CONFIG_LOG_BUF_SHIFT=18 13CONFIG_LOG_BUF_SHIFT=18
12CONFIG_CGROUPS=y 14CONFIG_CGROUPS=y
13CONFIG_CGROUP_FREEZER=y 15CONFIG_CGROUP_FREEZER=y
@@ -34,8 +36,6 @@ CONFIG_SGI_PARTITION=y
34CONFIG_SUN_PARTITION=y 36CONFIG_SUN_PARTITION=y
35CONFIG_KARMA_PARTITION=y 37CONFIG_KARMA_PARTITION=y
36CONFIG_EFI_PARTITION=y 38CONFIG_EFI_PARTITION=y
37CONFIG_NO_HZ=y
38CONFIG_HIGH_RES_TIMERS=y
39CONFIG_SMP=y 39CONFIG_SMP=y
40CONFIG_CALGARY_IOMMU=y 40CONFIG_CALGARY_IOMMU=y
41CONFIG_NR_CPUS=64 41CONFIG_NR_CPUS=64
@@ -144,8 +144,6 @@ CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
144CONFIG_DEBUG_DEVRES=y 144CONFIG_DEBUG_DEVRES=y
145CONFIG_CONNECTOR=y 145CONFIG_CONNECTOR=y
146CONFIG_BLK_DEV_LOOP=y 146CONFIG_BLK_DEV_LOOP=y
147CONFIG_BLK_DEV_RAM=y
148CONFIG_BLK_DEV_RAM_SIZE=16384
149CONFIG_BLK_DEV_SD=y 147CONFIG_BLK_DEV_SD=y
150CONFIG_BLK_DEV_SR=y 148CONFIG_BLK_DEV_SR=y
151CONFIG_BLK_DEV_SR_VENDOR=y 149CONFIG_BLK_DEV_SR_VENDOR=y
@@ -227,8 +225,6 @@ CONFIG_SND_HRTIMER=y
227CONFIG_SND_HDA_INTEL=y 225CONFIG_SND_HDA_INTEL=y
228CONFIG_SND_HDA_HWDEP=y 226CONFIG_SND_HDA_HWDEP=y
229CONFIG_HIDRAW=y 227CONFIG_HIDRAW=y
230CONFIG_HID_PID=y
231CONFIG_USB_HIDDEV=y
232CONFIG_HID_GYRATION=y 228CONFIG_HID_GYRATION=y
233CONFIG_LOGITECH_FF=y 229CONFIG_LOGITECH_FF=y
234CONFIG_HID_NTRIG=y 230CONFIG_HID_NTRIG=y
@@ -239,11 +235,11 @@ CONFIG_HID_SAMSUNG=y
239CONFIG_HID_SONY=y 235CONFIG_HID_SONY=y
240CONFIG_HID_SUNPLUS=y 236CONFIG_HID_SUNPLUS=y
241CONFIG_HID_TOPSEED=y 237CONFIG_HID_TOPSEED=y
238CONFIG_HID_PID=y
239CONFIG_USB_HIDDEV=y
242CONFIG_USB=y 240CONFIG_USB=y
243CONFIG_USB_DEBUG=y 241CONFIG_USB_DEBUG=y
244CONFIG_USB_ANNOUNCE_NEW_DEVICES=y 242CONFIG_USB_ANNOUNCE_NEW_DEVICES=y
245CONFIG_USB_DEVICEFS=y
246# CONFIG_USB_DEVICE_CLASS is not set
247CONFIG_USB_MON=y 243CONFIG_USB_MON=y
248CONFIG_USB_EHCI_HCD=y 244CONFIG_USB_EHCI_HCD=y
249# CONFIG_USB_EHCI_TT_NEWSCHED is not set 245# CONFIG_USB_EHCI_TT_NEWSCHED is not set
@@ -262,10 +258,9 @@ CONFIG_AMD_IOMMU_STATS=y
262CONFIG_INTEL_IOMMU=y 258CONFIG_INTEL_IOMMU=y
263# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set 259# CONFIG_INTEL_IOMMU_DEFAULT_ON is not set
264CONFIG_EFI_VARS=y 260CONFIG_EFI_VARS=y
265CONFIG_EXT3_FS=y 261CONFIG_EXT4_FS=y
266# CONFIG_EXT3_DEFAULTS_TO_ORDERED is not set 262CONFIG_EXT4_FS_POSIX_ACL=y
267CONFIG_EXT3_FS_POSIX_ACL=y 263CONFIG_EXT4_FS_SECURITY=y
268CONFIG_EXT3_FS_SECURITY=y
269CONFIG_QUOTA=y 264CONFIG_QUOTA=y
270CONFIG_QUOTA_NETLINK_INTERFACE=y 265CONFIG_QUOTA_NETLINK_INTERFACE=y
271# CONFIG_PRINT_QUOTA_WARNING is not set 266# CONFIG_PRINT_QUOTA_WARNING is not set
@@ -280,7 +275,6 @@ CONFIG_PROC_KCORE=y
280CONFIG_TMPFS_POSIX_ACL=y 275CONFIG_TMPFS_POSIX_ACL=y
281CONFIG_HUGETLBFS=y 276CONFIG_HUGETLBFS=y
282CONFIG_NFS_FS=y 277CONFIG_NFS_FS=y
283CONFIG_NFS_V3=y
284CONFIG_NFS_V3_ACL=y 278CONFIG_NFS_V3_ACL=y
285CONFIG_NFS_V4=y 279CONFIG_NFS_V4=y
286CONFIG_ROOT_NFS=y 280CONFIG_ROOT_NFS=y
@@ -298,13 +292,11 @@ CONFIG_DEBUG_KERNEL=y
298CONFIG_SCHEDSTATS=y 292CONFIG_SCHEDSTATS=y
299CONFIG_TIMER_STATS=y 293CONFIG_TIMER_STATS=y
300CONFIG_DEBUG_STACK_USAGE=y 294CONFIG_DEBUG_STACK_USAGE=y
301CONFIG_SYSCTL_SYSCALL_CHECK=y
302CONFIG_BLK_DEV_IO_TRACE=y 295CONFIG_BLK_DEV_IO_TRACE=y
303CONFIG_PROVIDE_OHCI1394_DMA_INIT=y 296CONFIG_PROVIDE_OHCI1394_DMA_INIT=y
304CONFIG_EARLY_PRINTK_DBGP=y 297CONFIG_EARLY_PRINTK_DBGP=y
305CONFIG_DEBUG_STACKOVERFLOW=y 298CONFIG_DEBUG_STACKOVERFLOW=y
306# CONFIG_DEBUG_RODATA_TEST is not set 299# CONFIG_DEBUG_RODATA_TEST is not set
307CONFIG_DEBUG_NX_TEST=m
308CONFIG_DEBUG_BOOT_PARAMS=y 300CONFIG_DEBUG_BOOT_PARAMS=y
309CONFIG_OPTIMIZE_INLINING=y 301CONFIG_OPTIMIZE_INLINING=y
310CONFIG_KEYS_DEBUG_PROC_KEYS=y 302CONFIG_KEYS_DEBUG_PROC_KEYS=y
@@ -314,4 +306,3 @@ CONFIG_SECURITY_SELINUX=y
314CONFIG_SECURITY_SELINUX_BOOTPARAM=y 306CONFIG_SECURITY_SELINUX_BOOTPARAM=y
315CONFIG_SECURITY_SELINUX_DISABLE=y 307CONFIG_SECURITY_SELINUX_DISABLE=y
316# CONFIG_CRYPTO_ANSI_CPRNG is not set 308# CONFIG_CRYPTO_ANSI_CPRNG is not set
317CONFIG_CRC_T10DIF=y
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e908e5de82d3..5bacb4a226ac 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -12,6 +12,8 @@ obj-$(CONFIG_CRYPTO_SERPENT_SSE2_586) += serpent-sse2-i586.o
12 12
13obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o 13obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o
14obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o 14obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o
15obj-$(CONFIG_CRYPTO_CAST5_AVX_X86_64) += cast5-avx-x86_64.o
16obj-$(CONFIG_CRYPTO_CAST6_AVX_X86_64) += cast6-avx-x86_64.o
15obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o 17obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
16obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o 18obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
17obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o 19obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o
@@ -32,6 +34,8 @@ serpent-sse2-i586-y := serpent-sse2-i586-asm_32.o serpent_sse2_glue.o
32 34
33aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o 35aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o
34camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o 36camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o
37cast5-avx-x86_64-y := cast5-avx-x86_64-asm_64.o cast5_avx_glue.o
38cast6-avx-x86_64-y := cast6-avx-x86_64-asm_64.o cast6_avx_glue.o
35blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o 39blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
36twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o 40twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
37twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o 41twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o
diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c
index 59b37deb8c8d..aafe8ce0d65d 100644
--- a/arch/x86/crypto/aes_glue.c
+++ b/arch/x86/crypto/aes_glue.c
@@ -40,7 +40,6 @@ static struct crypto_alg aes_alg = {
40 .cra_blocksize = AES_BLOCK_SIZE, 40 .cra_blocksize = AES_BLOCK_SIZE,
41 .cra_ctxsize = sizeof(struct crypto_aes_ctx), 41 .cra_ctxsize = sizeof(struct crypto_aes_ctx),
42 .cra_module = THIS_MODULE, 42 .cra_module = THIS_MODULE,
43 .cra_list = LIST_HEAD_INIT(aes_alg.cra_list),
44 .cra_u = { 43 .cra_u = {
45 .cipher = { 44 .cipher = {
46 .cia_min_keysize = AES_MIN_KEY_SIZE, 45 .cia_min_keysize = AES_MIN_KEY_SIZE,
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index 34fdcff4d2c8..7c04d0da709b 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -28,6 +28,9 @@
28#include <crypto/aes.h> 28#include <crypto/aes.h>
29#include <crypto/cryptd.h> 29#include <crypto/cryptd.h>
30#include <crypto/ctr.h> 30#include <crypto/ctr.h>
31#include <crypto/b128ops.h>
32#include <crypto/lrw.h>
33#include <crypto/xts.h>
31#include <asm/cpu_device_id.h> 34#include <asm/cpu_device_id.h>
32#include <asm/i387.h> 35#include <asm/i387.h>
33#include <asm/crypto/aes.h> 36#include <asm/crypto/aes.h>
@@ -41,18 +44,10 @@
41#define HAS_CTR 44#define HAS_CTR
42#endif 45#endif
43 46
44#if defined(CONFIG_CRYPTO_LRW) || defined(CONFIG_CRYPTO_LRW_MODULE)
45#define HAS_LRW
46#endif
47
48#if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE) 47#if defined(CONFIG_CRYPTO_PCBC) || defined(CONFIG_CRYPTO_PCBC_MODULE)
49#define HAS_PCBC 48#define HAS_PCBC
50#endif 49#endif
51 50
52#if defined(CONFIG_CRYPTO_XTS) || defined(CONFIG_CRYPTO_XTS_MODULE)
53#define HAS_XTS
54#endif
55
56/* This data is stored at the end of the crypto_tfm struct. 51/* This data is stored at the end of the crypto_tfm struct.
57 * It's a type of per "session" data storage location. 52 * It's a type of per "session" data storage location.
58 * This needs to be 16 byte aligned. 53 * This needs to be 16 byte aligned.
@@ -79,6 +74,16 @@ struct aesni_hash_subkey_req_data {
79#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) 74#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1))
80#define RFC4106_HASH_SUBKEY_SIZE 16 75#define RFC4106_HASH_SUBKEY_SIZE 16
81 76
77struct aesni_lrw_ctx {
78 struct lrw_table_ctx lrw_table;
79 u8 raw_aes_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
80};
81
82struct aesni_xts_ctx {
83 u8 raw_tweak_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
84 u8 raw_crypt_ctx[sizeof(struct crypto_aes_ctx) + AESNI_ALIGN - 1];
85};
86
82asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, 87asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
83 unsigned int key_len); 88 unsigned int key_len);
84asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, 89asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out,
@@ -398,13 +403,6 @@ static int ablk_rfc3686_ctr_init(struct crypto_tfm *tfm)
398#endif 403#endif
399#endif 404#endif
400 405
401#ifdef HAS_LRW
402static int ablk_lrw_init(struct crypto_tfm *tfm)
403{
404 return ablk_init_common(tfm, "fpu(lrw(__driver-aes-aesni))");
405}
406#endif
407
408#ifdef HAS_PCBC 406#ifdef HAS_PCBC
409static int ablk_pcbc_init(struct crypto_tfm *tfm) 407static int ablk_pcbc_init(struct crypto_tfm *tfm)
410{ 408{
@@ -412,12 +410,160 @@ static int ablk_pcbc_init(struct crypto_tfm *tfm)
412} 410}
413#endif 411#endif
414 412
415#ifdef HAS_XTS 413static void lrw_xts_encrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
416static int ablk_xts_init(struct crypto_tfm *tfm)
417{ 414{
418 return ablk_init_common(tfm, "fpu(xts(__driver-aes-aesni))"); 415 aesni_ecb_enc(ctx, blks, blks, nbytes);
416}
417
418static void lrw_xts_decrypt_callback(void *ctx, u8 *blks, unsigned int nbytes)
419{
420 aesni_ecb_dec(ctx, blks, blks, nbytes);
421}
422
423static int lrw_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
424 unsigned int keylen)
425{
426 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
427 int err;
428
429 err = aes_set_key_common(tfm, ctx->raw_aes_ctx, key,
430 keylen - AES_BLOCK_SIZE);
431 if (err)
432 return err;
433
434 return lrw_init_table(&ctx->lrw_table, key + keylen - AES_BLOCK_SIZE);
435}
436
437static void lrw_aesni_exit_tfm(struct crypto_tfm *tfm)
438{
439 struct aesni_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
440
441 lrw_free_table(&ctx->lrw_table);
442}
443
444static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
445 struct scatterlist *src, unsigned int nbytes)
446{
447 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
448 be128 buf[8];
449 struct lrw_crypt_req req = {
450 .tbuf = buf,
451 .tbuflen = sizeof(buf),
452
453 .table_ctx = &ctx->lrw_table,
454 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
455 .crypt_fn = lrw_xts_encrypt_callback,
456 };
457 int ret;
458
459 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
460
461 kernel_fpu_begin();
462 ret = lrw_crypt(desc, dst, src, nbytes, &req);
463 kernel_fpu_end();
464
465 return ret;
466}
467
468static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
469 struct scatterlist *src, unsigned int nbytes)
470{
471 struct aesni_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
472 be128 buf[8];
473 struct lrw_crypt_req req = {
474 .tbuf = buf,
475 .tbuflen = sizeof(buf),
476
477 .table_ctx = &ctx->lrw_table,
478 .crypt_ctx = aes_ctx(ctx->raw_aes_ctx),
479 .crypt_fn = lrw_xts_decrypt_callback,
480 };
481 int ret;
482
483 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
484
485 kernel_fpu_begin();
486 ret = lrw_crypt(desc, dst, src, nbytes, &req);
487 kernel_fpu_end();
488
489 return ret;
490}
491
492static int xts_aesni_setkey(struct crypto_tfm *tfm, const u8 *key,
493 unsigned int keylen)
494{
495 struct aesni_xts_ctx *ctx = crypto_tfm_ctx(tfm);
496 u32 *flags = &tfm->crt_flags;
497 int err;
498
499 /* key consists of keys of equal size concatenated, therefore
500 * the length must be even
501 */
502 if (keylen % 2) {
503 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
504 return -EINVAL;
505 }
506
507 /* first half of xts-key is for crypt */
508 err = aes_set_key_common(tfm, ctx->raw_crypt_ctx, key, keylen / 2);
509 if (err)
510 return err;
511
512 /* second half of xts-key is for tweak */
513 return aes_set_key_common(tfm, ctx->raw_tweak_ctx, key + keylen / 2,
514 keylen / 2);
515}
516
517
518static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
519 struct scatterlist *src, unsigned int nbytes)
520{
521 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
522 be128 buf[8];
523 struct xts_crypt_req req = {
524 .tbuf = buf,
525 .tbuflen = sizeof(buf),
526
527 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
528 .tweak_fn = XTS_TWEAK_CAST(aesni_enc),
529 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
530 .crypt_fn = lrw_xts_encrypt_callback,
531 };
532 int ret;
533
534 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
535
536 kernel_fpu_begin();
537 ret = xts_crypt(desc, dst, src, nbytes, &req);
538 kernel_fpu_end();
539
540 return ret;
541}
542
543static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
544 struct scatterlist *src, unsigned int nbytes)
545{
546 struct aesni_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
547 be128 buf[8];
548 struct xts_crypt_req req = {
549 .tbuf = buf,
550 .tbuflen = sizeof(buf),
551
552 .tweak_ctx = aes_ctx(ctx->raw_tweak_ctx),
553 .tweak_fn = XTS_TWEAK_CAST(aesni_enc),
554 .crypt_ctx = aes_ctx(ctx->raw_crypt_ctx),
555 .crypt_fn = lrw_xts_decrypt_callback,
556 };
557 int ret;
558
559 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
560
561 kernel_fpu_begin();
562 ret = xts_crypt(desc, dst, src, nbytes, &req);
563 kernel_fpu_end();
564
565 return ret;
419} 566}
420#endif
421 567
422#ifdef CONFIG_X86_64 568#ifdef CONFIG_X86_64
423static int rfc4106_init(struct crypto_tfm *tfm) 569static int rfc4106_init(struct crypto_tfm *tfm)
@@ -1035,10 +1181,10 @@ static struct crypto_alg aesni_algs[] = { {
1035 }, 1181 },
1036#endif 1182#endif
1037#endif 1183#endif
1038#ifdef HAS_LRW 1184#ifdef HAS_PCBC
1039}, { 1185}, {
1040 .cra_name = "lrw(aes)", 1186 .cra_name = "pcbc(aes)",
1041 .cra_driver_name = "lrw-aes-aesni", 1187 .cra_driver_name = "pcbc-aes-aesni",
1042 .cra_priority = 400, 1188 .cra_priority = 400,
1043 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1189 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1044 .cra_blocksize = AES_BLOCK_SIZE, 1190 .cra_blocksize = AES_BLOCK_SIZE,
@@ -1046,12 +1192,12 @@ static struct crypto_alg aesni_algs[] = { {
1046 .cra_alignmask = 0, 1192 .cra_alignmask = 0,
1047 .cra_type = &crypto_ablkcipher_type, 1193 .cra_type = &crypto_ablkcipher_type,
1048 .cra_module = THIS_MODULE, 1194 .cra_module = THIS_MODULE,
1049 .cra_init = ablk_lrw_init, 1195 .cra_init = ablk_pcbc_init,
1050 .cra_exit = ablk_exit, 1196 .cra_exit = ablk_exit,
1051 .cra_u = { 1197 .cra_u = {
1052 .ablkcipher = { 1198 .ablkcipher = {
1053 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE, 1199 .min_keysize = AES_MIN_KEY_SIZE,
1054 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE, 1200 .max_keysize = AES_MAX_KEY_SIZE,
1055 .ivsize = AES_BLOCK_SIZE, 1201 .ivsize = AES_BLOCK_SIZE,
1056 .setkey = ablk_set_key, 1202 .setkey = ablk_set_key,
1057 .encrypt = ablk_encrypt, 1203 .encrypt = ablk_encrypt,
@@ -1059,10 +1205,50 @@ static struct crypto_alg aesni_algs[] = { {
1059 }, 1205 },
1060 }, 1206 },
1061#endif 1207#endif
1062#ifdef HAS_PCBC
1063}, { 1208}, {
1064 .cra_name = "pcbc(aes)", 1209 .cra_name = "__lrw-aes-aesni",
1065 .cra_driver_name = "pcbc-aes-aesni", 1210 .cra_driver_name = "__driver-lrw-aes-aesni",
1211 .cra_priority = 0,
1212 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1213 .cra_blocksize = AES_BLOCK_SIZE,
1214 .cra_ctxsize = sizeof(struct aesni_lrw_ctx),
1215 .cra_alignmask = 0,
1216 .cra_type = &crypto_blkcipher_type,
1217 .cra_module = THIS_MODULE,
1218 .cra_exit = lrw_aesni_exit_tfm,
1219 .cra_u = {
1220 .blkcipher = {
1221 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1222 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1223 .ivsize = AES_BLOCK_SIZE,
1224 .setkey = lrw_aesni_setkey,
1225 .encrypt = lrw_encrypt,
1226 .decrypt = lrw_decrypt,
1227 },
1228 },
1229}, {
1230 .cra_name = "__xts-aes-aesni",
1231 .cra_driver_name = "__driver-xts-aes-aesni",
1232 .cra_priority = 0,
1233 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
1234 .cra_blocksize = AES_BLOCK_SIZE,
1235 .cra_ctxsize = sizeof(struct aesni_xts_ctx),
1236 .cra_alignmask = 0,
1237 .cra_type = &crypto_blkcipher_type,
1238 .cra_module = THIS_MODULE,
1239 .cra_u = {
1240 .blkcipher = {
1241 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1242 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1243 .ivsize = AES_BLOCK_SIZE,
1244 .setkey = xts_aesni_setkey,
1245 .encrypt = xts_encrypt,
1246 .decrypt = xts_decrypt,
1247 },
1248 },
1249}, {
1250 .cra_name = "lrw(aes)",
1251 .cra_driver_name = "lrw-aes-aesni",
1066 .cra_priority = 400, 1252 .cra_priority = 400,
1067 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1253 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1068 .cra_blocksize = AES_BLOCK_SIZE, 1254 .cra_blocksize = AES_BLOCK_SIZE,
@@ -1070,20 +1256,18 @@ static struct crypto_alg aesni_algs[] = { {
1070 .cra_alignmask = 0, 1256 .cra_alignmask = 0,
1071 .cra_type = &crypto_ablkcipher_type, 1257 .cra_type = &crypto_ablkcipher_type,
1072 .cra_module = THIS_MODULE, 1258 .cra_module = THIS_MODULE,
1073 .cra_init = ablk_pcbc_init, 1259 .cra_init = ablk_init,
1074 .cra_exit = ablk_exit, 1260 .cra_exit = ablk_exit,
1075 .cra_u = { 1261 .cra_u = {
1076 .ablkcipher = { 1262 .ablkcipher = {
1077 .min_keysize = AES_MIN_KEY_SIZE, 1263 .min_keysize = AES_MIN_KEY_SIZE + AES_BLOCK_SIZE,
1078 .max_keysize = AES_MAX_KEY_SIZE, 1264 .max_keysize = AES_MAX_KEY_SIZE + AES_BLOCK_SIZE,
1079 .ivsize = AES_BLOCK_SIZE, 1265 .ivsize = AES_BLOCK_SIZE,
1080 .setkey = ablk_set_key, 1266 .setkey = ablk_set_key,
1081 .encrypt = ablk_encrypt, 1267 .encrypt = ablk_encrypt,
1082 .decrypt = ablk_decrypt, 1268 .decrypt = ablk_decrypt,
1083 }, 1269 },
1084 }, 1270 },
1085#endif
1086#ifdef HAS_XTS
1087}, { 1271}, {
1088 .cra_name = "xts(aes)", 1272 .cra_name = "xts(aes)",
1089 .cra_driver_name = "xts-aes-aesni", 1273 .cra_driver_name = "xts-aes-aesni",
@@ -1094,7 +1278,7 @@ static struct crypto_alg aesni_algs[] = { {
1094 .cra_alignmask = 0, 1278 .cra_alignmask = 0,
1095 .cra_type = &crypto_ablkcipher_type, 1279 .cra_type = &crypto_ablkcipher_type,
1096 .cra_module = THIS_MODULE, 1280 .cra_module = THIS_MODULE,
1097 .cra_init = ablk_xts_init, 1281 .cra_init = ablk_init,
1098 .cra_exit = ablk_exit, 1282 .cra_exit = ablk_exit,
1099 .cra_u = { 1283 .cra_u = {
1100 .ablkcipher = { 1284 .ablkcipher = {
@@ -1106,7 +1290,6 @@ static struct crypto_alg aesni_algs[] = { {
1106 .decrypt = ablk_decrypt, 1290 .decrypt = ablk_decrypt,
1107 }, 1291 },
1108 }, 1292 },
1109#endif
1110} }; 1293} };
1111 1294
1112 1295
@@ -1118,7 +1301,7 @@ MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1118 1301
1119static int __init aesni_init(void) 1302static int __init aesni_init(void)
1120{ 1303{
1121 int err, i; 1304 int err;
1122 1305
1123 if (!x86_match_cpu(aesni_cpu_id)) 1306 if (!x86_match_cpu(aesni_cpu_id))
1124 return -ENODEV; 1307 return -ENODEV;
@@ -1127,9 +1310,6 @@ static int __init aesni_init(void)
1127 if (err) 1310 if (err)
1128 return err; 1311 return err;
1129 1312
1130 for (i = 0; i < ARRAY_SIZE(aesni_algs); i++)
1131 INIT_LIST_HEAD(&aesni_algs[i].cra_list);
1132
1133 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); 1313 return crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs));
1134} 1314}
1135 1315
diff --git a/arch/x86/crypto/blowfish_glue.c b/arch/x86/crypto/blowfish_glue.c
index 7967474de8f7..50ec333b70e6 100644
--- a/arch/x86/crypto/blowfish_glue.c
+++ b/arch/x86/crypto/blowfish_glue.c
@@ -367,7 +367,6 @@ static struct crypto_alg bf_algs[4] = { {
367 .cra_ctxsize = sizeof(struct bf_ctx), 367 .cra_ctxsize = sizeof(struct bf_ctx),
368 .cra_alignmask = 0, 368 .cra_alignmask = 0,
369 .cra_module = THIS_MODULE, 369 .cra_module = THIS_MODULE,
370 .cra_list = LIST_HEAD_INIT(bf_algs[0].cra_list),
371 .cra_u = { 370 .cra_u = {
372 .cipher = { 371 .cipher = {
373 .cia_min_keysize = BF_MIN_KEY_SIZE, 372 .cia_min_keysize = BF_MIN_KEY_SIZE,
@@ -387,7 +386,6 @@ static struct crypto_alg bf_algs[4] = { {
387 .cra_alignmask = 0, 386 .cra_alignmask = 0,
388 .cra_type = &crypto_blkcipher_type, 387 .cra_type = &crypto_blkcipher_type,
389 .cra_module = THIS_MODULE, 388 .cra_module = THIS_MODULE,
390 .cra_list = LIST_HEAD_INIT(bf_algs[1].cra_list),
391 .cra_u = { 389 .cra_u = {
392 .blkcipher = { 390 .blkcipher = {
393 .min_keysize = BF_MIN_KEY_SIZE, 391 .min_keysize = BF_MIN_KEY_SIZE,
@@ -407,7 +405,6 @@ static struct crypto_alg bf_algs[4] = { {
407 .cra_alignmask = 0, 405 .cra_alignmask = 0,
408 .cra_type = &crypto_blkcipher_type, 406 .cra_type = &crypto_blkcipher_type,
409 .cra_module = THIS_MODULE, 407 .cra_module = THIS_MODULE,
410 .cra_list = LIST_HEAD_INIT(bf_algs[2].cra_list),
411 .cra_u = { 408 .cra_u = {
412 .blkcipher = { 409 .blkcipher = {
413 .min_keysize = BF_MIN_KEY_SIZE, 410 .min_keysize = BF_MIN_KEY_SIZE,
@@ -428,7 +425,6 @@ static struct crypto_alg bf_algs[4] = { {
428 .cra_alignmask = 0, 425 .cra_alignmask = 0,
429 .cra_type = &crypto_blkcipher_type, 426 .cra_type = &crypto_blkcipher_type,
430 .cra_module = THIS_MODULE, 427 .cra_module = THIS_MODULE,
431 .cra_list = LIST_HEAD_INIT(bf_algs[3].cra_list),
432 .cra_u = { 428 .cra_u = {
433 .blkcipher = { 429 .blkcipher = {
434 .min_keysize = BF_MIN_KEY_SIZE, 430 .min_keysize = BF_MIN_KEY_SIZE,
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index eeb2b3b743e9..42ffd2bbab5b 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -92,715 +92,715 @@ static void camellia_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
92 92
93/* camellia sboxes */ 93/* camellia sboxes */
94const u64 camellia_sp10011110[256] = { 94const u64 camellia_sp10011110[256] = {
95 0x7000007070707000, 0x8200008282828200, 0x2c00002c2c2c2c00, 95 0x7000007070707000ULL, 0x8200008282828200ULL, 0x2c00002c2c2c2c00ULL,
96 0xec0000ecececec00, 0xb30000b3b3b3b300, 0x2700002727272700, 96 0xec0000ecececec00ULL, 0xb30000b3b3b3b300ULL, 0x2700002727272700ULL,
97 0xc00000c0c0c0c000, 0xe50000e5e5e5e500, 0xe40000e4e4e4e400, 97 0xc00000c0c0c0c000ULL, 0xe50000e5e5e5e500ULL, 0xe40000e4e4e4e400ULL,
98 0x8500008585858500, 0x5700005757575700, 0x3500003535353500, 98 0x8500008585858500ULL, 0x5700005757575700ULL, 0x3500003535353500ULL,
99 0xea0000eaeaeaea00, 0x0c00000c0c0c0c00, 0xae0000aeaeaeae00, 99 0xea0000eaeaeaea00ULL, 0x0c00000c0c0c0c00ULL, 0xae0000aeaeaeae00ULL,
100 0x4100004141414100, 0x2300002323232300, 0xef0000efefefef00, 100 0x4100004141414100ULL, 0x2300002323232300ULL, 0xef0000efefefef00ULL,
101 0x6b00006b6b6b6b00, 0x9300009393939300, 0x4500004545454500, 101 0x6b00006b6b6b6b00ULL, 0x9300009393939300ULL, 0x4500004545454500ULL,
102 0x1900001919191900, 0xa50000a5a5a5a500, 0x2100002121212100, 102 0x1900001919191900ULL, 0xa50000a5a5a5a500ULL, 0x2100002121212100ULL,
103 0xed0000edededed00, 0x0e00000e0e0e0e00, 0x4f00004f4f4f4f00, 103 0xed0000edededed00ULL, 0x0e00000e0e0e0e00ULL, 0x4f00004f4f4f4f00ULL,
104 0x4e00004e4e4e4e00, 0x1d00001d1d1d1d00, 0x6500006565656500, 104 0x4e00004e4e4e4e00ULL, 0x1d00001d1d1d1d00ULL, 0x6500006565656500ULL,
105 0x9200009292929200, 0xbd0000bdbdbdbd00, 0x8600008686868600, 105 0x9200009292929200ULL, 0xbd0000bdbdbdbd00ULL, 0x8600008686868600ULL,
106 0xb80000b8b8b8b800, 0xaf0000afafafaf00, 0x8f00008f8f8f8f00, 106 0xb80000b8b8b8b800ULL, 0xaf0000afafafaf00ULL, 0x8f00008f8f8f8f00ULL,
107 0x7c00007c7c7c7c00, 0xeb0000ebebebeb00, 0x1f00001f1f1f1f00, 107 0x7c00007c7c7c7c00ULL, 0xeb0000ebebebeb00ULL, 0x1f00001f1f1f1f00ULL,
108 0xce0000cececece00, 0x3e00003e3e3e3e00, 0x3000003030303000, 108 0xce0000cececece00ULL, 0x3e00003e3e3e3e00ULL, 0x3000003030303000ULL,
109 0xdc0000dcdcdcdc00, 0x5f00005f5f5f5f00, 0x5e00005e5e5e5e00, 109 0xdc0000dcdcdcdc00ULL, 0x5f00005f5f5f5f00ULL, 0x5e00005e5e5e5e00ULL,
110 0xc50000c5c5c5c500, 0x0b00000b0b0b0b00, 0x1a00001a1a1a1a00, 110 0xc50000c5c5c5c500ULL, 0x0b00000b0b0b0b00ULL, 0x1a00001a1a1a1a00ULL,
111 0xa60000a6a6a6a600, 0xe10000e1e1e1e100, 0x3900003939393900, 111 0xa60000a6a6a6a600ULL, 0xe10000e1e1e1e100ULL, 0x3900003939393900ULL,
112 0xca0000cacacaca00, 0xd50000d5d5d5d500, 0x4700004747474700, 112 0xca0000cacacaca00ULL, 0xd50000d5d5d5d500ULL, 0x4700004747474700ULL,
113 0x5d00005d5d5d5d00, 0x3d00003d3d3d3d00, 0xd90000d9d9d9d900, 113 0x5d00005d5d5d5d00ULL, 0x3d00003d3d3d3d00ULL, 0xd90000d9d9d9d900ULL,
114 0x0100000101010100, 0x5a00005a5a5a5a00, 0xd60000d6d6d6d600, 114 0x0100000101010100ULL, 0x5a00005a5a5a5a00ULL, 0xd60000d6d6d6d600ULL,
115 0x5100005151515100, 0x5600005656565600, 0x6c00006c6c6c6c00, 115 0x5100005151515100ULL, 0x5600005656565600ULL, 0x6c00006c6c6c6c00ULL,
116 0x4d00004d4d4d4d00, 0x8b00008b8b8b8b00, 0x0d00000d0d0d0d00, 116 0x4d00004d4d4d4d00ULL, 0x8b00008b8b8b8b00ULL, 0x0d00000d0d0d0d00ULL,
117 0x9a00009a9a9a9a00, 0x6600006666666600, 0xfb0000fbfbfbfb00, 117 0x9a00009a9a9a9a00ULL, 0x6600006666666600ULL, 0xfb0000fbfbfbfb00ULL,
118 0xcc0000cccccccc00, 0xb00000b0b0b0b000, 0x2d00002d2d2d2d00, 118 0xcc0000cccccccc00ULL, 0xb00000b0b0b0b000ULL, 0x2d00002d2d2d2d00ULL,
119 0x7400007474747400, 0x1200001212121200, 0x2b00002b2b2b2b00, 119 0x7400007474747400ULL, 0x1200001212121200ULL, 0x2b00002b2b2b2b00ULL,
120 0x2000002020202000, 0xf00000f0f0f0f000, 0xb10000b1b1b1b100, 120 0x2000002020202000ULL, 0xf00000f0f0f0f000ULL, 0xb10000b1b1b1b100ULL,
121 0x8400008484848400, 0x9900009999999900, 0xdf0000dfdfdfdf00, 121 0x8400008484848400ULL, 0x9900009999999900ULL, 0xdf0000dfdfdfdf00ULL,
122 0x4c00004c4c4c4c00, 0xcb0000cbcbcbcb00, 0xc20000c2c2c2c200, 122 0x4c00004c4c4c4c00ULL, 0xcb0000cbcbcbcb00ULL, 0xc20000c2c2c2c200ULL,
123 0x3400003434343400, 0x7e00007e7e7e7e00, 0x7600007676767600, 123 0x3400003434343400ULL, 0x7e00007e7e7e7e00ULL, 0x7600007676767600ULL,
124 0x0500000505050500, 0x6d00006d6d6d6d00, 0xb70000b7b7b7b700, 124 0x0500000505050500ULL, 0x6d00006d6d6d6d00ULL, 0xb70000b7b7b7b700ULL,
125 0xa90000a9a9a9a900, 0x3100003131313100, 0xd10000d1d1d1d100, 125 0xa90000a9a9a9a900ULL, 0x3100003131313100ULL, 0xd10000d1d1d1d100ULL,
126 0x1700001717171700, 0x0400000404040400, 0xd70000d7d7d7d700, 126 0x1700001717171700ULL, 0x0400000404040400ULL, 0xd70000d7d7d7d700ULL,
127 0x1400001414141400, 0x5800005858585800, 0x3a00003a3a3a3a00, 127 0x1400001414141400ULL, 0x5800005858585800ULL, 0x3a00003a3a3a3a00ULL,
128 0x6100006161616100, 0xde0000dededede00, 0x1b00001b1b1b1b00, 128 0x6100006161616100ULL, 0xde0000dededede00ULL, 0x1b00001b1b1b1b00ULL,
129 0x1100001111111100, 0x1c00001c1c1c1c00, 0x3200003232323200, 129 0x1100001111111100ULL, 0x1c00001c1c1c1c00ULL, 0x3200003232323200ULL,
130 0x0f00000f0f0f0f00, 0x9c00009c9c9c9c00, 0x1600001616161600, 130 0x0f00000f0f0f0f00ULL, 0x9c00009c9c9c9c00ULL, 0x1600001616161600ULL,
131 0x5300005353535300, 0x1800001818181800, 0xf20000f2f2f2f200, 131 0x5300005353535300ULL, 0x1800001818181800ULL, 0xf20000f2f2f2f200ULL,
132 0x2200002222222200, 0xfe0000fefefefe00, 0x4400004444444400, 132 0x2200002222222200ULL, 0xfe0000fefefefe00ULL, 0x4400004444444400ULL,
133 0xcf0000cfcfcfcf00, 0xb20000b2b2b2b200, 0xc30000c3c3c3c300, 133 0xcf0000cfcfcfcf00ULL, 0xb20000b2b2b2b200ULL, 0xc30000c3c3c3c300ULL,
134 0xb50000b5b5b5b500, 0x7a00007a7a7a7a00, 0x9100009191919100, 134 0xb50000b5b5b5b500ULL, 0x7a00007a7a7a7a00ULL, 0x9100009191919100ULL,
135 0x2400002424242400, 0x0800000808080800, 0xe80000e8e8e8e800, 135 0x2400002424242400ULL, 0x0800000808080800ULL, 0xe80000e8e8e8e800ULL,
136 0xa80000a8a8a8a800, 0x6000006060606000, 0xfc0000fcfcfcfc00, 136 0xa80000a8a8a8a800ULL, 0x6000006060606000ULL, 0xfc0000fcfcfcfc00ULL,
137 0x6900006969696900, 0x5000005050505000, 0xaa0000aaaaaaaa00, 137 0x6900006969696900ULL, 0x5000005050505000ULL, 0xaa0000aaaaaaaa00ULL,
138 0xd00000d0d0d0d000, 0xa00000a0a0a0a000, 0x7d00007d7d7d7d00, 138 0xd00000d0d0d0d000ULL, 0xa00000a0a0a0a000ULL, 0x7d00007d7d7d7d00ULL,
139 0xa10000a1a1a1a100, 0x8900008989898900, 0x6200006262626200, 139 0xa10000a1a1a1a100ULL, 0x8900008989898900ULL, 0x6200006262626200ULL,
140 0x9700009797979700, 0x5400005454545400, 0x5b00005b5b5b5b00, 140 0x9700009797979700ULL, 0x5400005454545400ULL, 0x5b00005b5b5b5b00ULL,
141 0x1e00001e1e1e1e00, 0x9500009595959500, 0xe00000e0e0e0e000, 141 0x1e00001e1e1e1e00ULL, 0x9500009595959500ULL, 0xe00000e0e0e0e000ULL,
142 0xff0000ffffffff00, 0x6400006464646400, 0xd20000d2d2d2d200, 142 0xff0000ffffffff00ULL, 0x6400006464646400ULL, 0xd20000d2d2d2d200ULL,
143 0x1000001010101000, 0xc40000c4c4c4c400, 0x0000000000000000, 143 0x1000001010101000ULL, 0xc40000c4c4c4c400ULL, 0x0000000000000000ULL,
144 0x4800004848484800, 0xa30000a3a3a3a300, 0xf70000f7f7f7f700, 144 0x4800004848484800ULL, 0xa30000a3a3a3a300ULL, 0xf70000f7f7f7f700ULL,
145 0x7500007575757500, 0xdb0000dbdbdbdb00, 0x8a00008a8a8a8a00, 145 0x7500007575757500ULL, 0xdb0000dbdbdbdb00ULL, 0x8a00008a8a8a8a00ULL,
146 0x0300000303030300, 0xe60000e6e6e6e600, 0xda0000dadadada00, 146 0x0300000303030300ULL, 0xe60000e6e6e6e600ULL, 0xda0000dadadada00ULL,
147 0x0900000909090900, 0x3f00003f3f3f3f00, 0xdd0000dddddddd00, 147 0x0900000909090900ULL, 0x3f00003f3f3f3f00ULL, 0xdd0000dddddddd00ULL,
148 0x9400009494949400, 0x8700008787878700, 0x5c00005c5c5c5c00, 148 0x9400009494949400ULL, 0x8700008787878700ULL, 0x5c00005c5c5c5c00ULL,
149 0x8300008383838300, 0x0200000202020200, 0xcd0000cdcdcdcd00, 149 0x8300008383838300ULL, 0x0200000202020200ULL, 0xcd0000cdcdcdcd00ULL,
150 0x4a00004a4a4a4a00, 0x9000009090909000, 0x3300003333333300, 150 0x4a00004a4a4a4a00ULL, 0x9000009090909000ULL, 0x3300003333333300ULL,
151 0x7300007373737300, 0x6700006767676700, 0xf60000f6f6f6f600, 151 0x7300007373737300ULL, 0x6700006767676700ULL, 0xf60000f6f6f6f600ULL,
152 0xf30000f3f3f3f300, 0x9d00009d9d9d9d00, 0x7f00007f7f7f7f00, 152 0xf30000f3f3f3f300ULL, 0x9d00009d9d9d9d00ULL, 0x7f00007f7f7f7f00ULL,
153 0xbf0000bfbfbfbf00, 0xe20000e2e2e2e200, 0x5200005252525200, 153 0xbf0000bfbfbfbf00ULL, 0xe20000e2e2e2e200ULL, 0x5200005252525200ULL,
154 0x9b00009b9b9b9b00, 0xd80000d8d8d8d800, 0x2600002626262600, 154 0x9b00009b9b9b9b00ULL, 0xd80000d8d8d8d800ULL, 0x2600002626262600ULL,
155 0xc80000c8c8c8c800, 0x3700003737373700, 0xc60000c6c6c6c600, 155 0xc80000c8c8c8c800ULL, 0x3700003737373700ULL, 0xc60000c6c6c6c600ULL,
156 0x3b00003b3b3b3b00, 0x8100008181818100, 0x9600009696969600, 156 0x3b00003b3b3b3b00ULL, 0x8100008181818100ULL, 0x9600009696969600ULL,
157 0x6f00006f6f6f6f00, 0x4b00004b4b4b4b00, 0x1300001313131300, 157 0x6f00006f6f6f6f00ULL, 0x4b00004b4b4b4b00ULL, 0x1300001313131300ULL,
158 0xbe0000bebebebe00, 0x6300006363636300, 0x2e00002e2e2e2e00, 158 0xbe0000bebebebe00ULL, 0x6300006363636300ULL, 0x2e00002e2e2e2e00ULL,
159 0xe90000e9e9e9e900, 0x7900007979797900, 0xa70000a7a7a7a700, 159 0xe90000e9e9e9e900ULL, 0x7900007979797900ULL, 0xa70000a7a7a7a700ULL,
160 0x8c00008c8c8c8c00, 0x9f00009f9f9f9f00, 0x6e00006e6e6e6e00, 160 0x8c00008c8c8c8c00ULL, 0x9f00009f9f9f9f00ULL, 0x6e00006e6e6e6e00ULL,
161 0xbc0000bcbcbcbc00, 0x8e00008e8e8e8e00, 0x2900002929292900, 161 0xbc0000bcbcbcbc00ULL, 0x8e00008e8e8e8e00ULL, 0x2900002929292900ULL,
162 0xf50000f5f5f5f500, 0xf90000f9f9f9f900, 0xb60000b6b6b6b600, 162 0xf50000f5f5f5f500ULL, 0xf90000f9f9f9f900ULL, 0xb60000b6b6b6b600ULL,
163 0x2f00002f2f2f2f00, 0xfd0000fdfdfdfd00, 0xb40000b4b4b4b400, 163 0x2f00002f2f2f2f00ULL, 0xfd0000fdfdfdfd00ULL, 0xb40000b4b4b4b400ULL,
164 0x5900005959595900, 0x7800007878787800, 0x9800009898989800, 164 0x5900005959595900ULL, 0x7800007878787800ULL, 0x9800009898989800ULL,
165 0x0600000606060600, 0x6a00006a6a6a6a00, 0xe70000e7e7e7e700, 165 0x0600000606060600ULL, 0x6a00006a6a6a6a00ULL, 0xe70000e7e7e7e700ULL,
166 0x4600004646464600, 0x7100007171717100, 0xba0000babababa00, 166 0x4600004646464600ULL, 0x7100007171717100ULL, 0xba0000babababa00ULL,
167 0xd40000d4d4d4d400, 0x2500002525252500, 0xab0000abababab00, 167 0xd40000d4d4d4d400ULL, 0x2500002525252500ULL, 0xab0000abababab00ULL,
168 0x4200004242424200, 0x8800008888888800, 0xa20000a2a2a2a200, 168 0x4200004242424200ULL, 0x8800008888888800ULL, 0xa20000a2a2a2a200ULL,
169 0x8d00008d8d8d8d00, 0xfa0000fafafafa00, 0x7200007272727200, 169 0x8d00008d8d8d8d00ULL, 0xfa0000fafafafa00ULL, 0x7200007272727200ULL,
170 0x0700000707070700, 0xb90000b9b9b9b900, 0x5500005555555500, 170 0x0700000707070700ULL, 0xb90000b9b9b9b900ULL, 0x5500005555555500ULL,
171 0xf80000f8f8f8f800, 0xee0000eeeeeeee00, 0xac0000acacacac00, 171 0xf80000f8f8f8f800ULL, 0xee0000eeeeeeee00ULL, 0xac0000acacacac00ULL,
172 0x0a00000a0a0a0a00, 0x3600003636363600, 0x4900004949494900, 172 0x0a00000a0a0a0a00ULL, 0x3600003636363600ULL, 0x4900004949494900ULL,
173 0x2a00002a2a2a2a00, 0x6800006868686800, 0x3c00003c3c3c3c00, 173 0x2a00002a2a2a2a00ULL, 0x6800006868686800ULL, 0x3c00003c3c3c3c00ULL,
174 0x3800003838383800, 0xf10000f1f1f1f100, 0xa40000a4a4a4a400, 174 0x3800003838383800ULL, 0xf10000f1f1f1f100ULL, 0xa40000a4a4a4a400ULL,
175 0x4000004040404000, 0x2800002828282800, 0xd30000d3d3d3d300, 175 0x4000004040404000ULL, 0x2800002828282800ULL, 0xd30000d3d3d3d300ULL,
176 0x7b00007b7b7b7b00, 0xbb0000bbbbbbbb00, 0xc90000c9c9c9c900, 176 0x7b00007b7b7b7b00ULL, 0xbb0000bbbbbbbb00ULL, 0xc90000c9c9c9c900ULL,
177 0x4300004343434300, 0xc10000c1c1c1c100, 0x1500001515151500, 177 0x4300004343434300ULL, 0xc10000c1c1c1c100ULL, 0x1500001515151500ULL,
178 0xe30000e3e3e3e300, 0xad0000adadadad00, 0xf40000f4f4f4f400, 178 0xe30000e3e3e3e300ULL, 0xad0000adadadad00ULL, 0xf40000f4f4f4f400ULL,
179 0x7700007777777700, 0xc70000c7c7c7c700, 0x8000008080808000, 179 0x7700007777777700ULL, 0xc70000c7c7c7c700ULL, 0x8000008080808000ULL,
180 0x9e00009e9e9e9e00, 180 0x9e00009e9e9e9e00ULL,
181}; 181};
182 182
183const u64 camellia_sp22000222[256] = { 183const u64 camellia_sp22000222[256] = {
184 0xe0e0000000e0e0e0, 0x0505000000050505, 0x5858000000585858, 184 0xe0e0000000e0e0e0ULL, 0x0505000000050505ULL, 0x5858000000585858ULL,
185 0xd9d9000000d9d9d9, 0x6767000000676767, 0x4e4e0000004e4e4e, 185 0xd9d9000000d9d9d9ULL, 0x6767000000676767ULL, 0x4e4e0000004e4e4eULL,
186 0x8181000000818181, 0xcbcb000000cbcbcb, 0xc9c9000000c9c9c9, 186 0x8181000000818181ULL, 0xcbcb000000cbcbcbULL, 0xc9c9000000c9c9c9ULL,
187 0x0b0b0000000b0b0b, 0xaeae000000aeaeae, 0x6a6a0000006a6a6a, 187 0x0b0b0000000b0b0bULL, 0xaeae000000aeaeaeULL, 0x6a6a0000006a6a6aULL,
188 0xd5d5000000d5d5d5, 0x1818000000181818, 0x5d5d0000005d5d5d, 188 0xd5d5000000d5d5d5ULL, 0x1818000000181818ULL, 0x5d5d0000005d5d5dULL,
189 0x8282000000828282, 0x4646000000464646, 0xdfdf000000dfdfdf, 189 0x8282000000828282ULL, 0x4646000000464646ULL, 0xdfdf000000dfdfdfULL,
190 0xd6d6000000d6d6d6, 0x2727000000272727, 0x8a8a0000008a8a8a, 190 0xd6d6000000d6d6d6ULL, 0x2727000000272727ULL, 0x8a8a0000008a8a8aULL,
191 0x3232000000323232, 0x4b4b0000004b4b4b, 0x4242000000424242, 191 0x3232000000323232ULL, 0x4b4b0000004b4b4bULL, 0x4242000000424242ULL,
192 0xdbdb000000dbdbdb, 0x1c1c0000001c1c1c, 0x9e9e0000009e9e9e, 192 0xdbdb000000dbdbdbULL, 0x1c1c0000001c1c1cULL, 0x9e9e0000009e9e9eULL,
193 0x9c9c0000009c9c9c, 0x3a3a0000003a3a3a, 0xcaca000000cacaca, 193 0x9c9c0000009c9c9cULL, 0x3a3a0000003a3a3aULL, 0xcaca000000cacacaULL,
194 0x2525000000252525, 0x7b7b0000007b7b7b, 0x0d0d0000000d0d0d, 194 0x2525000000252525ULL, 0x7b7b0000007b7b7bULL, 0x0d0d0000000d0d0dULL,
195 0x7171000000717171, 0x5f5f0000005f5f5f, 0x1f1f0000001f1f1f, 195 0x7171000000717171ULL, 0x5f5f0000005f5f5fULL, 0x1f1f0000001f1f1fULL,
196 0xf8f8000000f8f8f8, 0xd7d7000000d7d7d7, 0x3e3e0000003e3e3e, 196 0xf8f8000000f8f8f8ULL, 0xd7d7000000d7d7d7ULL, 0x3e3e0000003e3e3eULL,
197 0x9d9d0000009d9d9d, 0x7c7c0000007c7c7c, 0x6060000000606060, 197 0x9d9d0000009d9d9dULL, 0x7c7c0000007c7c7cULL, 0x6060000000606060ULL,
198 0xb9b9000000b9b9b9, 0xbebe000000bebebe, 0xbcbc000000bcbcbc, 198 0xb9b9000000b9b9b9ULL, 0xbebe000000bebebeULL, 0xbcbc000000bcbcbcULL,
199 0x8b8b0000008b8b8b, 0x1616000000161616, 0x3434000000343434, 199 0x8b8b0000008b8b8bULL, 0x1616000000161616ULL, 0x3434000000343434ULL,
200 0x4d4d0000004d4d4d, 0xc3c3000000c3c3c3, 0x7272000000727272, 200 0x4d4d0000004d4d4dULL, 0xc3c3000000c3c3c3ULL, 0x7272000000727272ULL,
201 0x9595000000959595, 0xabab000000ababab, 0x8e8e0000008e8e8e, 201 0x9595000000959595ULL, 0xabab000000abababULL, 0x8e8e0000008e8e8eULL,
202 0xbaba000000bababa, 0x7a7a0000007a7a7a, 0xb3b3000000b3b3b3, 202 0xbaba000000bababaULL, 0x7a7a0000007a7a7aULL, 0xb3b3000000b3b3b3ULL,
203 0x0202000000020202, 0xb4b4000000b4b4b4, 0xadad000000adadad, 203 0x0202000000020202ULL, 0xb4b4000000b4b4b4ULL, 0xadad000000adadadULL,
204 0xa2a2000000a2a2a2, 0xacac000000acacac, 0xd8d8000000d8d8d8, 204 0xa2a2000000a2a2a2ULL, 0xacac000000acacacULL, 0xd8d8000000d8d8d8ULL,
205 0x9a9a0000009a9a9a, 0x1717000000171717, 0x1a1a0000001a1a1a, 205 0x9a9a0000009a9a9aULL, 0x1717000000171717ULL, 0x1a1a0000001a1a1aULL,
206 0x3535000000353535, 0xcccc000000cccccc, 0xf7f7000000f7f7f7, 206 0x3535000000353535ULL, 0xcccc000000ccccccULL, 0xf7f7000000f7f7f7ULL,
207 0x9999000000999999, 0x6161000000616161, 0x5a5a0000005a5a5a, 207 0x9999000000999999ULL, 0x6161000000616161ULL, 0x5a5a0000005a5a5aULL,
208 0xe8e8000000e8e8e8, 0x2424000000242424, 0x5656000000565656, 208 0xe8e8000000e8e8e8ULL, 0x2424000000242424ULL, 0x5656000000565656ULL,
209 0x4040000000404040, 0xe1e1000000e1e1e1, 0x6363000000636363, 209 0x4040000000404040ULL, 0xe1e1000000e1e1e1ULL, 0x6363000000636363ULL,
210 0x0909000000090909, 0x3333000000333333, 0xbfbf000000bfbfbf, 210 0x0909000000090909ULL, 0x3333000000333333ULL, 0xbfbf000000bfbfbfULL,
211 0x9898000000989898, 0x9797000000979797, 0x8585000000858585, 211 0x9898000000989898ULL, 0x9797000000979797ULL, 0x8585000000858585ULL,
212 0x6868000000686868, 0xfcfc000000fcfcfc, 0xecec000000ececec, 212 0x6868000000686868ULL, 0xfcfc000000fcfcfcULL, 0xecec000000ecececULL,
213 0x0a0a0000000a0a0a, 0xdada000000dadada, 0x6f6f0000006f6f6f, 213 0x0a0a0000000a0a0aULL, 0xdada000000dadadaULL, 0x6f6f0000006f6f6fULL,
214 0x5353000000535353, 0x6262000000626262, 0xa3a3000000a3a3a3, 214 0x5353000000535353ULL, 0x6262000000626262ULL, 0xa3a3000000a3a3a3ULL,
215 0x2e2e0000002e2e2e, 0x0808000000080808, 0xafaf000000afafaf, 215 0x2e2e0000002e2e2eULL, 0x0808000000080808ULL, 0xafaf000000afafafULL,
216 0x2828000000282828, 0xb0b0000000b0b0b0, 0x7474000000747474, 216 0x2828000000282828ULL, 0xb0b0000000b0b0b0ULL, 0x7474000000747474ULL,
217 0xc2c2000000c2c2c2, 0xbdbd000000bdbdbd, 0x3636000000363636, 217 0xc2c2000000c2c2c2ULL, 0xbdbd000000bdbdbdULL, 0x3636000000363636ULL,
218 0x2222000000222222, 0x3838000000383838, 0x6464000000646464, 218 0x2222000000222222ULL, 0x3838000000383838ULL, 0x6464000000646464ULL,
219 0x1e1e0000001e1e1e, 0x3939000000393939, 0x2c2c0000002c2c2c, 219 0x1e1e0000001e1e1eULL, 0x3939000000393939ULL, 0x2c2c0000002c2c2cULL,
220 0xa6a6000000a6a6a6, 0x3030000000303030, 0xe5e5000000e5e5e5, 220 0xa6a6000000a6a6a6ULL, 0x3030000000303030ULL, 0xe5e5000000e5e5e5ULL,
221 0x4444000000444444, 0xfdfd000000fdfdfd, 0x8888000000888888, 221 0x4444000000444444ULL, 0xfdfd000000fdfdfdULL, 0x8888000000888888ULL,
222 0x9f9f0000009f9f9f, 0x6565000000656565, 0x8787000000878787, 222 0x9f9f0000009f9f9fULL, 0x6565000000656565ULL, 0x8787000000878787ULL,
223 0x6b6b0000006b6b6b, 0xf4f4000000f4f4f4, 0x2323000000232323, 223 0x6b6b0000006b6b6bULL, 0xf4f4000000f4f4f4ULL, 0x2323000000232323ULL,
224 0x4848000000484848, 0x1010000000101010, 0xd1d1000000d1d1d1, 224 0x4848000000484848ULL, 0x1010000000101010ULL, 0xd1d1000000d1d1d1ULL,
225 0x5151000000515151, 0xc0c0000000c0c0c0, 0xf9f9000000f9f9f9, 225 0x5151000000515151ULL, 0xc0c0000000c0c0c0ULL, 0xf9f9000000f9f9f9ULL,
226 0xd2d2000000d2d2d2, 0xa0a0000000a0a0a0, 0x5555000000555555, 226 0xd2d2000000d2d2d2ULL, 0xa0a0000000a0a0a0ULL, 0x5555000000555555ULL,
227 0xa1a1000000a1a1a1, 0x4141000000414141, 0xfafa000000fafafa, 227 0xa1a1000000a1a1a1ULL, 0x4141000000414141ULL, 0xfafa000000fafafaULL,
228 0x4343000000434343, 0x1313000000131313, 0xc4c4000000c4c4c4, 228 0x4343000000434343ULL, 0x1313000000131313ULL, 0xc4c4000000c4c4c4ULL,
229 0x2f2f0000002f2f2f, 0xa8a8000000a8a8a8, 0xb6b6000000b6b6b6, 229 0x2f2f0000002f2f2fULL, 0xa8a8000000a8a8a8ULL, 0xb6b6000000b6b6b6ULL,
230 0x3c3c0000003c3c3c, 0x2b2b0000002b2b2b, 0xc1c1000000c1c1c1, 230 0x3c3c0000003c3c3cULL, 0x2b2b0000002b2b2bULL, 0xc1c1000000c1c1c1ULL,
231 0xffff000000ffffff, 0xc8c8000000c8c8c8, 0xa5a5000000a5a5a5, 231 0xffff000000ffffffULL, 0xc8c8000000c8c8c8ULL, 0xa5a5000000a5a5a5ULL,
232 0x2020000000202020, 0x8989000000898989, 0x0000000000000000, 232 0x2020000000202020ULL, 0x8989000000898989ULL, 0x0000000000000000ULL,
233 0x9090000000909090, 0x4747000000474747, 0xefef000000efefef, 233 0x9090000000909090ULL, 0x4747000000474747ULL, 0xefef000000efefefULL,
234 0xeaea000000eaeaea, 0xb7b7000000b7b7b7, 0x1515000000151515, 234 0xeaea000000eaeaeaULL, 0xb7b7000000b7b7b7ULL, 0x1515000000151515ULL,
235 0x0606000000060606, 0xcdcd000000cdcdcd, 0xb5b5000000b5b5b5, 235 0x0606000000060606ULL, 0xcdcd000000cdcdcdULL, 0xb5b5000000b5b5b5ULL,
236 0x1212000000121212, 0x7e7e0000007e7e7e, 0xbbbb000000bbbbbb, 236 0x1212000000121212ULL, 0x7e7e0000007e7e7eULL, 0xbbbb000000bbbbbbULL,
237 0x2929000000292929, 0x0f0f0000000f0f0f, 0xb8b8000000b8b8b8, 237 0x2929000000292929ULL, 0x0f0f0000000f0f0fULL, 0xb8b8000000b8b8b8ULL,
238 0x0707000000070707, 0x0404000000040404, 0x9b9b0000009b9b9b, 238 0x0707000000070707ULL, 0x0404000000040404ULL, 0x9b9b0000009b9b9bULL,
239 0x9494000000949494, 0x2121000000212121, 0x6666000000666666, 239 0x9494000000949494ULL, 0x2121000000212121ULL, 0x6666000000666666ULL,
240 0xe6e6000000e6e6e6, 0xcece000000cecece, 0xeded000000ededed, 240 0xe6e6000000e6e6e6ULL, 0xcece000000cececeULL, 0xeded000000edededULL,
241 0xe7e7000000e7e7e7, 0x3b3b0000003b3b3b, 0xfefe000000fefefe, 241 0xe7e7000000e7e7e7ULL, 0x3b3b0000003b3b3bULL, 0xfefe000000fefefeULL,
242 0x7f7f0000007f7f7f, 0xc5c5000000c5c5c5, 0xa4a4000000a4a4a4, 242 0x7f7f0000007f7f7fULL, 0xc5c5000000c5c5c5ULL, 0xa4a4000000a4a4a4ULL,
243 0x3737000000373737, 0xb1b1000000b1b1b1, 0x4c4c0000004c4c4c, 243 0x3737000000373737ULL, 0xb1b1000000b1b1b1ULL, 0x4c4c0000004c4c4cULL,
244 0x9191000000919191, 0x6e6e0000006e6e6e, 0x8d8d0000008d8d8d, 244 0x9191000000919191ULL, 0x6e6e0000006e6e6eULL, 0x8d8d0000008d8d8dULL,
245 0x7676000000767676, 0x0303000000030303, 0x2d2d0000002d2d2d, 245 0x7676000000767676ULL, 0x0303000000030303ULL, 0x2d2d0000002d2d2dULL,
246 0xdede000000dedede, 0x9696000000969696, 0x2626000000262626, 246 0xdede000000dededeULL, 0x9696000000969696ULL, 0x2626000000262626ULL,
247 0x7d7d0000007d7d7d, 0xc6c6000000c6c6c6, 0x5c5c0000005c5c5c, 247 0x7d7d0000007d7d7dULL, 0xc6c6000000c6c6c6ULL, 0x5c5c0000005c5c5cULL,
248 0xd3d3000000d3d3d3, 0xf2f2000000f2f2f2, 0x4f4f0000004f4f4f, 248 0xd3d3000000d3d3d3ULL, 0xf2f2000000f2f2f2ULL, 0x4f4f0000004f4f4fULL,
249 0x1919000000191919, 0x3f3f0000003f3f3f, 0xdcdc000000dcdcdc, 249 0x1919000000191919ULL, 0x3f3f0000003f3f3fULL, 0xdcdc000000dcdcdcULL,
250 0x7979000000797979, 0x1d1d0000001d1d1d, 0x5252000000525252, 250 0x7979000000797979ULL, 0x1d1d0000001d1d1dULL, 0x5252000000525252ULL,
251 0xebeb000000ebebeb, 0xf3f3000000f3f3f3, 0x6d6d0000006d6d6d, 251 0xebeb000000ebebebULL, 0xf3f3000000f3f3f3ULL, 0x6d6d0000006d6d6dULL,
252 0x5e5e0000005e5e5e, 0xfbfb000000fbfbfb, 0x6969000000696969, 252 0x5e5e0000005e5e5eULL, 0xfbfb000000fbfbfbULL, 0x6969000000696969ULL,
253 0xb2b2000000b2b2b2, 0xf0f0000000f0f0f0, 0x3131000000313131, 253 0xb2b2000000b2b2b2ULL, 0xf0f0000000f0f0f0ULL, 0x3131000000313131ULL,
254 0x0c0c0000000c0c0c, 0xd4d4000000d4d4d4, 0xcfcf000000cfcfcf, 254 0x0c0c0000000c0c0cULL, 0xd4d4000000d4d4d4ULL, 0xcfcf000000cfcfcfULL,
255 0x8c8c0000008c8c8c, 0xe2e2000000e2e2e2, 0x7575000000757575, 255 0x8c8c0000008c8c8cULL, 0xe2e2000000e2e2e2ULL, 0x7575000000757575ULL,
256 0xa9a9000000a9a9a9, 0x4a4a0000004a4a4a, 0x5757000000575757, 256 0xa9a9000000a9a9a9ULL, 0x4a4a0000004a4a4aULL, 0x5757000000575757ULL,
257 0x8484000000848484, 0x1111000000111111, 0x4545000000454545, 257 0x8484000000848484ULL, 0x1111000000111111ULL, 0x4545000000454545ULL,
258 0x1b1b0000001b1b1b, 0xf5f5000000f5f5f5, 0xe4e4000000e4e4e4, 258 0x1b1b0000001b1b1bULL, 0xf5f5000000f5f5f5ULL, 0xe4e4000000e4e4e4ULL,
259 0x0e0e0000000e0e0e, 0x7373000000737373, 0xaaaa000000aaaaaa, 259 0x0e0e0000000e0e0eULL, 0x7373000000737373ULL, 0xaaaa000000aaaaaaULL,
260 0xf1f1000000f1f1f1, 0xdddd000000dddddd, 0x5959000000595959, 260 0xf1f1000000f1f1f1ULL, 0xdddd000000ddddddULL, 0x5959000000595959ULL,
261 0x1414000000141414, 0x6c6c0000006c6c6c, 0x9292000000929292, 261 0x1414000000141414ULL, 0x6c6c0000006c6c6cULL, 0x9292000000929292ULL,
262 0x5454000000545454, 0xd0d0000000d0d0d0, 0x7878000000787878, 262 0x5454000000545454ULL, 0xd0d0000000d0d0d0ULL, 0x7878000000787878ULL,
263 0x7070000000707070, 0xe3e3000000e3e3e3, 0x4949000000494949, 263 0x7070000000707070ULL, 0xe3e3000000e3e3e3ULL, 0x4949000000494949ULL,
264 0x8080000000808080, 0x5050000000505050, 0xa7a7000000a7a7a7, 264 0x8080000000808080ULL, 0x5050000000505050ULL, 0xa7a7000000a7a7a7ULL,
265 0xf6f6000000f6f6f6, 0x7777000000777777, 0x9393000000939393, 265 0xf6f6000000f6f6f6ULL, 0x7777000000777777ULL, 0x9393000000939393ULL,
266 0x8686000000868686, 0x8383000000838383, 0x2a2a0000002a2a2a, 266 0x8686000000868686ULL, 0x8383000000838383ULL, 0x2a2a0000002a2a2aULL,
267 0xc7c7000000c7c7c7, 0x5b5b0000005b5b5b, 0xe9e9000000e9e9e9, 267 0xc7c7000000c7c7c7ULL, 0x5b5b0000005b5b5bULL, 0xe9e9000000e9e9e9ULL,
268 0xeeee000000eeeeee, 0x8f8f0000008f8f8f, 0x0101000000010101, 268 0xeeee000000eeeeeeULL, 0x8f8f0000008f8f8fULL, 0x0101000000010101ULL,
269 0x3d3d0000003d3d3d, 269 0x3d3d0000003d3d3dULL,
270}; 270};
271 271
272const u64 camellia_sp03303033[256] = { 272const u64 camellia_sp03303033[256] = {
273 0x0038380038003838, 0x0041410041004141, 0x0016160016001616, 273 0x0038380038003838ULL, 0x0041410041004141ULL, 0x0016160016001616ULL,
274 0x0076760076007676, 0x00d9d900d900d9d9, 0x0093930093009393, 274 0x0076760076007676ULL, 0x00d9d900d900d9d9ULL, 0x0093930093009393ULL,
275 0x0060600060006060, 0x00f2f200f200f2f2, 0x0072720072007272, 275 0x0060600060006060ULL, 0x00f2f200f200f2f2ULL, 0x0072720072007272ULL,
276 0x00c2c200c200c2c2, 0x00abab00ab00abab, 0x009a9a009a009a9a, 276 0x00c2c200c200c2c2ULL, 0x00abab00ab00ababULL, 0x009a9a009a009a9aULL,
277 0x0075750075007575, 0x0006060006000606, 0x0057570057005757, 277 0x0075750075007575ULL, 0x0006060006000606ULL, 0x0057570057005757ULL,
278 0x00a0a000a000a0a0, 0x0091910091009191, 0x00f7f700f700f7f7, 278 0x00a0a000a000a0a0ULL, 0x0091910091009191ULL, 0x00f7f700f700f7f7ULL,
279 0x00b5b500b500b5b5, 0x00c9c900c900c9c9, 0x00a2a200a200a2a2, 279 0x00b5b500b500b5b5ULL, 0x00c9c900c900c9c9ULL, 0x00a2a200a200a2a2ULL,
280 0x008c8c008c008c8c, 0x00d2d200d200d2d2, 0x0090900090009090, 280 0x008c8c008c008c8cULL, 0x00d2d200d200d2d2ULL, 0x0090900090009090ULL,
281 0x00f6f600f600f6f6, 0x0007070007000707, 0x00a7a700a700a7a7, 281 0x00f6f600f600f6f6ULL, 0x0007070007000707ULL, 0x00a7a700a700a7a7ULL,
282 0x0027270027002727, 0x008e8e008e008e8e, 0x00b2b200b200b2b2, 282 0x0027270027002727ULL, 0x008e8e008e008e8eULL, 0x00b2b200b200b2b2ULL,
283 0x0049490049004949, 0x00dede00de00dede, 0x0043430043004343, 283 0x0049490049004949ULL, 0x00dede00de00dedeULL, 0x0043430043004343ULL,
284 0x005c5c005c005c5c, 0x00d7d700d700d7d7, 0x00c7c700c700c7c7, 284 0x005c5c005c005c5cULL, 0x00d7d700d700d7d7ULL, 0x00c7c700c700c7c7ULL,
285 0x003e3e003e003e3e, 0x00f5f500f500f5f5, 0x008f8f008f008f8f, 285 0x003e3e003e003e3eULL, 0x00f5f500f500f5f5ULL, 0x008f8f008f008f8fULL,
286 0x0067670067006767, 0x001f1f001f001f1f, 0x0018180018001818, 286 0x0067670067006767ULL, 0x001f1f001f001f1fULL, 0x0018180018001818ULL,
287 0x006e6e006e006e6e, 0x00afaf00af00afaf, 0x002f2f002f002f2f, 287 0x006e6e006e006e6eULL, 0x00afaf00af00afafULL, 0x002f2f002f002f2fULL,
288 0x00e2e200e200e2e2, 0x0085850085008585, 0x000d0d000d000d0d, 288 0x00e2e200e200e2e2ULL, 0x0085850085008585ULL, 0x000d0d000d000d0dULL,
289 0x0053530053005353, 0x00f0f000f000f0f0, 0x009c9c009c009c9c, 289 0x0053530053005353ULL, 0x00f0f000f000f0f0ULL, 0x009c9c009c009c9cULL,
290 0x0065650065006565, 0x00eaea00ea00eaea, 0x00a3a300a300a3a3, 290 0x0065650065006565ULL, 0x00eaea00ea00eaeaULL, 0x00a3a300a300a3a3ULL,
291 0x00aeae00ae00aeae, 0x009e9e009e009e9e, 0x00ecec00ec00ecec, 291 0x00aeae00ae00aeaeULL, 0x009e9e009e009e9eULL, 0x00ecec00ec00ececULL,
292 0x0080800080008080, 0x002d2d002d002d2d, 0x006b6b006b006b6b, 292 0x0080800080008080ULL, 0x002d2d002d002d2dULL, 0x006b6b006b006b6bULL,
293 0x00a8a800a800a8a8, 0x002b2b002b002b2b, 0x0036360036003636, 293 0x00a8a800a800a8a8ULL, 0x002b2b002b002b2bULL, 0x0036360036003636ULL,
294 0x00a6a600a600a6a6, 0x00c5c500c500c5c5, 0x0086860086008686, 294 0x00a6a600a600a6a6ULL, 0x00c5c500c500c5c5ULL, 0x0086860086008686ULL,
295 0x004d4d004d004d4d, 0x0033330033003333, 0x00fdfd00fd00fdfd, 295 0x004d4d004d004d4dULL, 0x0033330033003333ULL, 0x00fdfd00fd00fdfdULL,
296 0x0066660066006666, 0x0058580058005858, 0x0096960096009696, 296 0x0066660066006666ULL, 0x0058580058005858ULL, 0x0096960096009696ULL,
297 0x003a3a003a003a3a, 0x0009090009000909, 0x0095950095009595, 297 0x003a3a003a003a3aULL, 0x0009090009000909ULL, 0x0095950095009595ULL,
298 0x0010100010001010, 0x0078780078007878, 0x00d8d800d800d8d8, 298 0x0010100010001010ULL, 0x0078780078007878ULL, 0x00d8d800d800d8d8ULL,
299 0x0042420042004242, 0x00cccc00cc00cccc, 0x00efef00ef00efef, 299 0x0042420042004242ULL, 0x00cccc00cc00ccccULL, 0x00efef00ef00efefULL,
300 0x0026260026002626, 0x00e5e500e500e5e5, 0x0061610061006161, 300 0x0026260026002626ULL, 0x00e5e500e500e5e5ULL, 0x0061610061006161ULL,
301 0x001a1a001a001a1a, 0x003f3f003f003f3f, 0x003b3b003b003b3b, 301 0x001a1a001a001a1aULL, 0x003f3f003f003f3fULL, 0x003b3b003b003b3bULL,
302 0x0082820082008282, 0x00b6b600b600b6b6, 0x00dbdb00db00dbdb, 302 0x0082820082008282ULL, 0x00b6b600b600b6b6ULL, 0x00dbdb00db00dbdbULL,
303 0x00d4d400d400d4d4, 0x0098980098009898, 0x00e8e800e800e8e8, 303 0x00d4d400d400d4d4ULL, 0x0098980098009898ULL, 0x00e8e800e800e8e8ULL,
304 0x008b8b008b008b8b, 0x0002020002000202, 0x00ebeb00eb00ebeb, 304 0x008b8b008b008b8bULL, 0x0002020002000202ULL, 0x00ebeb00eb00ebebULL,
305 0x000a0a000a000a0a, 0x002c2c002c002c2c, 0x001d1d001d001d1d, 305 0x000a0a000a000a0aULL, 0x002c2c002c002c2cULL, 0x001d1d001d001d1dULL,
306 0x00b0b000b000b0b0, 0x006f6f006f006f6f, 0x008d8d008d008d8d, 306 0x00b0b000b000b0b0ULL, 0x006f6f006f006f6fULL, 0x008d8d008d008d8dULL,
307 0x0088880088008888, 0x000e0e000e000e0e, 0x0019190019001919, 307 0x0088880088008888ULL, 0x000e0e000e000e0eULL, 0x0019190019001919ULL,
308 0x0087870087008787, 0x004e4e004e004e4e, 0x000b0b000b000b0b, 308 0x0087870087008787ULL, 0x004e4e004e004e4eULL, 0x000b0b000b000b0bULL,
309 0x00a9a900a900a9a9, 0x000c0c000c000c0c, 0x0079790079007979, 309 0x00a9a900a900a9a9ULL, 0x000c0c000c000c0cULL, 0x0079790079007979ULL,
310 0x0011110011001111, 0x007f7f007f007f7f, 0x0022220022002222, 310 0x0011110011001111ULL, 0x007f7f007f007f7fULL, 0x0022220022002222ULL,
311 0x00e7e700e700e7e7, 0x0059590059005959, 0x00e1e100e100e1e1, 311 0x00e7e700e700e7e7ULL, 0x0059590059005959ULL, 0x00e1e100e100e1e1ULL,
312 0x00dada00da00dada, 0x003d3d003d003d3d, 0x00c8c800c800c8c8, 312 0x00dada00da00dadaULL, 0x003d3d003d003d3dULL, 0x00c8c800c800c8c8ULL,
313 0x0012120012001212, 0x0004040004000404, 0x0074740074007474, 313 0x0012120012001212ULL, 0x0004040004000404ULL, 0x0074740074007474ULL,
314 0x0054540054005454, 0x0030300030003030, 0x007e7e007e007e7e, 314 0x0054540054005454ULL, 0x0030300030003030ULL, 0x007e7e007e007e7eULL,
315 0x00b4b400b400b4b4, 0x0028280028002828, 0x0055550055005555, 315 0x00b4b400b400b4b4ULL, 0x0028280028002828ULL, 0x0055550055005555ULL,
316 0x0068680068006868, 0x0050500050005050, 0x00bebe00be00bebe, 316 0x0068680068006868ULL, 0x0050500050005050ULL, 0x00bebe00be00bebeULL,
317 0x00d0d000d000d0d0, 0x00c4c400c400c4c4, 0x0031310031003131, 317 0x00d0d000d000d0d0ULL, 0x00c4c400c400c4c4ULL, 0x0031310031003131ULL,
318 0x00cbcb00cb00cbcb, 0x002a2a002a002a2a, 0x00adad00ad00adad, 318 0x00cbcb00cb00cbcbULL, 0x002a2a002a002a2aULL, 0x00adad00ad00adadULL,
319 0x000f0f000f000f0f, 0x00caca00ca00caca, 0x0070700070007070, 319 0x000f0f000f000f0fULL, 0x00caca00ca00cacaULL, 0x0070700070007070ULL,
320 0x00ffff00ff00ffff, 0x0032320032003232, 0x0069690069006969, 320 0x00ffff00ff00ffffULL, 0x0032320032003232ULL, 0x0069690069006969ULL,
321 0x0008080008000808, 0x0062620062006262, 0x0000000000000000, 321 0x0008080008000808ULL, 0x0062620062006262ULL, 0x0000000000000000ULL,
322 0x0024240024002424, 0x00d1d100d100d1d1, 0x00fbfb00fb00fbfb, 322 0x0024240024002424ULL, 0x00d1d100d100d1d1ULL, 0x00fbfb00fb00fbfbULL,
323 0x00baba00ba00baba, 0x00eded00ed00eded, 0x0045450045004545, 323 0x00baba00ba00babaULL, 0x00eded00ed00ededULL, 0x0045450045004545ULL,
324 0x0081810081008181, 0x0073730073007373, 0x006d6d006d006d6d, 324 0x0081810081008181ULL, 0x0073730073007373ULL, 0x006d6d006d006d6dULL,
325 0x0084840084008484, 0x009f9f009f009f9f, 0x00eeee00ee00eeee, 325 0x0084840084008484ULL, 0x009f9f009f009f9fULL, 0x00eeee00ee00eeeeULL,
326 0x004a4a004a004a4a, 0x00c3c300c300c3c3, 0x002e2e002e002e2e, 326 0x004a4a004a004a4aULL, 0x00c3c300c300c3c3ULL, 0x002e2e002e002e2eULL,
327 0x00c1c100c100c1c1, 0x0001010001000101, 0x00e6e600e600e6e6, 327 0x00c1c100c100c1c1ULL, 0x0001010001000101ULL, 0x00e6e600e600e6e6ULL,
328 0x0025250025002525, 0x0048480048004848, 0x0099990099009999, 328 0x0025250025002525ULL, 0x0048480048004848ULL, 0x0099990099009999ULL,
329 0x00b9b900b900b9b9, 0x00b3b300b300b3b3, 0x007b7b007b007b7b, 329 0x00b9b900b900b9b9ULL, 0x00b3b300b300b3b3ULL, 0x007b7b007b007b7bULL,
330 0x00f9f900f900f9f9, 0x00cece00ce00cece, 0x00bfbf00bf00bfbf, 330 0x00f9f900f900f9f9ULL, 0x00cece00ce00ceceULL, 0x00bfbf00bf00bfbfULL,
331 0x00dfdf00df00dfdf, 0x0071710071007171, 0x0029290029002929, 331 0x00dfdf00df00dfdfULL, 0x0071710071007171ULL, 0x0029290029002929ULL,
332 0x00cdcd00cd00cdcd, 0x006c6c006c006c6c, 0x0013130013001313, 332 0x00cdcd00cd00cdcdULL, 0x006c6c006c006c6cULL, 0x0013130013001313ULL,
333 0x0064640064006464, 0x009b9b009b009b9b, 0x0063630063006363, 333 0x0064640064006464ULL, 0x009b9b009b009b9bULL, 0x0063630063006363ULL,
334 0x009d9d009d009d9d, 0x00c0c000c000c0c0, 0x004b4b004b004b4b, 334 0x009d9d009d009d9dULL, 0x00c0c000c000c0c0ULL, 0x004b4b004b004b4bULL,
335 0x00b7b700b700b7b7, 0x00a5a500a500a5a5, 0x0089890089008989, 335 0x00b7b700b700b7b7ULL, 0x00a5a500a500a5a5ULL, 0x0089890089008989ULL,
336 0x005f5f005f005f5f, 0x00b1b100b100b1b1, 0x0017170017001717, 336 0x005f5f005f005f5fULL, 0x00b1b100b100b1b1ULL, 0x0017170017001717ULL,
337 0x00f4f400f400f4f4, 0x00bcbc00bc00bcbc, 0x00d3d300d300d3d3, 337 0x00f4f400f400f4f4ULL, 0x00bcbc00bc00bcbcULL, 0x00d3d300d300d3d3ULL,
338 0x0046460046004646, 0x00cfcf00cf00cfcf, 0x0037370037003737, 338 0x0046460046004646ULL, 0x00cfcf00cf00cfcfULL, 0x0037370037003737ULL,
339 0x005e5e005e005e5e, 0x0047470047004747, 0x0094940094009494, 339 0x005e5e005e005e5eULL, 0x0047470047004747ULL, 0x0094940094009494ULL,
340 0x00fafa00fa00fafa, 0x00fcfc00fc00fcfc, 0x005b5b005b005b5b, 340 0x00fafa00fa00fafaULL, 0x00fcfc00fc00fcfcULL, 0x005b5b005b005b5bULL,
341 0x0097970097009797, 0x00fefe00fe00fefe, 0x005a5a005a005a5a, 341 0x0097970097009797ULL, 0x00fefe00fe00fefeULL, 0x005a5a005a005a5aULL,
342 0x00acac00ac00acac, 0x003c3c003c003c3c, 0x004c4c004c004c4c, 342 0x00acac00ac00acacULL, 0x003c3c003c003c3cULL, 0x004c4c004c004c4cULL,
343 0x0003030003000303, 0x0035350035003535, 0x00f3f300f300f3f3, 343 0x0003030003000303ULL, 0x0035350035003535ULL, 0x00f3f300f300f3f3ULL,
344 0x0023230023002323, 0x00b8b800b800b8b8, 0x005d5d005d005d5d, 344 0x0023230023002323ULL, 0x00b8b800b800b8b8ULL, 0x005d5d005d005d5dULL,
345 0x006a6a006a006a6a, 0x0092920092009292, 0x00d5d500d500d5d5, 345 0x006a6a006a006a6aULL, 0x0092920092009292ULL, 0x00d5d500d500d5d5ULL,
346 0x0021210021002121, 0x0044440044004444, 0x0051510051005151, 346 0x0021210021002121ULL, 0x0044440044004444ULL, 0x0051510051005151ULL,
347 0x00c6c600c600c6c6, 0x007d7d007d007d7d, 0x0039390039003939, 347 0x00c6c600c600c6c6ULL, 0x007d7d007d007d7dULL, 0x0039390039003939ULL,
348 0x0083830083008383, 0x00dcdc00dc00dcdc, 0x00aaaa00aa00aaaa, 348 0x0083830083008383ULL, 0x00dcdc00dc00dcdcULL, 0x00aaaa00aa00aaaaULL,
349 0x007c7c007c007c7c, 0x0077770077007777, 0x0056560056005656, 349 0x007c7c007c007c7cULL, 0x0077770077007777ULL, 0x0056560056005656ULL,
350 0x0005050005000505, 0x001b1b001b001b1b, 0x00a4a400a400a4a4, 350 0x0005050005000505ULL, 0x001b1b001b001b1bULL, 0x00a4a400a400a4a4ULL,
351 0x0015150015001515, 0x0034340034003434, 0x001e1e001e001e1e, 351 0x0015150015001515ULL, 0x0034340034003434ULL, 0x001e1e001e001e1eULL,
352 0x001c1c001c001c1c, 0x00f8f800f800f8f8, 0x0052520052005252, 352 0x001c1c001c001c1cULL, 0x00f8f800f800f8f8ULL, 0x0052520052005252ULL,
353 0x0020200020002020, 0x0014140014001414, 0x00e9e900e900e9e9, 353 0x0020200020002020ULL, 0x0014140014001414ULL, 0x00e9e900e900e9e9ULL,
354 0x00bdbd00bd00bdbd, 0x00dddd00dd00dddd, 0x00e4e400e400e4e4, 354 0x00bdbd00bd00bdbdULL, 0x00dddd00dd00ddddULL, 0x00e4e400e400e4e4ULL,
355 0x00a1a100a100a1a1, 0x00e0e000e000e0e0, 0x008a8a008a008a8a, 355 0x00a1a100a100a1a1ULL, 0x00e0e000e000e0e0ULL, 0x008a8a008a008a8aULL,
356 0x00f1f100f100f1f1, 0x00d6d600d600d6d6, 0x007a7a007a007a7a, 356 0x00f1f100f100f1f1ULL, 0x00d6d600d600d6d6ULL, 0x007a7a007a007a7aULL,
357 0x00bbbb00bb00bbbb, 0x00e3e300e300e3e3, 0x0040400040004040, 357 0x00bbbb00bb00bbbbULL, 0x00e3e300e300e3e3ULL, 0x0040400040004040ULL,
358 0x004f4f004f004f4f, 358 0x004f4f004f004f4fULL,
359}; 359};
360 360
361const u64 camellia_sp00444404[256] = { 361const u64 camellia_sp00444404[256] = {
362 0x0000707070700070, 0x00002c2c2c2c002c, 0x0000b3b3b3b300b3, 362 0x0000707070700070ULL, 0x00002c2c2c2c002cULL, 0x0000b3b3b3b300b3ULL,
363 0x0000c0c0c0c000c0, 0x0000e4e4e4e400e4, 0x0000575757570057, 363 0x0000c0c0c0c000c0ULL, 0x0000e4e4e4e400e4ULL, 0x0000575757570057ULL,
364 0x0000eaeaeaea00ea, 0x0000aeaeaeae00ae, 0x0000232323230023, 364 0x0000eaeaeaea00eaULL, 0x0000aeaeaeae00aeULL, 0x0000232323230023ULL,
365 0x00006b6b6b6b006b, 0x0000454545450045, 0x0000a5a5a5a500a5, 365 0x00006b6b6b6b006bULL, 0x0000454545450045ULL, 0x0000a5a5a5a500a5ULL,
366 0x0000edededed00ed, 0x00004f4f4f4f004f, 0x00001d1d1d1d001d, 366 0x0000edededed00edULL, 0x00004f4f4f4f004fULL, 0x00001d1d1d1d001dULL,
367 0x0000929292920092, 0x0000868686860086, 0x0000afafafaf00af, 367 0x0000929292920092ULL, 0x0000868686860086ULL, 0x0000afafafaf00afULL,
368 0x00007c7c7c7c007c, 0x00001f1f1f1f001f, 0x00003e3e3e3e003e, 368 0x00007c7c7c7c007cULL, 0x00001f1f1f1f001fULL, 0x00003e3e3e3e003eULL,
369 0x0000dcdcdcdc00dc, 0x00005e5e5e5e005e, 0x00000b0b0b0b000b, 369 0x0000dcdcdcdc00dcULL, 0x00005e5e5e5e005eULL, 0x00000b0b0b0b000bULL,
370 0x0000a6a6a6a600a6, 0x0000393939390039, 0x0000d5d5d5d500d5, 370 0x0000a6a6a6a600a6ULL, 0x0000393939390039ULL, 0x0000d5d5d5d500d5ULL,
371 0x00005d5d5d5d005d, 0x0000d9d9d9d900d9, 0x00005a5a5a5a005a, 371 0x00005d5d5d5d005dULL, 0x0000d9d9d9d900d9ULL, 0x00005a5a5a5a005aULL,
372 0x0000515151510051, 0x00006c6c6c6c006c, 0x00008b8b8b8b008b, 372 0x0000515151510051ULL, 0x00006c6c6c6c006cULL, 0x00008b8b8b8b008bULL,
373 0x00009a9a9a9a009a, 0x0000fbfbfbfb00fb, 0x0000b0b0b0b000b0, 373 0x00009a9a9a9a009aULL, 0x0000fbfbfbfb00fbULL, 0x0000b0b0b0b000b0ULL,
374 0x0000747474740074, 0x00002b2b2b2b002b, 0x0000f0f0f0f000f0, 374 0x0000747474740074ULL, 0x00002b2b2b2b002bULL, 0x0000f0f0f0f000f0ULL,
375 0x0000848484840084, 0x0000dfdfdfdf00df, 0x0000cbcbcbcb00cb, 375 0x0000848484840084ULL, 0x0000dfdfdfdf00dfULL, 0x0000cbcbcbcb00cbULL,
376 0x0000343434340034, 0x0000767676760076, 0x00006d6d6d6d006d, 376 0x0000343434340034ULL, 0x0000767676760076ULL, 0x00006d6d6d6d006dULL,
377 0x0000a9a9a9a900a9, 0x0000d1d1d1d100d1, 0x0000040404040004, 377 0x0000a9a9a9a900a9ULL, 0x0000d1d1d1d100d1ULL, 0x0000040404040004ULL,
378 0x0000141414140014, 0x00003a3a3a3a003a, 0x0000dededede00de, 378 0x0000141414140014ULL, 0x00003a3a3a3a003aULL, 0x0000dededede00deULL,
379 0x0000111111110011, 0x0000323232320032, 0x00009c9c9c9c009c, 379 0x0000111111110011ULL, 0x0000323232320032ULL, 0x00009c9c9c9c009cULL,
380 0x0000535353530053, 0x0000f2f2f2f200f2, 0x0000fefefefe00fe, 380 0x0000535353530053ULL, 0x0000f2f2f2f200f2ULL, 0x0000fefefefe00feULL,
381 0x0000cfcfcfcf00cf, 0x0000c3c3c3c300c3, 0x00007a7a7a7a007a, 381 0x0000cfcfcfcf00cfULL, 0x0000c3c3c3c300c3ULL, 0x00007a7a7a7a007aULL,
382 0x0000242424240024, 0x0000e8e8e8e800e8, 0x0000606060600060, 382 0x0000242424240024ULL, 0x0000e8e8e8e800e8ULL, 0x0000606060600060ULL,
383 0x0000696969690069, 0x0000aaaaaaaa00aa, 0x0000a0a0a0a000a0, 383 0x0000696969690069ULL, 0x0000aaaaaaaa00aaULL, 0x0000a0a0a0a000a0ULL,
384 0x0000a1a1a1a100a1, 0x0000626262620062, 0x0000545454540054, 384 0x0000a1a1a1a100a1ULL, 0x0000626262620062ULL, 0x0000545454540054ULL,
385 0x00001e1e1e1e001e, 0x0000e0e0e0e000e0, 0x0000646464640064, 385 0x00001e1e1e1e001eULL, 0x0000e0e0e0e000e0ULL, 0x0000646464640064ULL,
386 0x0000101010100010, 0x0000000000000000, 0x0000a3a3a3a300a3, 386 0x0000101010100010ULL, 0x0000000000000000ULL, 0x0000a3a3a3a300a3ULL,
387 0x0000757575750075, 0x00008a8a8a8a008a, 0x0000e6e6e6e600e6, 387 0x0000757575750075ULL, 0x00008a8a8a8a008aULL, 0x0000e6e6e6e600e6ULL,
388 0x0000090909090009, 0x0000dddddddd00dd, 0x0000878787870087, 388 0x0000090909090009ULL, 0x0000dddddddd00ddULL, 0x0000878787870087ULL,
389 0x0000838383830083, 0x0000cdcdcdcd00cd, 0x0000909090900090, 389 0x0000838383830083ULL, 0x0000cdcdcdcd00cdULL, 0x0000909090900090ULL,
390 0x0000737373730073, 0x0000f6f6f6f600f6, 0x00009d9d9d9d009d, 390 0x0000737373730073ULL, 0x0000f6f6f6f600f6ULL, 0x00009d9d9d9d009dULL,
391 0x0000bfbfbfbf00bf, 0x0000525252520052, 0x0000d8d8d8d800d8, 391 0x0000bfbfbfbf00bfULL, 0x0000525252520052ULL, 0x0000d8d8d8d800d8ULL,
392 0x0000c8c8c8c800c8, 0x0000c6c6c6c600c6, 0x0000818181810081, 392 0x0000c8c8c8c800c8ULL, 0x0000c6c6c6c600c6ULL, 0x0000818181810081ULL,
393 0x00006f6f6f6f006f, 0x0000131313130013, 0x0000636363630063, 393 0x00006f6f6f6f006fULL, 0x0000131313130013ULL, 0x0000636363630063ULL,
394 0x0000e9e9e9e900e9, 0x0000a7a7a7a700a7, 0x00009f9f9f9f009f, 394 0x0000e9e9e9e900e9ULL, 0x0000a7a7a7a700a7ULL, 0x00009f9f9f9f009fULL,
395 0x0000bcbcbcbc00bc, 0x0000292929290029, 0x0000f9f9f9f900f9, 395 0x0000bcbcbcbc00bcULL, 0x0000292929290029ULL, 0x0000f9f9f9f900f9ULL,
396 0x00002f2f2f2f002f, 0x0000b4b4b4b400b4, 0x0000787878780078, 396 0x00002f2f2f2f002fULL, 0x0000b4b4b4b400b4ULL, 0x0000787878780078ULL,
397 0x0000060606060006, 0x0000e7e7e7e700e7, 0x0000717171710071, 397 0x0000060606060006ULL, 0x0000e7e7e7e700e7ULL, 0x0000717171710071ULL,
398 0x0000d4d4d4d400d4, 0x0000abababab00ab, 0x0000888888880088, 398 0x0000d4d4d4d400d4ULL, 0x0000abababab00abULL, 0x0000888888880088ULL,
399 0x00008d8d8d8d008d, 0x0000727272720072, 0x0000b9b9b9b900b9, 399 0x00008d8d8d8d008dULL, 0x0000727272720072ULL, 0x0000b9b9b9b900b9ULL,
400 0x0000f8f8f8f800f8, 0x0000acacacac00ac, 0x0000363636360036, 400 0x0000f8f8f8f800f8ULL, 0x0000acacacac00acULL, 0x0000363636360036ULL,
401 0x00002a2a2a2a002a, 0x00003c3c3c3c003c, 0x0000f1f1f1f100f1, 401 0x00002a2a2a2a002aULL, 0x00003c3c3c3c003cULL, 0x0000f1f1f1f100f1ULL,
402 0x0000404040400040, 0x0000d3d3d3d300d3, 0x0000bbbbbbbb00bb, 402 0x0000404040400040ULL, 0x0000d3d3d3d300d3ULL, 0x0000bbbbbbbb00bbULL,
403 0x0000434343430043, 0x0000151515150015, 0x0000adadadad00ad, 403 0x0000434343430043ULL, 0x0000151515150015ULL, 0x0000adadadad00adULL,
404 0x0000777777770077, 0x0000808080800080, 0x0000828282820082, 404 0x0000777777770077ULL, 0x0000808080800080ULL, 0x0000828282820082ULL,
405 0x0000ecececec00ec, 0x0000272727270027, 0x0000e5e5e5e500e5, 405 0x0000ecececec00ecULL, 0x0000272727270027ULL, 0x0000e5e5e5e500e5ULL,
406 0x0000858585850085, 0x0000353535350035, 0x00000c0c0c0c000c, 406 0x0000858585850085ULL, 0x0000353535350035ULL, 0x00000c0c0c0c000cULL,
407 0x0000414141410041, 0x0000efefefef00ef, 0x0000939393930093, 407 0x0000414141410041ULL, 0x0000efefefef00efULL, 0x0000939393930093ULL,
408 0x0000191919190019, 0x0000212121210021, 0x00000e0e0e0e000e, 408 0x0000191919190019ULL, 0x0000212121210021ULL, 0x00000e0e0e0e000eULL,
409 0x00004e4e4e4e004e, 0x0000656565650065, 0x0000bdbdbdbd00bd, 409 0x00004e4e4e4e004eULL, 0x0000656565650065ULL, 0x0000bdbdbdbd00bdULL,
410 0x0000b8b8b8b800b8, 0x00008f8f8f8f008f, 0x0000ebebebeb00eb, 410 0x0000b8b8b8b800b8ULL, 0x00008f8f8f8f008fULL, 0x0000ebebebeb00ebULL,
411 0x0000cececece00ce, 0x0000303030300030, 0x00005f5f5f5f005f, 411 0x0000cececece00ceULL, 0x0000303030300030ULL, 0x00005f5f5f5f005fULL,
412 0x0000c5c5c5c500c5, 0x00001a1a1a1a001a, 0x0000e1e1e1e100e1, 412 0x0000c5c5c5c500c5ULL, 0x00001a1a1a1a001aULL, 0x0000e1e1e1e100e1ULL,
413 0x0000cacacaca00ca, 0x0000474747470047, 0x00003d3d3d3d003d, 413 0x0000cacacaca00caULL, 0x0000474747470047ULL, 0x00003d3d3d3d003dULL,
414 0x0000010101010001, 0x0000d6d6d6d600d6, 0x0000565656560056, 414 0x0000010101010001ULL, 0x0000d6d6d6d600d6ULL, 0x0000565656560056ULL,
415 0x00004d4d4d4d004d, 0x00000d0d0d0d000d, 0x0000666666660066, 415 0x00004d4d4d4d004dULL, 0x00000d0d0d0d000dULL, 0x0000666666660066ULL,
416 0x0000cccccccc00cc, 0x00002d2d2d2d002d, 0x0000121212120012, 416 0x0000cccccccc00ccULL, 0x00002d2d2d2d002dULL, 0x0000121212120012ULL,
417 0x0000202020200020, 0x0000b1b1b1b100b1, 0x0000999999990099, 417 0x0000202020200020ULL, 0x0000b1b1b1b100b1ULL, 0x0000999999990099ULL,
418 0x00004c4c4c4c004c, 0x0000c2c2c2c200c2, 0x00007e7e7e7e007e, 418 0x00004c4c4c4c004cULL, 0x0000c2c2c2c200c2ULL, 0x00007e7e7e7e007eULL,
419 0x0000050505050005, 0x0000b7b7b7b700b7, 0x0000313131310031, 419 0x0000050505050005ULL, 0x0000b7b7b7b700b7ULL, 0x0000313131310031ULL,
420 0x0000171717170017, 0x0000d7d7d7d700d7, 0x0000585858580058, 420 0x0000171717170017ULL, 0x0000d7d7d7d700d7ULL, 0x0000585858580058ULL,
421 0x0000616161610061, 0x00001b1b1b1b001b, 0x00001c1c1c1c001c, 421 0x0000616161610061ULL, 0x00001b1b1b1b001bULL, 0x00001c1c1c1c001cULL,
422 0x00000f0f0f0f000f, 0x0000161616160016, 0x0000181818180018, 422 0x00000f0f0f0f000fULL, 0x0000161616160016ULL, 0x0000181818180018ULL,
423 0x0000222222220022, 0x0000444444440044, 0x0000b2b2b2b200b2, 423 0x0000222222220022ULL, 0x0000444444440044ULL, 0x0000b2b2b2b200b2ULL,
424 0x0000b5b5b5b500b5, 0x0000919191910091, 0x0000080808080008, 424 0x0000b5b5b5b500b5ULL, 0x0000919191910091ULL, 0x0000080808080008ULL,
425 0x0000a8a8a8a800a8, 0x0000fcfcfcfc00fc, 0x0000505050500050, 425 0x0000a8a8a8a800a8ULL, 0x0000fcfcfcfc00fcULL, 0x0000505050500050ULL,
426 0x0000d0d0d0d000d0, 0x00007d7d7d7d007d, 0x0000898989890089, 426 0x0000d0d0d0d000d0ULL, 0x00007d7d7d7d007dULL, 0x0000898989890089ULL,
427 0x0000979797970097, 0x00005b5b5b5b005b, 0x0000959595950095, 427 0x0000979797970097ULL, 0x00005b5b5b5b005bULL, 0x0000959595950095ULL,
428 0x0000ffffffff00ff, 0x0000d2d2d2d200d2, 0x0000c4c4c4c400c4, 428 0x0000ffffffff00ffULL, 0x0000d2d2d2d200d2ULL, 0x0000c4c4c4c400c4ULL,
429 0x0000484848480048, 0x0000f7f7f7f700f7, 0x0000dbdbdbdb00db, 429 0x0000484848480048ULL, 0x0000f7f7f7f700f7ULL, 0x0000dbdbdbdb00dbULL,
430 0x0000030303030003, 0x0000dadadada00da, 0x00003f3f3f3f003f, 430 0x0000030303030003ULL, 0x0000dadadada00daULL, 0x00003f3f3f3f003fULL,
431 0x0000949494940094, 0x00005c5c5c5c005c, 0x0000020202020002, 431 0x0000949494940094ULL, 0x00005c5c5c5c005cULL, 0x0000020202020002ULL,
432 0x00004a4a4a4a004a, 0x0000333333330033, 0x0000676767670067, 432 0x00004a4a4a4a004aULL, 0x0000333333330033ULL, 0x0000676767670067ULL,
433 0x0000f3f3f3f300f3, 0x00007f7f7f7f007f, 0x0000e2e2e2e200e2, 433 0x0000f3f3f3f300f3ULL, 0x00007f7f7f7f007fULL, 0x0000e2e2e2e200e2ULL,
434 0x00009b9b9b9b009b, 0x0000262626260026, 0x0000373737370037, 434 0x00009b9b9b9b009bULL, 0x0000262626260026ULL, 0x0000373737370037ULL,
435 0x00003b3b3b3b003b, 0x0000969696960096, 0x00004b4b4b4b004b, 435 0x00003b3b3b3b003bULL, 0x0000969696960096ULL, 0x00004b4b4b4b004bULL,
436 0x0000bebebebe00be, 0x00002e2e2e2e002e, 0x0000797979790079, 436 0x0000bebebebe00beULL, 0x00002e2e2e2e002eULL, 0x0000797979790079ULL,
437 0x00008c8c8c8c008c, 0x00006e6e6e6e006e, 0x00008e8e8e8e008e, 437 0x00008c8c8c8c008cULL, 0x00006e6e6e6e006eULL, 0x00008e8e8e8e008eULL,
438 0x0000f5f5f5f500f5, 0x0000b6b6b6b600b6, 0x0000fdfdfdfd00fd, 438 0x0000f5f5f5f500f5ULL, 0x0000b6b6b6b600b6ULL, 0x0000fdfdfdfd00fdULL,
439 0x0000595959590059, 0x0000989898980098, 0x00006a6a6a6a006a, 439 0x0000595959590059ULL, 0x0000989898980098ULL, 0x00006a6a6a6a006aULL,
440 0x0000464646460046, 0x0000babababa00ba, 0x0000252525250025, 440 0x0000464646460046ULL, 0x0000babababa00baULL, 0x0000252525250025ULL,
441 0x0000424242420042, 0x0000a2a2a2a200a2, 0x0000fafafafa00fa, 441 0x0000424242420042ULL, 0x0000a2a2a2a200a2ULL, 0x0000fafafafa00faULL,
442 0x0000070707070007, 0x0000555555550055, 0x0000eeeeeeee00ee, 442 0x0000070707070007ULL, 0x0000555555550055ULL, 0x0000eeeeeeee00eeULL,
443 0x00000a0a0a0a000a, 0x0000494949490049, 0x0000686868680068, 443 0x00000a0a0a0a000aULL, 0x0000494949490049ULL, 0x0000686868680068ULL,
444 0x0000383838380038, 0x0000a4a4a4a400a4, 0x0000282828280028, 444 0x0000383838380038ULL, 0x0000a4a4a4a400a4ULL, 0x0000282828280028ULL,
445 0x00007b7b7b7b007b, 0x0000c9c9c9c900c9, 0x0000c1c1c1c100c1, 445 0x00007b7b7b7b007bULL, 0x0000c9c9c9c900c9ULL, 0x0000c1c1c1c100c1ULL,
446 0x0000e3e3e3e300e3, 0x0000f4f4f4f400f4, 0x0000c7c7c7c700c7, 446 0x0000e3e3e3e300e3ULL, 0x0000f4f4f4f400f4ULL, 0x0000c7c7c7c700c7ULL,
447 0x00009e9e9e9e009e, 447 0x00009e9e9e9e009eULL,
448}; 448};
449 449
450const u64 camellia_sp02220222[256] = { 450const u64 camellia_sp02220222[256] = {
451 0x00e0e0e000e0e0e0, 0x0005050500050505, 0x0058585800585858, 451 0x00e0e0e000e0e0e0ULL, 0x0005050500050505ULL, 0x0058585800585858ULL,
452 0x00d9d9d900d9d9d9, 0x0067676700676767, 0x004e4e4e004e4e4e, 452 0x00d9d9d900d9d9d9ULL, 0x0067676700676767ULL, 0x004e4e4e004e4e4eULL,
453 0x0081818100818181, 0x00cbcbcb00cbcbcb, 0x00c9c9c900c9c9c9, 453 0x0081818100818181ULL, 0x00cbcbcb00cbcbcbULL, 0x00c9c9c900c9c9c9ULL,
454 0x000b0b0b000b0b0b, 0x00aeaeae00aeaeae, 0x006a6a6a006a6a6a, 454 0x000b0b0b000b0b0bULL, 0x00aeaeae00aeaeaeULL, 0x006a6a6a006a6a6aULL,
455 0x00d5d5d500d5d5d5, 0x0018181800181818, 0x005d5d5d005d5d5d, 455 0x00d5d5d500d5d5d5ULL, 0x0018181800181818ULL, 0x005d5d5d005d5d5dULL,
456 0x0082828200828282, 0x0046464600464646, 0x00dfdfdf00dfdfdf, 456 0x0082828200828282ULL, 0x0046464600464646ULL, 0x00dfdfdf00dfdfdfULL,
457 0x00d6d6d600d6d6d6, 0x0027272700272727, 0x008a8a8a008a8a8a, 457 0x00d6d6d600d6d6d6ULL, 0x0027272700272727ULL, 0x008a8a8a008a8a8aULL,
458 0x0032323200323232, 0x004b4b4b004b4b4b, 0x0042424200424242, 458 0x0032323200323232ULL, 0x004b4b4b004b4b4bULL, 0x0042424200424242ULL,
459 0x00dbdbdb00dbdbdb, 0x001c1c1c001c1c1c, 0x009e9e9e009e9e9e, 459 0x00dbdbdb00dbdbdbULL, 0x001c1c1c001c1c1cULL, 0x009e9e9e009e9e9eULL,
460 0x009c9c9c009c9c9c, 0x003a3a3a003a3a3a, 0x00cacaca00cacaca, 460 0x009c9c9c009c9c9cULL, 0x003a3a3a003a3a3aULL, 0x00cacaca00cacacaULL,
461 0x0025252500252525, 0x007b7b7b007b7b7b, 0x000d0d0d000d0d0d, 461 0x0025252500252525ULL, 0x007b7b7b007b7b7bULL, 0x000d0d0d000d0d0dULL,
462 0x0071717100717171, 0x005f5f5f005f5f5f, 0x001f1f1f001f1f1f, 462 0x0071717100717171ULL, 0x005f5f5f005f5f5fULL, 0x001f1f1f001f1f1fULL,
463 0x00f8f8f800f8f8f8, 0x00d7d7d700d7d7d7, 0x003e3e3e003e3e3e, 463 0x00f8f8f800f8f8f8ULL, 0x00d7d7d700d7d7d7ULL, 0x003e3e3e003e3e3eULL,
464 0x009d9d9d009d9d9d, 0x007c7c7c007c7c7c, 0x0060606000606060, 464 0x009d9d9d009d9d9dULL, 0x007c7c7c007c7c7cULL, 0x0060606000606060ULL,
465 0x00b9b9b900b9b9b9, 0x00bebebe00bebebe, 0x00bcbcbc00bcbcbc, 465 0x00b9b9b900b9b9b9ULL, 0x00bebebe00bebebeULL, 0x00bcbcbc00bcbcbcULL,
466 0x008b8b8b008b8b8b, 0x0016161600161616, 0x0034343400343434, 466 0x008b8b8b008b8b8bULL, 0x0016161600161616ULL, 0x0034343400343434ULL,
467 0x004d4d4d004d4d4d, 0x00c3c3c300c3c3c3, 0x0072727200727272, 467 0x004d4d4d004d4d4dULL, 0x00c3c3c300c3c3c3ULL, 0x0072727200727272ULL,
468 0x0095959500959595, 0x00ababab00ababab, 0x008e8e8e008e8e8e, 468 0x0095959500959595ULL, 0x00ababab00abababULL, 0x008e8e8e008e8e8eULL,
469 0x00bababa00bababa, 0x007a7a7a007a7a7a, 0x00b3b3b300b3b3b3, 469 0x00bababa00bababaULL, 0x007a7a7a007a7a7aULL, 0x00b3b3b300b3b3b3ULL,
470 0x0002020200020202, 0x00b4b4b400b4b4b4, 0x00adadad00adadad, 470 0x0002020200020202ULL, 0x00b4b4b400b4b4b4ULL, 0x00adadad00adadadULL,
471 0x00a2a2a200a2a2a2, 0x00acacac00acacac, 0x00d8d8d800d8d8d8, 471 0x00a2a2a200a2a2a2ULL, 0x00acacac00acacacULL, 0x00d8d8d800d8d8d8ULL,
472 0x009a9a9a009a9a9a, 0x0017171700171717, 0x001a1a1a001a1a1a, 472 0x009a9a9a009a9a9aULL, 0x0017171700171717ULL, 0x001a1a1a001a1a1aULL,
473 0x0035353500353535, 0x00cccccc00cccccc, 0x00f7f7f700f7f7f7, 473 0x0035353500353535ULL, 0x00cccccc00ccccccULL, 0x00f7f7f700f7f7f7ULL,
474 0x0099999900999999, 0x0061616100616161, 0x005a5a5a005a5a5a, 474 0x0099999900999999ULL, 0x0061616100616161ULL, 0x005a5a5a005a5a5aULL,
475 0x00e8e8e800e8e8e8, 0x0024242400242424, 0x0056565600565656, 475 0x00e8e8e800e8e8e8ULL, 0x0024242400242424ULL, 0x0056565600565656ULL,
476 0x0040404000404040, 0x00e1e1e100e1e1e1, 0x0063636300636363, 476 0x0040404000404040ULL, 0x00e1e1e100e1e1e1ULL, 0x0063636300636363ULL,
477 0x0009090900090909, 0x0033333300333333, 0x00bfbfbf00bfbfbf, 477 0x0009090900090909ULL, 0x0033333300333333ULL, 0x00bfbfbf00bfbfbfULL,
478 0x0098989800989898, 0x0097979700979797, 0x0085858500858585, 478 0x0098989800989898ULL, 0x0097979700979797ULL, 0x0085858500858585ULL,
479 0x0068686800686868, 0x00fcfcfc00fcfcfc, 0x00ececec00ececec, 479 0x0068686800686868ULL, 0x00fcfcfc00fcfcfcULL, 0x00ececec00ecececULL,
480 0x000a0a0a000a0a0a, 0x00dadada00dadada, 0x006f6f6f006f6f6f, 480 0x000a0a0a000a0a0aULL, 0x00dadada00dadadaULL, 0x006f6f6f006f6f6fULL,
481 0x0053535300535353, 0x0062626200626262, 0x00a3a3a300a3a3a3, 481 0x0053535300535353ULL, 0x0062626200626262ULL, 0x00a3a3a300a3a3a3ULL,
482 0x002e2e2e002e2e2e, 0x0008080800080808, 0x00afafaf00afafaf, 482 0x002e2e2e002e2e2eULL, 0x0008080800080808ULL, 0x00afafaf00afafafULL,
483 0x0028282800282828, 0x00b0b0b000b0b0b0, 0x0074747400747474, 483 0x0028282800282828ULL, 0x00b0b0b000b0b0b0ULL, 0x0074747400747474ULL,
484 0x00c2c2c200c2c2c2, 0x00bdbdbd00bdbdbd, 0x0036363600363636, 484 0x00c2c2c200c2c2c2ULL, 0x00bdbdbd00bdbdbdULL, 0x0036363600363636ULL,
485 0x0022222200222222, 0x0038383800383838, 0x0064646400646464, 485 0x0022222200222222ULL, 0x0038383800383838ULL, 0x0064646400646464ULL,
486 0x001e1e1e001e1e1e, 0x0039393900393939, 0x002c2c2c002c2c2c, 486 0x001e1e1e001e1e1eULL, 0x0039393900393939ULL, 0x002c2c2c002c2c2cULL,
487 0x00a6a6a600a6a6a6, 0x0030303000303030, 0x00e5e5e500e5e5e5, 487 0x00a6a6a600a6a6a6ULL, 0x0030303000303030ULL, 0x00e5e5e500e5e5e5ULL,
488 0x0044444400444444, 0x00fdfdfd00fdfdfd, 0x0088888800888888, 488 0x0044444400444444ULL, 0x00fdfdfd00fdfdfdULL, 0x0088888800888888ULL,
489 0x009f9f9f009f9f9f, 0x0065656500656565, 0x0087878700878787, 489 0x009f9f9f009f9f9fULL, 0x0065656500656565ULL, 0x0087878700878787ULL,
490 0x006b6b6b006b6b6b, 0x00f4f4f400f4f4f4, 0x0023232300232323, 490 0x006b6b6b006b6b6bULL, 0x00f4f4f400f4f4f4ULL, 0x0023232300232323ULL,
491 0x0048484800484848, 0x0010101000101010, 0x00d1d1d100d1d1d1, 491 0x0048484800484848ULL, 0x0010101000101010ULL, 0x00d1d1d100d1d1d1ULL,
492 0x0051515100515151, 0x00c0c0c000c0c0c0, 0x00f9f9f900f9f9f9, 492 0x0051515100515151ULL, 0x00c0c0c000c0c0c0ULL, 0x00f9f9f900f9f9f9ULL,
493 0x00d2d2d200d2d2d2, 0x00a0a0a000a0a0a0, 0x0055555500555555, 493 0x00d2d2d200d2d2d2ULL, 0x00a0a0a000a0a0a0ULL, 0x0055555500555555ULL,
494 0x00a1a1a100a1a1a1, 0x0041414100414141, 0x00fafafa00fafafa, 494 0x00a1a1a100a1a1a1ULL, 0x0041414100414141ULL, 0x00fafafa00fafafaULL,
495 0x0043434300434343, 0x0013131300131313, 0x00c4c4c400c4c4c4, 495 0x0043434300434343ULL, 0x0013131300131313ULL, 0x00c4c4c400c4c4c4ULL,
496 0x002f2f2f002f2f2f, 0x00a8a8a800a8a8a8, 0x00b6b6b600b6b6b6, 496 0x002f2f2f002f2f2fULL, 0x00a8a8a800a8a8a8ULL, 0x00b6b6b600b6b6b6ULL,
497 0x003c3c3c003c3c3c, 0x002b2b2b002b2b2b, 0x00c1c1c100c1c1c1, 497 0x003c3c3c003c3c3cULL, 0x002b2b2b002b2b2bULL, 0x00c1c1c100c1c1c1ULL,
498 0x00ffffff00ffffff, 0x00c8c8c800c8c8c8, 0x00a5a5a500a5a5a5, 498 0x00ffffff00ffffffULL, 0x00c8c8c800c8c8c8ULL, 0x00a5a5a500a5a5a5ULL,
499 0x0020202000202020, 0x0089898900898989, 0x0000000000000000, 499 0x0020202000202020ULL, 0x0089898900898989ULL, 0x0000000000000000ULL,
500 0x0090909000909090, 0x0047474700474747, 0x00efefef00efefef, 500 0x0090909000909090ULL, 0x0047474700474747ULL, 0x00efefef00efefefULL,
501 0x00eaeaea00eaeaea, 0x00b7b7b700b7b7b7, 0x0015151500151515, 501 0x00eaeaea00eaeaeaULL, 0x00b7b7b700b7b7b7ULL, 0x0015151500151515ULL,
502 0x0006060600060606, 0x00cdcdcd00cdcdcd, 0x00b5b5b500b5b5b5, 502 0x0006060600060606ULL, 0x00cdcdcd00cdcdcdULL, 0x00b5b5b500b5b5b5ULL,
503 0x0012121200121212, 0x007e7e7e007e7e7e, 0x00bbbbbb00bbbbbb, 503 0x0012121200121212ULL, 0x007e7e7e007e7e7eULL, 0x00bbbbbb00bbbbbbULL,
504 0x0029292900292929, 0x000f0f0f000f0f0f, 0x00b8b8b800b8b8b8, 504 0x0029292900292929ULL, 0x000f0f0f000f0f0fULL, 0x00b8b8b800b8b8b8ULL,
505 0x0007070700070707, 0x0004040400040404, 0x009b9b9b009b9b9b, 505 0x0007070700070707ULL, 0x0004040400040404ULL, 0x009b9b9b009b9b9bULL,
506 0x0094949400949494, 0x0021212100212121, 0x0066666600666666, 506 0x0094949400949494ULL, 0x0021212100212121ULL, 0x0066666600666666ULL,
507 0x00e6e6e600e6e6e6, 0x00cecece00cecece, 0x00ededed00ededed, 507 0x00e6e6e600e6e6e6ULL, 0x00cecece00cececeULL, 0x00ededed00edededULL,
508 0x00e7e7e700e7e7e7, 0x003b3b3b003b3b3b, 0x00fefefe00fefefe, 508 0x00e7e7e700e7e7e7ULL, 0x003b3b3b003b3b3bULL, 0x00fefefe00fefefeULL,
509 0x007f7f7f007f7f7f, 0x00c5c5c500c5c5c5, 0x00a4a4a400a4a4a4, 509 0x007f7f7f007f7f7fULL, 0x00c5c5c500c5c5c5ULL, 0x00a4a4a400a4a4a4ULL,
510 0x0037373700373737, 0x00b1b1b100b1b1b1, 0x004c4c4c004c4c4c, 510 0x0037373700373737ULL, 0x00b1b1b100b1b1b1ULL, 0x004c4c4c004c4c4cULL,
511 0x0091919100919191, 0x006e6e6e006e6e6e, 0x008d8d8d008d8d8d, 511 0x0091919100919191ULL, 0x006e6e6e006e6e6eULL, 0x008d8d8d008d8d8dULL,
512 0x0076767600767676, 0x0003030300030303, 0x002d2d2d002d2d2d, 512 0x0076767600767676ULL, 0x0003030300030303ULL, 0x002d2d2d002d2d2dULL,
513 0x00dedede00dedede, 0x0096969600969696, 0x0026262600262626, 513 0x00dedede00dededeULL, 0x0096969600969696ULL, 0x0026262600262626ULL,
514 0x007d7d7d007d7d7d, 0x00c6c6c600c6c6c6, 0x005c5c5c005c5c5c, 514 0x007d7d7d007d7d7dULL, 0x00c6c6c600c6c6c6ULL, 0x005c5c5c005c5c5cULL,
515 0x00d3d3d300d3d3d3, 0x00f2f2f200f2f2f2, 0x004f4f4f004f4f4f, 515 0x00d3d3d300d3d3d3ULL, 0x00f2f2f200f2f2f2ULL, 0x004f4f4f004f4f4fULL,
516 0x0019191900191919, 0x003f3f3f003f3f3f, 0x00dcdcdc00dcdcdc, 516 0x0019191900191919ULL, 0x003f3f3f003f3f3fULL, 0x00dcdcdc00dcdcdcULL,
517 0x0079797900797979, 0x001d1d1d001d1d1d, 0x0052525200525252, 517 0x0079797900797979ULL, 0x001d1d1d001d1d1dULL, 0x0052525200525252ULL,
518 0x00ebebeb00ebebeb, 0x00f3f3f300f3f3f3, 0x006d6d6d006d6d6d, 518 0x00ebebeb00ebebebULL, 0x00f3f3f300f3f3f3ULL, 0x006d6d6d006d6d6dULL,
519 0x005e5e5e005e5e5e, 0x00fbfbfb00fbfbfb, 0x0069696900696969, 519 0x005e5e5e005e5e5eULL, 0x00fbfbfb00fbfbfbULL, 0x0069696900696969ULL,
520 0x00b2b2b200b2b2b2, 0x00f0f0f000f0f0f0, 0x0031313100313131, 520 0x00b2b2b200b2b2b2ULL, 0x00f0f0f000f0f0f0ULL, 0x0031313100313131ULL,
521 0x000c0c0c000c0c0c, 0x00d4d4d400d4d4d4, 0x00cfcfcf00cfcfcf, 521 0x000c0c0c000c0c0cULL, 0x00d4d4d400d4d4d4ULL, 0x00cfcfcf00cfcfcfULL,
522 0x008c8c8c008c8c8c, 0x00e2e2e200e2e2e2, 0x0075757500757575, 522 0x008c8c8c008c8c8cULL, 0x00e2e2e200e2e2e2ULL, 0x0075757500757575ULL,
523 0x00a9a9a900a9a9a9, 0x004a4a4a004a4a4a, 0x0057575700575757, 523 0x00a9a9a900a9a9a9ULL, 0x004a4a4a004a4a4aULL, 0x0057575700575757ULL,
524 0x0084848400848484, 0x0011111100111111, 0x0045454500454545, 524 0x0084848400848484ULL, 0x0011111100111111ULL, 0x0045454500454545ULL,
525 0x001b1b1b001b1b1b, 0x00f5f5f500f5f5f5, 0x00e4e4e400e4e4e4, 525 0x001b1b1b001b1b1bULL, 0x00f5f5f500f5f5f5ULL, 0x00e4e4e400e4e4e4ULL,
526 0x000e0e0e000e0e0e, 0x0073737300737373, 0x00aaaaaa00aaaaaa, 526 0x000e0e0e000e0e0eULL, 0x0073737300737373ULL, 0x00aaaaaa00aaaaaaULL,
527 0x00f1f1f100f1f1f1, 0x00dddddd00dddddd, 0x0059595900595959, 527 0x00f1f1f100f1f1f1ULL, 0x00dddddd00ddddddULL, 0x0059595900595959ULL,
528 0x0014141400141414, 0x006c6c6c006c6c6c, 0x0092929200929292, 528 0x0014141400141414ULL, 0x006c6c6c006c6c6cULL, 0x0092929200929292ULL,
529 0x0054545400545454, 0x00d0d0d000d0d0d0, 0x0078787800787878, 529 0x0054545400545454ULL, 0x00d0d0d000d0d0d0ULL, 0x0078787800787878ULL,
530 0x0070707000707070, 0x00e3e3e300e3e3e3, 0x0049494900494949, 530 0x0070707000707070ULL, 0x00e3e3e300e3e3e3ULL, 0x0049494900494949ULL,
531 0x0080808000808080, 0x0050505000505050, 0x00a7a7a700a7a7a7, 531 0x0080808000808080ULL, 0x0050505000505050ULL, 0x00a7a7a700a7a7a7ULL,
532 0x00f6f6f600f6f6f6, 0x0077777700777777, 0x0093939300939393, 532 0x00f6f6f600f6f6f6ULL, 0x0077777700777777ULL, 0x0093939300939393ULL,
533 0x0086868600868686, 0x0083838300838383, 0x002a2a2a002a2a2a, 533 0x0086868600868686ULL, 0x0083838300838383ULL, 0x002a2a2a002a2a2aULL,
534 0x00c7c7c700c7c7c7, 0x005b5b5b005b5b5b, 0x00e9e9e900e9e9e9, 534 0x00c7c7c700c7c7c7ULL, 0x005b5b5b005b5b5bULL, 0x00e9e9e900e9e9e9ULL,
535 0x00eeeeee00eeeeee, 0x008f8f8f008f8f8f, 0x0001010100010101, 535 0x00eeeeee00eeeeeeULL, 0x008f8f8f008f8f8fULL, 0x0001010100010101ULL,
536 0x003d3d3d003d3d3d, 536 0x003d3d3d003d3d3dULL,
537}; 537};
538 538
539const u64 camellia_sp30333033[256] = { 539const u64 camellia_sp30333033[256] = {
540 0x3800383838003838, 0x4100414141004141, 0x1600161616001616, 540 0x3800383838003838ULL, 0x4100414141004141ULL, 0x1600161616001616ULL,
541 0x7600767676007676, 0xd900d9d9d900d9d9, 0x9300939393009393, 541 0x7600767676007676ULL, 0xd900d9d9d900d9d9ULL, 0x9300939393009393ULL,
542 0x6000606060006060, 0xf200f2f2f200f2f2, 0x7200727272007272, 542 0x6000606060006060ULL, 0xf200f2f2f200f2f2ULL, 0x7200727272007272ULL,
543 0xc200c2c2c200c2c2, 0xab00ababab00abab, 0x9a009a9a9a009a9a, 543 0xc200c2c2c200c2c2ULL, 0xab00ababab00ababULL, 0x9a009a9a9a009a9aULL,
544 0x7500757575007575, 0x0600060606000606, 0x5700575757005757, 544 0x7500757575007575ULL, 0x0600060606000606ULL, 0x5700575757005757ULL,
545 0xa000a0a0a000a0a0, 0x9100919191009191, 0xf700f7f7f700f7f7, 545 0xa000a0a0a000a0a0ULL, 0x9100919191009191ULL, 0xf700f7f7f700f7f7ULL,
546 0xb500b5b5b500b5b5, 0xc900c9c9c900c9c9, 0xa200a2a2a200a2a2, 546 0xb500b5b5b500b5b5ULL, 0xc900c9c9c900c9c9ULL, 0xa200a2a2a200a2a2ULL,
547 0x8c008c8c8c008c8c, 0xd200d2d2d200d2d2, 0x9000909090009090, 547 0x8c008c8c8c008c8cULL, 0xd200d2d2d200d2d2ULL, 0x9000909090009090ULL,
548 0xf600f6f6f600f6f6, 0x0700070707000707, 0xa700a7a7a700a7a7, 548 0xf600f6f6f600f6f6ULL, 0x0700070707000707ULL, 0xa700a7a7a700a7a7ULL,
549 0x2700272727002727, 0x8e008e8e8e008e8e, 0xb200b2b2b200b2b2, 549 0x2700272727002727ULL, 0x8e008e8e8e008e8eULL, 0xb200b2b2b200b2b2ULL,
550 0x4900494949004949, 0xde00dedede00dede, 0x4300434343004343, 550 0x4900494949004949ULL, 0xde00dedede00dedeULL, 0x4300434343004343ULL,
551 0x5c005c5c5c005c5c, 0xd700d7d7d700d7d7, 0xc700c7c7c700c7c7, 551 0x5c005c5c5c005c5cULL, 0xd700d7d7d700d7d7ULL, 0xc700c7c7c700c7c7ULL,
552 0x3e003e3e3e003e3e, 0xf500f5f5f500f5f5, 0x8f008f8f8f008f8f, 552 0x3e003e3e3e003e3eULL, 0xf500f5f5f500f5f5ULL, 0x8f008f8f8f008f8fULL,
553 0x6700676767006767, 0x1f001f1f1f001f1f, 0x1800181818001818, 553 0x6700676767006767ULL, 0x1f001f1f1f001f1fULL, 0x1800181818001818ULL,
554 0x6e006e6e6e006e6e, 0xaf00afafaf00afaf, 0x2f002f2f2f002f2f, 554 0x6e006e6e6e006e6eULL, 0xaf00afafaf00afafULL, 0x2f002f2f2f002f2fULL,
555 0xe200e2e2e200e2e2, 0x8500858585008585, 0x0d000d0d0d000d0d, 555 0xe200e2e2e200e2e2ULL, 0x8500858585008585ULL, 0x0d000d0d0d000d0dULL,
556 0x5300535353005353, 0xf000f0f0f000f0f0, 0x9c009c9c9c009c9c, 556 0x5300535353005353ULL, 0xf000f0f0f000f0f0ULL, 0x9c009c9c9c009c9cULL,
557 0x6500656565006565, 0xea00eaeaea00eaea, 0xa300a3a3a300a3a3, 557 0x6500656565006565ULL, 0xea00eaeaea00eaeaULL, 0xa300a3a3a300a3a3ULL,
558 0xae00aeaeae00aeae, 0x9e009e9e9e009e9e, 0xec00ececec00ecec, 558 0xae00aeaeae00aeaeULL, 0x9e009e9e9e009e9eULL, 0xec00ececec00ececULL,
559 0x8000808080008080, 0x2d002d2d2d002d2d, 0x6b006b6b6b006b6b, 559 0x8000808080008080ULL, 0x2d002d2d2d002d2dULL, 0x6b006b6b6b006b6bULL,
560 0xa800a8a8a800a8a8, 0x2b002b2b2b002b2b, 0x3600363636003636, 560 0xa800a8a8a800a8a8ULL, 0x2b002b2b2b002b2bULL, 0x3600363636003636ULL,
561 0xa600a6a6a600a6a6, 0xc500c5c5c500c5c5, 0x8600868686008686, 561 0xa600a6a6a600a6a6ULL, 0xc500c5c5c500c5c5ULL, 0x8600868686008686ULL,
562 0x4d004d4d4d004d4d, 0x3300333333003333, 0xfd00fdfdfd00fdfd, 562 0x4d004d4d4d004d4dULL, 0x3300333333003333ULL, 0xfd00fdfdfd00fdfdULL,
563 0x6600666666006666, 0x5800585858005858, 0x9600969696009696, 563 0x6600666666006666ULL, 0x5800585858005858ULL, 0x9600969696009696ULL,
564 0x3a003a3a3a003a3a, 0x0900090909000909, 0x9500959595009595, 564 0x3a003a3a3a003a3aULL, 0x0900090909000909ULL, 0x9500959595009595ULL,
565 0x1000101010001010, 0x7800787878007878, 0xd800d8d8d800d8d8, 565 0x1000101010001010ULL, 0x7800787878007878ULL, 0xd800d8d8d800d8d8ULL,
566 0x4200424242004242, 0xcc00cccccc00cccc, 0xef00efefef00efef, 566 0x4200424242004242ULL, 0xcc00cccccc00ccccULL, 0xef00efefef00efefULL,
567 0x2600262626002626, 0xe500e5e5e500e5e5, 0x6100616161006161, 567 0x2600262626002626ULL, 0xe500e5e5e500e5e5ULL, 0x6100616161006161ULL,
568 0x1a001a1a1a001a1a, 0x3f003f3f3f003f3f, 0x3b003b3b3b003b3b, 568 0x1a001a1a1a001a1aULL, 0x3f003f3f3f003f3fULL, 0x3b003b3b3b003b3bULL,
569 0x8200828282008282, 0xb600b6b6b600b6b6, 0xdb00dbdbdb00dbdb, 569 0x8200828282008282ULL, 0xb600b6b6b600b6b6ULL, 0xdb00dbdbdb00dbdbULL,
570 0xd400d4d4d400d4d4, 0x9800989898009898, 0xe800e8e8e800e8e8, 570 0xd400d4d4d400d4d4ULL, 0x9800989898009898ULL, 0xe800e8e8e800e8e8ULL,
571 0x8b008b8b8b008b8b, 0x0200020202000202, 0xeb00ebebeb00ebeb, 571 0x8b008b8b8b008b8bULL, 0x0200020202000202ULL, 0xeb00ebebeb00ebebULL,
572 0x0a000a0a0a000a0a, 0x2c002c2c2c002c2c, 0x1d001d1d1d001d1d, 572 0x0a000a0a0a000a0aULL, 0x2c002c2c2c002c2cULL, 0x1d001d1d1d001d1dULL,
573 0xb000b0b0b000b0b0, 0x6f006f6f6f006f6f, 0x8d008d8d8d008d8d, 573 0xb000b0b0b000b0b0ULL, 0x6f006f6f6f006f6fULL, 0x8d008d8d8d008d8dULL,
574 0x8800888888008888, 0x0e000e0e0e000e0e, 0x1900191919001919, 574 0x8800888888008888ULL, 0x0e000e0e0e000e0eULL, 0x1900191919001919ULL,
575 0x8700878787008787, 0x4e004e4e4e004e4e, 0x0b000b0b0b000b0b, 575 0x8700878787008787ULL, 0x4e004e4e4e004e4eULL, 0x0b000b0b0b000b0bULL,
576 0xa900a9a9a900a9a9, 0x0c000c0c0c000c0c, 0x7900797979007979, 576 0xa900a9a9a900a9a9ULL, 0x0c000c0c0c000c0cULL, 0x7900797979007979ULL,
577 0x1100111111001111, 0x7f007f7f7f007f7f, 0x2200222222002222, 577 0x1100111111001111ULL, 0x7f007f7f7f007f7fULL, 0x2200222222002222ULL,
578 0xe700e7e7e700e7e7, 0x5900595959005959, 0xe100e1e1e100e1e1, 578 0xe700e7e7e700e7e7ULL, 0x5900595959005959ULL, 0xe100e1e1e100e1e1ULL,
579 0xda00dadada00dada, 0x3d003d3d3d003d3d, 0xc800c8c8c800c8c8, 579 0xda00dadada00dadaULL, 0x3d003d3d3d003d3dULL, 0xc800c8c8c800c8c8ULL,
580 0x1200121212001212, 0x0400040404000404, 0x7400747474007474, 580 0x1200121212001212ULL, 0x0400040404000404ULL, 0x7400747474007474ULL,
581 0x5400545454005454, 0x3000303030003030, 0x7e007e7e7e007e7e, 581 0x5400545454005454ULL, 0x3000303030003030ULL, 0x7e007e7e7e007e7eULL,
582 0xb400b4b4b400b4b4, 0x2800282828002828, 0x5500555555005555, 582 0xb400b4b4b400b4b4ULL, 0x2800282828002828ULL, 0x5500555555005555ULL,
583 0x6800686868006868, 0x5000505050005050, 0xbe00bebebe00bebe, 583 0x6800686868006868ULL, 0x5000505050005050ULL, 0xbe00bebebe00bebeULL,
584 0xd000d0d0d000d0d0, 0xc400c4c4c400c4c4, 0x3100313131003131, 584 0xd000d0d0d000d0d0ULL, 0xc400c4c4c400c4c4ULL, 0x3100313131003131ULL,
585 0xcb00cbcbcb00cbcb, 0x2a002a2a2a002a2a, 0xad00adadad00adad, 585 0xcb00cbcbcb00cbcbULL, 0x2a002a2a2a002a2aULL, 0xad00adadad00adadULL,
586 0x0f000f0f0f000f0f, 0xca00cacaca00caca, 0x7000707070007070, 586 0x0f000f0f0f000f0fULL, 0xca00cacaca00cacaULL, 0x7000707070007070ULL,
587 0xff00ffffff00ffff, 0x3200323232003232, 0x6900696969006969, 587 0xff00ffffff00ffffULL, 0x3200323232003232ULL, 0x6900696969006969ULL,
588 0x0800080808000808, 0x6200626262006262, 0x0000000000000000, 588 0x0800080808000808ULL, 0x6200626262006262ULL, 0x0000000000000000ULL,
589 0x2400242424002424, 0xd100d1d1d100d1d1, 0xfb00fbfbfb00fbfb, 589 0x2400242424002424ULL, 0xd100d1d1d100d1d1ULL, 0xfb00fbfbfb00fbfbULL,
590 0xba00bababa00baba, 0xed00ededed00eded, 0x4500454545004545, 590 0xba00bababa00babaULL, 0xed00ededed00ededULL, 0x4500454545004545ULL,
591 0x8100818181008181, 0x7300737373007373, 0x6d006d6d6d006d6d, 591 0x8100818181008181ULL, 0x7300737373007373ULL, 0x6d006d6d6d006d6dULL,
592 0x8400848484008484, 0x9f009f9f9f009f9f, 0xee00eeeeee00eeee, 592 0x8400848484008484ULL, 0x9f009f9f9f009f9fULL, 0xee00eeeeee00eeeeULL,
593 0x4a004a4a4a004a4a, 0xc300c3c3c300c3c3, 0x2e002e2e2e002e2e, 593 0x4a004a4a4a004a4aULL, 0xc300c3c3c300c3c3ULL, 0x2e002e2e2e002e2eULL,
594 0xc100c1c1c100c1c1, 0x0100010101000101, 0xe600e6e6e600e6e6, 594 0xc100c1c1c100c1c1ULL, 0x0100010101000101ULL, 0xe600e6e6e600e6e6ULL,
595 0x2500252525002525, 0x4800484848004848, 0x9900999999009999, 595 0x2500252525002525ULL, 0x4800484848004848ULL, 0x9900999999009999ULL,
596 0xb900b9b9b900b9b9, 0xb300b3b3b300b3b3, 0x7b007b7b7b007b7b, 596 0xb900b9b9b900b9b9ULL, 0xb300b3b3b300b3b3ULL, 0x7b007b7b7b007b7bULL,
597 0xf900f9f9f900f9f9, 0xce00cecece00cece, 0xbf00bfbfbf00bfbf, 597 0xf900f9f9f900f9f9ULL, 0xce00cecece00ceceULL, 0xbf00bfbfbf00bfbfULL,
598 0xdf00dfdfdf00dfdf, 0x7100717171007171, 0x2900292929002929, 598 0xdf00dfdfdf00dfdfULL, 0x7100717171007171ULL, 0x2900292929002929ULL,
599 0xcd00cdcdcd00cdcd, 0x6c006c6c6c006c6c, 0x1300131313001313, 599 0xcd00cdcdcd00cdcdULL, 0x6c006c6c6c006c6cULL, 0x1300131313001313ULL,
600 0x6400646464006464, 0x9b009b9b9b009b9b, 0x6300636363006363, 600 0x6400646464006464ULL, 0x9b009b9b9b009b9bULL, 0x6300636363006363ULL,
601 0x9d009d9d9d009d9d, 0xc000c0c0c000c0c0, 0x4b004b4b4b004b4b, 601 0x9d009d9d9d009d9dULL, 0xc000c0c0c000c0c0ULL, 0x4b004b4b4b004b4bULL,
602 0xb700b7b7b700b7b7, 0xa500a5a5a500a5a5, 0x8900898989008989, 602 0xb700b7b7b700b7b7ULL, 0xa500a5a5a500a5a5ULL, 0x8900898989008989ULL,
603 0x5f005f5f5f005f5f, 0xb100b1b1b100b1b1, 0x1700171717001717, 603 0x5f005f5f5f005f5fULL, 0xb100b1b1b100b1b1ULL, 0x1700171717001717ULL,
604 0xf400f4f4f400f4f4, 0xbc00bcbcbc00bcbc, 0xd300d3d3d300d3d3, 604 0xf400f4f4f400f4f4ULL, 0xbc00bcbcbc00bcbcULL, 0xd300d3d3d300d3d3ULL,
605 0x4600464646004646, 0xcf00cfcfcf00cfcf, 0x3700373737003737, 605 0x4600464646004646ULL, 0xcf00cfcfcf00cfcfULL, 0x3700373737003737ULL,
606 0x5e005e5e5e005e5e, 0x4700474747004747, 0x9400949494009494, 606 0x5e005e5e5e005e5eULL, 0x4700474747004747ULL, 0x9400949494009494ULL,
607 0xfa00fafafa00fafa, 0xfc00fcfcfc00fcfc, 0x5b005b5b5b005b5b, 607 0xfa00fafafa00fafaULL, 0xfc00fcfcfc00fcfcULL, 0x5b005b5b5b005b5bULL,
608 0x9700979797009797, 0xfe00fefefe00fefe, 0x5a005a5a5a005a5a, 608 0x9700979797009797ULL, 0xfe00fefefe00fefeULL, 0x5a005a5a5a005a5aULL,
609 0xac00acacac00acac, 0x3c003c3c3c003c3c, 0x4c004c4c4c004c4c, 609 0xac00acacac00acacULL, 0x3c003c3c3c003c3cULL, 0x4c004c4c4c004c4cULL,
610 0x0300030303000303, 0x3500353535003535, 0xf300f3f3f300f3f3, 610 0x0300030303000303ULL, 0x3500353535003535ULL, 0xf300f3f3f300f3f3ULL,
611 0x2300232323002323, 0xb800b8b8b800b8b8, 0x5d005d5d5d005d5d, 611 0x2300232323002323ULL, 0xb800b8b8b800b8b8ULL, 0x5d005d5d5d005d5dULL,
612 0x6a006a6a6a006a6a, 0x9200929292009292, 0xd500d5d5d500d5d5, 612 0x6a006a6a6a006a6aULL, 0x9200929292009292ULL, 0xd500d5d5d500d5d5ULL,
613 0x2100212121002121, 0x4400444444004444, 0x5100515151005151, 613 0x2100212121002121ULL, 0x4400444444004444ULL, 0x5100515151005151ULL,
614 0xc600c6c6c600c6c6, 0x7d007d7d7d007d7d, 0x3900393939003939, 614 0xc600c6c6c600c6c6ULL, 0x7d007d7d7d007d7dULL, 0x3900393939003939ULL,
615 0x8300838383008383, 0xdc00dcdcdc00dcdc, 0xaa00aaaaaa00aaaa, 615 0x8300838383008383ULL, 0xdc00dcdcdc00dcdcULL, 0xaa00aaaaaa00aaaaULL,
616 0x7c007c7c7c007c7c, 0x7700777777007777, 0x5600565656005656, 616 0x7c007c7c7c007c7cULL, 0x7700777777007777ULL, 0x5600565656005656ULL,
617 0x0500050505000505, 0x1b001b1b1b001b1b, 0xa400a4a4a400a4a4, 617 0x0500050505000505ULL, 0x1b001b1b1b001b1bULL, 0xa400a4a4a400a4a4ULL,
618 0x1500151515001515, 0x3400343434003434, 0x1e001e1e1e001e1e, 618 0x1500151515001515ULL, 0x3400343434003434ULL, 0x1e001e1e1e001e1eULL,
619 0x1c001c1c1c001c1c, 0xf800f8f8f800f8f8, 0x5200525252005252, 619 0x1c001c1c1c001c1cULL, 0xf800f8f8f800f8f8ULL, 0x5200525252005252ULL,
620 0x2000202020002020, 0x1400141414001414, 0xe900e9e9e900e9e9, 620 0x2000202020002020ULL, 0x1400141414001414ULL, 0xe900e9e9e900e9e9ULL,
621 0xbd00bdbdbd00bdbd, 0xdd00dddddd00dddd, 0xe400e4e4e400e4e4, 621 0xbd00bdbdbd00bdbdULL, 0xdd00dddddd00ddddULL, 0xe400e4e4e400e4e4ULL,
622 0xa100a1a1a100a1a1, 0xe000e0e0e000e0e0, 0x8a008a8a8a008a8a, 622 0xa100a1a1a100a1a1ULL, 0xe000e0e0e000e0e0ULL, 0x8a008a8a8a008a8aULL,
623 0xf100f1f1f100f1f1, 0xd600d6d6d600d6d6, 0x7a007a7a7a007a7a, 623 0xf100f1f1f100f1f1ULL, 0xd600d6d6d600d6d6ULL, 0x7a007a7a7a007a7aULL,
624 0xbb00bbbbbb00bbbb, 0xe300e3e3e300e3e3, 0x4000404040004040, 624 0xbb00bbbbbb00bbbbULL, 0xe300e3e3e300e3e3ULL, 0x4000404040004040ULL,
625 0x4f004f4f4f004f4f, 625 0x4f004f4f4f004f4fULL,
626}; 626};
627 627
628const u64 camellia_sp44044404[256] = { 628const u64 camellia_sp44044404[256] = {
629 0x7070007070700070, 0x2c2c002c2c2c002c, 0xb3b300b3b3b300b3, 629 0x7070007070700070ULL, 0x2c2c002c2c2c002cULL, 0xb3b300b3b3b300b3ULL,
630 0xc0c000c0c0c000c0, 0xe4e400e4e4e400e4, 0x5757005757570057, 630 0xc0c000c0c0c000c0ULL, 0xe4e400e4e4e400e4ULL, 0x5757005757570057ULL,
631 0xeaea00eaeaea00ea, 0xaeae00aeaeae00ae, 0x2323002323230023, 631 0xeaea00eaeaea00eaULL, 0xaeae00aeaeae00aeULL, 0x2323002323230023ULL,
632 0x6b6b006b6b6b006b, 0x4545004545450045, 0xa5a500a5a5a500a5, 632 0x6b6b006b6b6b006bULL, 0x4545004545450045ULL, 0xa5a500a5a5a500a5ULL,
633 0xeded00ededed00ed, 0x4f4f004f4f4f004f, 0x1d1d001d1d1d001d, 633 0xeded00ededed00edULL, 0x4f4f004f4f4f004fULL, 0x1d1d001d1d1d001dULL,
634 0x9292009292920092, 0x8686008686860086, 0xafaf00afafaf00af, 634 0x9292009292920092ULL, 0x8686008686860086ULL, 0xafaf00afafaf00afULL,
635 0x7c7c007c7c7c007c, 0x1f1f001f1f1f001f, 0x3e3e003e3e3e003e, 635 0x7c7c007c7c7c007cULL, 0x1f1f001f1f1f001fULL, 0x3e3e003e3e3e003eULL,
636 0xdcdc00dcdcdc00dc, 0x5e5e005e5e5e005e, 0x0b0b000b0b0b000b, 636 0xdcdc00dcdcdc00dcULL, 0x5e5e005e5e5e005eULL, 0x0b0b000b0b0b000bULL,
637 0xa6a600a6a6a600a6, 0x3939003939390039, 0xd5d500d5d5d500d5, 637 0xa6a600a6a6a600a6ULL, 0x3939003939390039ULL, 0xd5d500d5d5d500d5ULL,
638 0x5d5d005d5d5d005d, 0xd9d900d9d9d900d9, 0x5a5a005a5a5a005a, 638 0x5d5d005d5d5d005dULL, 0xd9d900d9d9d900d9ULL, 0x5a5a005a5a5a005aULL,
639 0x5151005151510051, 0x6c6c006c6c6c006c, 0x8b8b008b8b8b008b, 639 0x5151005151510051ULL, 0x6c6c006c6c6c006cULL, 0x8b8b008b8b8b008bULL,
640 0x9a9a009a9a9a009a, 0xfbfb00fbfbfb00fb, 0xb0b000b0b0b000b0, 640 0x9a9a009a9a9a009aULL, 0xfbfb00fbfbfb00fbULL, 0xb0b000b0b0b000b0ULL,
641 0x7474007474740074, 0x2b2b002b2b2b002b, 0xf0f000f0f0f000f0, 641 0x7474007474740074ULL, 0x2b2b002b2b2b002bULL, 0xf0f000f0f0f000f0ULL,
642 0x8484008484840084, 0xdfdf00dfdfdf00df, 0xcbcb00cbcbcb00cb, 642 0x8484008484840084ULL, 0xdfdf00dfdfdf00dfULL, 0xcbcb00cbcbcb00cbULL,
643 0x3434003434340034, 0x7676007676760076, 0x6d6d006d6d6d006d, 643 0x3434003434340034ULL, 0x7676007676760076ULL, 0x6d6d006d6d6d006dULL,
644 0xa9a900a9a9a900a9, 0xd1d100d1d1d100d1, 0x0404000404040004, 644 0xa9a900a9a9a900a9ULL, 0xd1d100d1d1d100d1ULL, 0x0404000404040004ULL,
645 0x1414001414140014, 0x3a3a003a3a3a003a, 0xdede00dedede00de, 645 0x1414001414140014ULL, 0x3a3a003a3a3a003aULL, 0xdede00dedede00deULL,
646 0x1111001111110011, 0x3232003232320032, 0x9c9c009c9c9c009c, 646 0x1111001111110011ULL, 0x3232003232320032ULL, 0x9c9c009c9c9c009cULL,
647 0x5353005353530053, 0xf2f200f2f2f200f2, 0xfefe00fefefe00fe, 647 0x5353005353530053ULL, 0xf2f200f2f2f200f2ULL, 0xfefe00fefefe00feULL,
648 0xcfcf00cfcfcf00cf, 0xc3c300c3c3c300c3, 0x7a7a007a7a7a007a, 648 0xcfcf00cfcfcf00cfULL, 0xc3c300c3c3c300c3ULL, 0x7a7a007a7a7a007aULL,
649 0x2424002424240024, 0xe8e800e8e8e800e8, 0x6060006060600060, 649 0x2424002424240024ULL, 0xe8e800e8e8e800e8ULL, 0x6060006060600060ULL,
650 0x6969006969690069, 0xaaaa00aaaaaa00aa, 0xa0a000a0a0a000a0, 650 0x6969006969690069ULL, 0xaaaa00aaaaaa00aaULL, 0xa0a000a0a0a000a0ULL,
651 0xa1a100a1a1a100a1, 0x6262006262620062, 0x5454005454540054, 651 0xa1a100a1a1a100a1ULL, 0x6262006262620062ULL, 0x5454005454540054ULL,
652 0x1e1e001e1e1e001e, 0xe0e000e0e0e000e0, 0x6464006464640064, 652 0x1e1e001e1e1e001eULL, 0xe0e000e0e0e000e0ULL, 0x6464006464640064ULL,
653 0x1010001010100010, 0x0000000000000000, 0xa3a300a3a3a300a3, 653 0x1010001010100010ULL, 0x0000000000000000ULL, 0xa3a300a3a3a300a3ULL,
654 0x7575007575750075, 0x8a8a008a8a8a008a, 0xe6e600e6e6e600e6, 654 0x7575007575750075ULL, 0x8a8a008a8a8a008aULL, 0xe6e600e6e6e600e6ULL,
655 0x0909000909090009, 0xdddd00dddddd00dd, 0x8787008787870087, 655 0x0909000909090009ULL, 0xdddd00dddddd00ddULL, 0x8787008787870087ULL,
656 0x8383008383830083, 0xcdcd00cdcdcd00cd, 0x9090009090900090, 656 0x8383008383830083ULL, 0xcdcd00cdcdcd00cdULL, 0x9090009090900090ULL,
657 0x7373007373730073, 0xf6f600f6f6f600f6, 0x9d9d009d9d9d009d, 657 0x7373007373730073ULL, 0xf6f600f6f6f600f6ULL, 0x9d9d009d9d9d009dULL,
658 0xbfbf00bfbfbf00bf, 0x5252005252520052, 0xd8d800d8d8d800d8, 658 0xbfbf00bfbfbf00bfULL, 0x5252005252520052ULL, 0xd8d800d8d8d800d8ULL,
659 0xc8c800c8c8c800c8, 0xc6c600c6c6c600c6, 0x8181008181810081, 659 0xc8c800c8c8c800c8ULL, 0xc6c600c6c6c600c6ULL, 0x8181008181810081ULL,
660 0x6f6f006f6f6f006f, 0x1313001313130013, 0x6363006363630063, 660 0x6f6f006f6f6f006fULL, 0x1313001313130013ULL, 0x6363006363630063ULL,
661 0xe9e900e9e9e900e9, 0xa7a700a7a7a700a7, 0x9f9f009f9f9f009f, 661 0xe9e900e9e9e900e9ULL, 0xa7a700a7a7a700a7ULL, 0x9f9f009f9f9f009fULL,
662 0xbcbc00bcbcbc00bc, 0x2929002929290029, 0xf9f900f9f9f900f9, 662 0xbcbc00bcbcbc00bcULL, 0x2929002929290029ULL, 0xf9f900f9f9f900f9ULL,
663 0x2f2f002f2f2f002f, 0xb4b400b4b4b400b4, 0x7878007878780078, 663 0x2f2f002f2f2f002fULL, 0xb4b400b4b4b400b4ULL, 0x7878007878780078ULL,
664 0x0606000606060006, 0xe7e700e7e7e700e7, 0x7171007171710071, 664 0x0606000606060006ULL, 0xe7e700e7e7e700e7ULL, 0x7171007171710071ULL,
665 0xd4d400d4d4d400d4, 0xabab00ababab00ab, 0x8888008888880088, 665 0xd4d400d4d4d400d4ULL, 0xabab00ababab00abULL, 0x8888008888880088ULL,
666 0x8d8d008d8d8d008d, 0x7272007272720072, 0xb9b900b9b9b900b9, 666 0x8d8d008d8d8d008dULL, 0x7272007272720072ULL, 0xb9b900b9b9b900b9ULL,
667 0xf8f800f8f8f800f8, 0xacac00acacac00ac, 0x3636003636360036, 667 0xf8f800f8f8f800f8ULL, 0xacac00acacac00acULL, 0x3636003636360036ULL,
668 0x2a2a002a2a2a002a, 0x3c3c003c3c3c003c, 0xf1f100f1f1f100f1, 668 0x2a2a002a2a2a002aULL, 0x3c3c003c3c3c003cULL, 0xf1f100f1f1f100f1ULL,
669 0x4040004040400040, 0xd3d300d3d3d300d3, 0xbbbb00bbbbbb00bb, 669 0x4040004040400040ULL, 0xd3d300d3d3d300d3ULL, 0xbbbb00bbbbbb00bbULL,
670 0x4343004343430043, 0x1515001515150015, 0xadad00adadad00ad, 670 0x4343004343430043ULL, 0x1515001515150015ULL, 0xadad00adadad00adULL,
671 0x7777007777770077, 0x8080008080800080, 0x8282008282820082, 671 0x7777007777770077ULL, 0x8080008080800080ULL, 0x8282008282820082ULL,
672 0xecec00ececec00ec, 0x2727002727270027, 0xe5e500e5e5e500e5, 672 0xecec00ececec00ecULL, 0x2727002727270027ULL, 0xe5e500e5e5e500e5ULL,
673 0x8585008585850085, 0x3535003535350035, 0x0c0c000c0c0c000c, 673 0x8585008585850085ULL, 0x3535003535350035ULL, 0x0c0c000c0c0c000cULL,
674 0x4141004141410041, 0xefef00efefef00ef, 0x9393009393930093, 674 0x4141004141410041ULL, 0xefef00efefef00efULL, 0x9393009393930093ULL,
675 0x1919001919190019, 0x2121002121210021, 0x0e0e000e0e0e000e, 675 0x1919001919190019ULL, 0x2121002121210021ULL, 0x0e0e000e0e0e000eULL,
676 0x4e4e004e4e4e004e, 0x6565006565650065, 0xbdbd00bdbdbd00bd, 676 0x4e4e004e4e4e004eULL, 0x6565006565650065ULL, 0xbdbd00bdbdbd00bdULL,
677 0xb8b800b8b8b800b8, 0x8f8f008f8f8f008f, 0xebeb00ebebeb00eb, 677 0xb8b800b8b8b800b8ULL, 0x8f8f008f8f8f008fULL, 0xebeb00ebebeb00ebULL,
678 0xcece00cecece00ce, 0x3030003030300030, 0x5f5f005f5f5f005f, 678 0xcece00cecece00ceULL, 0x3030003030300030ULL, 0x5f5f005f5f5f005fULL,
679 0xc5c500c5c5c500c5, 0x1a1a001a1a1a001a, 0xe1e100e1e1e100e1, 679 0xc5c500c5c5c500c5ULL, 0x1a1a001a1a1a001aULL, 0xe1e100e1e1e100e1ULL,
680 0xcaca00cacaca00ca, 0x4747004747470047, 0x3d3d003d3d3d003d, 680 0xcaca00cacaca00caULL, 0x4747004747470047ULL, 0x3d3d003d3d3d003dULL,
681 0x0101000101010001, 0xd6d600d6d6d600d6, 0x5656005656560056, 681 0x0101000101010001ULL, 0xd6d600d6d6d600d6ULL, 0x5656005656560056ULL,
682 0x4d4d004d4d4d004d, 0x0d0d000d0d0d000d, 0x6666006666660066, 682 0x4d4d004d4d4d004dULL, 0x0d0d000d0d0d000dULL, 0x6666006666660066ULL,
683 0xcccc00cccccc00cc, 0x2d2d002d2d2d002d, 0x1212001212120012, 683 0xcccc00cccccc00ccULL, 0x2d2d002d2d2d002dULL, 0x1212001212120012ULL,
684 0x2020002020200020, 0xb1b100b1b1b100b1, 0x9999009999990099, 684 0x2020002020200020ULL, 0xb1b100b1b1b100b1ULL, 0x9999009999990099ULL,
685 0x4c4c004c4c4c004c, 0xc2c200c2c2c200c2, 0x7e7e007e7e7e007e, 685 0x4c4c004c4c4c004cULL, 0xc2c200c2c2c200c2ULL, 0x7e7e007e7e7e007eULL,
686 0x0505000505050005, 0xb7b700b7b7b700b7, 0x3131003131310031, 686 0x0505000505050005ULL, 0xb7b700b7b7b700b7ULL, 0x3131003131310031ULL,
687 0x1717001717170017, 0xd7d700d7d7d700d7, 0x5858005858580058, 687 0x1717001717170017ULL, 0xd7d700d7d7d700d7ULL, 0x5858005858580058ULL,
688 0x6161006161610061, 0x1b1b001b1b1b001b, 0x1c1c001c1c1c001c, 688 0x6161006161610061ULL, 0x1b1b001b1b1b001bULL, 0x1c1c001c1c1c001cULL,
689 0x0f0f000f0f0f000f, 0x1616001616160016, 0x1818001818180018, 689 0x0f0f000f0f0f000fULL, 0x1616001616160016ULL, 0x1818001818180018ULL,
690 0x2222002222220022, 0x4444004444440044, 0xb2b200b2b2b200b2, 690 0x2222002222220022ULL, 0x4444004444440044ULL, 0xb2b200b2b2b200b2ULL,
691 0xb5b500b5b5b500b5, 0x9191009191910091, 0x0808000808080008, 691 0xb5b500b5b5b500b5ULL, 0x9191009191910091ULL, 0x0808000808080008ULL,
692 0xa8a800a8a8a800a8, 0xfcfc00fcfcfc00fc, 0x5050005050500050, 692 0xa8a800a8a8a800a8ULL, 0xfcfc00fcfcfc00fcULL, 0x5050005050500050ULL,
693 0xd0d000d0d0d000d0, 0x7d7d007d7d7d007d, 0x8989008989890089, 693 0xd0d000d0d0d000d0ULL, 0x7d7d007d7d7d007dULL, 0x8989008989890089ULL,
694 0x9797009797970097, 0x5b5b005b5b5b005b, 0x9595009595950095, 694 0x9797009797970097ULL, 0x5b5b005b5b5b005bULL, 0x9595009595950095ULL,
695 0xffff00ffffff00ff, 0xd2d200d2d2d200d2, 0xc4c400c4c4c400c4, 695 0xffff00ffffff00ffULL, 0xd2d200d2d2d200d2ULL, 0xc4c400c4c4c400c4ULL,
696 0x4848004848480048, 0xf7f700f7f7f700f7, 0xdbdb00dbdbdb00db, 696 0x4848004848480048ULL, 0xf7f700f7f7f700f7ULL, 0xdbdb00dbdbdb00dbULL,
697 0x0303000303030003, 0xdada00dadada00da, 0x3f3f003f3f3f003f, 697 0x0303000303030003ULL, 0xdada00dadada00daULL, 0x3f3f003f3f3f003fULL,
698 0x9494009494940094, 0x5c5c005c5c5c005c, 0x0202000202020002, 698 0x9494009494940094ULL, 0x5c5c005c5c5c005cULL, 0x0202000202020002ULL,
699 0x4a4a004a4a4a004a, 0x3333003333330033, 0x6767006767670067, 699 0x4a4a004a4a4a004aULL, 0x3333003333330033ULL, 0x6767006767670067ULL,
700 0xf3f300f3f3f300f3, 0x7f7f007f7f7f007f, 0xe2e200e2e2e200e2, 700 0xf3f300f3f3f300f3ULL, 0x7f7f007f7f7f007fULL, 0xe2e200e2e2e200e2ULL,
701 0x9b9b009b9b9b009b, 0x2626002626260026, 0x3737003737370037, 701 0x9b9b009b9b9b009bULL, 0x2626002626260026ULL, 0x3737003737370037ULL,
702 0x3b3b003b3b3b003b, 0x9696009696960096, 0x4b4b004b4b4b004b, 702 0x3b3b003b3b3b003bULL, 0x9696009696960096ULL, 0x4b4b004b4b4b004bULL,
703 0xbebe00bebebe00be, 0x2e2e002e2e2e002e, 0x7979007979790079, 703 0xbebe00bebebe00beULL, 0x2e2e002e2e2e002eULL, 0x7979007979790079ULL,
704 0x8c8c008c8c8c008c, 0x6e6e006e6e6e006e, 0x8e8e008e8e8e008e, 704 0x8c8c008c8c8c008cULL, 0x6e6e006e6e6e006eULL, 0x8e8e008e8e8e008eULL,
705 0xf5f500f5f5f500f5, 0xb6b600b6b6b600b6, 0xfdfd00fdfdfd00fd, 705 0xf5f500f5f5f500f5ULL, 0xb6b600b6b6b600b6ULL, 0xfdfd00fdfdfd00fdULL,
706 0x5959005959590059, 0x9898009898980098, 0x6a6a006a6a6a006a, 706 0x5959005959590059ULL, 0x9898009898980098ULL, 0x6a6a006a6a6a006aULL,
707 0x4646004646460046, 0xbaba00bababa00ba, 0x2525002525250025, 707 0x4646004646460046ULL, 0xbaba00bababa00baULL, 0x2525002525250025ULL,
708 0x4242004242420042, 0xa2a200a2a2a200a2, 0xfafa00fafafa00fa, 708 0x4242004242420042ULL, 0xa2a200a2a2a200a2ULL, 0xfafa00fafafa00faULL,
709 0x0707000707070007, 0x5555005555550055, 0xeeee00eeeeee00ee, 709 0x0707000707070007ULL, 0x5555005555550055ULL, 0xeeee00eeeeee00eeULL,
710 0x0a0a000a0a0a000a, 0x4949004949490049, 0x6868006868680068, 710 0x0a0a000a0a0a000aULL, 0x4949004949490049ULL, 0x6868006868680068ULL,
711 0x3838003838380038, 0xa4a400a4a4a400a4, 0x2828002828280028, 711 0x3838003838380038ULL, 0xa4a400a4a4a400a4ULL, 0x2828002828280028ULL,
712 0x7b7b007b7b7b007b, 0xc9c900c9c9c900c9, 0xc1c100c1c1c100c1, 712 0x7b7b007b7b7b007bULL, 0xc9c900c9c9c900c9ULL, 0xc1c100c1c1c100c1ULL,
713 0xe3e300e3e3e300e3, 0xf4f400f4f4f400f4, 0xc7c700c7c7c700c7, 713 0xe3e300e3e3e300e3ULL, 0xf4f400f4f4f400f4ULL, 0xc7c700c7c7c700c7ULL,
714 0x9e9e009e9e9e009e, 714 0x9e9e009e9e9e009eULL,
715}; 715};
716 716
717const u64 camellia_sp11101110[256] = { 717const u64 camellia_sp11101110[256] = {
718 0x7070700070707000, 0x8282820082828200, 0x2c2c2c002c2c2c00, 718 0x7070700070707000ULL, 0x8282820082828200ULL, 0x2c2c2c002c2c2c00ULL,
719 0xececec00ececec00, 0xb3b3b300b3b3b300, 0x2727270027272700, 719 0xececec00ececec00ULL, 0xb3b3b300b3b3b300ULL, 0x2727270027272700ULL,
720 0xc0c0c000c0c0c000, 0xe5e5e500e5e5e500, 0xe4e4e400e4e4e400, 720 0xc0c0c000c0c0c000ULL, 0xe5e5e500e5e5e500ULL, 0xe4e4e400e4e4e400ULL,
721 0x8585850085858500, 0x5757570057575700, 0x3535350035353500, 721 0x8585850085858500ULL, 0x5757570057575700ULL, 0x3535350035353500ULL,
722 0xeaeaea00eaeaea00, 0x0c0c0c000c0c0c00, 0xaeaeae00aeaeae00, 722 0xeaeaea00eaeaea00ULL, 0x0c0c0c000c0c0c00ULL, 0xaeaeae00aeaeae00ULL,
723 0x4141410041414100, 0x2323230023232300, 0xefefef00efefef00, 723 0x4141410041414100ULL, 0x2323230023232300ULL, 0xefefef00efefef00ULL,
724 0x6b6b6b006b6b6b00, 0x9393930093939300, 0x4545450045454500, 724 0x6b6b6b006b6b6b00ULL, 0x9393930093939300ULL, 0x4545450045454500ULL,
725 0x1919190019191900, 0xa5a5a500a5a5a500, 0x2121210021212100, 725 0x1919190019191900ULL, 0xa5a5a500a5a5a500ULL, 0x2121210021212100ULL,
726 0xededed00ededed00, 0x0e0e0e000e0e0e00, 0x4f4f4f004f4f4f00, 726 0xededed00ededed00ULL, 0x0e0e0e000e0e0e00ULL, 0x4f4f4f004f4f4f00ULL,
727 0x4e4e4e004e4e4e00, 0x1d1d1d001d1d1d00, 0x6565650065656500, 727 0x4e4e4e004e4e4e00ULL, 0x1d1d1d001d1d1d00ULL, 0x6565650065656500ULL,
728 0x9292920092929200, 0xbdbdbd00bdbdbd00, 0x8686860086868600, 728 0x9292920092929200ULL, 0xbdbdbd00bdbdbd00ULL, 0x8686860086868600ULL,
729 0xb8b8b800b8b8b800, 0xafafaf00afafaf00, 0x8f8f8f008f8f8f00, 729 0xb8b8b800b8b8b800ULL, 0xafafaf00afafaf00ULL, 0x8f8f8f008f8f8f00ULL,
730 0x7c7c7c007c7c7c00, 0xebebeb00ebebeb00, 0x1f1f1f001f1f1f00, 730 0x7c7c7c007c7c7c00ULL, 0xebebeb00ebebeb00ULL, 0x1f1f1f001f1f1f00ULL,
731 0xcecece00cecece00, 0x3e3e3e003e3e3e00, 0x3030300030303000, 731 0xcecece00cecece00ULL, 0x3e3e3e003e3e3e00ULL, 0x3030300030303000ULL,
732 0xdcdcdc00dcdcdc00, 0x5f5f5f005f5f5f00, 0x5e5e5e005e5e5e00, 732 0xdcdcdc00dcdcdc00ULL, 0x5f5f5f005f5f5f00ULL, 0x5e5e5e005e5e5e00ULL,
733 0xc5c5c500c5c5c500, 0x0b0b0b000b0b0b00, 0x1a1a1a001a1a1a00, 733 0xc5c5c500c5c5c500ULL, 0x0b0b0b000b0b0b00ULL, 0x1a1a1a001a1a1a00ULL,
734 0xa6a6a600a6a6a600, 0xe1e1e100e1e1e100, 0x3939390039393900, 734 0xa6a6a600a6a6a600ULL, 0xe1e1e100e1e1e100ULL, 0x3939390039393900ULL,
735 0xcacaca00cacaca00, 0xd5d5d500d5d5d500, 0x4747470047474700, 735 0xcacaca00cacaca00ULL, 0xd5d5d500d5d5d500ULL, 0x4747470047474700ULL,
736 0x5d5d5d005d5d5d00, 0x3d3d3d003d3d3d00, 0xd9d9d900d9d9d900, 736 0x5d5d5d005d5d5d00ULL, 0x3d3d3d003d3d3d00ULL, 0xd9d9d900d9d9d900ULL,
737 0x0101010001010100, 0x5a5a5a005a5a5a00, 0xd6d6d600d6d6d600, 737 0x0101010001010100ULL, 0x5a5a5a005a5a5a00ULL, 0xd6d6d600d6d6d600ULL,
738 0x5151510051515100, 0x5656560056565600, 0x6c6c6c006c6c6c00, 738 0x5151510051515100ULL, 0x5656560056565600ULL, 0x6c6c6c006c6c6c00ULL,
739 0x4d4d4d004d4d4d00, 0x8b8b8b008b8b8b00, 0x0d0d0d000d0d0d00, 739 0x4d4d4d004d4d4d00ULL, 0x8b8b8b008b8b8b00ULL, 0x0d0d0d000d0d0d00ULL,
740 0x9a9a9a009a9a9a00, 0x6666660066666600, 0xfbfbfb00fbfbfb00, 740 0x9a9a9a009a9a9a00ULL, 0x6666660066666600ULL, 0xfbfbfb00fbfbfb00ULL,
741 0xcccccc00cccccc00, 0xb0b0b000b0b0b000, 0x2d2d2d002d2d2d00, 741 0xcccccc00cccccc00ULL, 0xb0b0b000b0b0b000ULL, 0x2d2d2d002d2d2d00ULL,
742 0x7474740074747400, 0x1212120012121200, 0x2b2b2b002b2b2b00, 742 0x7474740074747400ULL, 0x1212120012121200ULL, 0x2b2b2b002b2b2b00ULL,
743 0x2020200020202000, 0xf0f0f000f0f0f000, 0xb1b1b100b1b1b100, 743 0x2020200020202000ULL, 0xf0f0f000f0f0f000ULL, 0xb1b1b100b1b1b100ULL,
744 0x8484840084848400, 0x9999990099999900, 0xdfdfdf00dfdfdf00, 744 0x8484840084848400ULL, 0x9999990099999900ULL, 0xdfdfdf00dfdfdf00ULL,
745 0x4c4c4c004c4c4c00, 0xcbcbcb00cbcbcb00, 0xc2c2c200c2c2c200, 745 0x4c4c4c004c4c4c00ULL, 0xcbcbcb00cbcbcb00ULL, 0xc2c2c200c2c2c200ULL,
746 0x3434340034343400, 0x7e7e7e007e7e7e00, 0x7676760076767600, 746 0x3434340034343400ULL, 0x7e7e7e007e7e7e00ULL, 0x7676760076767600ULL,
747 0x0505050005050500, 0x6d6d6d006d6d6d00, 0xb7b7b700b7b7b700, 747 0x0505050005050500ULL, 0x6d6d6d006d6d6d00ULL, 0xb7b7b700b7b7b700ULL,
748 0xa9a9a900a9a9a900, 0x3131310031313100, 0xd1d1d100d1d1d100, 748 0xa9a9a900a9a9a900ULL, 0x3131310031313100ULL, 0xd1d1d100d1d1d100ULL,
749 0x1717170017171700, 0x0404040004040400, 0xd7d7d700d7d7d700, 749 0x1717170017171700ULL, 0x0404040004040400ULL, 0xd7d7d700d7d7d700ULL,
750 0x1414140014141400, 0x5858580058585800, 0x3a3a3a003a3a3a00, 750 0x1414140014141400ULL, 0x5858580058585800ULL, 0x3a3a3a003a3a3a00ULL,
751 0x6161610061616100, 0xdedede00dedede00, 0x1b1b1b001b1b1b00, 751 0x6161610061616100ULL, 0xdedede00dedede00ULL, 0x1b1b1b001b1b1b00ULL,
752 0x1111110011111100, 0x1c1c1c001c1c1c00, 0x3232320032323200, 752 0x1111110011111100ULL, 0x1c1c1c001c1c1c00ULL, 0x3232320032323200ULL,
753 0x0f0f0f000f0f0f00, 0x9c9c9c009c9c9c00, 0x1616160016161600, 753 0x0f0f0f000f0f0f00ULL, 0x9c9c9c009c9c9c00ULL, 0x1616160016161600ULL,
754 0x5353530053535300, 0x1818180018181800, 0xf2f2f200f2f2f200, 754 0x5353530053535300ULL, 0x1818180018181800ULL, 0xf2f2f200f2f2f200ULL,
755 0x2222220022222200, 0xfefefe00fefefe00, 0x4444440044444400, 755 0x2222220022222200ULL, 0xfefefe00fefefe00ULL, 0x4444440044444400ULL,
756 0xcfcfcf00cfcfcf00, 0xb2b2b200b2b2b200, 0xc3c3c300c3c3c300, 756 0xcfcfcf00cfcfcf00ULL, 0xb2b2b200b2b2b200ULL, 0xc3c3c300c3c3c300ULL,
757 0xb5b5b500b5b5b500, 0x7a7a7a007a7a7a00, 0x9191910091919100, 757 0xb5b5b500b5b5b500ULL, 0x7a7a7a007a7a7a00ULL, 0x9191910091919100ULL,
758 0x2424240024242400, 0x0808080008080800, 0xe8e8e800e8e8e800, 758 0x2424240024242400ULL, 0x0808080008080800ULL, 0xe8e8e800e8e8e800ULL,
759 0xa8a8a800a8a8a800, 0x6060600060606000, 0xfcfcfc00fcfcfc00, 759 0xa8a8a800a8a8a800ULL, 0x6060600060606000ULL, 0xfcfcfc00fcfcfc00ULL,
760 0x6969690069696900, 0x5050500050505000, 0xaaaaaa00aaaaaa00, 760 0x6969690069696900ULL, 0x5050500050505000ULL, 0xaaaaaa00aaaaaa00ULL,
761 0xd0d0d000d0d0d000, 0xa0a0a000a0a0a000, 0x7d7d7d007d7d7d00, 761 0xd0d0d000d0d0d000ULL, 0xa0a0a000a0a0a000ULL, 0x7d7d7d007d7d7d00ULL,
762 0xa1a1a100a1a1a100, 0x8989890089898900, 0x6262620062626200, 762 0xa1a1a100a1a1a100ULL, 0x8989890089898900ULL, 0x6262620062626200ULL,
763 0x9797970097979700, 0x5454540054545400, 0x5b5b5b005b5b5b00, 763 0x9797970097979700ULL, 0x5454540054545400ULL, 0x5b5b5b005b5b5b00ULL,
764 0x1e1e1e001e1e1e00, 0x9595950095959500, 0xe0e0e000e0e0e000, 764 0x1e1e1e001e1e1e00ULL, 0x9595950095959500ULL, 0xe0e0e000e0e0e000ULL,
765 0xffffff00ffffff00, 0x6464640064646400, 0xd2d2d200d2d2d200, 765 0xffffff00ffffff00ULL, 0x6464640064646400ULL, 0xd2d2d200d2d2d200ULL,
766 0x1010100010101000, 0xc4c4c400c4c4c400, 0x0000000000000000, 766 0x1010100010101000ULL, 0xc4c4c400c4c4c400ULL, 0x0000000000000000ULL,
767 0x4848480048484800, 0xa3a3a300a3a3a300, 0xf7f7f700f7f7f700, 767 0x4848480048484800ULL, 0xa3a3a300a3a3a300ULL, 0xf7f7f700f7f7f700ULL,
768 0x7575750075757500, 0xdbdbdb00dbdbdb00, 0x8a8a8a008a8a8a00, 768 0x7575750075757500ULL, 0xdbdbdb00dbdbdb00ULL, 0x8a8a8a008a8a8a00ULL,
769 0x0303030003030300, 0xe6e6e600e6e6e600, 0xdadada00dadada00, 769 0x0303030003030300ULL, 0xe6e6e600e6e6e600ULL, 0xdadada00dadada00ULL,
770 0x0909090009090900, 0x3f3f3f003f3f3f00, 0xdddddd00dddddd00, 770 0x0909090009090900ULL, 0x3f3f3f003f3f3f00ULL, 0xdddddd00dddddd00ULL,
771 0x9494940094949400, 0x8787870087878700, 0x5c5c5c005c5c5c00, 771 0x9494940094949400ULL, 0x8787870087878700ULL, 0x5c5c5c005c5c5c00ULL,
772 0x8383830083838300, 0x0202020002020200, 0xcdcdcd00cdcdcd00, 772 0x8383830083838300ULL, 0x0202020002020200ULL, 0xcdcdcd00cdcdcd00ULL,
773 0x4a4a4a004a4a4a00, 0x9090900090909000, 0x3333330033333300, 773 0x4a4a4a004a4a4a00ULL, 0x9090900090909000ULL, 0x3333330033333300ULL,
774 0x7373730073737300, 0x6767670067676700, 0xf6f6f600f6f6f600, 774 0x7373730073737300ULL, 0x6767670067676700ULL, 0xf6f6f600f6f6f600ULL,
775 0xf3f3f300f3f3f300, 0x9d9d9d009d9d9d00, 0x7f7f7f007f7f7f00, 775 0xf3f3f300f3f3f300ULL, 0x9d9d9d009d9d9d00ULL, 0x7f7f7f007f7f7f00ULL,
776 0xbfbfbf00bfbfbf00, 0xe2e2e200e2e2e200, 0x5252520052525200, 776 0xbfbfbf00bfbfbf00ULL, 0xe2e2e200e2e2e200ULL, 0x5252520052525200ULL,
777 0x9b9b9b009b9b9b00, 0xd8d8d800d8d8d800, 0x2626260026262600, 777 0x9b9b9b009b9b9b00ULL, 0xd8d8d800d8d8d800ULL, 0x2626260026262600ULL,
778 0xc8c8c800c8c8c800, 0x3737370037373700, 0xc6c6c600c6c6c600, 778 0xc8c8c800c8c8c800ULL, 0x3737370037373700ULL, 0xc6c6c600c6c6c600ULL,
779 0x3b3b3b003b3b3b00, 0x8181810081818100, 0x9696960096969600, 779 0x3b3b3b003b3b3b00ULL, 0x8181810081818100ULL, 0x9696960096969600ULL,
780 0x6f6f6f006f6f6f00, 0x4b4b4b004b4b4b00, 0x1313130013131300, 780 0x6f6f6f006f6f6f00ULL, 0x4b4b4b004b4b4b00ULL, 0x1313130013131300ULL,
781 0xbebebe00bebebe00, 0x6363630063636300, 0x2e2e2e002e2e2e00, 781 0xbebebe00bebebe00ULL, 0x6363630063636300ULL, 0x2e2e2e002e2e2e00ULL,
782 0xe9e9e900e9e9e900, 0x7979790079797900, 0xa7a7a700a7a7a700, 782 0xe9e9e900e9e9e900ULL, 0x7979790079797900ULL, 0xa7a7a700a7a7a700ULL,
783 0x8c8c8c008c8c8c00, 0x9f9f9f009f9f9f00, 0x6e6e6e006e6e6e00, 783 0x8c8c8c008c8c8c00ULL, 0x9f9f9f009f9f9f00ULL, 0x6e6e6e006e6e6e00ULL,
784 0xbcbcbc00bcbcbc00, 0x8e8e8e008e8e8e00, 0x2929290029292900, 784 0xbcbcbc00bcbcbc00ULL, 0x8e8e8e008e8e8e00ULL, 0x2929290029292900ULL,
785 0xf5f5f500f5f5f500, 0xf9f9f900f9f9f900, 0xb6b6b600b6b6b600, 785 0xf5f5f500f5f5f500ULL, 0xf9f9f900f9f9f900ULL, 0xb6b6b600b6b6b600ULL,
786 0x2f2f2f002f2f2f00, 0xfdfdfd00fdfdfd00, 0xb4b4b400b4b4b400, 786 0x2f2f2f002f2f2f00ULL, 0xfdfdfd00fdfdfd00ULL, 0xb4b4b400b4b4b400ULL,
787 0x5959590059595900, 0x7878780078787800, 0x9898980098989800, 787 0x5959590059595900ULL, 0x7878780078787800ULL, 0x9898980098989800ULL,
788 0x0606060006060600, 0x6a6a6a006a6a6a00, 0xe7e7e700e7e7e700, 788 0x0606060006060600ULL, 0x6a6a6a006a6a6a00ULL, 0xe7e7e700e7e7e700ULL,
789 0x4646460046464600, 0x7171710071717100, 0xbababa00bababa00, 789 0x4646460046464600ULL, 0x7171710071717100ULL, 0xbababa00bababa00ULL,
790 0xd4d4d400d4d4d400, 0x2525250025252500, 0xababab00ababab00, 790 0xd4d4d400d4d4d400ULL, 0x2525250025252500ULL, 0xababab00ababab00ULL,
791 0x4242420042424200, 0x8888880088888800, 0xa2a2a200a2a2a200, 791 0x4242420042424200ULL, 0x8888880088888800ULL, 0xa2a2a200a2a2a200ULL,
792 0x8d8d8d008d8d8d00, 0xfafafa00fafafa00, 0x7272720072727200, 792 0x8d8d8d008d8d8d00ULL, 0xfafafa00fafafa00ULL, 0x7272720072727200ULL,
793 0x0707070007070700, 0xb9b9b900b9b9b900, 0x5555550055555500, 793 0x0707070007070700ULL, 0xb9b9b900b9b9b900ULL, 0x5555550055555500ULL,
794 0xf8f8f800f8f8f800, 0xeeeeee00eeeeee00, 0xacacac00acacac00, 794 0xf8f8f800f8f8f800ULL, 0xeeeeee00eeeeee00ULL, 0xacacac00acacac00ULL,
795 0x0a0a0a000a0a0a00, 0x3636360036363600, 0x4949490049494900, 795 0x0a0a0a000a0a0a00ULL, 0x3636360036363600ULL, 0x4949490049494900ULL,
796 0x2a2a2a002a2a2a00, 0x6868680068686800, 0x3c3c3c003c3c3c00, 796 0x2a2a2a002a2a2a00ULL, 0x6868680068686800ULL, 0x3c3c3c003c3c3c00ULL,
797 0x3838380038383800, 0xf1f1f100f1f1f100, 0xa4a4a400a4a4a400, 797 0x3838380038383800ULL, 0xf1f1f100f1f1f100ULL, 0xa4a4a400a4a4a400ULL,
798 0x4040400040404000, 0x2828280028282800, 0xd3d3d300d3d3d300, 798 0x4040400040404000ULL, 0x2828280028282800ULL, 0xd3d3d300d3d3d300ULL,
799 0x7b7b7b007b7b7b00, 0xbbbbbb00bbbbbb00, 0xc9c9c900c9c9c900, 799 0x7b7b7b007b7b7b00ULL, 0xbbbbbb00bbbbbb00ULL, 0xc9c9c900c9c9c900ULL,
800 0x4343430043434300, 0xc1c1c100c1c1c100, 0x1515150015151500, 800 0x4343430043434300ULL, 0xc1c1c100c1c1c100ULL, 0x1515150015151500ULL,
801 0xe3e3e300e3e3e300, 0xadadad00adadad00, 0xf4f4f400f4f4f400, 801 0xe3e3e300e3e3e300ULL, 0xadadad00adadad00ULL, 0xf4f4f400f4f4f400ULL,
802 0x7777770077777700, 0xc7c7c700c7c7c700, 0x8080800080808000, 802 0x7777770077777700ULL, 0xc7c7c700c7c7c700ULL, 0x8080800080808000ULL,
803 0x9e9e9e009e9e9e00, 803 0x9e9e9e009e9e9e00ULL,
804}; 804};
805 805
806/* key constants */ 806/* key constants */
@@ -1601,7 +1601,6 @@ static struct crypto_alg camellia_algs[6] = { {
1601 .cra_ctxsize = sizeof(struct camellia_ctx), 1601 .cra_ctxsize = sizeof(struct camellia_ctx),
1602 .cra_alignmask = 0, 1602 .cra_alignmask = 0,
1603 .cra_module = THIS_MODULE, 1603 .cra_module = THIS_MODULE,
1604 .cra_list = LIST_HEAD_INIT(camellia_algs[0].cra_list),
1605 .cra_u = { 1604 .cra_u = {
1606 .cipher = { 1605 .cipher = {
1607 .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE, 1606 .cia_min_keysize = CAMELLIA_MIN_KEY_SIZE,
@@ -1621,7 +1620,6 @@ static struct crypto_alg camellia_algs[6] = { {
1621 .cra_alignmask = 0, 1620 .cra_alignmask = 0,
1622 .cra_type = &crypto_blkcipher_type, 1621 .cra_type = &crypto_blkcipher_type,
1623 .cra_module = THIS_MODULE, 1622 .cra_module = THIS_MODULE,
1624 .cra_list = LIST_HEAD_INIT(camellia_algs[1].cra_list),
1625 .cra_u = { 1623 .cra_u = {
1626 .blkcipher = { 1624 .blkcipher = {
1627 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 1625 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
@@ -1641,7 +1639,6 @@ static struct crypto_alg camellia_algs[6] = { {
1641 .cra_alignmask = 0, 1639 .cra_alignmask = 0,
1642 .cra_type = &crypto_blkcipher_type, 1640 .cra_type = &crypto_blkcipher_type,
1643 .cra_module = THIS_MODULE, 1641 .cra_module = THIS_MODULE,
1644 .cra_list = LIST_HEAD_INIT(camellia_algs[2].cra_list),
1645 .cra_u = { 1642 .cra_u = {
1646 .blkcipher = { 1643 .blkcipher = {
1647 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 1644 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
@@ -1662,7 +1659,6 @@ static struct crypto_alg camellia_algs[6] = { {
1662 .cra_alignmask = 0, 1659 .cra_alignmask = 0,
1663 .cra_type = &crypto_blkcipher_type, 1660 .cra_type = &crypto_blkcipher_type,
1664 .cra_module = THIS_MODULE, 1661 .cra_module = THIS_MODULE,
1665 .cra_list = LIST_HEAD_INIT(camellia_algs[3].cra_list),
1666 .cra_u = { 1662 .cra_u = {
1667 .blkcipher = { 1663 .blkcipher = {
1668 .min_keysize = CAMELLIA_MIN_KEY_SIZE, 1664 .min_keysize = CAMELLIA_MIN_KEY_SIZE,
@@ -1683,7 +1679,6 @@ static struct crypto_alg camellia_algs[6] = { {
1683 .cra_alignmask = 0, 1679 .cra_alignmask = 0,
1684 .cra_type = &crypto_blkcipher_type, 1680 .cra_type = &crypto_blkcipher_type,
1685 .cra_module = THIS_MODULE, 1681 .cra_module = THIS_MODULE,
1686 .cra_list = LIST_HEAD_INIT(camellia_algs[4].cra_list),
1687 .cra_exit = lrw_exit_tfm, 1682 .cra_exit = lrw_exit_tfm,
1688 .cra_u = { 1683 .cra_u = {
1689 .blkcipher = { 1684 .blkcipher = {
@@ -1707,7 +1702,6 @@ static struct crypto_alg camellia_algs[6] = { {
1707 .cra_alignmask = 0, 1702 .cra_alignmask = 0,
1708 .cra_type = &crypto_blkcipher_type, 1703 .cra_type = &crypto_blkcipher_type,
1709 .cra_module = THIS_MODULE, 1704 .cra_module = THIS_MODULE,
1710 .cra_list = LIST_HEAD_INIT(camellia_algs[5].cra_list),
1711 .cra_u = { 1705 .cra_u = {
1712 .blkcipher = { 1706 .blkcipher = {
1713 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2, 1707 .min_keysize = CAMELLIA_MIN_KEY_SIZE * 2,
diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
new file mode 100644
index 000000000000..a41a3aaba220
--- /dev/null
+++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S
@@ -0,0 +1,376 @@
1/*
2 * Cast5 Cipher 16-way parallel algorithm (AVX/x86_64)
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
22 * USA
23 *
24 */
25
26.file "cast5-avx-x86_64-asm_64.S"
27
28.extern cast5_s1
29.extern cast5_s2
30.extern cast5_s3
31.extern cast5_s4
32
33/* structure of crypto context */
34#define km 0
35#define kr (16*4)
36#define rr ((16*4)+16)
37
38/* s-boxes */
39#define s1 cast5_s1
40#define s2 cast5_s2
41#define s3 cast5_s3
42#define s4 cast5_s4
43
44/**********************************************************************
45 16-way AVX cast5
46 **********************************************************************/
47#define CTX %rdi
48
49#define RL1 %xmm0
50#define RR1 %xmm1
51#define RL2 %xmm2
52#define RR2 %xmm3
53#define RL3 %xmm4
54#define RR3 %xmm5
55#define RL4 %xmm6
56#define RR4 %xmm7
57
58#define RX %xmm8
59
60#define RKM %xmm9
61#define RKR %xmm10
62#define RKRF %xmm11
63#define RKRR %xmm12
64
65#define R32 %xmm13
66#define R1ST %xmm14
67
68#define RTMP %xmm15
69
70#define RID1 %rbp
71#define RID1d %ebp
72#define RID2 %rsi
73#define RID2d %esi
74
75#define RGI1 %rdx
76#define RGI1bl %dl
77#define RGI1bh %dh
78#define RGI2 %rcx
79#define RGI2bl %cl
80#define RGI2bh %ch
81
82#define RGI3 %rax
83#define RGI3bl %al
84#define RGI3bh %ah
85#define RGI4 %rbx
86#define RGI4bl %bl
87#define RGI4bh %bh
88
89#define RFS1 %r8
90#define RFS1d %r8d
91#define RFS2 %r9
92#define RFS2d %r9d
93#define RFS3 %r10
94#define RFS3d %r10d
95
96
97#define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \
98 movzbl src ## bh, RID1d; \
99 movzbl src ## bl, RID2d; \
100 shrq $16, src; \
101 movl s1(, RID1, 4), dst ## d; \
102 op1 s2(, RID2, 4), dst ## d; \
103 movzbl src ## bh, RID1d; \
104 movzbl src ## bl, RID2d; \
105 interleave_op(il_reg); \
106 op2 s3(, RID1, 4), dst ## d; \
107 op3 s4(, RID2, 4), dst ## d;
108
109#define dummy(d) /* do nothing */
110
111#define shr_next(reg) \
112 shrq $16, reg;
113
114#define F_head(a, x, gi1, gi2, op0) \
115 op0 a, RKM, x; \
116 vpslld RKRF, x, RTMP; \
117 vpsrld RKRR, x, x; \
118 vpor RTMP, x, x; \
119 \
120 vmovq x, gi1; \
121 vpextrq $1, x, gi2;
122
123#define F_tail(a, x, gi1, gi2, op1, op2, op3) \
124 lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \
125 lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \
126 \
127 lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none); \
128 shlq $32, RFS2; \
129 orq RFS1, RFS2; \
130 lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none); \
131 shlq $32, RFS1; \
132 orq RFS1, RFS3; \
133 \
134 vmovq RFS2, x; \
135 vpinsrq $1, RFS3, x, x;
136
137#define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \
138 F_head(b1, RX, RGI1, RGI2, op0); \
139 F_head(b2, RX, RGI3, RGI4, op0); \
140 \
141 F_tail(b1, RX, RGI1, RGI2, op1, op2, op3); \
142 F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3); \
143 \
144 vpxor a1, RX, a1; \
145 vpxor a2, RTMP, a2;
146
147#define F1_2(a1, b1, a2, b2) \
148 F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl)
149#define F2_2(a1, b1, a2, b2) \
150 F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl)
151#define F3_2(a1, b1, a2, b2) \
152 F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl)
153
154#define subround(a1, b1, a2, b2, f) \
155 F ## f ## _2(a1, b1, a2, b2);
156
157#define round(l, r, n, f) \
158 vbroadcastss (km+(4*n))(CTX), RKM; \
159 vpand R1ST, RKR, RKRF; \
160 vpsubq RKRF, R32, RKRR; \
161 vpsrldq $1, RKR, RKR; \
162 subround(l ## 1, r ## 1, l ## 2, r ## 2, f); \
163 subround(l ## 3, r ## 3, l ## 4, r ## 4, f);
164
165#define enc_preload_rkr() \
166 vbroadcastss .L16_mask, RKR; \
167 /* add 16-bit rotation to key rotations (mod 32) */ \
168 vpxor kr(CTX), RKR, RKR;
169
170#define dec_preload_rkr() \
171 vbroadcastss .L16_mask, RKR; \
172 /* add 16-bit rotation to key rotations (mod 32) */ \
173 vpxor kr(CTX), RKR, RKR; \
174 vpshufb .Lbswap128_mask, RKR, RKR;
175
176#define transpose_2x4(x0, x1, t0, t1) \
177 vpunpckldq x1, x0, t0; \
178 vpunpckhdq x1, x0, t1; \
179 \
180 vpunpcklqdq t1, t0, x0; \
181 vpunpckhqdq t1, t0, x1;
182
183#define inpack_blocks(in, x0, x1, t0, t1, rmask) \
184 vmovdqu (0*4*4)(in), x0; \
185 vmovdqu (1*4*4)(in), x1; \
186 vpshufb rmask, x0, x0; \
187 vpshufb rmask, x1, x1; \
188 \
189 transpose_2x4(x0, x1, t0, t1)
190
191#define outunpack_blocks(out, x0, x1, t0, t1, rmask) \
192 transpose_2x4(x0, x1, t0, t1) \
193 \
194 vpshufb rmask, x0, x0; \
195 vpshufb rmask, x1, x1; \
196 vmovdqu x0, (0*4*4)(out); \
197 vmovdqu x1, (1*4*4)(out);
198
199#define outunpack_xor_blocks(out, x0, x1, t0, t1, rmask) \
200 transpose_2x4(x0, x1, t0, t1) \
201 \
202 vpshufb rmask, x0, x0; \
203 vpshufb rmask, x1, x1; \
204 vpxor (0*4*4)(out), x0, x0; \
205 vmovdqu x0, (0*4*4)(out); \
206 vpxor (1*4*4)(out), x1, x1; \
207 vmovdqu x1, (1*4*4)(out);
208
209.data
210
211.align 16
212.Lbswap_mask:
213 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
214.Lbswap128_mask:
215 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
216.L16_mask:
217 .byte 16, 16, 16, 16
218.L32_mask:
219 .byte 32, 0, 0, 0
220.Lfirst_mask:
221 .byte 0x1f, 0, 0, 0
222
223.text
224
225.align 16
226.global __cast5_enc_blk_16way
227.type __cast5_enc_blk_16way,@function;
228
229__cast5_enc_blk_16way:
230 /* input:
231 * %rdi: ctx, CTX
232 * %rsi: dst
233 * %rdx: src
234 * %rcx: bool, if true: xor output
235 */
236
237 pushq %rbp;
238 pushq %rbx;
239 pushq %rcx;
240
241 vmovdqa .Lbswap_mask, RKM;
242 vmovd .Lfirst_mask, R1ST;
243 vmovd .L32_mask, R32;
244 enc_preload_rkr();
245
246 leaq 1*(2*4*4)(%rdx), %rax;
247 inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM);
248 inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM);
249 leaq 2*(2*4*4)(%rdx), %rax;
250 inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM);
251 leaq 3*(2*4*4)(%rdx), %rax;
252 inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM);
253
254 movq %rsi, %r11;
255
256 round(RL, RR, 0, 1);
257 round(RR, RL, 1, 2);
258 round(RL, RR, 2, 3);
259 round(RR, RL, 3, 1);
260 round(RL, RR, 4, 2);
261 round(RR, RL, 5, 3);
262 round(RL, RR, 6, 1);
263 round(RR, RL, 7, 2);
264 round(RL, RR, 8, 3);
265 round(RR, RL, 9, 1);
266 round(RL, RR, 10, 2);
267 round(RR, RL, 11, 3);
268
269 movzbl rr(CTX), %eax;
270 testl %eax, %eax;
271 jnz __skip_enc;
272
273 round(RL, RR, 12, 1);
274 round(RR, RL, 13, 2);
275 round(RL, RR, 14, 3);
276 round(RR, RL, 15, 1);
277
278__skip_enc:
279 popq %rcx;
280 popq %rbx;
281 popq %rbp;
282
283 vmovdqa .Lbswap_mask, RKM;
284 leaq 1*(2*4*4)(%r11), %rax;
285
286 testb %cl, %cl;
287 jnz __enc_xor16;
288
289 outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
290 outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
291 leaq 2*(2*4*4)(%r11), %rax;
292 outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
293 leaq 3*(2*4*4)(%r11), %rax;
294 outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
295
296 ret;
297
298__enc_xor16:
299 outunpack_xor_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
300 outunpack_xor_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
301 leaq 2*(2*4*4)(%r11), %rax;
302 outunpack_xor_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
303 leaq 3*(2*4*4)(%r11), %rax;
304 outunpack_xor_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
305
306 ret;
307
308.align 16
309.global cast5_dec_blk_16way
310.type cast5_dec_blk_16way,@function;
311
312cast5_dec_blk_16way:
313 /* input:
314 * %rdi: ctx, CTX
315 * %rsi: dst
316 * %rdx: src
317 */
318
319 pushq %rbp;
320 pushq %rbx;
321
322 vmovdqa .Lbswap_mask, RKM;
323 vmovd .Lfirst_mask, R1ST;
324 vmovd .L32_mask, R32;
325 dec_preload_rkr();
326
327 leaq 1*(2*4*4)(%rdx), %rax;
328 inpack_blocks(%rdx, RL1, RR1, RTMP, RX, RKM);
329 inpack_blocks(%rax, RL2, RR2, RTMP, RX, RKM);
330 leaq 2*(2*4*4)(%rdx), %rax;
331 inpack_blocks(%rax, RL3, RR3, RTMP, RX, RKM);
332 leaq 3*(2*4*4)(%rdx), %rax;
333 inpack_blocks(%rax, RL4, RR4, RTMP, RX, RKM);
334
335 movq %rsi, %r11;
336
337 movzbl rr(CTX), %eax;
338 testl %eax, %eax;
339 jnz __skip_dec;
340
341 round(RL, RR, 15, 1);
342 round(RR, RL, 14, 3);
343 round(RL, RR, 13, 2);
344 round(RR, RL, 12, 1);
345
346__dec_tail:
347 round(RL, RR, 11, 3);
348 round(RR, RL, 10, 2);
349 round(RL, RR, 9, 1);
350 round(RR, RL, 8, 3);
351 round(RL, RR, 7, 2);
352 round(RR, RL, 6, 1);
353 round(RL, RR, 5, 3);
354 round(RR, RL, 4, 2);
355 round(RL, RR, 3, 1);
356 round(RR, RL, 2, 3);
357 round(RL, RR, 1, 2);
358 round(RR, RL, 0, 1);
359
360 vmovdqa .Lbswap_mask, RKM;
361 popq %rbx;
362 popq %rbp;
363
364 leaq 1*(2*4*4)(%r11), %rax;
365 outunpack_blocks(%r11, RR1, RL1, RTMP, RX, RKM);
366 outunpack_blocks(%rax, RR2, RL2, RTMP, RX, RKM);
367 leaq 2*(2*4*4)(%r11), %rax;
368 outunpack_blocks(%rax, RR3, RL3, RTMP, RX, RKM);
369 leaq 3*(2*4*4)(%r11), %rax;
370 outunpack_blocks(%rax, RR4, RL4, RTMP, RX, RKM);
371
372 ret;
373
374__skip_dec:
375 vpsrldq $4, RKR, RKR;
376 jmp __dec_tail;
diff --git a/arch/x86/crypto/cast5_avx_glue.c b/arch/x86/crypto/cast5_avx_glue.c
new file mode 100644
index 000000000000..e0ea14f9547f
--- /dev/null
+++ b/arch/x86/crypto/cast5_avx_glue.c
@@ -0,0 +1,530 @@
1/*
2 * Glue Code for the AVX assembler implemention of the Cast5 Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
20 * USA
21 *
22 */
23
24#include <linux/module.h>
25#include <linux/hardirq.h>
26#include <linux/types.h>
27#include <linux/crypto.h>
28#include <linux/err.h>
29#include <crypto/algapi.h>
30#include <crypto/cast5.h>
31#include <crypto/cryptd.h>
32#include <crypto/ctr.h>
33#include <asm/xcr.h>
34#include <asm/xsave.h>
35#include <asm/crypto/ablk_helper.h>
36#include <asm/crypto/glue_helper.h>
37
38#define CAST5_PARALLEL_BLOCKS 16
39
40asmlinkage void __cast5_enc_blk_16way(struct cast5_ctx *ctx, u8 *dst,
41 const u8 *src, bool xor);
42asmlinkage void cast5_dec_blk_16way(struct cast5_ctx *ctx, u8 *dst,
43 const u8 *src);
44
45static inline void cast5_enc_blk_xway(struct cast5_ctx *ctx, u8 *dst,
46 const u8 *src)
47{
48 __cast5_enc_blk_16way(ctx, dst, src, false);
49}
50
51static inline void cast5_enc_blk_xway_xor(struct cast5_ctx *ctx, u8 *dst,
52 const u8 *src)
53{
54 __cast5_enc_blk_16way(ctx, dst, src, true);
55}
56
57static inline void cast5_dec_blk_xway(struct cast5_ctx *ctx, u8 *dst,
58 const u8 *src)
59{
60 cast5_dec_blk_16way(ctx, dst, src);
61}
62
63
64static inline bool cast5_fpu_begin(bool fpu_enabled, unsigned int nbytes)
65{
66 return glue_fpu_begin(CAST5_BLOCK_SIZE, CAST5_PARALLEL_BLOCKS,
67 NULL, fpu_enabled, nbytes);
68}
69
70static inline void cast5_fpu_end(bool fpu_enabled)
71{
72 return glue_fpu_end(fpu_enabled);
73}
74
75static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
76 bool enc)
77{
78 bool fpu_enabled = false;
79 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
80 const unsigned int bsize = CAST5_BLOCK_SIZE;
81 unsigned int nbytes;
82 int err;
83
84 err = blkcipher_walk_virt(desc, walk);
85 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
86
87 while ((nbytes = walk->nbytes)) {
88 u8 *wsrc = walk->src.virt.addr;
89 u8 *wdst = walk->dst.virt.addr;
90
91 fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
92
93 /* Process multi-block batch */
94 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
95 do {
96 if (enc)
97 cast5_enc_blk_xway(ctx, wdst, wsrc);
98 else
99 cast5_dec_blk_xway(ctx, wdst, wsrc);
100
101 wsrc += bsize * CAST5_PARALLEL_BLOCKS;
102 wdst += bsize * CAST5_PARALLEL_BLOCKS;
103 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
104 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
105
106 if (nbytes < bsize)
107 goto done;
108 }
109
110 /* Handle leftovers */
111 do {
112 if (enc)
113 __cast5_encrypt(ctx, wdst, wsrc);
114 else
115 __cast5_decrypt(ctx, wdst, wsrc);
116
117 wsrc += bsize;
118 wdst += bsize;
119 nbytes -= bsize;
120 } while (nbytes >= bsize);
121
122done:
123 err = blkcipher_walk_done(desc, walk, nbytes);
124 }
125
126 cast5_fpu_end(fpu_enabled);
127 return err;
128}
129
130static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
131 struct scatterlist *src, unsigned int nbytes)
132{
133 struct blkcipher_walk walk;
134
135 blkcipher_walk_init(&walk, dst, src, nbytes);
136 return ecb_crypt(desc, &walk, true);
137}
138
139static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
140 struct scatterlist *src, unsigned int nbytes)
141{
142 struct blkcipher_walk walk;
143
144 blkcipher_walk_init(&walk, dst, src, nbytes);
145 return ecb_crypt(desc, &walk, false);
146}
147
148static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
149 struct blkcipher_walk *walk)
150{
151 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
152 const unsigned int bsize = CAST5_BLOCK_SIZE;
153 unsigned int nbytes = walk->nbytes;
154 u64 *src = (u64 *)walk->src.virt.addr;
155 u64 *dst = (u64 *)walk->dst.virt.addr;
156 u64 *iv = (u64 *)walk->iv;
157
158 do {
159 *dst = *src ^ *iv;
160 __cast5_encrypt(ctx, (u8 *)dst, (u8 *)dst);
161 iv = dst;
162
163 src += 1;
164 dst += 1;
165 nbytes -= bsize;
166 } while (nbytes >= bsize);
167
168 *(u64 *)walk->iv = *iv;
169 return nbytes;
170}
171
172static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
173 struct scatterlist *src, unsigned int nbytes)
174{
175 struct blkcipher_walk walk;
176 int err;
177
178 blkcipher_walk_init(&walk, dst, src, nbytes);
179 err = blkcipher_walk_virt(desc, &walk);
180
181 while ((nbytes = walk.nbytes)) {
182 nbytes = __cbc_encrypt(desc, &walk);
183 err = blkcipher_walk_done(desc, &walk, nbytes);
184 }
185
186 return err;
187}
188
189static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
190 struct blkcipher_walk *walk)
191{
192 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
193 const unsigned int bsize = CAST5_BLOCK_SIZE;
194 unsigned int nbytes = walk->nbytes;
195 u64 *src = (u64 *)walk->src.virt.addr;
196 u64 *dst = (u64 *)walk->dst.virt.addr;
197 u64 ivs[CAST5_PARALLEL_BLOCKS - 1];
198 u64 last_iv;
199 int i;
200
201 /* Start of the last block. */
202 src += nbytes / bsize - 1;
203 dst += nbytes / bsize - 1;
204
205 last_iv = *src;
206
207 /* Process multi-block batch */
208 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
209 do {
210 nbytes -= bsize * (CAST5_PARALLEL_BLOCKS - 1);
211 src -= CAST5_PARALLEL_BLOCKS - 1;
212 dst -= CAST5_PARALLEL_BLOCKS - 1;
213
214 for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++)
215 ivs[i] = src[i];
216
217 cast5_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
218
219 for (i = 0; i < CAST5_PARALLEL_BLOCKS - 1; i++)
220 *(dst + (i + 1)) ^= *(ivs + i);
221
222 nbytes -= bsize;
223 if (nbytes < bsize)
224 goto done;
225
226 *dst ^= *(src - 1);
227 src -= 1;
228 dst -= 1;
229 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
230
231 if (nbytes < bsize)
232 goto done;
233 }
234
235 /* Handle leftovers */
236 for (;;) {
237 __cast5_decrypt(ctx, (u8 *)dst, (u8 *)src);
238
239 nbytes -= bsize;
240 if (nbytes < bsize)
241 break;
242
243 *dst ^= *(src - 1);
244 src -= 1;
245 dst -= 1;
246 }
247
248done:
249 *dst ^= *(u64 *)walk->iv;
250 *(u64 *)walk->iv = last_iv;
251
252 return nbytes;
253}
254
255static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
256 struct scatterlist *src, unsigned int nbytes)
257{
258 bool fpu_enabled = false;
259 struct blkcipher_walk walk;
260 int err;
261
262 blkcipher_walk_init(&walk, dst, src, nbytes);
263 err = blkcipher_walk_virt(desc, &walk);
264 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
265
266 while ((nbytes = walk.nbytes)) {
267 fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
268 nbytes = __cbc_decrypt(desc, &walk);
269 err = blkcipher_walk_done(desc, &walk, nbytes);
270 }
271
272 cast5_fpu_end(fpu_enabled);
273 return err;
274}
275
276static void ctr_crypt_final(struct blkcipher_desc *desc,
277 struct blkcipher_walk *walk)
278{
279 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
280 u8 *ctrblk = walk->iv;
281 u8 keystream[CAST5_BLOCK_SIZE];
282 u8 *src = walk->src.virt.addr;
283 u8 *dst = walk->dst.virt.addr;
284 unsigned int nbytes = walk->nbytes;
285
286 __cast5_encrypt(ctx, keystream, ctrblk);
287 crypto_xor(keystream, src, nbytes);
288 memcpy(dst, keystream, nbytes);
289
290 crypto_inc(ctrblk, CAST5_BLOCK_SIZE);
291}
292
293static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
294 struct blkcipher_walk *walk)
295{
296 struct cast5_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
297 const unsigned int bsize = CAST5_BLOCK_SIZE;
298 unsigned int nbytes = walk->nbytes;
299 u64 *src = (u64 *)walk->src.virt.addr;
300 u64 *dst = (u64 *)walk->dst.virt.addr;
301 u64 ctrblk = be64_to_cpu(*(__be64 *)walk->iv);
302 __be64 ctrblocks[CAST5_PARALLEL_BLOCKS];
303 int i;
304
305 /* Process multi-block batch */
306 if (nbytes >= bsize * CAST5_PARALLEL_BLOCKS) {
307 do {
308 /* create ctrblks for parallel encrypt */
309 for (i = 0; i < CAST5_PARALLEL_BLOCKS; i++) {
310 if (dst != src)
311 dst[i] = src[i];
312
313 ctrblocks[i] = cpu_to_be64(ctrblk++);
314 }
315
316 cast5_enc_blk_xway_xor(ctx, (u8 *)dst,
317 (u8 *)ctrblocks);
318
319 src += CAST5_PARALLEL_BLOCKS;
320 dst += CAST5_PARALLEL_BLOCKS;
321 nbytes -= bsize * CAST5_PARALLEL_BLOCKS;
322 } while (nbytes >= bsize * CAST5_PARALLEL_BLOCKS);
323
324 if (nbytes < bsize)
325 goto done;
326 }
327
328 /* Handle leftovers */
329 do {
330 if (dst != src)
331 *dst = *src;
332
333 ctrblocks[0] = cpu_to_be64(ctrblk++);
334
335 __cast5_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
336 *dst ^= ctrblocks[0];
337
338 src += 1;
339 dst += 1;
340 nbytes -= bsize;
341 } while (nbytes >= bsize);
342
343done:
344 *(__be64 *)walk->iv = cpu_to_be64(ctrblk);
345 return nbytes;
346}
347
348static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
349 struct scatterlist *src, unsigned int nbytes)
350{
351 bool fpu_enabled = false;
352 struct blkcipher_walk walk;
353 int err;
354
355 blkcipher_walk_init(&walk, dst, src, nbytes);
356 err = blkcipher_walk_virt_block(desc, &walk, CAST5_BLOCK_SIZE);
357 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
358
359 while ((nbytes = walk.nbytes) >= CAST5_BLOCK_SIZE) {
360 fpu_enabled = cast5_fpu_begin(fpu_enabled, nbytes);
361 nbytes = __ctr_crypt(desc, &walk);
362 err = blkcipher_walk_done(desc, &walk, nbytes);
363 }
364
365 cast5_fpu_end(fpu_enabled);
366
367 if (walk.nbytes) {
368 ctr_crypt_final(desc, &walk);
369 err = blkcipher_walk_done(desc, &walk, 0);
370 }
371
372 return err;
373}
374
375
376static struct crypto_alg cast5_algs[6] = { {
377 .cra_name = "__ecb-cast5-avx",
378 .cra_driver_name = "__driver-ecb-cast5-avx",
379 .cra_priority = 0,
380 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
381 .cra_blocksize = CAST5_BLOCK_SIZE,
382 .cra_ctxsize = sizeof(struct cast5_ctx),
383 .cra_alignmask = 0,
384 .cra_type = &crypto_blkcipher_type,
385 .cra_module = THIS_MODULE,
386 .cra_u = {
387 .blkcipher = {
388 .min_keysize = CAST5_MIN_KEY_SIZE,
389 .max_keysize = CAST5_MAX_KEY_SIZE,
390 .setkey = cast5_setkey,
391 .encrypt = ecb_encrypt,
392 .decrypt = ecb_decrypt,
393 },
394 },
395}, {
396 .cra_name = "__cbc-cast5-avx",
397 .cra_driver_name = "__driver-cbc-cast5-avx",
398 .cra_priority = 0,
399 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
400 .cra_blocksize = CAST5_BLOCK_SIZE,
401 .cra_ctxsize = sizeof(struct cast5_ctx),
402 .cra_alignmask = 0,
403 .cra_type = &crypto_blkcipher_type,
404 .cra_module = THIS_MODULE,
405 .cra_u = {
406 .blkcipher = {
407 .min_keysize = CAST5_MIN_KEY_SIZE,
408 .max_keysize = CAST5_MAX_KEY_SIZE,
409 .setkey = cast5_setkey,
410 .encrypt = cbc_encrypt,
411 .decrypt = cbc_decrypt,
412 },
413 },
414}, {
415 .cra_name = "__ctr-cast5-avx",
416 .cra_driver_name = "__driver-ctr-cast5-avx",
417 .cra_priority = 0,
418 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
419 .cra_blocksize = 1,
420 .cra_ctxsize = sizeof(struct cast5_ctx),
421 .cra_alignmask = 0,
422 .cra_type = &crypto_blkcipher_type,
423 .cra_module = THIS_MODULE,
424 .cra_u = {
425 .blkcipher = {
426 .min_keysize = CAST5_MIN_KEY_SIZE,
427 .max_keysize = CAST5_MAX_KEY_SIZE,
428 .ivsize = CAST5_BLOCK_SIZE,
429 .setkey = cast5_setkey,
430 .encrypt = ctr_crypt,
431 .decrypt = ctr_crypt,
432 },
433 },
434}, {
435 .cra_name = "ecb(cast5)",
436 .cra_driver_name = "ecb-cast5-avx",
437 .cra_priority = 200,
438 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
439 .cra_blocksize = CAST5_BLOCK_SIZE,
440 .cra_ctxsize = sizeof(struct async_helper_ctx),
441 .cra_alignmask = 0,
442 .cra_type = &crypto_ablkcipher_type,
443 .cra_module = THIS_MODULE,
444 .cra_init = ablk_init,
445 .cra_exit = ablk_exit,
446 .cra_u = {
447 .ablkcipher = {
448 .min_keysize = CAST5_MIN_KEY_SIZE,
449 .max_keysize = CAST5_MAX_KEY_SIZE,
450 .setkey = ablk_set_key,
451 .encrypt = ablk_encrypt,
452 .decrypt = ablk_decrypt,
453 },
454 },
455}, {
456 .cra_name = "cbc(cast5)",
457 .cra_driver_name = "cbc-cast5-avx",
458 .cra_priority = 200,
459 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
460 .cra_blocksize = CAST5_BLOCK_SIZE,
461 .cra_ctxsize = sizeof(struct async_helper_ctx),
462 .cra_alignmask = 0,
463 .cra_type = &crypto_ablkcipher_type,
464 .cra_module = THIS_MODULE,
465 .cra_init = ablk_init,
466 .cra_exit = ablk_exit,
467 .cra_u = {
468 .ablkcipher = {
469 .min_keysize = CAST5_MIN_KEY_SIZE,
470 .max_keysize = CAST5_MAX_KEY_SIZE,
471 .ivsize = CAST5_BLOCK_SIZE,
472 .setkey = ablk_set_key,
473 .encrypt = __ablk_encrypt,
474 .decrypt = ablk_decrypt,
475 },
476 },
477}, {
478 .cra_name = "ctr(cast5)",
479 .cra_driver_name = "ctr-cast5-avx",
480 .cra_priority = 200,
481 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
482 .cra_blocksize = 1,
483 .cra_ctxsize = sizeof(struct async_helper_ctx),
484 .cra_alignmask = 0,
485 .cra_type = &crypto_ablkcipher_type,
486 .cra_module = THIS_MODULE,
487 .cra_init = ablk_init,
488 .cra_exit = ablk_exit,
489 .cra_u = {
490 .ablkcipher = {
491 .min_keysize = CAST5_MIN_KEY_SIZE,
492 .max_keysize = CAST5_MAX_KEY_SIZE,
493 .ivsize = CAST5_BLOCK_SIZE,
494 .setkey = ablk_set_key,
495 .encrypt = ablk_encrypt,
496 .decrypt = ablk_encrypt,
497 .geniv = "chainiv",
498 },
499 },
500} };
501
502static int __init cast5_init(void)
503{
504 u64 xcr0;
505
506 if (!cpu_has_avx || !cpu_has_osxsave) {
507 pr_info("AVX instructions are not detected.\n");
508 return -ENODEV;
509 }
510
511 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
512 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
513 pr_info("AVX detected but unusable.\n");
514 return -ENODEV;
515 }
516
517 return crypto_register_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
518}
519
520static void __exit cast5_exit(void)
521{
522 crypto_unregister_algs(cast5_algs, ARRAY_SIZE(cast5_algs));
523}
524
525module_init(cast5_init);
526module_exit(cast5_exit);
527
528MODULE_DESCRIPTION("Cast5 Cipher Algorithm, AVX optimized");
529MODULE_LICENSE("GPL");
530MODULE_ALIAS("cast5");
diff --git a/arch/x86/crypto/cast6-avx-x86_64-asm_64.S b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
new file mode 100644
index 000000000000..218d283772f4
--- /dev/null
+++ b/arch/x86/crypto/cast6-avx-x86_64-asm_64.S
@@ -0,0 +1,383 @@
1/*
2 * Cast6 Cipher 8-way parallel algorithm (AVX/x86_64)
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
22 * USA
23 *
24 */
25
26.file "cast6-avx-x86_64-asm_64.S"
27
28.extern cast6_s1
29.extern cast6_s2
30.extern cast6_s3
31.extern cast6_s4
32
33/* structure of crypto context */
34#define km 0
35#define kr (12*4*4)
36
37/* s-boxes */
38#define s1 cast6_s1
39#define s2 cast6_s2
40#define s3 cast6_s3
41#define s4 cast6_s4
42
43/**********************************************************************
44 8-way AVX cast6
45 **********************************************************************/
46#define CTX %rdi
47
48#define RA1 %xmm0
49#define RB1 %xmm1
50#define RC1 %xmm2
51#define RD1 %xmm3
52
53#define RA2 %xmm4
54#define RB2 %xmm5
55#define RC2 %xmm6
56#define RD2 %xmm7
57
58#define RX %xmm8
59
60#define RKM %xmm9
61#define RKR %xmm10
62#define RKRF %xmm11
63#define RKRR %xmm12
64#define R32 %xmm13
65#define R1ST %xmm14
66
67#define RTMP %xmm15
68
69#define RID1 %rbp
70#define RID1d %ebp
71#define RID2 %rsi
72#define RID2d %esi
73
74#define RGI1 %rdx
75#define RGI1bl %dl
76#define RGI1bh %dh
77#define RGI2 %rcx
78#define RGI2bl %cl
79#define RGI2bh %ch
80
81#define RGI3 %rax
82#define RGI3bl %al
83#define RGI3bh %ah
84#define RGI4 %rbx
85#define RGI4bl %bl
86#define RGI4bh %bh
87
88#define RFS1 %r8
89#define RFS1d %r8d
90#define RFS2 %r9
91#define RFS2d %r9d
92#define RFS3 %r10
93#define RFS3d %r10d
94
95
96#define lookup_32bit(src, dst, op1, op2, op3, interleave_op, il_reg) \
97 movzbl src ## bh, RID1d; \
98 movzbl src ## bl, RID2d; \
99 shrq $16, src; \
100 movl s1(, RID1, 4), dst ## d; \
101 op1 s2(, RID2, 4), dst ## d; \
102 movzbl src ## bh, RID1d; \
103 movzbl src ## bl, RID2d; \
104 interleave_op(il_reg); \
105 op2 s3(, RID1, 4), dst ## d; \
106 op3 s4(, RID2, 4), dst ## d;
107
108#define dummy(d) /* do nothing */
109
110#define shr_next(reg) \
111 shrq $16, reg;
112
113#define F_head(a, x, gi1, gi2, op0) \
114 op0 a, RKM, x; \
115 vpslld RKRF, x, RTMP; \
116 vpsrld RKRR, x, x; \
117 vpor RTMP, x, x; \
118 \
119 vmovq x, gi1; \
120 vpextrq $1, x, gi2;
121
122#define F_tail(a, x, gi1, gi2, op1, op2, op3) \
123 lookup_32bit(##gi1, RFS1, op1, op2, op3, shr_next, ##gi1); \
124 lookup_32bit(##gi2, RFS3, op1, op2, op3, shr_next, ##gi2); \
125 \
126 lookup_32bit(##gi1, RFS2, op1, op2, op3, dummy, none); \
127 shlq $32, RFS2; \
128 orq RFS1, RFS2; \
129 lookup_32bit(##gi2, RFS1, op1, op2, op3, dummy, none); \
130 shlq $32, RFS1; \
131 orq RFS1, RFS3; \
132 \
133 vmovq RFS2, x; \
134 vpinsrq $1, RFS3, x, x;
135
136#define F_2(a1, b1, a2, b2, op0, op1, op2, op3) \
137 F_head(b1, RX, RGI1, RGI2, op0); \
138 F_head(b2, RX, RGI3, RGI4, op0); \
139 \
140 F_tail(b1, RX, RGI1, RGI2, op1, op2, op3); \
141 F_tail(b2, RTMP, RGI3, RGI4, op1, op2, op3); \
142 \
143 vpxor a1, RX, a1; \
144 vpxor a2, RTMP, a2;
145
146#define F1_2(a1, b1, a2, b2) \
147 F_2(a1, b1, a2, b2, vpaddd, xorl, subl, addl)
148#define F2_2(a1, b1, a2, b2) \
149 F_2(a1, b1, a2, b2, vpxor, subl, addl, xorl)
150#define F3_2(a1, b1, a2, b2) \
151 F_2(a1, b1, a2, b2, vpsubd, addl, xorl, subl)
152
153#define qop(in, out, f) \
154 F ## f ## _2(out ## 1, in ## 1, out ## 2, in ## 2);
155
156#define get_round_keys(nn) \
157 vbroadcastss (km+(4*(nn)))(CTX), RKM; \
158 vpand R1ST, RKR, RKRF; \
159 vpsubq RKRF, R32, RKRR; \
160 vpsrldq $1, RKR, RKR;
161
162#define Q(n) \
163 get_round_keys(4*n+0); \
164 qop(RD, RC, 1); \
165 \
166 get_round_keys(4*n+1); \
167 qop(RC, RB, 2); \
168 \
169 get_round_keys(4*n+2); \
170 qop(RB, RA, 3); \
171 \
172 get_round_keys(4*n+3); \
173 qop(RA, RD, 1);
174
175#define QBAR(n) \
176 get_round_keys(4*n+3); \
177 qop(RA, RD, 1); \
178 \
179 get_round_keys(4*n+2); \
180 qop(RB, RA, 3); \
181 \
182 get_round_keys(4*n+1); \
183 qop(RC, RB, 2); \
184 \
185 get_round_keys(4*n+0); \
186 qop(RD, RC, 1);
187
188#define shuffle(mask) \
189 vpshufb mask, RKR, RKR;
190
191#define preload_rkr(n, do_mask, mask) \
192 vbroadcastss .L16_mask, RKR; \
193 /* add 16-bit rotation to key rotations (mod 32) */ \
194 vpxor (kr+n*16)(CTX), RKR, RKR; \
195 do_mask(mask);
196
197#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
198 vpunpckldq x1, x0, t0; \
199 vpunpckhdq x1, x0, t2; \
200 vpunpckldq x3, x2, t1; \
201 vpunpckhdq x3, x2, x3; \
202 \
203 vpunpcklqdq t1, t0, x0; \
204 vpunpckhqdq t1, t0, x1; \
205 vpunpcklqdq x3, t2, x2; \
206 vpunpckhqdq x3, t2, x3;
207
208#define inpack_blocks(in, x0, x1, x2, x3, t0, t1, t2, rmask) \
209 vmovdqu (0*4*4)(in), x0; \
210 vmovdqu (1*4*4)(in), x1; \
211 vmovdqu (2*4*4)(in), x2; \
212 vmovdqu (3*4*4)(in), x3; \
213 vpshufb rmask, x0, x0; \
214 vpshufb rmask, x1, x1; \
215 vpshufb rmask, x2, x2; \
216 vpshufb rmask, x3, x3; \
217 \
218 transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
219
220#define outunpack_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \
221 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
222 \
223 vpshufb rmask, x0, x0; \
224 vpshufb rmask, x1, x1; \
225 vpshufb rmask, x2, x2; \
226 vpshufb rmask, x3, x3; \
227 vmovdqu x0, (0*4*4)(out); \
228 vmovdqu x1, (1*4*4)(out); \
229 vmovdqu x2, (2*4*4)(out); \
230 vmovdqu x3, (3*4*4)(out);
231
232#define outunpack_xor_blocks(out, x0, x1, x2, x3, t0, t1, t2, rmask) \
233 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
234 \
235 vpshufb rmask, x0, x0; \
236 vpshufb rmask, x1, x1; \
237 vpshufb rmask, x2, x2; \
238 vpshufb rmask, x3, x3; \
239 vpxor (0*4*4)(out), x0, x0; \
240 vmovdqu x0, (0*4*4)(out); \
241 vpxor (1*4*4)(out), x1, x1; \
242 vmovdqu x1, (1*4*4)(out); \
243 vpxor (2*4*4)(out), x2, x2; \
244 vmovdqu x2, (2*4*4)(out); \
245 vpxor (3*4*4)(out), x3, x3; \
246 vmovdqu x3, (3*4*4)(out);
247
248.data
249
250.align 16
251.Lbswap_mask:
252 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
253.Lrkr_enc_Q_Q_QBAR_QBAR:
254 .byte 0, 1, 2, 3, 4, 5, 6, 7, 11, 10, 9, 8, 15, 14, 13, 12
255.Lrkr_enc_QBAR_QBAR_QBAR_QBAR:
256 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
257.Lrkr_dec_Q_Q_Q_Q:
258 .byte 12, 13, 14, 15, 8, 9, 10, 11, 4, 5, 6, 7, 0, 1, 2, 3
259.Lrkr_dec_Q_Q_QBAR_QBAR:
260 .byte 12, 13, 14, 15, 8, 9, 10, 11, 7, 6, 5, 4, 3, 2, 1, 0
261.Lrkr_dec_QBAR_QBAR_QBAR_QBAR:
262 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
263.L16_mask:
264 .byte 16, 16, 16, 16
265.L32_mask:
266 .byte 32, 0, 0, 0
267.Lfirst_mask:
268 .byte 0x1f, 0, 0, 0
269
270.text
271
272.align 16
273.global __cast6_enc_blk_8way
274.type __cast6_enc_blk_8way,@function;
275
276__cast6_enc_blk_8way:
277 /* input:
278 * %rdi: ctx, CTX
279 * %rsi: dst
280 * %rdx: src
281 * %rcx: bool, if true: xor output
282 */
283
284 pushq %rbp;
285 pushq %rbx;
286 pushq %rcx;
287
288 vmovdqa .Lbswap_mask, RKM;
289 vmovd .Lfirst_mask, R1ST;
290 vmovd .L32_mask, R32;
291
292 leaq (4*4*4)(%rdx), %rax;
293 inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
294 inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
295
296 movq %rsi, %r11;
297
298 preload_rkr(0, dummy, none);
299 Q(0);
300 Q(1);
301 Q(2);
302 Q(3);
303 preload_rkr(1, shuffle, .Lrkr_enc_Q_Q_QBAR_QBAR);
304 Q(4);
305 Q(5);
306 QBAR(6);
307 QBAR(7);
308 preload_rkr(2, shuffle, .Lrkr_enc_QBAR_QBAR_QBAR_QBAR);
309 QBAR(8);
310 QBAR(9);
311 QBAR(10);
312 QBAR(11);
313
314 popq %rcx;
315 popq %rbx;
316 popq %rbp;
317
318 vmovdqa .Lbswap_mask, RKM;
319 leaq (4*4*4)(%r11), %rax;
320
321 testb %cl, %cl;
322 jnz __enc_xor8;
323
324 outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
325 outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
326
327 ret;
328
329__enc_xor8:
330 outunpack_xor_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
331 outunpack_xor_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
332
333 ret;
334
335.align 16
336.global cast6_dec_blk_8way
337.type cast6_dec_blk_8way,@function;
338
339cast6_dec_blk_8way:
340 /* input:
341 * %rdi: ctx, CTX
342 * %rsi: dst
343 * %rdx: src
344 */
345
346 pushq %rbp;
347 pushq %rbx;
348
349 vmovdqa .Lbswap_mask, RKM;
350 vmovd .Lfirst_mask, R1ST;
351 vmovd .L32_mask, R32;
352
353 leaq (4*4*4)(%rdx), %rax;
354 inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
355 inpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
356
357 movq %rsi, %r11;
358
359 preload_rkr(2, shuffle, .Lrkr_dec_Q_Q_Q_Q);
360 Q(11);
361 Q(10);
362 Q(9);
363 Q(8);
364 preload_rkr(1, shuffle, .Lrkr_dec_Q_Q_QBAR_QBAR);
365 Q(7);
366 Q(6);
367 QBAR(5);
368 QBAR(4);
369 preload_rkr(0, shuffle, .Lrkr_dec_QBAR_QBAR_QBAR_QBAR);
370 QBAR(3);
371 QBAR(2);
372 QBAR(1);
373 QBAR(0);
374
375 popq %rbx;
376 popq %rbp;
377
378 vmovdqa .Lbswap_mask, RKM;
379 leaq (4*4*4)(%r11), %rax;
380 outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RTMP, RX, RKRF, RKM);
381 outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RTMP, RX, RKRF, RKM);
382
383 ret;
diff --git a/arch/x86/crypto/cast6_avx_glue.c b/arch/x86/crypto/cast6_avx_glue.c
new file mode 100644
index 000000000000..15e5f85a5011
--- /dev/null
+++ b/arch/x86/crypto/cast6_avx_glue.c
@@ -0,0 +1,648 @@
1/*
2 * Glue Code for the AVX assembler implemention of the Cast6 Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
20 * USA
21 *
22 */
23
24#include <linux/module.h>
25#include <linux/hardirq.h>
26#include <linux/types.h>
27#include <linux/crypto.h>
28#include <linux/err.h>
29#include <crypto/algapi.h>
30#include <crypto/cast6.h>
31#include <crypto/cryptd.h>
32#include <crypto/b128ops.h>
33#include <crypto/ctr.h>
34#include <crypto/lrw.h>
35#include <crypto/xts.h>
36#include <asm/xcr.h>
37#include <asm/xsave.h>
38#include <asm/crypto/ablk_helper.h>
39#include <asm/crypto/glue_helper.h>
40
41#define CAST6_PARALLEL_BLOCKS 8
42
43asmlinkage void __cast6_enc_blk_8way(struct cast6_ctx *ctx, u8 *dst,
44 const u8 *src, bool xor);
45asmlinkage void cast6_dec_blk_8way(struct cast6_ctx *ctx, u8 *dst,
46 const u8 *src);
47
48static inline void cast6_enc_blk_xway(struct cast6_ctx *ctx, u8 *dst,
49 const u8 *src)
50{
51 __cast6_enc_blk_8way(ctx, dst, src, false);
52}
53
54static inline void cast6_enc_blk_xway_xor(struct cast6_ctx *ctx, u8 *dst,
55 const u8 *src)
56{
57 __cast6_enc_blk_8way(ctx, dst, src, true);
58}
59
60static inline void cast6_dec_blk_xway(struct cast6_ctx *ctx, u8 *dst,
61 const u8 *src)
62{
63 cast6_dec_blk_8way(ctx, dst, src);
64}
65
66
67static void cast6_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
68{
69 u128 ivs[CAST6_PARALLEL_BLOCKS - 1];
70 unsigned int j;
71
72 for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++)
73 ivs[j] = src[j];
74
75 cast6_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
76
77 for (j = 0; j < CAST6_PARALLEL_BLOCKS - 1; j++)
78 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
79}
80
81static void cast6_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
82{
83 be128 ctrblk;
84
85 u128_to_be128(&ctrblk, iv);
86 u128_inc(iv);
87
88 __cast6_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
89 u128_xor(dst, src, (u128 *)&ctrblk);
90}
91
92static void cast6_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
93 u128 *iv)
94{
95 be128 ctrblks[CAST6_PARALLEL_BLOCKS];
96 unsigned int i;
97
98 for (i = 0; i < CAST6_PARALLEL_BLOCKS; i++) {
99 if (dst != src)
100 dst[i] = src[i];
101
102 u128_to_be128(&ctrblks[i], iv);
103 u128_inc(iv);
104 }
105
106 cast6_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
107}
108
109static const struct common_glue_ctx cast6_enc = {
110 .num_funcs = 2,
111 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
112
113 .funcs = { {
114 .num_blocks = CAST6_PARALLEL_BLOCKS,
115 .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_enc_blk_xway) }
116 }, {
117 .num_blocks = 1,
118 .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_encrypt) }
119 } }
120};
121
122static const struct common_glue_ctx cast6_ctr = {
123 .num_funcs = 2,
124 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
125
126 .funcs = { {
127 .num_blocks = CAST6_PARALLEL_BLOCKS,
128 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr_xway) }
129 }, {
130 .num_blocks = 1,
131 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(cast6_crypt_ctr) }
132 } }
133};
134
135static const struct common_glue_ctx cast6_dec = {
136 .num_funcs = 2,
137 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
138
139 .funcs = { {
140 .num_blocks = CAST6_PARALLEL_BLOCKS,
141 .fn_u = { .ecb = GLUE_FUNC_CAST(cast6_dec_blk_xway) }
142 }, {
143 .num_blocks = 1,
144 .fn_u = { .ecb = GLUE_FUNC_CAST(__cast6_decrypt) }
145 } }
146};
147
148static const struct common_glue_ctx cast6_dec_cbc = {
149 .num_funcs = 2,
150 .fpu_blocks_limit = CAST6_PARALLEL_BLOCKS,
151
152 .funcs = { {
153 .num_blocks = CAST6_PARALLEL_BLOCKS,
154 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(cast6_decrypt_cbc_xway) }
155 }, {
156 .num_blocks = 1,
157 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__cast6_decrypt) }
158 } }
159};
160
161static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
162 struct scatterlist *src, unsigned int nbytes)
163{
164 return glue_ecb_crypt_128bit(&cast6_enc, desc, dst, src, nbytes);
165}
166
167static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
168 struct scatterlist *src, unsigned int nbytes)
169{
170 return glue_ecb_crypt_128bit(&cast6_dec, desc, dst, src, nbytes);
171}
172
173static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
174 struct scatterlist *src, unsigned int nbytes)
175{
176 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__cast6_encrypt), desc,
177 dst, src, nbytes);
178}
179
180static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
181 struct scatterlist *src, unsigned int nbytes)
182{
183 return glue_cbc_decrypt_128bit(&cast6_dec_cbc, desc, dst, src,
184 nbytes);
185}
186
187static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
188 struct scatterlist *src, unsigned int nbytes)
189{
190 return glue_ctr_crypt_128bit(&cast6_ctr, desc, dst, src, nbytes);
191}
192
193static inline bool cast6_fpu_begin(bool fpu_enabled, unsigned int nbytes)
194{
195 return glue_fpu_begin(CAST6_BLOCK_SIZE, CAST6_PARALLEL_BLOCKS,
196 NULL, fpu_enabled, nbytes);
197}
198
199static inline void cast6_fpu_end(bool fpu_enabled)
200{
201 glue_fpu_end(fpu_enabled);
202}
203
204struct crypt_priv {
205 struct cast6_ctx *ctx;
206 bool fpu_enabled;
207};
208
209static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
210{
211 const unsigned int bsize = CAST6_BLOCK_SIZE;
212 struct crypt_priv *ctx = priv;
213 int i;
214
215 ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
216
217 if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
218 cast6_enc_blk_xway(ctx->ctx, srcdst, srcdst);
219 return;
220 }
221
222 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
223 __cast6_encrypt(ctx->ctx, srcdst, srcdst);
224}
225
226static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
227{
228 const unsigned int bsize = CAST6_BLOCK_SIZE;
229 struct crypt_priv *ctx = priv;
230 int i;
231
232 ctx->fpu_enabled = cast6_fpu_begin(ctx->fpu_enabled, nbytes);
233
234 if (nbytes == bsize * CAST6_PARALLEL_BLOCKS) {
235 cast6_dec_blk_xway(ctx->ctx, srcdst, srcdst);
236 return;
237 }
238
239 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
240 __cast6_decrypt(ctx->ctx, srcdst, srcdst);
241}
242
243struct cast6_lrw_ctx {
244 struct lrw_table_ctx lrw_table;
245 struct cast6_ctx cast6_ctx;
246};
247
248static int lrw_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
249 unsigned int keylen)
250{
251 struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
252 int err;
253
254 err = __cast6_setkey(&ctx->cast6_ctx, key, keylen - CAST6_BLOCK_SIZE,
255 &tfm->crt_flags);
256 if (err)
257 return err;
258
259 return lrw_init_table(&ctx->lrw_table, key + keylen - CAST6_BLOCK_SIZE);
260}
261
262static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
263 struct scatterlist *src, unsigned int nbytes)
264{
265 struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
266 be128 buf[CAST6_PARALLEL_BLOCKS];
267 struct crypt_priv crypt_ctx = {
268 .ctx = &ctx->cast6_ctx,
269 .fpu_enabled = false,
270 };
271 struct lrw_crypt_req req = {
272 .tbuf = buf,
273 .tbuflen = sizeof(buf),
274
275 .table_ctx = &ctx->lrw_table,
276 .crypt_ctx = &crypt_ctx,
277 .crypt_fn = encrypt_callback,
278 };
279 int ret;
280
281 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
282 ret = lrw_crypt(desc, dst, src, nbytes, &req);
283 cast6_fpu_end(crypt_ctx.fpu_enabled);
284
285 return ret;
286}
287
288static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
289 struct scatterlist *src, unsigned int nbytes)
290{
291 struct cast6_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
292 be128 buf[CAST6_PARALLEL_BLOCKS];
293 struct crypt_priv crypt_ctx = {
294 .ctx = &ctx->cast6_ctx,
295 .fpu_enabled = false,
296 };
297 struct lrw_crypt_req req = {
298 .tbuf = buf,
299 .tbuflen = sizeof(buf),
300
301 .table_ctx = &ctx->lrw_table,
302 .crypt_ctx = &crypt_ctx,
303 .crypt_fn = decrypt_callback,
304 };
305 int ret;
306
307 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
308 ret = lrw_crypt(desc, dst, src, nbytes, &req);
309 cast6_fpu_end(crypt_ctx.fpu_enabled);
310
311 return ret;
312}
313
314static void lrw_exit_tfm(struct crypto_tfm *tfm)
315{
316 struct cast6_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
317
318 lrw_free_table(&ctx->lrw_table);
319}
320
321struct cast6_xts_ctx {
322 struct cast6_ctx tweak_ctx;
323 struct cast6_ctx crypt_ctx;
324};
325
326static int xts_cast6_setkey(struct crypto_tfm *tfm, const u8 *key,
327 unsigned int keylen)
328{
329 struct cast6_xts_ctx *ctx = crypto_tfm_ctx(tfm);
330 u32 *flags = &tfm->crt_flags;
331 int err;
332
333 /* key consists of keys of equal size concatenated, therefore
334 * the length must be even
335 */
336 if (keylen % 2) {
337 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
338 return -EINVAL;
339 }
340
341 /* first half of xts-key is for crypt */
342 err = __cast6_setkey(&ctx->crypt_ctx, key, keylen / 2, flags);
343 if (err)
344 return err;
345
346 /* second half of xts-key is for tweak */
347 return __cast6_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
348 flags);
349}
350
351static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
352 struct scatterlist *src, unsigned int nbytes)
353{
354 struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
355 be128 buf[CAST6_PARALLEL_BLOCKS];
356 struct crypt_priv crypt_ctx = {
357 .ctx = &ctx->crypt_ctx,
358 .fpu_enabled = false,
359 };
360 struct xts_crypt_req req = {
361 .tbuf = buf,
362 .tbuflen = sizeof(buf),
363
364 .tweak_ctx = &ctx->tweak_ctx,
365 .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt),
366 .crypt_ctx = &crypt_ctx,
367 .crypt_fn = encrypt_callback,
368 };
369 int ret;
370
371 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
372 ret = xts_crypt(desc, dst, src, nbytes, &req);
373 cast6_fpu_end(crypt_ctx.fpu_enabled);
374
375 return ret;
376}
377
378static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
379 struct scatterlist *src, unsigned int nbytes)
380{
381 struct cast6_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
382 be128 buf[CAST6_PARALLEL_BLOCKS];
383 struct crypt_priv crypt_ctx = {
384 .ctx = &ctx->crypt_ctx,
385 .fpu_enabled = false,
386 };
387 struct xts_crypt_req req = {
388 .tbuf = buf,
389 .tbuflen = sizeof(buf),
390
391 .tweak_ctx = &ctx->tweak_ctx,
392 .tweak_fn = XTS_TWEAK_CAST(__cast6_encrypt),
393 .crypt_ctx = &crypt_ctx,
394 .crypt_fn = decrypt_callback,
395 };
396 int ret;
397
398 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
399 ret = xts_crypt(desc, dst, src, nbytes, &req);
400 cast6_fpu_end(crypt_ctx.fpu_enabled);
401
402 return ret;
403}
404
405static struct crypto_alg cast6_algs[10] = { {
406 .cra_name = "__ecb-cast6-avx",
407 .cra_driver_name = "__driver-ecb-cast6-avx",
408 .cra_priority = 0,
409 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
410 .cra_blocksize = CAST6_BLOCK_SIZE,
411 .cra_ctxsize = sizeof(struct cast6_ctx),
412 .cra_alignmask = 0,
413 .cra_type = &crypto_blkcipher_type,
414 .cra_module = THIS_MODULE,
415 .cra_u = {
416 .blkcipher = {
417 .min_keysize = CAST6_MIN_KEY_SIZE,
418 .max_keysize = CAST6_MAX_KEY_SIZE,
419 .setkey = cast6_setkey,
420 .encrypt = ecb_encrypt,
421 .decrypt = ecb_decrypt,
422 },
423 },
424}, {
425 .cra_name = "__cbc-cast6-avx",
426 .cra_driver_name = "__driver-cbc-cast6-avx",
427 .cra_priority = 0,
428 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
429 .cra_blocksize = CAST6_BLOCK_SIZE,
430 .cra_ctxsize = sizeof(struct cast6_ctx),
431 .cra_alignmask = 0,
432 .cra_type = &crypto_blkcipher_type,
433 .cra_module = THIS_MODULE,
434 .cra_u = {
435 .blkcipher = {
436 .min_keysize = CAST6_MIN_KEY_SIZE,
437 .max_keysize = CAST6_MAX_KEY_SIZE,
438 .setkey = cast6_setkey,
439 .encrypt = cbc_encrypt,
440 .decrypt = cbc_decrypt,
441 },
442 },
443}, {
444 .cra_name = "__ctr-cast6-avx",
445 .cra_driver_name = "__driver-ctr-cast6-avx",
446 .cra_priority = 0,
447 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
448 .cra_blocksize = 1,
449 .cra_ctxsize = sizeof(struct cast6_ctx),
450 .cra_alignmask = 0,
451 .cra_type = &crypto_blkcipher_type,
452 .cra_module = THIS_MODULE,
453 .cra_u = {
454 .blkcipher = {
455 .min_keysize = CAST6_MIN_KEY_SIZE,
456 .max_keysize = CAST6_MAX_KEY_SIZE,
457 .ivsize = CAST6_BLOCK_SIZE,
458 .setkey = cast6_setkey,
459 .encrypt = ctr_crypt,
460 .decrypt = ctr_crypt,
461 },
462 },
463}, {
464 .cra_name = "__lrw-cast6-avx",
465 .cra_driver_name = "__driver-lrw-cast6-avx",
466 .cra_priority = 0,
467 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
468 .cra_blocksize = CAST6_BLOCK_SIZE,
469 .cra_ctxsize = sizeof(struct cast6_lrw_ctx),
470 .cra_alignmask = 0,
471 .cra_type = &crypto_blkcipher_type,
472 .cra_module = THIS_MODULE,
473 .cra_exit = lrw_exit_tfm,
474 .cra_u = {
475 .blkcipher = {
476 .min_keysize = CAST6_MIN_KEY_SIZE +
477 CAST6_BLOCK_SIZE,
478 .max_keysize = CAST6_MAX_KEY_SIZE +
479 CAST6_BLOCK_SIZE,
480 .ivsize = CAST6_BLOCK_SIZE,
481 .setkey = lrw_cast6_setkey,
482 .encrypt = lrw_encrypt,
483 .decrypt = lrw_decrypt,
484 },
485 },
486}, {
487 .cra_name = "__xts-cast6-avx",
488 .cra_driver_name = "__driver-xts-cast6-avx",
489 .cra_priority = 0,
490 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
491 .cra_blocksize = CAST6_BLOCK_SIZE,
492 .cra_ctxsize = sizeof(struct cast6_xts_ctx),
493 .cra_alignmask = 0,
494 .cra_type = &crypto_blkcipher_type,
495 .cra_module = THIS_MODULE,
496 .cra_u = {
497 .blkcipher = {
498 .min_keysize = CAST6_MIN_KEY_SIZE * 2,
499 .max_keysize = CAST6_MAX_KEY_SIZE * 2,
500 .ivsize = CAST6_BLOCK_SIZE,
501 .setkey = xts_cast6_setkey,
502 .encrypt = xts_encrypt,
503 .decrypt = xts_decrypt,
504 },
505 },
506}, {
507 .cra_name = "ecb(cast6)",
508 .cra_driver_name = "ecb-cast6-avx",
509 .cra_priority = 200,
510 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
511 .cra_blocksize = CAST6_BLOCK_SIZE,
512 .cra_ctxsize = sizeof(struct async_helper_ctx),
513 .cra_alignmask = 0,
514 .cra_type = &crypto_ablkcipher_type,
515 .cra_module = THIS_MODULE,
516 .cra_init = ablk_init,
517 .cra_exit = ablk_exit,
518 .cra_u = {
519 .ablkcipher = {
520 .min_keysize = CAST6_MIN_KEY_SIZE,
521 .max_keysize = CAST6_MAX_KEY_SIZE,
522 .setkey = ablk_set_key,
523 .encrypt = ablk_encrypt,
524 .decrypt = ablk_decrypt,
525 },
526 },
527}, {
528 .cra_name = "cbc(cast6)",
529 .cra_driver_name = "cbc-cast6-avx",
530 .cra_priority = 200,
531 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
532 .cra_blocksize = CAST6_BLOCK_SIZE,
533 .cra_ctxsize = sizeof(struct async_helper_ctx),
534 .cra_alignmask = 0,
535 .cra_type = &crypto_ablkcipher_type,
536 .cra_module = THIS_MODULE,
537 .cra_init = ablk_init,
538 .cra_exit = ablk_exit,
539 .cra_u = {
540 .ablkcipher = {
541 .min_keysize = CAST6_MIN_KEY_SIZE,
542 .max_keysize = CAST6_MAX_KEY_SIZE,
543 .ivsize = CAST6_BLOCK_SIZE,
544 .setkey = ablk_set_key,
545 .encrypt = __ablk_encrypt,
546 .decrypt = ablk_decrypt,
547 },
548 },
549}, {
550 .cra_name = "ctr(cast6)",
551 .cra_driver_name = "ctr-cast6-avx",
552 .cra_priority = 200,
553 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
554 .cra_blocksize = 1,
555 .cra_ctxsize = sizeof(struct async_helper_ctx),
556 .cra_alignmask = 0,
557 .cra_type = &crypto_ablkcipher_type,
558 .cra_module = THIS_MODULE,
559 .cra_init = ablk_init,
560 .cra_exit = ablk_exit,
561 .cra_u = {
562 .ablkcipher = {
563 .min_keysize = CAST6_MIN_KEY_SIZE,
564 .max_keysize = CAST6_MAX_KEY_SIZE,
565 .ivsize = CAST6_BLOCK_SIZE,
566 .setkey = ablk_set_key,
567 .encrypt = ablk_encrypt,
568 .decrypt = ablk_encrypt,
569 .geniv = "chainiv",
570 },
571 },
572}, {
573 .cra_name = "lrw(cast6)",
574 .cra_driver_name = "lrw-cast6-avx",
575 .cra_priority = 200,
576 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
577 .cra_blocksize = CAST6_BLOCK_SIZE,
578 .cra_ctxsize = sizeof(struct async_helper_ctx),
579 .cra_alignmask = 0,
580 .cra_type = &crypto_ablkcipher_type,
581 .cra_module = THIS_MODULE,
582 .cra_init = ablk_init,
583 .cra_exit = ablk_exit,
584 .cra_u = {
585 .ablkcipher = {
586 .min_keysize = CAST6_MIN_KEY_SIZE +
587 CAST6_BLOCK_SIZE,
588 .max_keysize = CAST6_MAX_KEY_SIZE +
589 CAST6_BLOCK_SIZE,
590 .ivsize = CAST6_BLOCK_SIZE,
591 .setkey = ablk_set_key,
592 .encrypt = ablk_encrypt,
593 .decrypt = ablk_decrypt,
594 },
595 },
596}, {
597 .cra_name = "xts(cast6)",
598 .cra_driver_name = "xts-cast6-avx",
599 .cra_priority = 200,
600 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
601 .cra_blocksize = CAST6_BLOCK_SIZE,
602 .cra_ctxsize = sizeof(struct async_helper_ctx),
603 .cra_alignmask = 0,
604 .cra_type = &crypto_ablkcipher_type,
605 .cra_module = THIS_MODULE,
606 .cra_init = ablk_init,
607 .cra_exit = ablk_exit,
608 .cra_u = {
609 .ablkcipher = {
610 .min_keysize = CAST6_MIN_KEY_SIZE * 2,
611 .max_keysize = CAST6_MAX_KEY_SIZE * 2,
612 .ivsize = CAST6_BLOCK_SIZE,
613 .setkey = ablk_set_key,
614 .encrypt = ablk_encrypt,
615 .decrypt = ablk_decrypt,
616 },
617 },
618} };
619
620static int __init cast6_init(void)
621{
622 u64 xcr0;
623
624 if (!cpu_has_avx || !cpu_has_osxsave) {
625 pr_info("AVX instructions are not detected.\n");
626 return -ENODEV;
627 }
628
629 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
630 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
631 pr_info("AVX detected but unusable.\n");
632 return -ENODEV;
633 }
634
635 return crypto_register_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
636}
637
638static void __exit cast6_exit(void)
639{
640 crypto_unregister_algs(cast6_algs, ARRAY_SIZE(cast6_algs));
641}
642
643module_init(cast6_init);
644module_exit(cast6_exit);
645
646MODULE_DESCRIPTION("Cast6 Cipher Algorithm, AVX optimized");
647MODULE_LICENSE("GPL");
648MODULE_ALIAS("cast6");
diff --git a/arch/x86/crypto/ghash-clmulni-intel_glue.c b/arch/x86/crypto/ghash-clmulni-intel_glue.c
index b4bf0a63b520..6759dd1135be 100644
--- a/arch/x86/crypto/ghash-clmulni-intel_glue.c
+++ b/arch/x86/crypto/ghash-clmulni-intel_glue.c
@@ -150,7 +150,6 @@ static struct shash_alg ghash_alg = {
150 .cra_blocksize = GHASH_BLOCK_SIZE, 150 .cra_blocksize = GHASH_BLOCK_SIZE,
151 .cra_ctxsize = sizeof(struct ghash_ctx), 151 .cra_ctxsize = sizeof(struct ghash_ctx),
152 .cra_module = THIS_MODULE, 152 .cra_module = THIS_MODULE,
153 .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list),
154 }, 153 },
155}; 154};
156 155
@@ -288,7 +287,6 @@ static struct ahash_alg ghash_async_alg = {
288 .cra_blocksize = GHASH_BLOCK_SIZE, 287 .cra_blocksize = GHASH_BLOCK_SIZE,
289 .cra_type = &crypto_ahash_type, 288 .cra_type = &crypto_ahash_type,
290 .cra_module = THIS_MODULE, 289 .cra_module = THIS_MODULE,
291 .cra_list = LIST_HEAD_INIT(ghash_async_alg.halg.base.cra_list),
292 .cra_init = ghash_async_init_tfm, 290 .cra_init = ghash_async_init_tfm,
293 .cra_exit = ghash_async_exit_tfm, 291 .cra_exit = ghash_async_exit_tfm,
294 }, 292 },
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 4854f0f31e4f..30b3927bd733 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -110,7 +110,7 @@ static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
110 nbytes -= bsize; 110 nbytes -= bsize;
111 } while (nbytes >= bsize); 111 } while (nbytes >= bsize);
112 112
113 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); 113 *(u128 *)walk->iv = *iv;
114 return nbytes; 114 return nbytes;
115} 115}
116 116
diff --git a/arch/x86/crypto/salsa20_glue.c b/arch/x86/crypto/salsa20_glue.c
index bccb76d80987..a3a3c0205c16 100644
--- a/arch/x86/crypto/salsa20_glue.c
+++ b/arch/x86/crypto/salsa20_glue.c
@@ -97,7 +97,6 @@ static struct crypto_alg alg = {
97 .cra_ctxsize = sizeof(struct salsa20_ctx), 97 .cra_ctxsize = sizeof(struct salsa20_ctx),
98 .cra_alignmask = 3, 98 .cra_alignmask = 3,
99 .cra_module = THIS_MODULE, 99 .cra_module = THIS_MODULE,
100 .cra_list = LIST_HEAD_INIT(alg.cra_list),
101 .cra_u = { 100 .cra_u = {
102 .blkcipher = { 101 .blkcipher = {
103 .setkey = setkey, 102 .setkey = setkey,
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
index b36bdac237eb..3f543a04cf1e 100644
--- a/arch/x86/crypto/serpent_avx_glue.c
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -390,7 +390,6 @@ static struct crypto_alg serpent_algs[10] = { {
390 .cra_alignmask = 0, 390 .cra_alignmask = 0,
391 .cra_type = &crypto_blkcipher_type, 391 .cra_type = &crypto_blkcipher_type,
392 .cra_module = THIS_MODULE, 392 .cra_module = THIS_MODULE,
393 .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list),
394 .cra_u = { 393 .cra_u = {
395 .blkcipher = { 394 .blkcipher = {
396 .min_keysize = SERPENT_MIN_KEY_SIZE, 395 .min_keysize = SERPENT_MIN_KEY_SIZE,
@@ -410,7 +409,6 @@ static struct crypto_alg serpent_algs[10] = { {
410 .cra_alignmask = 0, 409 .cra_alignmask = 0,
411 .cra_type = &crypto_blkcipher_type, 410 .cra_type = &crypto_blkcipher_type,
412 .cra_module = THIS_MODULE, 411 .cra_module = THIS_MODULE,
413 .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list),
414 .cra_u = { 412 .cra_u = {
415 .blkcipher = { 413 .blkcipher = {
416 .min_keysize = SERPENT_MIN_KEY_SIZE, 414 .min_keysize = SERPENT_MIN_KEY_SIZE,
@@ -430,7 +428,6 @@ static struct crypto_alg serpent_algs[10] = { {
430 .cra_alignmask = 0, 428 .cra_alignmask = 0,
431 .cra_type = &crypto_blkcipher_type, 429 .cra_type = &crypto_blkcipher_type,
432 .cra_module = THIS_MODULE, 430 .cra_module = THIS_MODULE,
433 .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list),
434 .cra_u = { 431 .cra_u = {
435 .blkcipher = { 432 .blkcipher = {
436 .min_keysize = SERPENT_MIN_KEY_SIZE, 433 .min_keysize = SERPENT_MIN_KEY_SIZE,
@@ -451,7 +448,6 @@ static struct crypto_alg serpent_algs[10] = { {
451 .cra_alignmask = 0, 448 .cra_alignmask = 0,
452 .cra_type = &crypto_blkcipher_type, 449 .cra_type = &crypto_blkcipher_type,
453 .cra_module = THIS_MODULE, 450 .cra_module = THIS_MODULE,
454 .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list),
455 .cra_exit = lrw_exit_tfm, 451 .cra_exit = lrw_exit_tfm,
456 .cra_u = { 452 .cra_u = {
457 .blkcipher = { 453 .blkcipher = {
@@ -475,7 +471,6 @@ static struct crypto_alg serpent_algs[10] = { {
475 .cra_alignmask = 0, 471 .cra_alignmask = 0,
476 .cra_type = &crypto_blkcipher_type, 472 .cra_type = &crypto_blkcipher_type,
477 .cra_module = THIS_MODULE, 473 .cra_module = THIS_MODULE,
478 .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list),
479 .cra_u = { 474 .cra_u = {
480 .blkcipher = { 475 .blkcipher = {
481 .min_keysize = SERPENT_MIN_KEY_SIZE * 2, 476 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
@@ -496,7 +491,6 @@ static struct crypto_alg serpent_algs[10] = { {
496 .cra_alignmask = 0, 491 .cra_alignmask = 0,
497 .cra_type = &crypto_ablkcipher_type, 492 .cra_type = &crypto_ablkcipher_type,
498 .cra_module = THIS_MODULE, 493 .cra_module = THIS_MODULE,
499 .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list),
500 .cra_init = ablk_init, 494 .cra_init = ablk_init,
501 .cra_exit = ablk_exit, 495 .cra_exit = ablk_exit,
502 .cra_u = { 496 .cra_u = {
@@ -518,7 +512,6 @@ static struct crypto_alg serpent_algs[10] = { {
518 .cra_alignmask = 0, 512 .cra_alignmask = 0,
519 .cra_type = &crypto_ablkcipher_type, 513 .cra_type = &crypto_ablkcipher_type,
520 .cra_module = THIS_MODULE, 514 .cra_module = THIS_MODULE,
521 .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list),
522 .cra_init = ablk_init, 515 .cra_init = ablk_init,
523 .cra_exit = ablk_exit, 516 .cra_exit = ablk_exit,
524 .cra_u = { 517 .cra_u = {
@@ -541,7 +534,6 @@ static struct crypto_alg serpent_algs[10] = { {
541 .cra_alignmask = 0, 534 .cra_alignmask = 0,
542 .cra_type = &crypto_ablkcipher_type, 535 .cra_type = &crypto_ablkcipher_type,
543 .cra_module = THIS_MODULE, 536 .cra_module = THIS_MODULE,
544 .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list),
545 .cra_init = ablk_init, 537 .cra_init = ablk_init,
546 .cra_exit = ablk_exit, 538 .cra_exit = ablk_exit,
547 .cra_u = { 539 .cra_u = {
@@ -565,7 +557,6 @@ static struct crypto_alg serpent_algs[10] = { {
565 .cra_alignmask = 0, 557 .cra_alignmask = 0,
566 .cra_type = &crypto_ablkcipher_type, 558 .cra_type = &crypto_ablkcipher_type,
567 .cra_module = THIS_MODULE, 559 .cra_module = THIS_MODULE,
568 .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list),
569 .cra_init = ablk_init, 560 .cra_init = ablk_init,
570 .cra_exit = ablk_exit, 561 .cra_exit = ablk_exit,
571 .cra_u = { 562 .cra_u = {
@@ -590,7 +581,6 @@ static struct crypto_alg serpent_algs[10] = { {
590 .cra_alignmask = 0, 581 .cra_alignmask = 0,
591 .cra_type = &crypto_ablkcipher_type, 582 .cra_type = &crypto_ablkcipher_type,
592 .cra_module = THIS_MODULE, 583 .cra_module = THIS_MODULE,
593 .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list),
594 .cra_init = ablk_init, 584 .cra_init = ablk_init,
595 .cra_exit = ablk_exit, 585 .cra_exit = ablk_exit,
596 .cra_u = { 586 .cra_u = {
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index d679c8675f4a..9107a9908c41 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -393,7 +393,6 @@ static struct crypto_alg serpent_algs[10] = { {
393 .cra_alignmask = 0, 393 .cra_alignmask = 0,
394 .cra_type = &crypto_blkcipher_type, 394 .cra_type = &crypto_blkcipher_type,
395 .cra_module = THIS_MODULE, 395 .cra_module = THIS_MODULE,
396 .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list),
397 .cra_u = { 396 .cra_u = {
398 .blkcipher = { 397 .blkcipher = {
399 .min_keysize = SERPENT_MIN_KEY_SIZE, 398 .min_keysize = SERPENT_MIN_KEY_SIZE,
@@ -413,7 +412,6 @@ static struct crypto_alg serpent_algs[10] = { {
413 .cra_alignmask = 0, 412 .cra_alignmask = 0,
414 .cra_type = &crypto_blkcipher_type, 413 .cra_type = &crypto_blkcipher_type,
415 .cra_module = THIS_MODULE, 414 .cra_module = THIS_MODULE,
416 .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list),
417 .cra_u = { 415 .cra_u = {
418 .blkcipher = { 416 .blkcipher = {
419 .min_keysize = SERPENT_MIN_KEY_SIZE, 417 .min_keysize = SERPENT_MIN_KEY_SIZE,
@@ -433,7 +431,6 @@ static struct crypto_alg serpent_algs[10] = { {
433 .cra_alignmask = 0, 431 .cra_alignmask = 0,
434 .cra_type = &crypto_blkcipher_type, 432 .cra_type = &crypto_blkcipher_type,
435 .cra_module = THIS_MODULE, 433 .cra_module = THIS_MODULE,
436 .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list),
437 .cra_u = { 434 .cra_u = {
438 .blkcipher = { 435 .blkcipher = {
439 .min_keysize = SERPENT_MIN_KEY_SIZE, 436 .min_keysize = SERPENT_MIN_KEY_SIZE,
@@ -454,7 +451,6 @@ static struct crypto_alg serpent_algs[10] = { {
454 .cra_alignmask = 0, 451 .cra_alignmask = 0,
455 .cra_type = &crypto_blkcipher_type, 452 .cra_type = &crypto_blkcipher_type,
456 .cra_module = THIS_MODULE, 453 .cra_module = THIS_MODULE,
457 .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list),
458 .cra_exit = lrw_exit_tfm, 454 .cra_exit = lrw_exit_tfm,
459 .cra_u = { 455 .cra_u = {
460 .blkcipher = { 456 .blkcipher = {
@@ -478,7 +474,6 @@ static struct crypto_alg serpent_algs[10] = { {
478 .cra_alignmask = 0, 474 .cra_alignmask = 0,
479 .cra_type = &crypto_blkcipher_type, 475 .cra_type = &crypto_blkcipher_type,
480 .cra_module = THIS_MODULE, 476 .cra_module = THIS_MODULE,
481 .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list),
482 .cra_u = { 477 .cra_u = {
483 .blkcipher = { 478 .blkcipher = {
484 .min_keysize = SERPENT_MIN_KEY_SIZE * 2, 479 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
@@ -499,7 +494,6 @@ static struct crypto_alg serpent_algs[10] = { {
499 .cra_alignmask = 0, 494 .cra_alignmask = 0,
500 .cra_type = &crypto_ablkcipher_type, 495 .cra_type = &crypto_ablkcipher_type,
501 .cra_module = THIS_MODULE, 496 .cra_module = THIS_MODULE,
502 .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list),
503 .cra_init = ablk_init, 497 .cra_init = ablk_init,
504 .cra_exit = ablk_exit, 498 .cra_exit = ablk_exit,
505 .cra_u = { 499 .cra_u = {
@@ -521,7 +515,6 @@ static struct crypto_alg serpent_algs[10] = { {
521 .cra_alignmask = 0, 515 .cra_alignmask = 0,
522 .cra_type = &crypto_ablkcipher_type, 516 .cra_type = &crypto_ablkcipher_type,
523 .cra_module = THIS_MODULE, 517 .cra_module = THIS_MODULE,
524 .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list),
525 .cra_init = ablk_init, 518 .cra_init = ablk_init,
526 .cra_exit = ablk_exit, 519 .cra_exit = ablk_exit,
527 .cra_u = { 520 .cra_u = {
@@ -544,7 +537,6 @@ static struct crypto_alg serpent_algs[10] = { {
544 .cra_alignmask = 0, 537 .cra_alignmask = 0,
545 .cra_type = &crypto_ablkcipher_type, 538 .cra_type = &crypto_ablkcipher_type,
546 .cra_module = THIS_MODULE, 539 .cra_module = THIS_MODULE,
547 .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list),
548 .cra_init = ablk_init, 540 .cra_init = ablk_init,
549 .cra_exit = ablk_exit, 541 .cra_exit = ablk_exit,
550 .cra_u = { 542 .cra_u = {
@@ -568,7 +560,6 @@ static struct crypto_alg serpent_algs[10] = { {
568 .cra_alignmask = 0, 560 .cra_alignmask = 0,
569 .cra_type = &crypto_ablkcipher_type, 561 .cra_type = &crypto_ablkcipher_type,
570 .cra_module = THIS_MODULE, 562 .cra_module = THIS_MODULE,
571 .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list),
572 .cra_init = ablk_init, 563 .cra_init = ablk_init,
573 .cra_exit = ablk_exit, 564 .cra_exit = ablk_exit,
574 .cra_u = { 565 .cra_u = {
@@ -593,7 +584,6 @@ static struct crypto_alg serpent_algs[10] = { {
593 .cra_alignmask = 0, 584 .cra_alignmask = 0,
594 .cra_type = &crypto_ablkcipher_type, 585 .cra_type = &crypto_ablkcipher_type,
595 .cra_module = THIS_MODULE, 586 .cra_module = THIS_MODULE,
596 .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list),
597 .cra_init = ablk_init, 587 .cra_init = ablk_init,
598 .cra_exit = ablk_exit, 588 .cra_exit = ablk_exit,
599 .cra_u = { 589 .cra_u = {
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
index 35f45574390d..1585abb13dde 100644
--- a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -4,6 +4,8 @@
4 * Copyright (C) 2012 Johannes Goetzfried 4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> 5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 * 6 *
7 * Copyright © 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
8 *
7 * This program is free software; you can redistribute it and/or modify 9 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by 10 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or 11 * the Free Software Foundation; either version 2 of the License, or
@@ -47,16 +49,22 @@
47#define RC2 %xmm6 49#define RC2 %xmm6
48#define RD2 %xmm7 50#define RD2 %xmm7
49 51
50#define RX %xmm8 52#define RX0 %xmm8
51#define RY %xmm9 53#define RY0 %xmm9
54
55#define RX1 %xmm10
56#define RY1 %xmm11
52 57
53#define RK1 %xmm10 58#define RK1 %xmm12
54#define RK2 %xmm11 59#define RK2 %xmm13
55 60
56#define RID1 %rax 61#define RT %xmm14
57#define RID1b %al 62#define RR %xmm15
58#define RID2 %rbx 63
59#define RID2b %bl 64#define RID1 %rbp
65#define RID1d %ebp
66#define RID2 %rsi
67#define RID2d %esi
60 68
61#define RGI1 %rdx 69#define RGI1 %rdx
62#define RGI1bl %dl 70#define RGI1bl %dl
@@ -65,6 +73,13 @@
65#define RGI2bl %cl 73#define RGI2bl %cl
66#define RGI2bh %ch 74#define RGI2bh %ch
67 75
76#define RGI3 %rax
77#define RGI3bl %al
78#define RGI3bh %ah
79#define RGI4 %rbx
80#define RGI4bl %bl
81#define RGI4bh %bh
82
68#define RGS1 %r8 83#define RGS1 %r8
69#define RGS1d %r8d 84#define RGS1d %r8d
70#define RGS2 %r9 85#define RGS2 %r9
@@ -73,89 +88,123 @@
73#define RGS3d %r10d 88#define RGS3d %r10d
74 89
75 90
76#define lookup_32bit(t0, t1, t2, t3, src, dst) \ 91#define lookup_32bit(t0, t1, t2, t3, src, dst, interleave_op, il_reg) \
77 movb src ## bl, RID1b; \ 92 movzbl src ## bl, RID1d; \
78 movb src ## bh, RID2b; \ 93 movzbl src ## bh, RID2d; \
79 movl t0(CTX, RID1, 4), dst ## d; \
80 xorl t1(CTX, RID2, 4), dst ## d; \
81 shrq $16, src; \ 94 shrq $16, src; \
82 movb src ## bl, RID1b; \ 95 movl t0(CTX, RID1, 4), dst ## d; \
83 movb src ## bh, RID2b; \ 96 movl t1(CTX, RID2, 4), RID2d; \
97 movzbl src ## bl, RID1d; \
98 xorl RID2d, dst ## d; \
99 movzbl src ## bh, RID2d; \
100 interleave_op(il_reg); \
84 xorl t2(CTX, RID1, 4), dst ## d; \ 101 xorl t2(CTX, RID1, 4), dst ## d; \
85 xorl t3(CTX, RID2, 4), dst ## d; 102 xorl t3(CTX, RID2, 4), dst ## d;
86 103
87#define G(a, x, t0, t1, t2, t3) \ 104#define dummy(d) /* do nothing */
88 vmovq a, RGI1; \ 105
89 vpsrldq $8, a, x; \ 106#define shr_next(reg) \
90 vmovq x, RGI2; \ 107 shrq $16, reg;
108
109#define G(gi1, gi2, x, t0, t1, t2, t3) \
110 lookup_32bit(t0, t1, t2, t3, ##gi1, RGS1, shr_next, ##gi1); \
111 lookup_32bit(t0, t1, t2, t3, ##gi2, RGS3, shr_next, ##gi2); \
112 \
113 lookup_32bit(t0, t1, t2, t3, ##gi1, RGS2, dummy, none); \
114 shlq $32, RGS2; \
115 orq RGS1, RGS2; \
116 lookup_32bit(t0, t1, t2, t3, ##gi2, RGS1, dummy, none); \
117 shlq $32, RGS1; \
118 orq RGS1, RGS3;
119
120#define round_head_2(a, b, x1, y1, x2, y2) \
121 vmovq b ## 1, RGI3; \
122 vpextrq $1, b ## 1, RGI4; \
91 \ 123 \
92 lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \ 124 G(RGI1, RGI2, x1, s0, s1, s2, s3); \
93 shrq $16, RGI1; \ 125 vmovq a ## 2, RGI1; \
94 lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \ 126 vpextrq $1, a ## 2, RGI2; \
95 shlq $32, RGS2; \ 127 vmovq RGS2, x1; \
96 orq RGS1, RGS2; \ 128 vpinsrq $1, RGS3, x1, x1; \
97 \ 129 \
98 lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \ 130 G(RGI3, RGI4, y1, s1, s2, s3, s0); \
99 shrq $16, RGI2; \ 131 vmovq b ## 2, RGI3; \
100 lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \ 132 vpextrq $1, b ## 2, RGI4; \
101 shlq $32, RGS3; \ 133 vmovq RGS2, y1; \
102 orq RGS1, RGS3; \ 134 vpinsrq $1, RGS3, y1, y1; \
103 \ 135 \
104 vmovq RGS2, x; \ 136 G(RGI1, RGI2, x2, s0, s1, s2, s3); \
105 vpinsrq $1, RGS3, x, x; 137 vmovq RGS2, x2; \
138 vpinsrq $1, RGS3, x2, x2; \
139 \
140 G(RGI3, RGI4, y2, s1, s2, s3, s0); \
141 vmovq RGS2, y2; \
142 vpinsrq $1, RGS3, y2, y2;
106 143
107#define encround(a, b, c, d, x, y) \ 144#define encround_tail(a, b, c, d, x, y, prerotate) \
108 G(a, x, s0, s1, s2, s3); \
109 G(b, y, s1, s2, s3, s0); \
110 vpaddd x, y, x; \ 145 vpaddd x, y, x; \
146 vpaddd x, RK1, RT;\
147 prerotate(b); \
148 vpxor RT, c, c; \
111 vpaddd y, x, y; \ 149 vpaddd y, x, y; \
112 vpaddd x, RK1, x; \
113 vpaddd y, RK2, y; \ 150 vpaddd y, RK2, y; \
114 vpxor x, c, c; \ 151 vpsrld $1, c, RT; \
115 vpsrld $1, c, x; \
116 vpslld $(32 - 1), c, c; \ 152 vpslld $(32 - 1), c, c; \
117 vpor c, x, c; \ 153 vpor c, RT, c; \
118 vpslld $1, d, x; \ 154 vpxor d, y, d; \
119 vpsrld $(32 - 1), d, d; \ 155
120 vpor d, x, d; \ 156#define decround_tail(a, b, c, d, x, y, prerotate) \
121 vpxor d, y, d;
122
123#define decround(a, b, c, d, x, y) \
124 G(a, x, s0, s1, s2, s3); \
125 G(b, y, s1, s2, s3, s0); \
126 vpaddd x, y, x; \ 157 vpaddd x, y, x; \
158 vpaddd x, RK1, RT;\
159 prerotate(a); \
160 vpxor RT, c, c; \
127 vpaddd y, x, y; \ 161 vpaddd y, x, y; \
128 vpaddd y, RK2, y; \ 162 vpaddd y, RK2, y; \
129 vpxor d, y, d; \ 163 vpxor d, y, d; \
130 vpsrld $1, d, y; \ 164 vpsrld $1, d, y; \
131 vpslld $(32 - 1), d, d; \ 165 vpslld $(32 - 1), d, d; \
132 vpor d, y, d; \ 166 vpor d, y, d; \
133 vpslld $1, c, y; \ 167
134 vpsrld $(32 - 1), c, c; \ 168#define rotate_1l(x) \
135 vpor c, y, c; \ 169 vpslld $1, x, RR; \
136 vpaddd x, RK1, x; \ 170 vpsrld $(32 - 1), x, x; \
137 vpxor x, c, c; 171 vpor x, RR, x;
138 172
139#define encrypt_round(n, a, b, c, d) \ 173#define preload_rgi(c) \
140 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ 174 vmovq c, RGI1; \
141 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ 175 vpextrq $1, c, RGI2;
142 encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ 176
143 encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); 177#define encrypt_round(n, a, b, c, d, preload, prerotate) \
144 178 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \
145#define decrypt_round(n, a, b, c, d) \ 179 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \
146 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ 180 round_head_2(a, b, RX0, RY0, RX1, RY1); \
147 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ 181 encround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
148 decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ 182 preload(c ## 1); \
149 decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); 183 encround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
184
185#define decrypt_round(n, a, b, c, d, preload, prerotate) \
186 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \
187 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \
188 round_head_2(a, b, RX0, RY0, RX1, RY1); \
189 decround_tail(a ## 1, b ## 1, c ## 1, d ## 1, RX0, RY0, prerotate); \
190 preload(c ## 1); \
191 decround_tail(a ## 2, b ## 2, c ## 2, d ## 2, RX1, RY1, prerotate);
150 192
151#define encrypt_cycle(n) \ 193#define encrypt_cycle(n) \
152 encrypt_round((2*n), RA, RB, RC, RD); \ 194 encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
153 encrypt_round(((2*n) + 1), RC, RD, RA, RB); 195 encrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l);
196
197#define encrypt_cycle_last(n) \
198 encrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l); \
199 encrypt_round(((2*n) + 1), RC, RD, RA, RB, dummy, dummy);
154 200
155#define decrypt_cycle(n) \ 201#define decrypt_cycle(n) \
156 decrypt_round(((2*n) + 1), RC, RD, RA, RB); \ 202 decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
157 decrypt_round((2*n), RA, RB, RC, RD); 203 decrypt_round((2*n), RA, RB, RC, RD, preload_rgi, rotate_1l);
158 204
205#define decrypt_cycle_last(n) \
206 decrypt_round(((2*n) + 1), RC, RD, RA, RB, preload_rgi, rotate_1l); \
207 decrypt_round((2*n), RA, RB, RC, RD, dummy, dummy);
159 208
160#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ 209#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
161 vpunpckldq x1, x0, t0; \ 210 vpunpckldq x1, x0, t0; \
@@ -216,17 +265,20 @@ __twofish_enc_blk_8way:
216 * %rcx: bool, if true: xor output 265 * %rcx: bool, if true: xor output
217 */ 266 */
218 267
268 pushq %rbp;
219 pushq %rbx; 269 pushq %rbx;
220 pushq %rcx; 270 pushq %rcx;
221 271
222 vmovdqu w(CTX), RK1; 272 vmovdqu w(CTX), RK1;
223 273
224 leaq (4*4*4)(%rdx), %rax; 274 leaq (4*4*4)(%rdx), %rax;
225 inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); 275 inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
226 inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); 276 preload_rgi(RA1);
277 rotate_1l(RD1);
278 inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
279 rotate_1l(RD2);
227 280
228 xorq RID1, RID1; 281 movq %rsi, %r11;
229 xorq RID2, RID2;
230 282
231 encrypt_cycle(0); 283 encrypt_cycle(0);
232 encrypt_cycle(1); 284 encrypt_cycle(1);
@@ -235,26 +287,27 @@ __twofish_enc_blk_8way:
235 encrypt_cycle(4); 287 encrypt_cycle(4);
236 encrypt_cycle(5); 288 encrypt_cycle(5);
237 encrypt_cycle(6); 289 encrypt_cycle(6);
238 encrypt_cycle(7); 290 encrypt_cycle_last(7);
239 291
240 vmovdqu (w+4*4)(CTX), RK1; 292 vmovdqu (w+4*4)(CTX), RK1;
241 293
242 popq %rcx; 294 popq %rcx;
243 popq %rbx; 295 popq %rbx;
296 popq %rbp;
244 297
245 leaq (4*4*4)(%rsi), %rax; 298 leaq (4*4*4)(%r11), %rax;
246 299
247 testb %cl, %cl; 300 testb %cl, %cl;
248 jnz __enc_xor8; 301 jnz __enc_xor8;
249 302
250 outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); 303 outunpack_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
251 outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); 304 outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
252 305
253 ret; 306 ret;
254 307
255__enc_xor8: 308__enc_xor8:
256 outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); 309 outunpack_xor_blocks(%r11, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
257 outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); 310 outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
258 311
259 ret; 312 ret;
260 313
@@ -269,16 +322,19 @@ twofish_dec_blk_8way:
269 * %rdx: src 322 * %rdx: src
270 */ 323 */
271 324
325 pushq %rbp;
272 pushq %rbx; 326 pushq %rbx;
273 327
274 vmovdqu (w+4*4)(CTX), RK1; 328 vmovdqu (w+4*4)(CTX), RK1;
275 329
276 leaq (4*4*4)(%rdx), %rax; 330 leaq (4*4*4)(%rdx), %rax;
277 inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); 331 inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX0, RY0, RK2);
278 inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); 332 preload_rgi(RC1);
333 rotate_1l(RA1);
334 inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX0, RY0, RK2);
335 rotate_1l(RA2);
279 336
280 xorq RID1, RID1; 337 movq %rsi, %r11;
281 xorq RID2, RID2;
282 338
283 decrypt_cycle(7); 339 decrypt_cycle(7);
284 decrypt_cycle(6); 340 decrypt_cycle(6);
@@ -287,14 +343,15 @@ twofish_dec_blk_8way:
287 decrypt_cycle(3); 343 decrypt_cycle(3);
288 decrypt_cycle(2); 344 decrypt_cycle(2);
289 decrypt_cycle(1); 345 decrypt_cycle(1);
290 decrypt_cycle(0); 346 decrypt_cycle_last(0);
291 347
292 vmovdqu (w)(CTX), RK1; 348 vmovdqu (w)(CTX), RK1;
293 349
294 popq %rbx; 350 popq %rbx;
351 popq %rbp;
295 352
296 leaq (4*4*4)(%rsi), %rax; 353 leaq (4*4*4)(%r11), %rax;
297 outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); 354 outunpack_blocks(%r11, RA1, RB1, RC1, RD1, RK1, RX0, RY0, RK2);
298 outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); 355 outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX0, RY0, RK2);
299 356
300 ret; 357 ret;
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
index 782b67ddaf6a..e7708b5442e0 100644
--- a/arch/x86/crypto/twofish_avx_glue.c
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -378,7 +378,6 @@ static struct crypto_alg twofish_algs[10] = { {
378 .cra_alignmask = 0, 378 .cra_alignmask = 0,
379 .cra_type = &crypto_blkcipher_type, 379 .cra_type = &crypto_blkcipher_type,
380 .cra_module = THIS_MODULE, 380 .cra_module = THIS_MODULE,
381 .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list),
382 .cra_u = { 381 .cra_u = {
383 .blkcipher = { 382 .blkcipher = {
384 .min_keysize = TF_MIN_KEY_SIZE, 383 .min_keysize = TF_MIN_KEY_SIZE,
@@ -398,7 +397,6 @@ static struct crypto_alg twofish_algs[10] = { {
398 .cra_alignmask = 0, 397 .cra_alignmask = 0,
399 .cra_type = &crypto_blkcipher_type, 398 .cra_type = &crypto_blkcipher_type,
400 .cra_module = THIS_MODULE, 399 .cra_module = THIS_MODULE,
401 .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list),
402 .cra_u = { 400 .cra_u = {
403 .blkcipher = { 401 .blkcipher = {
404 .min_keysize = TF_MIN_KEY_SIZE, 402 .min_keysize = TF_MIN_KEY_SIZE,
@@ -418,7 +416,6 @@ static struct crypto_alg twofish_algs[10] = { {
418 .cra_alignmask = 0, 416 .cra_alignmask = 0,
419 .cra_type = &crypto_blkcipher_type, 417 .cra_type = &crypto_blkcipher_type,
420 .cra_module = THIS_MODULE, 418 .cra_module = THIS_MODULE,
421 .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list),
422 .cra_u = { 419 .cra_u = {
423 .blkcipher = { 420 .blkcipher = {
424 .min_keysize = TF_MIN_KEY_SIZE, 421 .min_keysize = TF_MIN_KEY_SIZE,
@@ -439,7 +436,6 @@ static struct crypto_alg twofish_algs[10] = { {
439 .cra_alignmask = 0, 436 .cra_alignmask = 0,
440 .cra_type = &crypto_blkcipher_type, 437 .cra_type = &crypto_blkcipher_type,
441 .cra_module = THIS_MODULE, 438 .cra_module = THIS_MODULE,
442 .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list),
443 .cra_exit = lrw_twofish_exit_tfm, 439 .cra_exit = lrw_twofish_exit_tfm,
444 .cra_u = { 440 .cra_u = {
445 .blkcipher = { 441 .blkcipher = {
@@ -463,7 +459,6 @@ static struct crypto_alg twofish_algs[10] = { {
463 .cra_alignmask = 0, 459 .cra_alignmask = 0,
464 .cra_type = &crypto_blkcipher_type, 460 .cra_type = &crypto_blkcipher_type,
465 .cra_module = THIS_MODULE, 461 .cra_module = THIS_MODULE,
466 .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list),
467 .cra_u = { 462 .cra_u = {
468 .blkcipher = { 463 .blkcipher = {
469 .min_keysize = TF_MIN_KEY_SIZE * 2, 464 .min_keysize = TF_MIN_KEY_SIZE * 2,
@@ -484,7 +479,6 @@ static struct crypto_alg twofish_algs[10] = { {
484 .cra_alignmask = 0, 479 .cra_alignmask = 0,
485 .cra_type = &crypto_ablkcipher_type, 480 .cra_type = &crypto_ablkcipher_type,
486 .cra_module = THIS_MODULE, 481 .cra_module = THIS_MODULE,
487 .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list),
488 .cra_init = ablk_init, 482 .cra_init = ablk_init,
489 .cra_exit = ablk_exit, 483 .cra_exit = ablk_exit,
490 .cra_u = { 484 .cra_u = {
@@ -506,7 +500,6 @@ static struct crypto_alg twofish_algs[10] = { {
506 .cra_alignmask = 0, 500 .cra_alignmask = 0,
507 .cra_type = &crypto_ablkcipher_type, 501 .cra_type = &crypto_ablkcipher_type,
508 .cra_module = THIS_MODULE, 502 .cra_module = THIS_MODULE,
509 .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list),
510 .cra_init = ablk_init, 503 .cra_init = ablk_init,
511 .cra_exit = ablk_exit, 504 .cra_exit = ablk_exit,
512 .cra_u = { 505 .cra_u = {
@@ -529,7 +522,6 @@ static struct crypto_alg twofish_algs[10] = { {
529 .cra_alignmask = 0, 522 .cra_alignmask = 0,
530 .cra_type = &crypto_ablkcipher_type, 523 .cra_type = &crypto_ablkcipher_type,
531 .cra_module = THIS_MODULE, 524 .cra_module = THIS_MODULE,
532 .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list),
533 .cra_init = ablk_init, 525 .cra_init = ablk_init,
534 .cra_exit = ablk_exit, 526 .cra_exit = ablk_exit,
535 .cra_u = { 527 .cra_u = {
@@ -553,7 +545,6 @@ static struct crypto_alg twofish_algs[10] = { {
553 .cra_alignmask = 0, 545 .cra_alignmask = 0,
554 .cra_type = &crypto_ablkcipher_type, 546 .cra_type = &crypto_ablkcipher_type,
555 .cra_module = THIS_MODULE, 547 .cra_module = THIS_MODULE,
556 .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list),
557 .cra_init = ablk_init, 548 .cra_init = ablk_init,
558 .cra_exit = ablk_exit, 549 .cra_exit = ablk_exit,
559 .cra_u = { 550 .cra_u = {
@@ -578,7 +569,6 @@ static struct crypto_alg twofish_algs[10] = { {
578 .cra_alignmask = 0, 569 .cra_alignmask = 0,
579 .cra_type = &crypto_ablkcipher_type, 570 .cra_type = &crypto_ablkcipher_type,
580 .cra_module = THIS_MODULE, 571 .cra_module = THIS_MODULE,
581 .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list),
582 .cra_init = ablk_init, 572 .cra_init = ablk_init,
583 .cra_exit = ablk_exit, 573 .cra_exit = ablk_exit,
584 .cra_u = { 574 .cra_u = {
diff --git a/arch/x86/crypto/twofish_glue.c b/arch/x86/crypto/twofish_glue.c
index 359ae084275c..0a5202303501 100644
--- a/arch/x86/crypto/twofish_glue.c
+++ b/arch/x86/crypto/twofish_glue.c
@@ -70,7 +70,6 @@ static struct crypto_alg alg = {
70 .cra_ctxsize = sizeof(struct twofish_ctx), 70 .cra_ctxsize = sizeof(struct twofish_ctx),
71 .cra_alignmask = 0, 71 .cra_alignmask = 0,
72 .cra_module = THIS_MODULE, 72 .cra_module = THIS_MODULE,
73 .cra_list = LIST_HEAD_INIT(alg.cra_list),
74 .cra_u = { 73 .cra_u = {
75 .cipher = { 74 .cipher = {
76 .cia_min_keysize = TF_MIN_KEY_SIZE, 75 .cia_min_keysize = TF_MIN_KEY_SIZE,
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 15f9347316c8..aa3eb358b7e8 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -342,7 +342,6 @@ static struct crypto_alg tf_algs[5] = { {
342 .cra_alignmask = 0, 342 .cra_alignmask = 0,
343 .cra_type = &crypto_blkcipher_type, 343 .cra_type = &crypto_blkcipher_type,
344 .cra_module = THIS_MODULE, 344 .cra_module = THIS_MODULE,
345 .cra_list = LIST_HEAD_INIT(tf_algs[0].cra_list),
346 .cra_u = { 345 .cra_u = {
347 .blkcipher = { 346 .blkcipher = {
348 .min_keysize = TF_MIN_KEY_SIZE, 347 .min_keysize = TF_MIN_KEY_SIZE,
@@ -362,7 +361,6 @@ static struct crypto_alg tf_algs[5] = { {
362 .cra_alignmask = 0, 361 .cra_alignmask = 0,
363 .cra_type = &crypto_blkcipher_type, 362 .cra_type = &crypto_blkcipher_type,
364 .cra_module = THIS_MODULE, 363 .cra_module = THIS_MODULE,
365 .cra_list = LIST_HEAD_INIT(tf_algs[1].cra_list),
366 .cra_u = { 364 .cra_u = {
367 .blkcipher = { 365 .blkcipher = {
368 .min_keysize = TF_MIN_KEY_SIZE, 366 .min_keysize = TF_MIN_KEY_SIZE,
@@ -383,7 +381,6 @@ static struct crypto_alg tf_algs[5] = { {
383 .cra_alignmask = 0, 381 .cra_alignmask = 0,
384 .cra_type = &crypto_blkcipher_type, 382 .cra_type = &crypto_blkcipher_type,
385 .cra_module = THIS_MODULE, 383 .cra_module = THIS_MODULE,
386 .cra_list = LIST_HEAD_INIT(tf_algs[2].cra_list),
387 .cra_u = { 384 .cra_u = {
388 .blkcipher = { 385 .blkcipher = {
389 .min_keysize = TF_MIN_KEY_SIZE, 386 .min_keysize = TF_MIN_KEY_SIZE,
@@ -404,7 +401,6 @@ static struct crypto_alg tf_algs[5] = { {
404 .cra_alignmask = 0, 401 .cra_alignmask = 0,
405 .cra_type = &crypto_blkcipher_type, 402 .cra_type = &crypto_blkcipher_type,
406 .cra_module = THIS_MODULE, 403 .cra_module = THIS_MODULE,
407 .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list),
408 .cra_exit = lrw_twofish_exit_tfm, 404 .cra_exit = lrw_twofish_exit_tfm,
409 .cra_u = { 405 .cra_u = {
410 .blkcipher = { 406 .blkcipher = {
@@ -426,7 +422,6 @@ static struct crypto_alg tf_algs[5] = { {
426 .cra_alignmask = 0, 422 .cra_alignmask = 0,
427 .cra_type = &crypto_blkcipher_type, 423 .cra_type = &crypto_blkcipher_type,
428 .cra_module = THIS_MODULE, 424 .cra_module = THIS_MODULE,
429 .cra_list = LIST_HEAD_INIT(tf_algs[4].cra_list),
430 .cra_u = { 425 .cra_u = {
431 .blkcipher = { 426 .blkcipher = {
432 .min_keysize = TF_MIN_KEY_SIZE * 2, 427 .min_keysize = TF_MIN_KEY_SIZE * 2,
diff --git a/arch/x86/ia32/ia32_signal.c b/arch/x86/ia32/ia32_signal.c
index 673ac9b63d6b..efc6a958b71d 100644
--- a/arch/x86/ia32/ia32_signal.c
+++ b/arch/x86/ia32/ia32_signal.c
@@ -32,6 +32,7 @@
32#include <asm/sigframe.h> 32#include <asm/sigframe.h>
33#include <asm/sighandling.h> 33#include <asm/sighandling.h>
34#include <asm/sys_ia32.h> 34#include <asm/sys_ia32.h>
35#include <asm/smap.h>
35 36
36#define FIX_EFLAGS __FIX_EFLAGS 37#define FIX_EFLAGS __FIX_EFLAGS
37 38
@@ -162,7 +163,8 @@ asmlinkage long sys32_sigaltstack(const stack_ia32_t __user *uss_ptr,
162 } 163 }
163 seg = get_fs(); 164 seg = get_fs();
164 set_fs(KERNEL_DS); 165 set_fs(KERNEL_DS);
165 ret = do_sigaltstack(uss_ptr ? &uss : NULL, &uoss, regs->sp); 166 ret = do_sigaltstack((stack_t __force __user *) (uss_ptr ? &uss : NULL),
167 (stack_t __force __user *) &uoss, regs->sp);
166 set_fs(seg); 168 set_fs(seg);
167 if (ret >= 0 && uoss_ptr) { 169 if (ret >= 0 && uoss_ptr) {
168 if (!access_ok(VERIFY_WRITE, uoss_ptr, sizeof(stack_ia32_t))) 170 if (!access_ok(VERIFY_WRITE, uoss_ptr, sizeof(stack_ia32_t)))
@@ -250,11 +252,12 @@ static int ia32_restore_sigcontext(struct pt_regs *regs,
250 252
251 get_user_ex(tmp, &sc->fpstate); 253 get_user_ex(tmp, &sc->fpstate);
252 buf = compat_ptr(tmp); 254 buf = compat_ptr(tmp);
253 err |= restore_i387_xstate_ia32(buf);
254 255
255 get_user_ex(*pax, &sc->ax); 256 get_user_ex(*pax, &sc->ax);
256 } get_user_catch(err); 257 } get_user_catch(err);
257 258
259 err |= restore_xstate_sig(buf, 1);
260
258 return err; 261 return err;
259} 262}
260 263
@@ -361,7 +364,7 @@ static int ia32_setup_sigcontext(struct sigcontext_ia32 __user *sc,
361 */ 364 */
362static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, 365static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
363 size_t frame_size, 366 size_t frame_size,
364 void **fpstate) 367 void __user **fpstate)
365{ 368{
366 unsigned long sp; 369 unsigned long sp;
367 370
@@ -381,9 +384,12 @@ static void __user *get_sigframe(struct k_sigaction *ka, struct pt_regs *regs,
381 sp = (unsigned long) ka->sa.sa_restorer; 384 sp = (unsigned long) ka->sa.sa_restorer;
382 385
383 if (used_math()) { 386 if (used_math()) {
384 sp = sp - sig_xstate_ia32_size; 387 unsigned long fx_aligned, math_size;
385 *fpstate = (struct _fpstate_ia32 *) sp; 388
386 if (save_i387_xstate_ia32(*fpstate) < 0) 389 sp = alloc_mathframe(sp, 1, &fx_aligned, &math_size);
390 *fpstate = (struct _fpstate_ia32 __user *) sp;
391 if (save_xstate_sig(*fpstate, (void __user *)fx_aligned,
392 math_size) < 0)
387 return (void __user *) -1L; 393 return (void __user *) -1L;
388 } 394 }
389 395
@@ -448,7 +454,7 @@ int ia32_setup_frame(int sig, struct k_sigaction *ka,
448 * These are actually not used anymore, but left because some 454 * These are actually not used anymore, but left because some
449 * gdb versions depend on them as a marker. 455 * gdb versions depend on them as a marker.
450 */ 456 */
451 put_user_ex(*((u64 *)&code), (u64 *)frame->retcode); 457 put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
452 } put_user_catch(err); 458 } put_user_catch(err);
453 459
454 if (err) 460 if (err)
@@ -502,7 +508,6 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
502 put_user_ex(sig, &frame->sig); 508 put_user_ex(sig, &frame->sig);
503 put_user_ex(ptr_to_compat(&frame->info), &frame->pinfo); 509 put_user_ex(ptr_to_compat(&frame->info), &frame->pinfo);
504 put_user_ex(ptr_to_compat(&frame->uc), &frame->puc); 510 put_user_ex(ptr_to_compat(&frame->uc), &frame->puc);
505 err |= copy_siginfo_to_user32(&frame->info, info);
506 511
507 /* Create the ucontext. */ 512 /* Create the ucontext. */
508 if (cpu_has_xsave) 513 if (cpu_has_xsave)
@@ -514,9 +519,6 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
514 put_user_ex(sas_ss_flags(regs->sp), 519 put_user_ex(sas_ss_flags(regs->sp),
515 &frame->uc.uc_stack.ss_flags); 520 &frame->uc.uc_stack.ss_flags);
516 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size); 521 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
517 err |= ia32_setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
518 regs, set->sig[0]);
519 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
520 522
521 if (ka->sa.sa_flags & SA_RESTORER) 523 if (ka->sa.sa_flags & SA_RESTORER)
522 restorer = ka->sa.sa_restorer; 524 restorer = ka->sa.sa_restorer;
@@ -529,9 +531,14 @@ int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
529 * Not actually used anymore, but left because some gdb 531 * Not actually used anymore, but left because some gdb
530 * versions need it. 532 * versions need it.
531 */ 533 */
532 put_user_ex(*((u64 *)&code), (u64 *)frame->retcode); 534 put_user_ex(*((u64 *)&code), (u64 __user *)frame->retcode);
533 } put_user_catch(err); 535 } put_user_catch(err);
534 536
537 err |= copy_siginfo_to_user32(&frame->info, info);
538 err |= ia32_setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
539 regs, set->sig[0]);
540 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
541
535 if (err) 542 if (err)
536 return -EFAULT; 543 return -EFAULT;
537 544
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S
index 20e5f7ba0e6b..076745fc8045 100644
--- a/arch/x86/ia32/ia32entry.S
+++ b/arch/x86/ia32/ia32entry.S
@@ -14,6 +14,7 @@
14#include <asm/segment.h> 14#include <asm/segment.h>
15#include <asm/irqflags.h> 15#include <asm/irqflags.h>
16#include <asm/asm.h> 16#include <asm/asm.h>
17#include <asm/smap.h>
17#include <linux/linkage.h> 18#include <linux/linkage.h>
18#include <linux/err.h> 19#include <linux/err.h>
19 20
@@ -146,8 +147,10 @@ ENTRY(ia32_sysenter_target)
146 SAVE_ARGS 0,1,0 147 SAVE_ARGS 0,1,0
147 /* no need to do an access_ok check here because rbp has been 148 /* no need to do an access_ok check here because rbp has been
148 32bit zero extended */ 149 32bit zero extended */
150 ASM_STAC
1491: movl (%rbp),%ebp 1511: movl (%rbp),%ebp
150 _ASM_EXTABLE(1b,ia32_badarg) 152 _ASM_EXTABLE(1b,ia32_badarg)
153 ASM_CLAC
151 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 154 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
152 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 155 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
153 CFI_REMEMBER_STATE 156 CFI_REMEMBER_STATE
@@ -301,8 +304,10 @@ ENTRY(ia32_cstar_target)
301 /* no need to do an access_ok check here because r8 has been 304 /* no need to do an access_ok check here because r8 has been
302 32bit zero extended */ 305 32bit zero extended */
303 /* hardware stack frame is complete now */ 306 /* hardware stack frame is complete now */
307 ASM_STAC
3041: movl (%r8),%r9d 3081: movl (%r8),%r9d
305 _ASM_EXTABLE(1b,ia32_badarg) 309 _ASM_EXTABLE(1b,ia32_badarg)
310 ASM_CLAC
306 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET) 311 orl $TS_COMPAT,TI_status+THREAD_INFO(%rsp,RIP-ARGOFFSET)
307 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET) 312 testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP-ARGOFFSET)
308 CFI_REMEMBER_STATE 313 CFI_REMEMBER_STATE
@@ -365,6 +370,7 @@ cstar_tracesys:
365END(ia32_cstar_target) 370END(ia32_cstar_target)
366 371
367ia32_badarg: 372ia32_badarg:
373 ASM_CLAC
368 movq $-EFAULT,%rax 374 movq $-EFAULT,%rax
369 jmp ia32_sysret 375 jmp ia32_sysret
370 CFI_ENDPROC 376 CFI_ENDPROC
@@ -459,7 +465,7 @@ GLOBAL(\label)
459 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi 465 PTREGSCALL stub32_rt_sigreturn, sys32_rt_sigreturn, %rdi
460 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi 466 PTREGSCALL stub32_sigreturn, sys32_sigreturn, %rdi
461 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx 467 PTREGSCALL stub32_sigaltstack, sys32_sigaltstack, %rdx
462 PTREGSCALL stub32_execve, sys32_execve, %rcx 468 PTREGSCALL stub32_execve, compat_sys_execve, %rcx
463 PTREGSCALL stub32_fork, sys_fork, %rdi 469 PTREGSCALL stub32_fork, sys_fork, %rdi
464 PTREGSCALL stub32_clone, sys32_clone, %rdx 470 PTREGSCALL stub32_clone, sys32_clone, %rdx
465 PTREGSCALL stub32_vfork, sys_vfork, %rdi 471 PTREGSCALL stub32_vfork, sys_vfork, %rdi
diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c
index 4540bece0946..86d68d1c8806 100644
--- a/arch/x86/ia32/sys_ia32.c
+++ b/arch/x86/ia32/sys_ia32.c
@@ -287,7 +287,7 @@ asmlinkage long sys32_sigaction(int sig, struct old_sigaction32 __user *act,
287 return ret; 287 return ret;
288} 288}
289 289
290asmlinkage long sys32_waitpid(compat_pid_t pid, unsigned int *stat_addr, 290asmlinkage long sys32_waitpid(compat_pid_t pid, unsigned int __user *stat_addr,
291 int options) 291 int options)
292{ 292{
293 return compat_sys_wait4(pid, stat_addr, options, NULL); 293 return compat_sys_wait4(pid, stat_addr, options, NULL);
@@ -385,21 +385,6 @@ asmlinkage long sys32_sendfile(int out_fd, int in_fd,
385 return ret; 385 return ret;
386} 386}
387 387
388asmlinkage long sys32_execve(const char __user *name, compat_uptr_t __user *argv,
389 compat_uptr_t __user *envp, struct pt_regs *regs)
390{
391 long error;
392 char *filename;
393
394 filename = getname(name);
395 error = PTR_ERR(filename);
396 if (IS_ERR(filename))
397 return error;
398 error = compat_do_execve(filename, argv, envp, regs);
399 putname(filename);
400 return error;
401}
402
403asmlinkage long sys32_clone(unsigned int clone_flags, unsigned int newsp, 388asmlinkage long sys32_clone(unsigned int clone_flags, unsigned int newsp,
404 struct pt_regs *regs) 389 struct pt_regs *regs)
405{ 390{
diff --git a/arch/x86/include/asm/Kbuild b/arch/x86/include/asm/Kbuild
index f9c0d3ba9e84..66e5f0ef0523 100644
--- a/arch/x86/include/asm/Kbuild
+++ b/arch/x86/include/asm/Kbuild
@@ -26,3 +26,5 @@ header-y += vsyscall.h
26genhdr-y += unistd_32.h 26genhdr-y += unistd_32.h
27genhdr-y += unistd_64.h 27genhdr-y += unistd_64.h
28genhdr-y += unistd_x32.h 28genhdr-y += unistd_x32.h
29
30generic-y += clkdev.h
diff --git a/arch/x86/include/asm/alternative-asm.h b/arch/x86/include/asm/alternative-asm.h
index 952bd0100c5c..372231c22a47 100644
--- a/arch/x86/include/asm/alternative-asm.h
+++ b/arch/x86/include/asm/alternative-asm.h
@@ -1,3 +1,6 @@
1#ifndef _ASM_X86_ALTERNATIVE_ASM_H
2#define _ASM_X86_ALTERNATIVE_ASM_H
3
1#ifdef __ASSEMBLY__ 4#ifdef __ASSEMBLY__
2 5
3#include <asm/asm.h> 6#include <asm/asm.h>
@@ -5,10 +8,10 @@
5#ifdef CONFIG_SMP 8#ifdef CONFIG_SMP
6 .macro LOCK_PREFIX 9 .macro LOCK_PREFIX
7672: lock 10672: lock
8 .section .smp_locks,"a" 11 .pushsection .smp_locks,"a"
9 .balign 4 12 .balign 4
10 .long 672b - . 13 .long 672b - .
11 .previous 14 .popsection
12 .endm 15 .endm
13#else 16#else
14 .macro LOCK_PREFIX 17 .macro LOCK_PREFIX
@@ -24,3 +27,5 @@
24.endm 27.endm
25 28
26#endif /* __ASSEMBLY__ */ 29#endif /* __ASSEMBLY__ */
30
31#endif /* _ASM_X86_ALTERNATIVE_ASM_H */
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h
index 70780689599a..58ed6d96a6ac 100644
--- a/arch/x86/include/asm/alternative.h
+++ b/arch/x86/include/asm/alternative.h
@@ -29,10 +29,10 @@
29 29
30#ifdef CONFIG_SMP 30#ifdef CONFIG_SMP
31#define LOCK_PREFIX_HERE \ 31#define LOCK_PREFIX_HERE \
32 ".section .smp_locks,\"a\"\n" \ 32 ".pushsection .smp_locks,\"a\"\n" \
33 ".balign 4\n" \ 33 ".balign 4\n" \
34 ".long 671f - .\n" /* offset */ \ 34 ".long 671f - .\n" /* offset */ \
35 ".previous\n" \ 35 ".popsection\n" \
36 "671:" 36 "671:"
37 37
38#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; " 38#define LOCK_PREFIX LOCK_PREFIX_HERE "\n\tlock; "
@@ -60,7 +60,7 @@ extern void alternatives_smp_module_add(struct module *mod, char *name,
60 void *locks, void *locks_end, 60 void *locks, void *locks_end,
61 void *text, void *text_end); 61 void *text, void *text_end);
62extern void alternatives_smp_module_del(struct module *mod); 62extern void alternatives_smp_module_del(struct module *mod);
63extern void alternatives_smp_switch(int smp); 63extern void alternatives_enable_smp(void);
64extern int alternatives_text_reserved(void *start, void *end); 64extern int alternatives_text_reserved(void *start, void *end);
65extern bool skip_smp_alternatives; 65extern bool skip_smp_alternatives;
66#else 66#else
@@ -68,7 +68,7 @@ static inline void alternatives_smp_module_add(struct module *mod, char *name,
68 void *locks, void *locks_end, 68 void *locks, void *locks_end,
69 void *text, void *text_end) {} 69 void *text, void *text_end) {}
70static inline void alternatives_smp_module_del(struct module *mod) {} 70static inline void alternatives_smp_module_del(struct module *mod) {}
71static inline void alternatives_smp_switch(int smp) {} 71static inline void alternatives_enable_smp(void) {}
72static inline int alternatives_text_reserved(void *start, void *end) 72static inline int alternatives_text_reserved(void *start, void *end)
73{ 73{
74 return 0; 74 return 0;
@@ -99,30 +99,30 @@ static inline int alternatives_text_reserved(void *start, void *end)
99/* alternative assembly primitive: */ 99/* alternative assembly primitive: */
100#define ALTERNATIVE(oldinstr, newinstr, feature) \ 100#define ALTERNATIVE(oldinstr, newinstr, feature) \
101 OLDINSTR(oldinstr) \ 101 OLDINSTR(oldinstr) \
102 ".section .altinstructions,\"a\"\n" \ 102 ".pushsection .altinstructions,\"a\"\n" \
103 ALTINSTR_ENTRY(feature, 1) \ 103 ALTINSTR_ENTRY(feature, 1) \
104 ".previous\n" \ 104 ".popsection\n" \
105 ".section .discard,\"aw\",@progbits\n" \ 105 ".pushsection .discard,\"aw\",@progbits\n" \
106 DISCARD_ENTRY(1) \ 106 DISCARD_ENTRY(1) \
107 ".previous\n" \ 107 ".popsection\n" \
108 ".section .altinstr_replacement, \"ax\"\n" \ 108 ".pushsection .altinstr_replacement, \"ax\"\n" \
109 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \ 109 ALTINSTR_REPLACEMENT(newinstr, feature, 1) \
110 ".previous" 110 ".popsection"
111 111
112#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\ 112#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
113 OLDINSTR(oldinstr) \ 113 OLDINSTR(oldinstr) \
114 ".section .altinstructions,\"a\"\n" \ 114 ".pushsection .altinstructions,\"a\"\n" \
115 ALTINSTR_ENTRY(feature1, 1) \ 115 ALTINSTR_ENTRY(feature1, 1) \
116 ALTINSTR_ENTRY(feature2, 2) \ 116 ALTINSTR_ENTRY(feature2, 2) \
117 ".previous\n" \ 117 ".popsection\n" \
118 ".section .discard,\"aw\",@progbits\n" \ 118 ".pushsection .discard,\"aw\",@progbits\n" \
119 DISCARD_ENTRY(1) \ 119 DISCARD_ENTRY(1) \
120 DISCARD_ENTRY(2) \ 120 DISCARD_ENTRY(2) \
121 ".previous\n" \ 121 ".popsection\n" \
122 ".section .altinstr_replacement, \"ax\"\n" \ 122 ".pushsection .altinstr_replacement, \"ax\"\n" \
123 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \ 123 ALTINSTR_REPLACEMENT(newinstr1, feature1, 1) \
124 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \ 124 ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
125 ".previous" 125 ".popsection"
126 126
127/* 127/*
128 * This must be included *after* the definition of ALTERNATIVE due to 128 * This must be included *after* the definition of ALTERNATIVE due to
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index f34261296ffb..338803422239 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -409,7 +409,7 @@ extern struct apic *apic;
409 * to enforce the order with in them. 409 * to enforce the order with in them.
410 */ 410 */
411#define apic_driver(sym) \ 411#define apic_driver(sym) \
412 static struct apic *__apicdrivers_##sym __used \ 412 static const struct apic *__apicdrivers_##sym __used \
413 __aligned(sizeof(struct apic *)) \ 413 __aligned(sizeof(struct apic *)) \
414 __section(.apicdrivers) = { &sym } 414 __section(.apicdrivers) = { &sym }
415 415
diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h
index 58cb6d4085f7..b6c3b821acf6 100644
--- a/arch/x86/include/asm/atomic.h
+++ b/arch/x86/include/asm/atomic.h
@@ -240,30 +240,6 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)
240 return c; 240 return c;
241} 241}
242 242
243
244/*
245 * atomic_dec_if_positive - decrement by 1 if old value positive
246 * @v: pointer of type atomic_t
247 *
248 * The function returns the old value of *v minus 1, even if
249 * the atomic variable, v, was not decremented.
250 */
251static inline int atomic_dec_if_positive(atomic_t *v)
252{
253 int c, old, dec;
254 c = atomic_read(v);
255 for (;;) {
256 dec = c - 1;
257 if (unlikely(dec < 0))
258 break;
259 old = atomic_cmpxchg((v), c, dec);
260 if (likely(old == c))
261 break;
262 c = old;
263 }
264 return dec;
265}
266
267/** 243/**
268 * atomic_inc_short - increment of a short integer 244 * atomic_inc_short - increment of a short integer
269 * @v: pointer to type int 245 * @v: pointer to type int
@@ -309,9 +285,9 @@ static inline void atomic_or_long(unsigned long *v1, unsigned long v2)
309#define smp_mb__after_atomic_inc() barrier() 285#define smp_mb__after_atomic_inc() barrier()
310 286
311#ifdef CONFIG_X86_32 287#ifdef CONFIG_X86_32
312# include "atomic64_32.h" 288# include <asm/atomic64_32.h>
313#else 289#else
314# include "atomic64_64.h" 290# include <asm/atomic64_64.h>
315#endif 291#endif
316 292
317#endif /* _ASM_X86_ATOMIC_H */ 293#endif /* _ASM_X86_ATOMIC_H */
diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h
index 72f5009deb5a..6dfd0195bb55 100644
--- a/arch/x86/include/asm/bitops.h
+++ b/arch/x86/include/asm/bitops.h
@@ -355,7 +355,7 @@ static int test_bit(int nr, const volatile unsigned long *addr);
355 */ 355 */
356static inline unsigned long __ffs(unsigned long word) 356static inline unsigned long __ffs(unsigned long word)
357{ 357{
358 asm("bsf %1,%0" 358 asm("rep; bsf %1,%0"
359 : "=r" (word) 359 : "=r" (word)
360 : "rm" (word)); 360 : "rm" (word));
361 return word; 361 return word;
@@ -369,7 +369,7 @@ static inline unsigned long __ffs(unsigned long word)
369 */ 369 */
370static inline unsigned long ffz(unsigned long word) 370static inline unsigned long ffz(unsigned long word)
371{ 371{
372 asm("bsf %1,%0" 372 asm("rep; bsf %1,%0"
373 : "=r" (word) 373 : "=r" (word)
374 : "r" (~word)); 374 : "r" (~word));
375 return word; 375 return word;
@@ -417,10 +417,9 @@ static inline int ffs(int x)
417 * We cannot do this on 32 bits because at the very least some 417 * We cannot do this on 32 bits because at the very least some
418 * 486 CPUs did not behave this way. 418 * 486 CPUs did not behave this way.
419 */ 419 */
420 long tmp = -1;
421 asm("bsfl %1,%0" 420 asm("bsfl %1,%0"
422 : "=r" (r) 421 : "=r" (r)
423 : "rm" (x), "0" (tmp)); 422 : "rm" (x), "0" (-1));
424#elif defined(CONFIG_X86_CMOV) 423#elif defined(CONFIG_X86_CMOV)
425 asm("bsfl %1,%0\n\t" 424 asm("bsfl %1,%0\n\t"
426 "cmovzl %2,%0" 425 "cmovzl %2,%0"
@@ -459,10 +458,9 @@ static inline int fls(int x)
459 * We cannot do this on 32 bits because at the very least some 458 * We cannot do this on 32 bits because at the very least some
460 * 486 CPUs did not behave this way. 459 * 486 CPUs did not behave this way.
461 */ 460 */
462 long tmp = -1;
463 asm("bsrl %1,%0" 461 asm("bsrl %1,%0"
464 : "=r" (r) 462 : "=r" (r)
465 : "rm" (x), "0" (tmp)); 463 : "rm" (x), "0" (-1));
466#elif defined(CONFIG_X86_CMOV) 464#elif defined(CONFIG_X86_CMOV)
467 asm("bsrl %1,%0\n\t" 465 asm("bsrl %1,%0\n\t"
468 "cmovzl %2,%0" 466 "cmovzl %2,%0"
@@ -490,13 +488,13 @@ static inline int fls(int x)
490#ifdef CONFIG_X86_64 488#ifdef CONFIG_X86_64
491static __always_inline int fls64(__u64 x) 489static __always_inline int fls64(__u64 x)
492{ 490{
493 long bitpos = -1; 491 int bitpos = -1;
494 /* 492 /*
495 * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the 493 * AMD64 says BSRQ won't clobber the dest reg if x==0; Intel64 says the
496 * dest reg is undefined if x==0, but their CPU architect says its 494 * dest reg is undefined if x==0, but their CPU architect says its
497 * value is written to set it to the same as before. 495 * value is written to set it to the same as before.
498 */ 496 */
499 asm("bsrq %1,%0" 497 asm("bsrq %1,%q0"
500 : "+r" (bitpos) 498 : "+r" (bitpos)
501 : "rm" (x)); 499 : "rm" (x));
502 return bitpos + 1; 500 return bitpos + 1;
diff --git a/arch/x86/include/asm/calling.h b/arch/x86/include/asm/calling.h
index a9e3a740f697..0fa675033912 100644
--- a/arch/x86/include/asm/calling.h
+++ b/arch/x86/include/asm/calling.h
@@ -46,41 +46,39 @@ For 32-bit we have the following conventions - kernel is built with
46 46
47*/ 47*/
48 48
49#include "dwarf2.h" 49#include <asm/dwarf2.h>
50 50
51/* 51/*
52 * 64-bit system call stack frame layout defines and helpers, for 52 * 64-bit system call stack frame layout defines and helpers,
53 * assembly code (note that the seemingly unnecessary parentheses 53 * for assembly code:
54 * are to prevent cpp from inserting spaces in expressions that get
55 * passed to macros):
56 */ 54 */
57 55
58#define R15 (0) 56#define R15 0
59#define R14 (8) 57#define R14 8
60#define R13 (16) 58#define R13 16
61#define R12 (24) 59#define R12 24
62#define RBP (32) 60#define RBP 32
63#define RBX (40) 61#define RBX 40
64 62
65/* arguments: interrupts/non tracing syscalls only save up to here: */ 63/* arguments: interrupts/non tracing syscalls only save up to here: */
66#define R11 (48) 64#define R11 48
67#define R10 (56) 65#define R10 56
68#define R9 (64) 66#define R9 64
69#define R8 (72) 67#define R8 72
70#define RAX (80) 68#define RAX 80
71#define RCX (88) 69#define RCX 88
72#define RDX (96) 70#define RDX 96
73#define RSI (104) 71#define RSI 104
74#define RDI (112) 72#define RDI 112
75#define ORIG_RAX (120) /* + error_code */ 73#define ORIG_RAX 120 /* + error_code */
76/* end of arguments */ 74/* end of arguments */
77 75
78/* cpu exception frame or undefined in case of fast syscall: */ 76/* cpu exception frame or undefined in case of fast syscall: */
79#define RIP (128) 77#define RIP 128
80#define CS (136) 78#define CS 136
81#define EFLAGS (144) 79#define EFLAGS 144
82#define RSP (152) 80#define RSP 152
83#define SS (160) 81#define SS 160
84 82
85#define ARGOFFSET R11 83#define ARGOFFSET R11
86#define SWFRAME ORIG_RAX 84#define SWFRAME ORIG_RAX
diff --git a/arch/x86/include/asm/checksum.h b/arch/x86/include/asm/checksum.h
index 848850fd7d62..5f5bb0f97361 100644
--- a/arch/x86/include/asm/checksum.h
+++ b/arch/x86/include/asm/checksum.h
@@ -1,5 +1,5 @@
1#ifdef CONFIG_X86_32 1#ifdef CONFIG_X86_32
2# include "checksum_32.h" 2# include <asm/checksum_32.h>
3#else 3#else
4# include "checksum_64.h" 4# include <asm/checksum_64.h>
5#endif 5#endif
diff --git a/arch/x86/include/asm/cmpxchg.h b/arch/x86/include/asm/cmpxchg.h
index 99480e55973d..8d871eaddb66 100644
--- a/arch/x86/include/asm/cmpxchg.h
+++ b/arch/x86/include/asm/cmpxchg.h
@@ -138,9 +138,9 @@ extern void __add_wrong_size(void)
138 __raw_cmpxchg((ptr), (old), (new), (size), "") 138 __raw_cmpxchg((ptr), (old), (new), (size), "")
139 139
140#ifdef CONFIG_X86_32 140#ifdef CONFIG_X86_32
141# include "cmpxchg_32.h" 141# include <asm/cmpxchg_32.h>
142#else 142#else
143# include "cmpxchg_64.h" 143# include <asm/cmpxchg_64.h>
144#endif 144#endif
145 145
146#ifdef __HAVE_ARCH_CMPXCHG 146#ifdef __HAVE_ARCH_CMPXCHG
diff --git a/arch/x86/include/asm/compat.h b/arch/x86/include/asm/compat.h
index fedf32b73e65..59c6c401f79f 100644
--- a/arch/x86/include/asm/compat.h
+++ b/arch/x86/include/asm/compat.h
@@ -41,6 +41,7 @@ typedef s64 __attribute__((aligned(4))) compat_s64;
41typedef u32 compat_uint_t; 41typedef u32 compat_uint_t;
42typedef u32 compat_ulong_t; 42typedef u32 compat_ulong_t;
43typedef u64 __attribute__((aligned(4))) compat_u64; 43typedef u64 __attribute__((aligned(4))) compat_u64;
44typedef u32 compat_uptr_t;
44 45
45struct compat_timespec { 46struct compat_timespec {
46 compat_time_t tv_sec; 47 compat_time_t tv_sec;
@@ -124,6 +125,78 @@ typedef u32 compat_old_sigset_t; /* at least 32 bits */
124 125
125typedef u32 compat_sigset_word; 126typedef u32 compat_sigset_word;
126 127
128typedef union compat_sigval {
129 compat_int_t sival_int;
130 compat_uptr_t sival_ptr;
131} compat_sigval_t;
132
133typedef struct compat_siginfo {
134 int si_signo;
135 int si_errno;
136 int si_code;
137
138 union {
139 int _pad[128/sizeof(int) - 3];
140
141 /* kill() */
142 struct {
143 unsigned int _pid; /* sender's pid */
144 unsigned int _uid; /* sender's uid */
145 } _kill;
146
147 /* POSIX.1b timers */
148 struct {
149 compat_timer_t _tid; /* timer id */
150 int _overrun; /* overrun count */
151 compat_sigval_t _sigval; /* same as below */
152 int _sys_private; /* not to be passed to user */
153 int _overrun_incr; /* amount to add to overrun */
154 } _timer;
155
156 /* POSIX.1b signals */
157 struct {
158 unsigned int _pid; /* sender's pid */
159 unsigned int _uid; /* sender's uid */
160 compat_sigval_t _sigval;
161 } _rt;
162
163 /* SIGCHLD */
164 struct {
165 unsigned int _pid; /* which child */
166 unsigned int _uid; /* sender's uid */
167 int _status; /* exit code */
168 compat_clock_t _utime;
169 compat_clock_t _stime;
170 } _sigchld;
171
172 /* SIGCHLD (x32 version) */
173 struct {
174 unsigned int _pid; /* which child */
175 unsigned int _uid; /* sender's uid */
176 int _status; /* exit code */
177 compat_s64 _utime;
178 compat_s64 _stime;
179 } _sigchld_x32;
180
181 /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
182 struct {
183 unsigned int _addr; /* faulting insn/memory ref. */
184 } _sigfault;
185
186 /* SIGPOLL */
187 struct {
188 int _band; /* POLL_IN, POLL_OUT, POLL_MSG */
189 int _fd;
190 } _sigpoll;
191
192 struct {
193 unsigned int _call_addr; /* calling insn */
194 int _syscall; /* triggering system call number */
195 unsigned int _arch; /* AUDIT_ARCH_* of syscall */
196 } _sigsys;
197 } _sifields;
198} compat_siginfo_t;
199
127#define COMPAT_OFF_T_MAX 0x7fffffff 200#define COMPAT_OFF_T_MAX 0x7fffffff
128#define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL 201#define COMPAT_LOFF_T_MAX 0x7fffffffffffffffL
129 202
@@ -209,7 +282,6 @@ typedef struct user_regs_struct32 compat_elf_gregset_t;
209 * as pointers because the syscall entry code will have 282 * as pointers because the syscall entry code will have
210 * appropriately converted them already. 283 * appropriately converted them already.
211 */ 284 */
212typedef u32 compat_uptr_t;
213 285
214static inline void __user *compat_ptr(compat_uptr_t uptr) 286static inline void __user *compat_ptr(compat_uptr_t uptr)
215{ 287{
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index 6b7ee5ff6820..8c297aa53eef 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -4,7 +4,9 @@
4#ifndef _ASM_X86_CPUFEATURE_H 4#ifndef _ASM_X86_CPUFEATURE_H
5#define _ASM_X86_CPUFEATURE_H 5#define _ASM_X86_CPUFEATURE_H
6 6
7#ifndef _ASM_X86_REQUIRED_FEATURES_H
7#include <asm/required-features.h> 8#include <asm/required-features.h>
9#endif
8 10
9#define NCAPINTS 10 /* N 32-bit words worth of info */ 11#define NCAPINTS 10 /* N 32-bit words worth of info */
10 12
@@ -97,6 +99,7 @@
97#define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */ 99#define X86_FEATURE_EXTD_APICID (3*32+26) /* has extended APICID (8 bits) */
98#define X86_FEATURE_AMD_DCM (3*32+27) /* multi-node processor */ 100#define X86_FEATURE_AMD_DCM (3*32+27) /* multi-node processor */
99#define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */ 101#define X86_FEATURE_APERFMPERF (3*32+28) /* APERFMPERF */
102#define X86_FEATURE_EAGER_FPU (3*32+29) /* "eagerfpu" Non lazy FPU restore */
100 103
101/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */ 104/* Intel-defined CPU features, CPUID level 0x00000001 (ecx), word 4 */
102#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ 105#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */
@@ -209,6 +212,7 @@
209#define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */ 212#define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */
210#define X86_FEATURE_RDSEED (9*32+18) /* The RDSEED instruction */ 213#define X86_FEATURE_RDSEED (9*32+18) /* The RDSEED instruction */
211#define X86_FEATURE_ADX (9*32+19) /* The ADCX and ADOX instructions */ 214#define X86_FEATURE_ADX (9*32+19) /* The ADCX and ADOX instructions */
215#define X86_FEATURE_SMAP (9*32+20) /* Supervisor Mode Access Prevention */
212 216
213#if defined(__KERNEL__) && !defined(__ASSEMBLY__) 217#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
214 218
@@ -299,12 +303,14 @@ extern const char * const x86_power_flags[32];
299#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2) 303#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2)
300#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC) 304#define cpu_has_x2apic boot_cpu_has(X86_FEATURE_X2APIC)
301#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE) 305#define cpu_has_xsave boot_cpu_has(X86_FEATURE_XSAVE)
306#define cpu_has_xsaveopt boot_cpu_has(X86_FEATURE_XSAVEOPT)
302#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE) 307#define cpu_has_osxsave boot_cpu_has(X86_FEATURE_OSXSAVE)
303#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR) 308#define cpu_has_hypervisor boot_cpu_has(X86_FEATURE_HYPERVISOR)
304#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ) 309#define cpu_has_pclmulqdq boot_cpu_has(X86_FEATURE_PCLMULQDQ)
305#define cpu_has_perfctr_core boot_cpu_has(X86_FEATURE_PERFCTR_CORE) 310#define cpu_has_perfctr_core boot_cpu_has(X86_FEATURE_PERFCTR_CORE)
306#define cpu_has_cx8 boot_cpu_has(X86_FEATURE_CX8) 311#define cpu_has_cx8 boot_cpu_has(X86_FEATURE_CX8)
307#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16) 312#define cpu_has_cx16 boot_cpu_has(X86_FEATURE_CX16)
313#define cpu_has_eager_fpu boot_cpu_has(X86_FEATURE_EAGER_FPU)
308 314
309#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64) 315#if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64)
310# define cpu_has_invlpg 1 316# define cpu_has_invlpg 1
diff --git a/arch/x86/include/asm/fpu-internal.h b/arch/x86/include/asm/fpu-internal.h
index 75f4c6d6a331..831dbb9c6c02 100644
--- a/arch/x86/include/asm/fpu-internal.h
+++ b/arch/x86/include/asm/fpu-internal.h
@@ -12,6 +12,7 @@
12 12
13#include <linux/kernel_stat.h> 13#include <linux/kernel_stat.h>
14#include <linux/regset.h> 14#include <linux/regset.h>
15#include <linux/compat.h>
15#include <linux/slab.h> 16#include <linux/slab.h>
16#include <asm/asm.h> 17#include <asm/asm.h>
17#include <asm/cpufeature.h> 18#include <asm/cpufeature.h>
@@ -20,43 +21,76 @@
20#include <asm/user.h> 21#include <asm/user.h>
21#include <asm/uaccess.h> 22#include <asm/uaccess.h>
22#include <asm/xsave.h> 23#include <asm/xsave.h>
24#include <asm/smap.h>
23 25
24extern unsigned int sig_xstate_size; 26#ifdef CONFIG_X86_64
27# include <asm/sigcontext32.h>
28# include <asm/user32.h>
29int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
30 compat_sigset_t *set, struct pt_regs *regs);
31int ia32_setup_frame(int sig, struct k_sigaction *ka,
32 compat_sigset_t *set, struct pt_regs *regs);
33#else
34# define user_i387_ia32_struct user_i387_struct
35# define user32_fxsr_struct user_fxsr_struct
36# define ia32_setup_frame __setup_frame
37# define ia32_setup_rt_frame __setup_rt_frame
38#endif
39
40extern unsigned int mxcsr_feature_mask;
25extern void fpu_init(void); 41extern void fpu_init(void);
42extern void eager_fpu_init(void);
26 43
27DECLARE_PER_CPU(struct task_struct *, fpu_owner_task); 44DECLARE_PER_CPU(struct task_struct *, fpu_owner_task);
28 45
46extern void convert_from_fxsr(struct user_i387_ia32_struct *env,
47 struct task_struct *tsk);
48extern void convert_to_fxsr(struct task_struct *tsk,
49 const struct user_i387_ia32_struct *env);
50
29extern user_regset_active_fn fpregs_active, xfpregs_active; 51extern user_regset_active_fn fpregs_active, xfpregs_active;
30extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, 52extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get,
31 xstateregs_get; 53 xstateregs_get;
32extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, 54extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
33 xstateregs_set; 55 xstateregs_set;
34 56
35
36/* 57/*
37 * xstateregs_active == fpregs_active. Please refer to the comment 58 * xstateregs_active == fpregs_active. Please refer to the comment
38 * at the definition of fpregs_active. 59 * at the definition of fpregs_active.
39 */ 60 */
40#define xstateregs_active fpregs_active 61#define xstateregs_active fpregs_active
41 62
42extern struct _fpx_sw_bytes fx_sw_reserved;
43#ifdef CONFIG_IA32_EMULATION
44extern unsigned int sig_xstate_ia32_size;
45extern struct _fpx_sw_bytes fx_sw_reserved_ia32;
46struct _fpstate_ia32;
47struct _xstate_ia32;
48extern int save_i387_xstate_ia32(void __user *buf);
49extern int restore_i387_xstate_ia32(void __user *buf);
50#endif
51
52#ifdef CONFIG_MATH_EMULATION 63#ifdef CONFIG_MATH_EMULATION
64# define HAVE_HWFP (boot_cpu_data.hard_math)
53extern void finit_soft_fpu(struct i387_soft_struct *soft); 65extern void finit_soft_fpu(struct i387_soft_struct *soft);
54#else 66#else
67# define HAVE_HWFP 1
55static inline void finit_soft_fpu(struct i387_soft_struct *soft) {} 68static inline void finit_soft_fpu(struct i387_soft_struct *soft) {}
56#endif 69#endif
57 70
71static inline int is_ia32_compat_frame(void)
72{
73 return config_enabled(CONFIG_IA32_EMULATION) &&
74 test_thread_flag(TIF_IA32);
75}
76
77static inline int is_ia32_frame(void)
78{
79 return config_enabled(CONFIG_X86_32) || is_ia32_compat_frame();
80}
81
82static inline int is_x32_frame(void)
83{
84 return config_enabled(CONFIG_X86_X32_ABI) && test_thread_flag(TIF_X32);
85}
86
58#define X87_FSW_ES (1 << 7) /* Exception Summary */ 87#define X87_FSW_ES (1 << 7) /* Exception Summary */
59 88
89static __always_inline __pure bool use_eager_fpu(void)
90{
91 return static_cpu_has(X86_FEATURE_EAGER_FPU);
92}
93
60static __always_inline __pure bool use_xsaveopt(void) 94static __always_inline __pure bool use_xsaveopt(void)
61{ 95{
62 return static_cpu_has(X86_FEATURE_XSAVEOPT); 96 return static_cpu_has(X86_FEATURE_XSAVEOPT);
@@ -72,6 +106,13 @@ static __always_inline __pure bool use_fxsr(void)
72 return static_cpu_has(X86_FEATURE_FXSR); 106 return static_cpu_has(X86_FEATURE_FXSR);
73} 107}
74 108
109static inline void fx_finit(struct i387_fxsave_struct *fx)
110{
111 memset(fx, 0, xstate_size);
112 fx->cwd = 0x37f;
113 fx->mxcsr = MXCSR_DEFAULT;
114}
115
75extern void __sanitize_i387_state(struct task_struct *); 116extern void __sanitize_i387_state(struct task_struct *);
76 117
77static inline void sanitize_i387_state(struct task_struct *tsk) 118static inline void sanitize_i387_state(struct task_struct *tsk)
@@ -81,131 +122,121 @@ static inline void sanitize_i387_state(struct task_struct *tsk)
81 __sanitize_i387_state(tsk); 122 __sanitize_i387_state(tsk);
82} 123}
83 124
84#ifdef CONFIG_X86_64 125#define user_insn(insn, output, input...) \
85static inline int fxrstor_checking(struct i387_fxsave_struct *fx) 126({ \
86{ 127 int err; \
87 int err; 128 asm volatile(ASM_STAC "\n" \
88 129 "1:" #insn "\n\t" \
89 /* See comment in fxsave() below. */ 130 "2: " ASM_CLAC "\n" \
90#ifdef CONFIG_AS_FXSAVEQ 131 ".section .fixup,\"ax\"\n" \
91 asm volatile("1: fxrstorq %[fx]\n\t" 132 "3: movl $-1,%[err]\n" \
92 "2:\n" 133 " jmp 2b\n" \
93 ".section .fixup,\"ax\"\n" 134 ".previous\n" \
94 "3: movl $-1,%[err]\n" 135 _ASM_EXTABLE(1b, 3b) \
95 " jmp 2b\n" 136 : [err] "=r" (err), output \
96 ".previous\n" 137 : "0"(0), input); \
97 _ASM_EXTABLE(1b, 3b) 138 err; \
98 : [err] "=r" (err) 139})
99 : [fx] "m" (*fx), "0" (0)); 140
100#else 141#define check_insn(insn, output, input...) \
101 asm volatile("1: rex64/fxrstor (%[fx])\n\t" 142({ \
102 "2:\n" 143 int err; \
103 ".section .fixup,\"ax\"\n" 144 asm volatile("1:" #insn "\n\t" \
104 "3: movl $-1,%[err]\n" 145 "2:\n" \
105 " jmp 2b\n" 146 ".section .fixup,\"ax\"\n" \
106 ".previous\n" 147 "3: movl $-1,%[err]\n" \
107 _ASM_EXTABLE(1b, 3b) 148 " jmp 2b\n" \
108 : [err] "=r" (err) 149 ".previous\n" \
109 : [fx] "R" (fx), "m" (*fx), "0" (0)); 150 _ASM_EXTABLE(1b, 3b) \
110#endif 151 : [err] "=r" (err), output \
111 return err; 152 : "0"(0), input); \
153 err; \
154})
155
156static inline int fsave_user(struct i387_fsave_struct __user *fx)
157{
158 return user_insn(fnsave %[fx]; fwait, [fx] "=m" (*fx), "m" (*fx));
112} 159}
113 160
114static inline int fxsave_user(struct i387_fxsave_struct __user *fx) 161static inline int fxsave_user(struct i387_fxsave_struct __user *fx)
115{ 162{
116 int err; 163 if (config_enabled(CONFIG_X86_32))
164 return user_insn(fxsave %[fx], [fx] "=m" (*fx), "m" (*fx));
165 else if (config_enabled(CONFIG_AS_FXSAVEQ))
166 return user_insn(fxsaveq %[fx], [fx] "=m" (*fx), "m" (*fx));
117 167
118 /* 168 /* See comment in fpu_fxsave() below. */
119 * Clear the bytes not touched by the fxsave and reserved 169 return user_insn(rex64/fxsave (%[fx]), "=m" (*fx), [fx] "R" (fx));
120 * for the SW usage.
121 */
122 err = __clear_user(&fx->sw_reserved,
123 sizeof(struct _fpx_sw_bytes));
124 if (unlikely(err))
125 return -EFAULT;
126
127 /* See comment in fxsave() below. */
128#ifdef CONFIG_AS_FXSAVEQ
129 asm volatile("1: fxsaveq %[fx]\n\t"
130 "2:\n"
131 ".section .fixup,\"ax\"\n"
132 "3: movl $-1,%[err]\n"
133 " jmp 2b\n"
134 ".previous\n"
135 _ASM_EXTABLE(1b, 3b)
136 : [err] "=r" (err), [fx] "=m" (*fx)
137 : "0" (0));
138#else
139 asm volatile("1: rex64/fxsave (%[fx])\n\t"
140 "2:\n"
141 ".section .fixup,\"ax\"\n"
142 "3: movl $-1,%[err]\n"
143 " jmp 2b\n"
144 ".previous\n"
145 _ASM_EXTABLE(1b, 3b)
146 : [err] "=r" (err), "=m" (*fx)
147 : [fx] "R" (fx), "0" (0));
148#endif
149 if (unlikely(err) &&
150 __clear_user(fx, sizeof(struct i387_fxsave_struct)))
151 err = -EFAULT;
152 /* No need to clear here because the caller clears USED_MATH */
153 return err;
154} 170}
155 171
156static inline void fpu_fxsave(struct fpu *fpu) 172static inline int fxrstor_checking(struct i387_fxsave_struct *fx)
157{ 173{
158 /* Using "rex64; fxsave %0" is broken because, if the memory operand 174 if (config_enabled(CONFIG_X86_32))
159 uses any extended registers for addressing, a second REX prefix 175 return check_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
160 will be generated (to the assembler, rex64 followed by semicolon 176 else if (config_enabled(CONFIG_AS_FXSAVEQ))
161 is a separate instruction), and hence the 64-bitness is lost. */ 177 return check_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
162 178
163#ifdef CONFIG_AS_FXSAVEQ 179 /* See comment in fpu_fxsave() below. */
164 /* Using "fxsaveq %0" would be the ideal choice, but is only supported 180 return check_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
165 starting with gas 2.16. */ 181 "m" (*fx));
166 __asm__ __volatile__("fxsaveq %0"
167 : "=m" (fpu->state->fxsave));
168#else
169 /* Using, as a workaround, the properly prefixed form below isn't
170 accepted by any binutils version so far released, complaining that
171 the same type of prefix is used twice if an extended register is
172 needed for addressing (fix submitted to mainline 2005-11-21).
173 asm volatile("rex64/fxsave %0"
174 : "=m" (fpu->state->fxsave));
175 This, however, we can work around by forcing the compiler to select
176 an addressing mode that doesn't require extended registers. */
177 asm volatile("rex64/fxsave (%[fx])"
178 : "=m" (fpu->state->fxsave)
179 : [fx] "R" (&fpu->state->fxsave));
180#endif
181} 182}
182 183
183#else /* CONFIG_X86_32 */ 184static inline int fxrstor_user(struct i387_fxsave_struct __user *fx)
185{
186 if (config_enabled(CONFIG_X86_32))
187 return user_insn(fxrstor %[fx], "=m" (*fx), [fx] "m" (*fx));
188 else if (config_enabled(CONFIG_AS_FXSAVEQ))
189 return user_insn(fxrstorq %[fx], "=m" (*fx), [fx] "m" (*fx));
184 190
185/* perform fxrstor iff the processor has extended states, otherwise frstor */ 191 /* See comment in fpu_fxsave() below. */
186static inline int fxrstor_checking(struct i387_fxsave_struct *fx) 192 return user_insn(rex64/fxrstor (%[fx]), "=m" (*fx), [fx] "R" (fx),
193 "m" (*fx));
194}
195
196static inline int frstor_checking(struct i387_fsave_struct *fx)
187{ 197{
188 /* 198 return check_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
189 * The "nop" is needed to make the instructions the same 199}
190 * length.
191 */
192 alternative_input(
193 "nop ; frstor %1",
194 "fxrstor %1",
195 X86_FEATURE_FXSR,
196 "m" (*fx));
197 200
198 return 0; 201static inline int frstor_user(struct i387_fsave_struct __user *fx)
202{
203 return user_insn(frstor %[fx], "=m" (*fx), [fx] "m" (*fx));
199} 204}
200 205
201static inline void fpu_fxsave(struct fpu *fpu) 206static inline void fpu_fxsave(struct fpu *fpu)
202{ 207{
203 asm volatile("fxsave %[fx]" 208 if (config_enabled(CONFIG_X86_32))
204 : [fx] "=m" (fpu->state->fxsave)); 209 asm volatile( "fxsave %[fx]" : [fx] "=m" (fpu->state->fxsave));
210 else if (config_enabled(CONFIG_AS_FXSAVEQ))
211 asm volatile("fxsaveq %0" : "=m" (fpu->state->fxsave));
212 else {
213 /* Using "rex64; fxsave %0" is broken because, if the memory
214 * operand uses any extended registers for addressing, a second
215 * REX prefix will be generated (to the assembler, rex64
216 * followed by semicolon is a separate instruction), and hence
217 * the 64-bitness is lost.
218 *
219 * Using "fxsaveq %0" would be the ideal choice, but is only
220 * supported starting with gas 2.16.
221 *
222 * Using, as a workaround, the properly prefixed form below
223 * isn't accepted by any binutils version so far released,
224 * complaining that the same type of prefix is used twice if
225 * an extended register is needed for addressing (fix submitted
226 * to mainline 2005-11-21).
227 *
228 * asm volatile("rex64/fxsave %0" : "=m" (fpu->state->fxsave));
229 *
230 * This, however, we can work around by forcing the compiler to
231 * select an addressing mode that doesn't require extended
232 * registers.
233 */
234 asm volatile( "rex64/fxsave (%[fx])"
235 : "=m" (fpu->state->fxsave)
236 : [fx] "R" (&fpu->state->fxsave));
237 }
205} 238}
206 239
207#endif /* CONFIG_X86_64 */
208
209/* 240/*
210 * These must be called with preempt disabled. Returns 241 * These must be called with preempt disabled. Returns
211 * 'true' if the FPU state is still intact. 242 * 'true' if the FPU state is still intact.
@@ -248,17 +279,14 @@ static inline int __save_init_fpu(struct task_struct *tsk)
248 return fpu_save_init(&tsk->thread.fpu); 279 return fpu_save_init(&tsk->thread.fpu);
249} 280}
250 281
251static inline int fpu_fxrstor_checking(struct fpu *fpu)
252{
253 return fxrstor_checking(&fpu->state->fxsave);
254}
255
256static inline int fpu_restore_checking(struct fpu *fpu) 282static inline int fpu_restore_checking(struct fpu *fpu)
257{ 283{
258 if (use_xsave()) 284 if (use_xsave())
259 return fpu_xrstor_checking(fpu); 285 return fpu_xrstor_checking(&fpu->state->xsave);
286 else if (use_fxsr())
287 return fxrstor_checking(&fpu->state->fxsave);
260 else 288 else
261 return fpu_fxrstor_checking(fpu); 289 return frstor_checking(&fpu->state->fsave);
262} 290}
263 291
264static inline int restore_fpu_checking(struct task_struct *tsk) 292static inline int restore_fpu_checking(struct task_struct *tsk)
@@ -310,15 +338,52 @@ static inline void __thread_set_has_fpu(struct task_struct *tsk)
310static inline void __thread_fpu_end(struct task_struct *tsk) 338static inline void __thread_fpu_end(struct task_struct *tsk)
311{ 339{
312 __thread_clear_has_fpu(tsk); 340 __thread_clear_has_fpu(tsk);
313 stts(); 341 if (!use_eager_fpu())
342 stts();
314} 343}
315 344
316static inline void __thread_fpu_begin(struct task_struct *tsk) 345static inline void __thread_fpu_begin(struct task_struct *tsk)
317{ 346{
318 clts(); 347 if (!use_eager_fpu())
348 clts();
319 __thread_set_has_fpu(tsk); 349 __thread_set_has_fpu(tsk);
320} 350}
321 351
352static inline void __drop_fpu(struct task_struct *tsk)
353{
354 if (__thread_has_fpu(tsk)) {
355 /* Ignore delayed exceptions from user space */
356 asm volatile("1: fwait\n"
357 "2:\n"
358 _ASM_EXTABLE(1b, 2b));
359 __thread_fpu_end(tsk);
360 }
361}
362
363static inline void drop_fpu(struct task_struct *tsk)
364{
365 /*
366 * Forget coprocessor state..
367 */
368 preempt_disable();
369 tsk->fpu_counter = 0;
370 __drop_fpu(tsk);
371 clear_used_math();
372 preempt_enable();
373}
374
375static inline void drop_init_fpu(struct task_struct *tsk)
376{
377 if (!use_eager_fpu())
378 drop_fpu(tsk);
379 else {
380 if (use_xsave())
381 xrstor_state(init_xstate_buf, -1);
382 else
383 fxrstor_checking(&init_xstate_buf->i387);
384 }
385}
386
322/* 387/*
323 * FPU state switching for scheduling. 388 * FPU state switching for scheduling.
324 * 389 *
@@ -352,7 +417,12 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta
352{ 417{
353 fpu_switch_t fpu; 418 fpu_switch_t fpu;
354 419
355 fpu.preload = tsk_used_math(new) && new->fpu_counter > 5; 420 /*
421 * If the task has used the math, pre-load the FPU on xsave processors
422 * or if the past 5 consecutive context-switches used math.
423 */
424 fpu.preload = tsk_used_math(new) && (use_eager_fpu() ||
425 new->fpu_counter > 5);
356 if (__thread_has_fpu(old)) { 426 if (__thread_has_fpu(old)) {
357 if (!__save_init_fpu(old)) 427 if (!__save_init_fpu(old))
358 cpu = ~0; 428 cpu = ~0;
@@ -364,14 +434,14 @@ static inline fpu_switch_t switch_fpu_prepare(struct task_struct *old, struct ta
364 new->fpu_counter++; 434 new->fpu_counter++;
365 __thread_set_has_fpu(new); 435 __thread_set_has_fpu(new);
366 prefetch(new->thread.fpu.state); 436 prefetch(new->thread.fpu.state);
367 } else 437 } else if (!use_eager_fpu())
368 stts(); 438 stts();
369 } else { 439 } else {
370 old->fpu_counter = 0; 440 old->fpu_counter = 0;
371 old->thread.fpu.last_cpu = ~0; 441 old->thread.fpu.last_cpu = ~0;
372 if (fpu.preload) { 442 if (fpu.preload) {
373 new->fpu_counter++; 443 new->fpu_counter++;
374 if (fpu_lazy_restore(new, cpu)) 444 if (!use_eager_fpu() && fpu_lazy_restore(new, cpu))
375 fpu.preload = 0; 445 fpu.preload = 0;
376 else 446 else
377 prefetch(new->thread.fpu.state); 447 prefetch(new->thread.fpu.state);
@@ -391,44 +461,40 @@ static inline void switch_fpu_finish(struct task_struct *new, fpu_switch_t fpu)
391{ 461{
392 if (fpu.preload) { 462 if (fpu.preload) {
393 if (unlikely(restore_fpu_checking(new))) 463 if (unlikely(restore_fpu_checking(new)))
394 __thread_fpu_end(new); 464 drop_init_fpu(new);
395 } 465 }
396} 466}
397 467
398/* 468/*
399 * Signal frame handlers... 469 * Signal frame handlers...
400 */ 470 */
401extern int save_i387_xstate(void __user *buf); 471extern int save_xstate_sig(void __user *buf, void __user *fx, int size);
402extern int restore_i387_xstate(void __user *buf); 472extern int __restore_xstate_sig(void __user *buf, void __user *fx, int size);
403 473
404static inline void __clear_fpu(struct task_struct *tsk) 474static inline int xstate_sigframe_size(void)
405{ 475{
406 if (__thread_has_fpu(tsk)) { 476 return use_xsave() ? xstate_size + FP_XSTATE_MAGIC2_SIZE : xstate_size;
407 /* Ignore delayed exceptions from user space */ 477}
408 asm volatile("1: fwait\n" 478
409 "2:\n" 479static inline int restore_xstate_sig(void __user *buf, int ia32_frame)
410 _ASM_EXTABLE(1b, 2b)); 480{
411 __thread_fpu_end(tsk); 481 void __user *buf_fx = buf;
482 int size = xstate_sigframe_size();
483
484 if (ia32_frame && use_fxsr()) {
485 buf_fx = buf + sizeof(struct i387_fsave_struct);
486 size += sizeof(struct i387_fsave_struct);
412 } 487 }
488
489 return __restore_xstate_sig(buf, buf_fx, size);
413} 490}
414 491
415/* 492/*
416 * The actual user_fpu_begin/end() functions 493 * Need to be preemption-safe.
417 * need to be preemption-safe.
418 * 494 *
419 * NOTE! user_fpu_end() must be used only after you 495 * NOTE! user_fpu_begin() must be used only immediately before restoring
420 * have saved the FP state, and user_fpu_begin() must 496 * it. This function does not do any save/restore on their own.
421 * be used only immediately before restoring it.
422 * These functions do not do any save/restore on
423 * their own.
424 */ 497 */
425static inline void user_fpu_end(void)
426{
427 preempt_disable();
428 __thread_fpu_end(current);
429 preempt_enable();
430}
431
432static inline void user_fpu_begin(void) 498static inline void user_fpu_begin(void)
433{ 499{
434 preempt_disable(); 500 preempt_disable();
@@ -437,25 +503,32 @@ static inline void user_fpu_begin(void)
437 preempt_enable(); 503 preempt_enable();
438} 504}
439 505
506static inline void __save_fpu(struct task_struct *tsk)
507{
508 if (use_xsave())
509 xsave_state(&tsk->thread.fpu.state->xsave, -1);
510 else
511 fpu_fxsave(&tsk->thread.fpu);
512}
513
440/* 514/*
441 * These disable preemption on their own and are safe 515 * These disable preemption on their own and are safe
442 */ 516 */
443static inline void save_init_fpu(struct task_struct *tsk) 517static inline void save_init_fpu(struct task_struct *tsk)
444{ 518{
445 WARN_ON_ONCE(!__thread_has_fpu(tsk)); 519 WARN_ON_ONCE(!__thread_has_fpu(tsk));
520
521 if (use_eager_fpu()) {
522 __save_fpu(tsk);
523 return;
524 }
525
446 preempt_disable(); 526 preempt_disable();
447 __save_init_fpu(tsk); 527 __save_init_fpu(tsk);
448 __thread_fpu_end(tsk); 528 __thread_fpu_end(tsk);
449 preempt_enable(); 529 preempt_enable();
450} 530}
451 531
452static inline void clear_fpu(struct task_struct *tsk)
453{
454 preempt_disable();
455 __clear_fpu(tsk);
456 preempt_enable();
457}
458
459/* 532/*
460 * i387 state interaction 533 * i387 state interaction
461 */ 534 */
@@ -510,11 +583,34 @@ static inline void fpu_free(struct fpu *fpu)
510 } 583 }
511} 584}
512 585
513static inline void fpu_copy(struct fpu *dst, struct fpu *src) 586static inline void fpu_copy(struct task_struct *dst, struct task_struct *src)
514{ 587{
515 memcpy(dst->state, src->state, xstate_size); 588 if (use_eager_fpu()) {
589 memset(&dst->thread.fpu.state->xsave, 0, xstate_size);
590 __save_fpu(dst);
591 } else {
592 struct fpu *dfpu = &dst->thread.fpu;
593 struct fpu *sfpu = &src->thread.fpu;
594
595 unlazy_fpu(src);
596 memcpy(dfpu->state, sfpu->state, xstate_size);
597 }
516} 598}
517 599
518extern void fpu_finit(struct fpu *fpu); 600static inline unsigned long
601alloc_mathframe(unsigned long sp, int ia32_frame, unsigned long *buf_fx,
602 unsigned long *size)
603{
604 unsigned long frame_size = xstate_sigframe_size();
605
606 *buf_fx = sp = round_down(sp - frame_size, 64);
607 if (ia32_frame && use_fxsr()) {
608 frame_size += sizeof(struct i387_fsave_struct);
609 sp -= sizeof(struct i387_fsave_struct);
610 }
611
612 *size = frame_size;
613 return sp;
614}
519 615
520#endif 616#endif
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
index b0767bc08740..9a25b522d377 100644
--- a/arch/x86/include/asm/ftrace.h
+++ b/arch/x86/include/asm/ftrace.h
@@ -3,38 +3,54 @@
3 3
4#ifdef __ASSEMBLY__ 4#ifdef __ASSEMBLY__
5 5
6 .macro MCOUNT_SAVE_FRAME 6 /* skip is set if the stack was already partially adjusted */
7 /* taken from glibc */ 7 .macro MCOUNT_SAVE_FRAME skip=0
8 subq $0x38, %rsp 8 /*
9 movq %rax, (%rsp) 9 * We add enough stack to save all regs.
10 movq %rcx, 8(%rsp) 10 */
11 movq %rdx, 16(%rsp) 11 subq $(SS+8-\skip), %rsp
12 movq %rsi, 24(%rsp) 12 movq %rax, RAX(%rsp)
13 movq %rdi, 32(%rsp) 13 movq %rcx, RCX(%rsp)
14 movq %r8, 40(%rsp) 14 movq %rdx, RDX(%rsp)
15 movq %r9, 48(%rsp) 15 movq %rsi, RSI(%rsp)
16 movq %rdi, RDI(%rsp)
17 movq %r8, R8(%rsp)
18 movq %r9, R9(%rsp)
19 /* Move RIP to its proper location */
20 movq SS+8(%rsp), %rdx
21 movq %rdx, RIP(%rsp)
16 .endm 22 .endm
17 23
18 .macro MCOUNT_RESTORE_FRAME 24 .macro MCOUNT_RESTORE_FRAME skip=0
19 movq 48(%rsp), %r9 25 movq R9(%rsp), %r9
20 movq 40(%rsp), %r8 26 movq R8(%rsp), %r8
21 movq 32(%rsp), %rdi 27 movq RDI(%rsp), %rdi
22 movq 24(%rsp), %rsi 28 movq RSI(%rsp), %rsi
23 movq 16(%rsp), %rdx 29 movq RDX(%rsp), %rdx
24 movq 8(%rsp), %rcx 30 movq RCX(%rsp), %rcx
25 movq (%rsp), %rax 31 movq RAX(%rsp), %rax
26 addq $0x38, %rsp 32 addq $(SS+8-\skip), %rsp
27 .endm 33 .endm
28 34
29#endif 35#endif
30 36
31#ifdef CONFIG_FUNCTION_TRACER 37#ifdef CONFIG_FUNCTION_TRACER
32#define MCOUNT_ADDR ((long)(mcount)) 38#ifdef CC_USING_FENTRY
39# define MCOUNT_ADDR ((long)(__fentry__))
40#else
41# define MCOUNT_ADDR ((long)(mcount))
42#endif
33#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */ 43#define MCOUNT_INSN_SIZE 5 /* sizeof mcount call */
34 44
45#ifdef CONFIG_DYNAMIC_FTRACE
46#define ARCH_SUPPORTS_FTRACE_OPS 1
47#define ARCH_SUPPORTS_FTRACE_SAVE_REGS
48#endif
49
35#ifndef __ASSEMBLY__ 50#ifndef __ASSEMBLY__
36extern void mcount(void); 51extern void mcount(void);
37extern atomic_t modifying_ftrace_code; 52extern atomic_t modifying_ftrace_code;
53extern void __fentry__(void);
38 54
39static inline unsigned long ftrace_call_adjust(unsigned long addr) 55static inline unsigned long ftrace_call_adjust(unsigned long addr)
40{ 56{
diff --git a/arch/x86/include/asm/futex.h b/arch/x86/include/asm/futex.h
index 71ecbcba1a4e..f373046e63ec 100644
--- a/arch/x86/include/asm/futex.h
+++ b/arch/x86/include/asm/futex.h
@@ -9,10 +9,13 @@
9#include <asm/asm.h> 9#include <asm/asm.h>
10#include <asm/errno.h> 10#include <asm/errno.h>
11#include <asm/processor.h> 11#include <asm/processor.h>
12#include <asm/smap.h>
12 13
13#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \ 14#define __futex_atomic_op1(insn, ret, oldval, uaddr, oparg) \
14 asm volatile("1:\t" insn "\n" \ 15 asm volatile("\t" ASM_STAC "\n" \
15 "2:\t.section .fixup,\"ax\"\n" \ 16 "1:\t" insn "\n" \
17 "2:\t" ASM_CLAC "\n" \
18 "\t.section .fixup,\"ax\"\n" \
16 "3:\tmov\t%3, %1\n" \ 19 "3:\tmov\t%3, %1\n" \
17 "\tjmp\t2b\n" \ 20 "\tjmp\t2b\n" \
18 "\t.previous\n" \ 21 "\t.previous\n" \
@@ -21,12 +24,14 @@
21 : "i" (-EFAULT), "0" (oparg), "1" (0)) 24 : "i" (-EFAULT), "0" (oparg), "1" (0))
22 25
23#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \ 26#define __futex_atomic_op2(insn, ret, oldval, uaddr, oparg) \
24 asm volatile("1:\tmovl %2, %0\n" \ 27 asm volatile("\t" ASM_STAC "\n" \
28 "1:\tmovl %2, %0\n" \
25 "\tmovl\t%0, %3\n" \ 29 "\tmovl\t%0, %3\n" \
26 "\t" insn "\n" \ 30 "\t" insn "\n" \
27 "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \ 31 "2:\t" LOCK_PREFIX "cmpxchgl %3, %2\n" \
28 "\tjnz\t1b\n" \ 32 "\tjnz\t1b\n" \
29 "3:\t.section .fixup,\"ax\"\n" \ 33 "3:\t" ASM_CLAC "\n" \
34 "\t.section .fixup,\"ax\"\n" \
30 "4:\tmov\t%5, %1\n" \ 35 "4:\tmov\t%5, %1\n" \
31 "\tjmp\t3b\n" \ 36 "\tjmp\t3b\n" \
32 "\t.previous\n" \ 37 "\t.previous\n" \
@@ -122,8 +127,10 @@ static inline int futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
122 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) 127 if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
123 return -EFAULT; 128 return -EFAULT;
124 129
125 asm volatile("1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" 130 asm volatile("\t" ASM_STAC "\n"
126 "2:\t.section .fixup, \"ax\"\n" 131 "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n"
132 "2:\t" ASM_CLAC "\n"
133 "\t.section .fixup, \"ax\"\n"
127 "3:\tmov %3, %0\n" 134 "3:\tmov %3, %0\n"
128 "\tjmp 2b\n" 135 "\tjmp 2b\n"
129 "\t.previous\n" 136 "\t.previous\n"
diff --git a/arch/x86/include/asm/hardirq.h b/arch/x86/include/asm/hardirq.h
index d3895dbf4ddb..81f04cee5f74 100644
--- a/arch/x86/include/asm/hardirq.h
+++ b/arch/x86/include/asm/hardirq.h
@@ -18,6 +18,10 @@ typedef struct {
18#ifdef CONFIG_SMP 18#ifdef CONFIG_SMP
19 unsigned int irq_resched_count; 19 unsigned int irq_resched_count;
20 unsigned int irq_call_count; 20 unsigned int irq_call_count;
21 /*
22 * irq_tlb_count is double-counted in irq_call_count, so it must be
23 * subtracted from irq_call_count when displaying irq_call_count
24 */
21 unsigned int irq_tlb_count; 25 unsigned int irq_tlb_count;
22#endif 26#endif
23#ifdef CONFIG_X86_THERMAL_VECTOR 27#ifdef CONFIG_X86_THERMAL_VECTOR
diff --git a/arch/x86/include/asm/hpet.h b/arch/x86/include/asm/hpet.h
index 2c392d663dce..434e2106cc87 100644
--- a/arch/x86/include/asm/hpet.h
+++ b/arch/x86/include/asm/hpet.h
@@ -35,8 +35,6 @@
35#define HPET_ID_NUMBER_SHIFT 8 35#define HPET_ID_NUMBER_SHIFT 8
36#define HPET_ID_VENDOR_SHIFT 16 36#define HPET_ID_VENDOR_SHIFT 16
37 37
38#define HPET_ID_VENDOR_8086 0x8086
39
40#define HPET_CFG_ENABLE 0x001 38#define HPET_CFG_ENABLE 0x001
41#define HPET_CFG_LEGACY 0x002 39#define HPET_CFG_LEGACY 0x002
42#define HPET_LEGACY_8254 2 40#define HPET_LEGACY_8254 2
diff --git a/arch/x86/include/asm/hugetlb.h b/arch/x86/include/asm/hugetlb.h
index 439a9acc132d..bdd35dbd0605 100644
--- a/arch/x86/include/asm/hugetlb.h
+++ b/arch/x86/include/asm/hugetlb.h
@@ -90,4 +90,8 @@ static inline void arch_release_hugepage(struct page *page)
90{ 90{
91} 91}
92 92
93static inline void arch_clear_hugepage_flags(struct page *page)
94{
95}
96
93#endif /* _ASM_X86_HUGETLB_H */ 97#endif /* _ASM_X86_HUGETLB_H */
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
index 257d9cca214f..ed8089d69094 100644
--- a/arch/x86/include/asm/i387.h
+++ b/arch/x86/include/asm/i387.h
@@ -19,12 +19,37 @@ struct pt_regs;
19struct user_i387_struct; 19struct user_i387_struct;
20 20
21extern int init_fpu(struct task_struct *child); 21extern int init_fpu(struct task_struct *child);
22extern void fpu_finit(struct fpu *fpu);
22extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); 23extern int dump_fpu(struct pt_regs *, struct user_i387_struct *);
23extern void math_state_restore(void); 24extern void math_state_restore(void);
24 25
25extern bool irq_fpu_usable(void); 26extern bool irq_fpu_usable(void);
26extern void kernel_fpu_begin(void); 27
27extern void kernel_fpu_end(void); 28/*
29 * Careful: __kernel_fpu_begin/end() must be called with preempt disabled
30 * and they don't touch the preempt state on their own.
31 * If you enable preemption after __kernel_fpu_begin(), preempt notifier
32 * should call the __kernel_fpu_end() to prevent the kernel/user FPU
33 * state from getting corrupted. KVM for example uses this model.
34 *
35 * All other cases use kernel_fpu_begin/end() which disable preemption
36 * during kernel FPU usage.
37 */
38extern void __kernel_fpu_begin(void);
39extern void __kernel_fpu_end(void);
40
41static inline void kernel_fpu_begin(void)
42{
43 WARN_ON_ONCE(!irq_fpu_usable());
44 preempt_disable();
45 __kernel_fpu_begin();
46}
47
48static inline void kernel_fpu_end(void)
49{
50 __kernel_fpu_end();
51 preempt_enable();
52}
28 53
29/* 54/*
30 * Some instructions like VIA's padlock instructions generate a spurious 55 * Some instructions like VIA's padlock instructions generate a spurious
diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h
index b04cbdb138cd..e6232773ce49 100644
--- a/arch/x86/include/asm/ia32.h
+++ b/arch/x86/include/asm/ia32.h
@@ -86,73 +86,6 @@ struct stat64 {
86 unsigned long long st_ino; 86 unsigned long long st_ino;
87} __attribute__((packed)); 87} __attribute__((packed));
88 88
89typedef struct compat_siginfo {
90 int si_signo;
91 int si_errno;
92 int si_code;
93
94 union {
95 int _pad[((128 / sizeof(int)) - 3)];
96
97 /* kill() */
98 struct {
99 unsigned int _pid; /* sender's pid */
100 unsigned int _uid; /* sender's uid */
101 } _kill;
102
103 /* POSIX.1b timers */
104 struct {
105 compat_timer_t _tid; /* timer id */
106 int _overrun; /* overrun count */
107 compat_sigval_t _sigval; /* same as below */
108 int _sys_private; /* not to be passed to user */
109 int _overrun_incr; /* amount to add to overrun */
110 } _timer;
111
112 /* POSIX.1b signals */
113 struct {
114 unsigned int _pid; /* sender's pid */
115 unsigned int _uid; /* sender's uid */
116 compat_sigval_t _sigval;
117 } _rt;
118
119 /* SIGCHLD */
120 struct {
121 unsigned int _pid; /* which child */
122 unsigned int _uid; /* sender's uid */
123 int _status; /* exit code */
124 compat_clock_t _utime;
125 compat_clock_t _stime;
126 } _sigchld;
127
128 /* SIGCHLD (x32 version) */
129 struct {
130 unsigned int _pid; /* which child */
131 unsigned int _uid; /* sender's uid */
132 int _status; /* exit code */
133 compat_s64 _utime;
134 compat_s64 _stime;
135 } _sigchld_x32;
136
137 /* SIGILL, SIGFPE, SIGSEGV, SIGBUS */
138 struct {
139 unsigned int _addr; /* faulting insn/memory ref. */
140 } _sigfault;
141
142 /* SIGPOLL */
143 struct {
144 int _band; /* POLL_IN, POLL_OUT, POLL_MSG */
145 int _fd;
146 } _sigpoll;
147
148 struct {
149 unsigned int _call_addr; /* calling insn */
150 int _syscall; /* triggering system call number */
151 unsigned int _arch; /* AUDIT_ARCH_* of syscall */
152 } _sigsys;
153 } _sifields;
154} compat_siginfo_t;
155
156#define IA32_STACK_TOP IA32_PAGE_OFFSET 89#define IA32_STACK_TOP IA32_PAGE_OFFSET
157 90
158#ifdef __KERNEL__ 91#ifdef __KERNEL__
diff --git a/arch/x86/include/asm/iommu_table.h b/arch/x86/include/asm/iommu_table.h
index f229b13a5f30..f42a04735a0a 100644
--- a/arch/x86/include/asm/iommu_table.h
+++ b/arch/x86/include/asm/iommu_table.h
@@ -48,7 +48,7 @@ struct iommu_table_entry {
48 48
49 49
50#define __IOMMU_INIT(_detect, _depend, _early_init, _late_init, _finish)\ 50#define __IOMMU_INIT(_detect, _depend, _early_init, _late_init, _finish)\
51 static const struct iommu_table_entry const \ 51 static const struct iommu_table_entry \
52 __iommu_entry_##_detect __used \ 52 __iommu_entry_##_detect __used \
53 __attribute__ ((unused, __section__(".iommu_table"), \ 53 __attribute__ ((unused, __section__(".iommu_table"), \
54 aligned((sizeof(void *))))) \ 54 aligned((sizeof(void *))))) \
@@ -63,10 +63,10 @@ struct iommu_table_entry {
63 * to stop detecting the other IOMMUs after yours has been detected. 63 * to stop detecting the other IOMMUs after yours has been detected.
64 */ 64 */
65#define IOMMU_INIT_POST(_detect) \ 65#define IOMMU_INIT_POST(_detect) \
66 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, 0, 0, 0) 66 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, NULL, NULL, 0)
67 67
68#define IOMMU_INIT_POST_FINISH(detect) \ 68#define IOMMU_INIT_POST_FINISH(detect) \
69 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, 0, 0, 1) 69 __IOMMU_INIT(_detect, pci_swiotlb_detect_4gb, NULL, NULL, 1)
70 70
71/* 71/*
72 * A more sophisticated version of IOMMU_INIT. This variant requires: 72 * A more sophisticated version of IOMMU_INIT. This variant requires:
diff --git a/arch/x86/include/asm/kprobes.h b/arch/x86/include/asm/kprobes.h
index 547882539157..d3ddd17405d0 100644
--- a/arch/x86/include/asm/kprobes.h
+++ b/arch/x86/include/asm/kprobes.h
@@ -27,6 +27,7 @@
27#include <asm/insn.h> 27#include <asm/insn.h>
28 28
29#define __ARCH_WANT_KPROBES_INSN_SLOT 29#define __ARCH_WANT_KPROBES_INSN_SLOT
30#define ARCH_SUPPORTS_KPROBES_ON_FTRACE
30 31
31struct pt_regs; 32struct pt_regs;
32struct kprobe; 33struct kprobe;
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h
index 246617efd67f..a65ec29e6ffb 100644
--- a/arch/x86/include/asm/kvm.h
+++ b/arch/x86/include/asm/kvm.h
@@ -9,6 +9,22 @@
9#include <linux/types.h> 9#include <linux/types.h>
10#include <linux/ioctl.h> 10#include <linux/ioctl.h>
11 11
12#define DE_VECTOR 0
13#define DB_VECTOR 1
14#define BP_VECTOR 3
15#define OF_VECTOR 4
16#define BR_VECTOR 5
17#define UD_VECTOR 6
18#define NM_VECTOR 7
19#define DF_VECTOR 8
20#define TS_VECTOR 10
21#define NP_VECTOR 11
22#define SS_VECTOR 12
23#define GP_VECTOR 13
24#define PF_VECTOR 14
25#define MF_VECTOR 16
26#define MC_VECTOR 18
27
12/* Select x86 specific features in <linux/kvm.h> */ 28/* Select x86 specific features in <linux/kvm.h> */
13#define __KVM_HAVE_PIT 29#define __KVM_HAVE_PIT
14#define __KVM_HAVE_IOAPIC 30#define __KVM_HAVE_IOAPIC
@@ -25,6 +41,7 @@
25#define __KVM_HAVE_DEBUGREGS 41#define __KVM_HAVE_DEBUGREGS
26#define __KVM_HAVE_XSAVE 42#define __KVM_HAVE_XSAVE
27#define __KVM_HAVE_XCRS 43#define __KVM_HAVE_XCRS
44#define __KVM_HAVE_READONLY_MEM
28 45
29/* Architectural interrupt line count. */ 46/* Architectural interrupt line count. */
30#define KVM_NR_INTERRUPTS 256 47#define KVM_NR_INTERRUPTS 256
diff --git a/arch/x86/include/asm/kvm_emulate.h b/arch/x86/include/asm/kvm_emulate.h
index c764f43b71c5..15f960c06ff7 100644
--- a/arch/x86/include/asm/kvm_emulate.h
+++ b/arch/x86/include/asm/kvm_emulate.h
@@ -86,6 +86,19 @@ struct x86_instruction_info {
86 86
87struct x86_emulate_ops { 87struct x86_emulate_ops {
88 /* 88 /*
89 * read_gpr: read a general purpose register (rax - r15)
90 *
91 * @reg: gpr number.
92 */
93 ulong (*read_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg);
94 /*
95 * write_gpr: write a general purpose register (rax - r15)
96 *
97 * @reg: gpr number.
98 * @val: value to write.
99 */
100 void (*write_gpr)(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val);
101 /*
89 * read_std: Read bytes of standard (non-emulated/special) memory. 102 * read_std: Read bytes of standard (non-emulated/special) memory.
90 * Used for descriptor reading. 103 * Used for descriptor reading.
91 * @addr: [IN ] Linear address from which to read. 104 * @addr: [IN ] Linear address from which to read.
@@ -200,8 +213,9 @@ typedef u32 __attribute__((vector_size(16))) sse128_t;
200 213
201/* Type, address-of, and value of an instruction's operand. */ 214/* Type, address-of, and value of an instruction's operand. */
202struct operand { 215struct operand {
203 enum { OP_REG, OP_MEM, OP_IMM, OP_XMM, OP_MM, OP_NONE } type; 216 enum { OP_REG, OP_MEM, OP_MEM_STR, OP_IMM, OP_XMM, OP_MM, OP_NONE } type;
204 unsigned int bytes; 217 unsigned int bytes;
218 unsigned int count;
205 union { 219 union {
206 unsigned long orig_val; 220 unsigned long orig_val;
207 u64 orig_val64; 221 u64 orig_val64;
@@ -221,6 +235,7 @@ struct operand {
221 char valptr[sizeof(unsigned long) + 2]; 235 char valptr[sizeof(unsigned long) + 2];
222 sse128_t vec_val; 236 sse128_t vec_val;
223 u64 mm_val; 237 u64 mm_val;
238 void *data;
224 }; 239 };
225}; 240};
226 241
@@ -236,14 +251,23 @@ struct read_cache {
236 unsigned long end; 251 unsigned long end;
237}; 252};
238 253
254/* Execution mode, passed to the emulator. */
255enum x86emul_mode {
256 X86EMUL_MODE_REAL, /* Real mode. */
257 X86EMUL_MODE_VM86, /* Virtual 8086 mode. */
258 X86EMUL_MODE_PROT16, /* 16-bit protected mode. */
259 X86EMUL_MODE_PROT32, /* 32-bit protected mode. */
260 X86EMUL_MODE_PROT64, /* 64-bit (long) mode. */
261};
262
239struct x86_emulate_ctxt { 263struct x86_emulate_ctxt {
240 struct x86_emulate_ops *ops; 264 const struct x86_emulate_ops *ops;
241 265
242 /* Register state before/after emulation. */ 266 /* Register state before/after emulation. */
243 unsigned long eflags; 267 unsigned long eflags;
244 unsigned long eip; /* eip before instruction emulation */ 268 unsigned long eip; /* eip before instruction emulation */
245 /* Emulated execution mode, represented by an X86EMUL_MODE value. */ 269 /* Emulated execution mode, represented by an X86EMUL_MODE value. */
246 int mode; 270 enum x86emul_mode mode;
247 271
248 /* interruptibility state, as a result of execution of STI or MOV SS */ 272 /* interruptibility state, as a result of execution of STI or MOV SS */
249 int interruptibility; 273 int interruptibility;
@@ -281,8 +305,10 @@ struct x86_emulate_ctxt {
281 bool rip_relative; 305 bool rip_relative;
282 unsigned long _eip; 306 unsigned long _eip;
283 struct operand memop; 307 struct operand memop;
308 u32 regs_valid; /* bitmaps of registers in _regs[] that can be read */
309 u32 regs_dirty; /* bitmaps of registers in _regs[] that have been written */
284 /* Fields above regs are cleared together. */ 310 /* Fields above regs are cleared together. */
285 unsigned long regs[NR_VCPU_REGS]; 311 unsigned long _regs[NR_VCPU_REGS];
286 struct operand *memopp; 312 struct operand *memopp;
287 struct fetch_cache fetch; 313 struct fetch_cache fetch;
288 struct read_cache io_read; 314 struct read_cache io_read;
@@ -293,17 +319,6 @@ struct x86_emulate_ctxt {
293#define REPE_PREFIX 0xf3 319#define REPE_PREFIX 0xf3
294#define REPNE_PREFIX 0xf2 320#define REPNE_PREFIX 0xf2
295 321
296/* Execution mode, passed to the emulator. */
297#define X86EMUL_MODE_REAL 0 /* Real mode. */
298#define X86EMUL_MODE_VM86 1 /* Virtual 8086 mode. */
299#define X86EMUL_MODE_PROT16 2 /* 16-bit protected mode. */
300#define X86EMUL_MODE_PROT32 4 /* 32-bit protected mode. */
301#define X86EMUL_MODE_PROT64 8 /* 64-bit (long) mode. */
302
303/* any protected mode */
304#define X86EMUL_MODE_PROT (X86EMUL_MODE_PROT16|X86EMUL_MODE_PROT32| \
305 X86EMUL_MODE_PROT64)
306
307/* CPUID vendors */ 322/* CPUID vendors */
308#define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541 323#define X86EMUL_CPUID_VENDOR_AuthenticAMD_ebx 0x68747541
309#define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163 324#define X86EMUL_CPUID_VENDOR_AuthenticAMD_ecx 0x444d4163
@@ -394,4 +409,7 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
394 u16 tss_selector, int idt_index, int reason, 409 u16 tss_selector, int idt_index, int reason,
395 bool has_error_code, u32 error_code); 410 bool has_error_code, u32 error_code);
396int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq); 411int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq);
412void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt);
413void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt);
414
397#endif /* _ASM_X86_KVM_X86_EMULATE_H */ 415#endif /* _ASM_X86_KVM_X86_EMULATE_H */
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 09155d64cf7e..b2e11f452435 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -75,22 +75,6 @@
75#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1)) 75#define KVM_HPAGE_MASK(x) (~(KVM_HPAGE_SIZE(x) - 1))
76#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE) 76#define KVM_PAGES_PER_HPAGE(x) (KVM_HPAGE_SIZE(x) / PAGE_SIZE)
77 77
78#define DE_VECTOR 0
79#define DB_VECTOR 1
80#define BP_VECTOR 3
81#define OF_VECTOR 4
82#define BR_VECTOR 5
83#define UD_VECTOR 6
84#define NM_VECTOR 7
85#define DF_VECTOR 8
86#define TS_VECTOR 10
87#define NP_VECTOR 11
88#define SS_VECTOR 12
89#define GP_VECTOR 13
90#define PF_VECTOR 14
91#define MF_VECTOR 16
92#define MC_VECTOR 18
93
94#define SELECTOR_TI_MASK (1 << 2) 78#define SELECTOR_TI_MASK (1 << 2)
95#define SELECTOR_RPL_MASK 0x03 79#define SELECTOR_RPL_MASK 0x03
96 80
@@ -287,10 +271,24 @@ struct kvm_mmu {
287 union kvm_mmu_page_role base_role; 271 union kvm_mmu_page_role base_role;
288 bool direct_map; 272 bool direct_map;
289 273
274 /*
275 * Bitmap; bit set = permission fault
276 * Byte index: page fault error code [4:1]
277 * Bit index: pte permissions in ACC_* format
278 */
279 u8 permissions[16];
280
290 u64 *pae_root; 281 u64 *pae_root;
291 u64 *lm_root; 282 u64 *lm_root;
292 u64 rsvd_bits_mask[2][4]; 283 u64 rsvd_bits_mask[2][4];
293 284
285 /*
286 * Bitmap: bit set = last pte in walk
287 * index[0:1]: level (zero-based)
288 * index[2]: pte.ps
289 */
290 u8 last_pte_bitmap;
291
294 bool nx; 292 bool nx;
295 293
296 u64 pdptrs[4]; /* pae */ 294 u64 pdptrs[4]; /* pae */
@@ -414,12 +412,15 @@ struct kvm_vcpu_arch {
414 struct x86_emulate_ctxt emulate_ctxt; 412 struct x86_emulate_ctxt emulate_ctxt;
415 bool emulate_regs_need_sync_to_vcpu; 413 bool emulate_regs_need_sync_to_vcpu;
416 bool emulate_regs_need_sync_from_vcpu; 414 bool emulate_regs_need_sync_from_vcpu;
415 int (*complete_userspace_io)(struct kvm_vcpu *vcpu);
417 416
418 gpa_t time; 417 gpa_t time;
419 struct pvclock_vcpu_time_info hv_clock; 418 struct pvclock_vcpu_time_info hv_clock;
420 unsigned int hw_tsc_khz; 419 unsigned int hw_tsc_khz;
421 unsigned int time_offset; 420 unsigned int time_offset;
422 struct page *time_page; 421 struct page *time_page;
422 /* set guest stopped flag in pvclock flags field */
423 bool pvclock_set_guest_stopped_request;
423 424
424 struct { 425 struct {
425 u64 msr_val; 426 u64 msr_val;
@@ -454,6 +455,7 @@ struct kvm_vcpu_arch {
454 unsigned long dr6; 455 unsigned long dr6;
455 unsigned long dr7; 456 unsigned long dr7;
456 unsigned long eff_db[KVM_NR_DB_REGS]; 457 unsigned long eff_db[KVM_NR_DB_REGS];
458 unsigned long guest_debug_dr7;
457 459
458 u64 mcg_cap; 460 u64 mcg_cap;
459 u64 mcg_status; 461 u64 mcg_status;
@@ -500,14 +502,24 @@ struct kvm_vcpu_arch {
500}; 502};
501 503
502struct kvm_lpage_info { 504struct kvm_lpage_info {
503 unsigned long rmap_pde;
504 int write_count; 505 int write_count;
505}; 506};
506 507
507struct kvm_arch_memory_slot { 508struct kvm_arch_memory_slot {
509 unsigned long *rmap[KVM_NR_PAGE_SIZES];
508 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1]; 510 struct kvm_lpage_info *lpage_info[KVM_NR_PAGE_SIZES - 1];
509}; 511};
510 512
513struct kvm_apic_map {
514 struct rcu_head rcu;
515 u8 ldr_bits;
516 /* fields bellow are used to decode ldr values in different modes */
517 u32 cid_shift, cid_mask, lid_mask;
518 struct kvm_lapic *phys_map[256];
519 /* first index is cluster id second is cpu id in a cluster */
520 struct kvm_lapic *logical_map[16][16];
521};
522
511struct kvm_arch { 523struct kvm_arch {
512 unsigned int n_used_mmu_pages; 524 unsigned int n_used_mmu_pages;
513 unsigned int n_requested_mmu_pages; 525 unsigned int n_requested_mmu_pages;
@@ -525,6 +537,8 @@ struct kvm_arch {
525 struct kvm_ioapic *vioapic; 537 struct kvm_ioapic *vioapic;
526 struct kvm_pit *vpit; 538 struct kvm_pit *vpit;
527 int vapics_in_nmi_mode; 539 int vapics_in_nmi_mode;
540 struct mutex apic_map_lock;
541 struct kvm_apic_map *apic_map;
528 542
529 unsigned int tss_addr; 543 unsigned int tss_addr;
530 struct page *apic_access_page; 544 struct page *apic_access_page;
@@ -618,8 +632,7 @@ struct kvm_x86_ops {
618 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu); 632 void (*vcpu_load)(struct kvm_vcpu *vcpu, int cpu);
619 void (*vcpu_put)(struct kvm_vcpu *vcpu); 633 void (*vcpu_put)(struct kvm_vcpu *vcpu);
620 634
621 void (*set_guest_debug)(struct kvm_vcpu *vcpu, 635 void (*update_db_bp_intercept)(struct kvm_vcpu *vcpu);
622 struct kvm_guest_debug *dbg);
623 int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); 636 int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata);
624 int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); 637 int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data);
625 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); 638 u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg);
@@ -957,6 +970,7 @@ extern bool kvm_rebooting;
957 970
958#define KVM_ARCH_WANT_MMU_NOTIFIER 971#define KVM_ARCH_WANT_MMU_NOTIFIER
959int kvm_unmap_hva(struct kvm *kvm, unsigned long hva); 972int kvm_unmap_hva(struct kvm *kvm, unsigned long hva);
973int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
960int kvm_age_hva(struct kvm *kvm, unsigned long hva); 974int kvm_age_hva(struct kvm *kvm, unsigned long hva);
961int kvm_test_age_hva(struct kvm *kvm, unsigned long hva); 975int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
962void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte); 976void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h
index 2f7712e08b1e..eb3e9d85e1f1 100644
--- a/arch/x86/include/asm/kvm_para.h
+++ b/arch/x86/include/asm/kvm_para.h
@@ -102,21 +102,21 @@ struct kvm_vcpu_pv_apf_data {
102extern void kvmclock_init(void); 102extern void kvmclock_init(void);
103extern int kvm_register_clock(char *txt); 103extern int kvm_register_clock(char *txt);
104 104
105#ifdef CONFIG_KVM_CLOCK 105#ifdef CONFIG_KVM_GUEST
106bool kvm_check_and_clear_guest_paused(void); 106bool kvm_check_and_clear_guest_paused(void);
107#else 107#else
108static inline bool kvm_check_and_clear_guest_paused(void) 108static inline bool kvm_check_and_clear_guest_paused(void)
109{ 109{
110 return false; 110 return false;
111} 111}
112#endif /* CONFIG_KVMCLOCK */ 112#endif /* CONFIG_KVM_GUEST */
113 113
114/* This instruction is vmcall. On non-VT architectures, it will generate a 114/* This instruction is vmcall. On non-VT architectures, it will generate a
115 * trap that we will then rewrite to the appropriate instruction. 115 * trap that we will then rewrite to the appropriate instruction.
116 */ 116 */
117#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1" 117#define KVM_HYPERCALL ".byte 0x0f,0x01,0xc1"
118 118
119/* For KVM hypercalls, a three-byte sequence of either the vmrun or the vmmrun 119/* For KVM hypercalls, a three-byte sequence of either the vmcall or the vmmcall
120 * instruction. The hypervisor may replace it with something else but only the 120 * instruction. The hypervisor may replace it with something else but only the
121 * instructions are guaranteed to be supported. 121 * instructions are guaranteed to be supported.
122 * 122 *
diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h
index a3ac52b29cbf..54d73b1f00a0 100644
--- a/arch/x86/include/asm/mce.h
+++ b/arch/x86/include/asm/mce.h
@@ -116,19 +116,9 @@ struct mce_log {
116/* Software defined banks */ 116/* Software defined banks */
117#define MCE_EXTENDED_BANK 128 117#define MCE_EXTENDED_BANK 128
118#define MCE_THERMAL_BANK MCE_EXTENDED_BANK + 0 118#define MCE_THERMAL_BANK MCE_EXTENDED_BANK + 0
119 119#define K8_MCE_THRESHOLD_BASE (MCE_EXTENDED_BANK + 1)
120#define K8_MCE_THRESHOLD_BASE (MCE_EXTENDED_BANK + 1) /* MCE_AMD */
121#define K8_MCE_THRESHOLD_BANK_0 (MCE_THRESHOLD_BASE + 0 * 9)
122#define K8_MCE_THRESHOLD_BANK_1 (MCE_THRESHOLD_BASE + 1 * 9)
123#define K8_MCE_THRESHOLD_BANK_2 (MCE_THRESHOLD_BASE + 2 * 9)
124#define K8_MCE_THRESHOLD_BANK_3 (MCE_THRESHOLD_BASE + 3 * 9)
125#define K8_MCE_THRESHOLD_BANK_4 (MCE_THRESHOLD_BASE + 4 * 9)
126#define K8_MCE_THRESHOLD_BANK_5 (MCE_THRESHOLD_BASE + 5 * 9)
127#define K8_MCE_THRESHOLD_DRAM_ECC (MCE_THRESHOLD_BANK_4 + 0)
128
129 120
130#ifdef __KERNEL__ 121#ifdef __KERNEL__
131
132extern void mce_register_decode_chain(struct notifier_block *nb); 122extern void mce_register_decode_chain(struct notifier_block *nb);
133extern void mce_unregister_decode_chain(struct notifier_block *nb); 123extern void mce_unregister_decode_chain(struct notifier_block *nb);
134 124
@@ -171,6 +161,7 @@ DECLARE_PER_CPU(struct device *, mce_device);
171#ifdef CONFIG_X86_MCE_INTEL 161#ifdef CONFIG_X86_MCE_INTEL
172extern int mce_cmci_disabled; 162extern int mce_cmci_disabled;
173extern int mce_ignore_ce; 163extern int mce_ignore_ce;
164extern int mce_bios_cmci_threshold;
174void mce_intel_feature_init(struct cpuinfo_x86 *c); 165void mce_intel_feature_init(struct cpuinfo_x86 *c);
175void cmci_clear(void); 166void cmci_clear(void);
176void cmci_reenable(void); 167void cmci_reenable(void);
diff --git a/arch/x86/include/asm/microcode.h b/arch/x86/include/asm/microcode.h
index 4ebe157bf73d..43d921b4752c 100644
--- a/arch/x86/include/asm/microcode.h
+++ b/arch/x86/include/asm/microcode.h
@@ -15,8 +15,8 @@ struct microcode_ops {
15 enum ucode_state (*request_microcode_user) (int cpu, 15 enum ucode_state (*request_microcode_user) (int cpu,
16 const void __user *buf, size_t size); 16 const void __user *buf, size_t size);
17 17
18 enum ucode_state (*request_microcode_fw) (int cpu, 18 enum ucode_state (*request_microcode_fw) (int cpu, struct device *,
19 struct device *device); 19 bool refresh_fw);
20 20
21 void (*microcode_fini_cpu) (int cpu); 21 void (*microcode_fini_cpu) (int cpu);
22 22
@@ -49,12 +49,6 @@ static inline struct microcode_ops * __init init_intel_microcode(void)
49#ifdef CONFIG_MICROCODE_AMD 49#ifdef CONFIG_MICROCODE_AMD
50extern struct microcode_ops * __init init_amd_microcode(void); 50extern struct microcode_ops * __init init_amd_microcode(void);
51extern void __exit exit_amd_microcode(void); 51extern void __exit exit_amd_microcode(void);
52
53static inline void get_ucode_data(void *to, const u8 *from, size_t n)
54{
55 memcpy(to, from, n);
56}
57
58#else 52#else
59static inline struct microcode_ops * __init init_amd_microcode(void) 53static inline struct microcode_ops * __init init_amd_microcode(void)
60{ 54{
diff --git a/arch/x86/include/asm/mmzone.h b/arch/x86/include/asm/mmzone.h
index 64217ea16a36..d497bc425cae 100644
--- a/arch/x86/include/asm/mmzone.h
+++ b/arch/x86/include/asm/mmzone.h
@@ -1,5 +1,5 @@
1#ifdef CONFIG_X86_32 1#ifdef CONFIG_X86_32
2# include "mmzone_32.h" 2# include <asm/mmzone_32.h>
3#else 3#else
4# include "mmzone_64.h" 4# include <asm/mmzone_64.h>
5#endif 5#endif
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 957ec87385af..7f0edceb7563 100644
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -121,6 +121,11 @@
121#define MSR_P6_EVNTSEL0 0x00000186 121#define MSR_P6_EVNTSEL0 0x00000186
122#define MSR_P6_EVNTSEL1 0x00000187 122#define MSR_P6_EVNTSEL1 0x00000187
123 123
124#define MSR_KNC_PERFCTR0 0x00000020
125#define MSR_KNC_PERFCTR1 0x00000021
126#define MSR_KNC_EVNTSEL0 0x00000028
127#define MSR_KNC_EVNTSEL1 0x00000029
128
124/* AMD64 MSRs. Not complete. See the architecture manual for a more 129/* AMD64 MSRs. Not complete. See the architecture manual for a more
125 complete list. */ 130 complete list. */
126 131
@@ -248,6 +253,9 @@
248 253
249#define MSR_IA32_PERF_STATUS 0x00000198 254#define MSR_IA32_PERF_STATUS 0x00000198
250#define MSR_IA32_PERF_CTL 0x00000199 255#define MSR_IA32_PERF_CTL 0x00000199
256#define MSR_AMD_PSTATE_DEF_BASE 0xc0010064
257#define MSR_AMD_PERF_STATUS 0xc0010063
258#define MSR_AMD_PERF_CTL 0xc0010062
251 259
252#define MSR_IA32_MPERF 0x000000e7 260#define MSR_IA32_MPERF 0x000000e7
253#define MSR_IA32_APERF 0x000000e8 261#define MSR_IA32_APERF 0x000000e8
diff --git a/arch/x86/include/asm/mutex.h b/arch/x86/include/asm/mutex.h
index a731b9c573a6..7d3a48275394 100644
--- a/arch/x86/include/asm/mutex.h
+++ b/arch/x86/include/asm/mutex.h
@@ -1,5 +1,5 @@
1#ifdef CONFIG_X86_32 1#ifdef CONFIG_X86_32
2# include "mutex_32.h" 2# include <asm/mutex_32.h>
3#else 3#else
4# include "mutex_64.h" 4# include <asm/mutex_64.h>
5#endif 5#endif
diff --git a/arch/x86/include/asm/numa.h b/arch/x86/include/asm/numa.h
index bfacd2ccf651..49119fcea2dc 100644
--- a/arch/x86/include/asm/numa.h
+++ b/arch/x86/include/asm/numa.h
@@ -53,9 +53,9 @@ static inline int numa_cpu_node(int cpu)
53#endif /* CONFIG_NUMA */ 53#endif /* CONFIG_NUMA */
54 54
55#ifdef CONFIG_X86_32 55#ifdef CONFIG_X86_32
56# include "numa_32.h" 56# include <asm/numa_32.h>
57#else 57#else
58# include "numa_64.h" 58# include <asm/numa_64.h>
59#endif 59#endif
60 60
61#ifdef CONFIG_NUMA 61#ifdef CONFIG_NUMA
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h
index df75d07571ce..6e41b9343928 100644
--- a/arch/x86/include/asm/pci.h
+++ b/arch/x86/include/asm/pci.h
@@ -141,7 +141,7 @@ void default_restore_msi_irqs(struct pci_dev *dev, int irq);
141#endif /* __KERNEL__ */ 141#endif /* __KERNEL__ */
142 142
143#ifdef CONFIG_X86_64 143#ifdef CONFIG_X86_64
144#include "pci_64.h" 144#include <asm/pci_64.h>
145#endif 145#endif
146 146
147/* implement the pci_ DMA API in terms of the generic device dma_ one */ 147/* implement the pci_ DMA API in terms of the generic device dma_ one */
diff --git a/arch/x86/include/asm/perf_event.h b/arch/x86/include/asm/perf_event.h
index cb4e43bce98a..4fabcdf1cfa7 100644
--- a/arch/x86/include/asm/perf_event.h
+++ b/arch/x86/include/asm/perf_event.h
@@ -262,4 +262,6 @@ static inline void perf_check_microcode(void) { }
262 static inline void amd_pmu_disable_virt(void) { } 262 static inline void amd_pmu_disable_virt(void) { }
263#endif 263#endif
264 264
265#define arch_perf_out_copy_user copy_from_user_nmi
266
265#endif /* _ASM_X86_PERF_EVENT_H */ 267#endif /* _ASM_X86_PERF_EVENT_H */
diff --git a/arch/x86/include/asm/perf_regs.h b/arch/x86/include/asm/perf_regs.h
new file mode 100644
index 000000000000..3f2207bfd17b
--- /dev/null
+++ b/arch/x86/include/asm/perf_regs.h
@@ -0,0 +1,33 @@
1#ifndef _ASM_X86_PERF_REGS_H
2#define _ASM_X86_PERF_REGS_H
3
4enum perf_event_x86_regs {
5 PERF_REG_X86_AX,
6 PERF_REG_X86_BX,
7 PERF_REG_X86_CX,
8 PERF_REG_X86_DX,
9 PERF_REG_X86_SI,
10 PERF_REG_X86_DI,
11 PERF_REG_X86_BP,
12 PERF_REG_X86_SP,
13 PERF_REG_X86_IP,
14 PERF_REG_X86_FLAGS,
15 PERF_REG_X86_CS,
16 PERF_REG_X86_SS,
17 PERF_REG_X86_DS,
18 PERF_REG_X86_ES,
19 PERF_REG_X86_FS,
20 PERF_REG_X86_GS,
21 PERF_REG_X86_R8,
22 PERF_REG_X86_R9,
23 PERF_REG_X86_R10,
24 PERF_REG_X86_R11,
25 PERF_REG_X86_R12,
26 PERF_REG_X86_R13,
27 PERF_REG_X86_R14,
28 PERF_REG_X86_R15,
29
30 PERF_REG_X86_32_MAX = PERF_REG_X86_GS + 1,
31 PERF_REG_X86_64_MAX = PERF_REG_X86_R15 + 1,
32};
33#endif /* _ASM_X86_PERF_REGS_H */
diff --git a/arch/x86/include/asm/pgtable.h b/arch/x86/include/asm/pgtable.h
index 49afb3f41eb6..a1f780d45f76 100644
--- a/arch/x86/include/asm/pgtable.h
+++ b/arch/x86/include/asm/pgtable.h
@@ -146,8 +146,7 @@ static inline unsigned long pmd_pfn(pmd_t pmd)
146 146
147static inline int pmd_large(pmd_t pte) 147static inline int pmd_large(pmd_t pte)
148{ 148{
149 return (pmd_flags(pte) & (_PAGE_PSE | _PAGE_PRESENT)) == 149 return pmd_flags(pte) & _PAGE_PSE;
150 (_PAGE_PSE | _PAGE_PRESENT);
151} 150}
152 151
153#ifdef CONFIG_TRANSPARENT_HUGEPAGE 152#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -384,9 +383,9 @@ pte_t *populate_extra_pte(unsigned long vaddr);
384#endif /* __ASSEMBLY__ */ 383#endif /* __ASSEMBLY__ */
385 384
386#ifdef CONFIG_X86_32 385#ifdef CONFIG_X86_32
387# include "pgtable_32.h" 386# include <asm/pgtable_32.h>
388#else 387#else
389# include "pgtable_64.h" 388# include <asm/pgtable_64.h>
390#endif 389#endif
391 390
392#ifndef __ASSEMBLY__ 391#ifndef __ASSEMBLY__
@@ -415,7 +414,13 @@ static inline int pte_hidden(pte_t pte)
415 414
416static inline int pmd_present(pmd_t pmd) 415static inline int pmd_present(pmd_t pmd)
417{ 416{
418 return pmd_flags(pmd) & _PAGE_PRESENT; 417 /*
418 * Checking for _PAGE_PSE is needed too because
419 * split_huge_page will temporarily clear the present bit (but
420 * the _PAGE_PSE flag will remain set at all times while the
421 * _PAGE_PRESENT bit is clear).
422 */
423 return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE);
419} 424}
420 425
421static inline int pmd_none(pmd_t pmd) 426static inline int pmd_none(pmd_t pmd)
diff --git a/arch/x86/include/asm/pgtable_32.h b/arch/x86/include/asm/pgtable_32.h
index 0c92113c4cb6..8faa215a503e 100644
--- a/arch/x86/include/asm/pgtable_32.h
+++ b/arch/x86/include/asm/pgtable_32.h
@@ -71,6 +71,7 @@ do { \
71 * tables contain all the necessary information. 71 * tables contain all the necessary information.
72 */ 72 */
73#define update_mmu_cache(vma, address, ptep) do { } while (0) 73#define update_mmu_cache(vma, address, ptep) do { } while (0)
74#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
74 75
75#endif /* !__ASSEMBLY__ */ 76#endif /* !__ASSEMBLY__ */
76 77
diff --git a/arch/x86/include/asm/pgtable_64.h b/arch/x86/include/asm/pgtable_64.h
index 8251be02301e..47356f9df82e 100644
--- a/arch/x86/include/asm/pgtable_64.h
+++ b/arch/x86/include/asm/pgtable_64.h
@@ -143,6 +143,7 @@ static inline int pgd_large(pgd_t pgd) { return 0; }
143#define pte_unmap(pte) ((void)(pte))/* NOP */ 143#define pte_unmap(pte) ((void)(pte))/* NOP */
144 144
145#define update_mmu_cache(vma, address, ptep) do { } while (0) 145#define update_mmu_cache(vma, address, ptep) do { } while (0)
146#define update_mmu_cache_pmd(vma, address, pmd) do { } while (0)
146 147
147/* Encode and de-code a swap entry */ 148/* Encode and de-code a swap entry */
148#if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE 149#if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE
diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index db8fec6d2953..ec8a1fc9505d 100644
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -174,9 +174,9 @@
174#endif 174#endif
175 175
176#ifdef CONFIG_X86_32 176#ifdef CONFIG_X86_32
177# include "pgtable_32_types.h" 177# include <asm/pgtable_32_types.h>
178#else 178#else
179# include "pgtable_64_types.h" 179# include <asm/pgtable_64_types.h>
180#endif 180#endif
181 181
182#ifndef __ASSEMBLY__ 182#ifndef __ASSEMBLY__
diff --git a/arch/x86/include/asm/posix_types.h b/arch/x86/include/asm/posix_types.h
index 7ef7c3020e5c..bad3665c25fc 100644
--- a/arch/x86/include/asm/posix_types.h
+++ b/arch/x86/include/asm/posix_types.h
@@ -1,15 +1,15 @@
1#ifdef __KERNEL__ 1#ifdef __KERNEL__
2# ifdef CONFIG_X86_32 2# ifdef CONFIG_X86_32
3# include "posix_types_32.h" 3# include <asm/posix_types_32.h>
4# else 4# else
5# include "posix_types_64.h" 5# include <asm/posix_types_64.h>
6# endif 6# endif
7#else 7#else
8# ifdef __i386__ 8# ifdef __i386__
9# include "posix_types_32.h" 9# include <asm/posix_types_32.h>
10# elif defined(__ILP32__) 10# elif defined(__ILP32__)
11# include "posix_types_x32.h" 11# include <asm/posix_types_x32.h>
12# else 12# else
13# include "posix_types_64.h" 13# include <asm/posix_types_64.h>
14# endif 14# endif
15#endif 15#endif
diff --git a/arch/x86/include/asm/processor-flags.h b/arch/x86/include/asm/processor-flags.h
index aea1d1d848c7..680cf09ed100 100644
--- a/arch/x86/include/asm/processor-flags.h
+++ b/arch/x86/include/asm/processor-flags.h
@@ -65,6 +65,7 @@
65#define X86_CR4_PCIDE 0x00020000 /* enable PCID support */ 65#define X86_CR4_PCIDE 0x00020000 /* enable PCID support */
66#define X86_CR4_OSXSAVE 0x00040000 /* enable xsave and xrestore */ 66#define X86_CR4_OSXSAVE 0x00040000 /* enable xsave and xrestore */
67#define X86_CR4_SMEP 0x00100000 /* enable SMEP support */ 67#define X86_CR4_SMEP 0x00100000 /* enable SMEP support */
68#define X86_CR4_SMAP 0x00200000 /* enable SMAP support */
68 69
69/* 70/*
70 * x86-64 Task Priority Register, CR8 71 * x86-64 Task Priority Register, CR8
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index d048cad9bcad..ad1fc8511674 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -423,7 +423,6 @@ DECLARE_INIT_PER_CPU(irq_stack_union);
423 423
424DECLARE_PER_CPU(char *, irq_stack_ptr); 424DECLARE_PER_CPU(char *, irq_stack_ptr);
425DECLARE_PER_CPU(unsigned int, irq_count); 425DECLARE_PER_CPU(unsigned int, irq_count);
426extern unsigned long kernel_eflags;
427extern asmlinkage void ignore_sysret(void); 426extern asmlinkage void ignore_sysret(void);
428#else /* X86_64 */ 427#else /* X86_64 */
429#ifdef CONFIG_CC_STACKPROTECTOR 428#ifdef CONFIG_CC_STACKPROTECTOR
@@ -589,11 +588,6 @@ typedef struct {
589} mm_segment_t; 588} mm_segment_t;
590 589
591 590
592/*
593 * create a kernel thread without removing it from tasklists
594 */
595extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags);
596
597/* Free all resources held by a thread. */ 591/* Free all resources held by a thread. */
598extern void release_thread(struct task_struct *); 592extern void release_thread(struct task_struct *);
599 593
@@ -759,6 +753,8 @@ static inline void update_debugctlmsr(unsigned long debugctlmsr)
759 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr); 753 wrmsrl(MSR_IA32_DEBUGCTLMSR, debugctlmsr);
760} 754}
761 755
756extern void set_task_blockstep(struct task_struct *task, bool on);
757
762/* 758/*
763 * from system description table in BIOS. Mostly for MCA use, but 759 * from system description table in BIOS. Mostly for MCA use, but
764 * others may find it useful: 760 * others may find it useful:
diff --git a/arch/x86/include/asm/rcu.h b/arch/x86/include/asm/rcu.h
new file mode 100644
index 000000000000..d1ac07a23979
--- /dev/null
+++ b/arch/x86/include/asm/rcu.h
@@ -0,0 +1,32 @@
1#ifndef _ASM_X86_RCU_H
2#define _ASM_X86_RCU_H
3
4#ifndef __ASSEMBLY__
5
6#include <linux/rcupdate.h>
7#include <asm/ptrace.h>
8
9static inline void exception_enter(struct pt_regs *regs)
10{
11 rcu_user_exit();
12}
13
14static inline void exception_exit(struct pt_regs *regs)
15{
16#ifdef CONFIG_RCU_USER_QS
17 if (user_mode(regs))
18 rcu_user_enter();
19#endif
20}
21
22#else /* __ASSEMBLY__ */
23
24#ifdef CONFIG_RCU_USER_QS
25# define SCHEDULE_USER call schedule_user
26#else
27# define SCHEDULE_USER call schedule
28#endif
29
30#endif /* !__ASSEMBLY__ */
31
32#endif
diff --git a/arch/x86/include/asm/seccomp.h b/arch/x86/include/asm/seccomp.h
index c62e58a5a90d..0f3d7f099224 100644
--- a/arch/x86/include/asm/seccomp.h
+++ b/arch/x86/include/asm/seccomp.h
@@ -1,5 +1,5 @@
1#ifdef CONFIG_X86_32 1#ifdef CONFIG_X86_32
2# include "seccomp_32.h" 2# include <asm/seccomp_32.h>
3#else 3#else
4# include "seccomp_64.h" 4# include <asm/seccomp_64.h>
5#endif 5#endif
diff --git a/arch/x86/include/asm/signal.h b/arch/x86/include/asm/signal.h
index 598457cbd0f8..323973f4abf1 100644
--- a/arch/x86/include/asm/signal.h
+++ b/arch/x86/include/asm/signal.h
@@ -31,6 +31,10 @@ typedef struct {
31 unsigned long sig[_NSIG_WORDS]; 31 unsigned long sig[_NSIG_WORDS];
32} sigset_t; 32} sigset_t;
33 33
34#ifndef CONFIG_COMPAT
35typedef sigset_t compat_sigset_t;
36#endif
37
34#else 38#else
35/* Here we must cater to libcs that poke about in kernel headers. */ 39/* Here we must cater to libcs that poke about in kernel headers. */
36 40
diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h
new file mode 100644
index 000000000000..8d3120f4e270
--- /dev/null
+++ b/arch/x86/include/asm/smap.h
@@ -0,0 +1,91 @@
1/*
2 * Supervisor Mode Access Prevention support
3 *
4 * Copyright (C) 2012 Intel Corporation
5 * Author: H. Peter Anvin <hpa@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
10 * of the License.
11 */
12
13#ifndef _ASM_X86_SMAP_H
14#define _ASM_X86_SMAP_H
15
16#include <linux/stringify.h>
17#include <asm/nops.h>
18#include <asm/cpufeature.h>
19
20/* "Raw" instruction opcodes */
21#define __ASM_CLAC .byte 0x0f,0x01,0xca
22#define __ASM_STAC .byte 0x0f,0x01,0xcb
23
24#ifdef __ASSEMBLY__
25
26#include <asm/alternative-asm.h>
27
28#ifdef CONFIG_X86_SMAP
29
30#define ASM_CLAC \
31 661: ASM_NOP3 ; \
32 .pushsection .altinstr_replacement, "ax" ; \
33 662: __ASM_CLAC ; \
34 .popsection ; \
35 .pushsection .altinstructions, "a" ; \
36 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ; \
37 .popsection
38
39#define ASM_STAC \
40 661: ASM_NOP3 ; \
41 .pushsection .altinstr_replacement, "ax" ; \
42 662: __ASM_STAC ; \
43 .popsection ; \
44 .pushsection .altinstructions, "a" ; \
45 altinstruction_entry 661b, 662b, X86_FEATURE_SMAP, 3, 3 ; \
46 .popsection
47
48#else /* CONFIG_X86_SMAP */
49
50#define ASM_CLAC
51#define ASM_STAC
52
53#endif /* CONFIG_X86_SMAP */
54
55#else /* __ASSEMBLY__ */
56
57#include <asm/alternative.h>
58
59#ifdef CONFIG_X86_SMAP
60
61static __always_inline void clac(void)
62{
63 /* Note: a barrier is implicit in alternative() */
64 alternative(ASM_NOP3, __stringify(__ASM_CLAC), X86_FEATURE_SMAP);
65}
66
67static __always_inline void stac(void)
68{
69 /* Note: a barrier is implicit in alternative() */
70 alternative(ASM_NOP3, __stringify(__ASM_STAC), X86_FEATURE_SMAP);
71}
72
73/* These macros can be used in asm() statements */
74#define ASM_CLAC \
75 ALTERNATIVE(ASM_NOP3, __stringify(__ASM_CLAC), X86_FEATURE_SMAP)
76#define ASM_STAC \
77 ALTERNATIVE(ASM_NOP3, __stringify(__ASM_STAC), X86_FEATURE_SMAP)
78
79#else /* CONFIG_X86_SMAP */
80
81static inline void clac(void) { }
82static inline void stac(void) { }
83
84#define ASM_CLAC
85#define ASM_STAC
86
87#endif /* CONFIG_X86_SMAP */
88
89#endif /* __ASSEMBLY__ */
90
91#endif /* _ASM_X86_SMAP_H */
diff --git a/arch/x86/include/asm/string.h b/arch/x86/include/asm/string.h
index 6dfd6d9373a0..09224d7a5862 100644
--- a/arch/x86/include/asm/string.h
+++ b/arch/x86/include/asm/string.h
@@ -1,5 +1,5 @@
1#ifdef CONFIG_X86_32 1#ifdef CONFIG_X86_32
2# include "string_32.h" 2# include <asm/string_32.h>
3#else 3#else
4# include "string_64.h" 4# include <asm/string_64.h>
5#endif 5#endif
diff --git a/arch/x86/include/asm/suspend.h b/arch/x86/include/asm/suspend.h
index 9bd521fe4570..2fab6c2c3575 100644
--- a/arch/x86/include/asm/suspend.h
+++ b/arch/x86/include/asm/suspend.h
@@ -1,5 +1,5 @@
1#ifdef CONFIG_X86_32 1#ifdef CONFIG_X86_32
2# include "suspend_32.h" 2# include <asm/suspend_32.h>
3#else 3#else
4# include "suspend_64.h" 4# include <asm/suspend_64.h>
5#endif 5#endif
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h
index f2b83bc7d784..cdf5674dd23a 100644
--- a/arch/x86/include/asm/svm.h
+++ b/arch/x86/include/asm/svm.h
@@ -1,6 +1,135 @@
1#ifndef __SVM_H 1#ifndef __SVM_H
2#define __SVM_H 2#define __SVM_H
3 3
4#define SVM_EXIT_READ_CR0 0x000
5#define SVM_EXIT_READ_CR3 0x003
6#define SVM_EXIT_READ_CR4 0x004
7#define SVM_EXIT_READ_CR8 0x008
8#define SVM_EXIT_WRITE_CR0 0x010
9#define SVM_EXIT_WRITE_CR3 0x013
10#define SVM_EXIT_WRITE_CR4 0x014
11#define SVM_EXIT_WRITE_CR8 0x018
12#define SVM_EXIT_READ_DR0 0x020
13#define SVM_EXIT_READ_DR1 0x021
14#define SVM_EXIT_READ_DR2 0x022
15#define SVM_EXIT_READ_DR3 0x023
16#define SVM_EXIT_READ_DR4 0x024
17#define SVM_EXIT_READ_DR5 0x025
18#define SVM_EXIT_READ_DR6 0x026
19#define SVM_EXIT_READ_DR7 0x027
20#define SVM_EXIT_WRITE_DR0 0x030
21#define SVM_EXIT_WRITE_DR1 0x031
22#define SVM_EXIT_WRITE_DR2 0x032
23#define SVM_EXIT_WRITE_DR3 0x033
24#define SVM_EXIT_WRITE_DR4 0x034
25#define SVM_EXIT_WRITE_DR5 0x035
26#define SVM_EXIT_WRITE_DR6 0x036
27#define SVM_EXIT_WRITE_DR7 0x037
28#define SVM_EXIT_EXCP_BASE 0x040
29#define SVM_EXIT_INTR 0x060
30#define SVM_EXIT_NMI 0x061
31#define SVM_EXIT_SMI 0x062
32#define SVM_EXIT_INIT 0x063
33#define SVM_EXIT_VINTR 0x064
34#define SVM_EXIT_CR0_SEL_WRITE 0x065
35#define SVM_EXIT_IDTR_READ 0x066
36#define SVM_EXIT_GDTR_READ 0x067
37#define SVM_EXIT_LDTR_READ 0x068
38#define SVM_EXIT_TR_READ 0x069
39#define SVM_EXIT_IDTR_WRITE 0x06a
40#define SVM_EXIT_GDTR_WRITE 0x06b
41#define SVM_EXIT_LDTR_WRITE 0x06c
42#define SVM_EXIT_TR_WRITE 0x06d
43#define SVM_EXIT_RDTSC 0x06e
44#define SVM_EXIT_RDPMC 0x06f
45#define SVM_EXIT_PUSHF 0x070
46#define SVM_EXIT_POPF 0x071
47#define SVM_EXIT_CPUID 0x072
48#define SVM_EXIT_RSM 0x073
49#define SVM_EXIT_IRET 0x074
50#define SVM_EXIT_SWINT 0x075
51#define SVM_EXIT_INVD 0x076
52#define SVM_EXIT_PAUSE 0x077
53#define SVM_EXIT_HLT 0x078
54#define SVM_EXIT_INVLPG 0x079
55#define SVM_EXIT_INVLPGA 0x07a
56#define SVM_EXIT_IOIO 0x07b
57#define SVM_EXIT_MSR 0x07c
58#define SVM_EXIT_TASK_SWITCH 0x07d
59#define SVM_EXIT_FERR_FREEZE 0x07e
60#define SVM_EXIT_SHUTDOWN 0x07f
61#define SVM_EXIT_VMRUN 0x080
62#define SVM_EXIT_VMMCALL 0x081
63#define SVM_EXIT_VMLOAD 0x082
64#define SVM_EXIT_VMSAVE 0x083
65#define SVM_EXIT_STGI 0x084
66#define SVM_EXIT_CLGI 0x085
67#define SVM_EXIT_SKINIT 0x086
68#define SVM_EXIT_RDTSCP 0x087
69#define SVM_EXIT_ICEBP 0x088
70#define SVM_EXIT_WBINVD 0x089
71#define SVM_EXIT_MONITOR 0x08a
72#define SVM_EXIT_MWAIT 0x08b
73#define SVM_EXIT_MWAIT_COND 0x08c
74#define SVM_EXIT_XSETBV 0x08d
75#define SVM_EXIT_NPF 0x400
76
77#define SVM_EXIT_ERR -1
78
79#define SVM_EXIT_REASONS \
80 { SVM_EXIT_READ_CR0, "read_cr0" }, \
81 { SVM_EXIT_READ_CR3, "read_cr3" }, \
82 { SVM_EXIT_READ_CR4, "read_cr4" }, \
83 { SVM_EXIT_READ_CR8, "read_cr8" }, \
84 { SVM_EXIT_WRITE_CR0, "write_cr0" }, \
85 { SVM_EXIT_WRITE_CR3, "write_cr3" }, \
86 { SVM_EXIT_WRITE_CR4, "write_cr4" }, \
87 { SVM_EXIT_WRITE_CR8, "write_cr8" }, \
88 { SVM_EXIT_READ_DR0, "read_dr0" }, \
89 { SVM_EXIT_READ_DR1, "read_dr1" }, \
90 { SVM_EXIT_READ_DR2, "read_dr2" }, \
91 { SVM_EXIT_READ_DR3, "read_dr3" }, \
92 { SVM_EXIT_WRITE_DR0, "write_dr0" }, \
93 { SVM_EXIT_WRITE_DR1, "write_dr1" }, \
94 { SVM_EXIT_WRITE_DR2, "write_dr2" }, \
95 { SVM_EXIT_WRITE_DR3, "write_dr3" }, \
96 { SVM_EXIT_WRITE_DR5, "write_dr5" }, \
97 { SVM_EXIT_WRITE_DR7, "write_dr7" }, \
98 { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" }, \
99 { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" }, \
100 { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" }, \
101 { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" }, \
102 { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" }, \
103 { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" }, \
104 { SVM_EXIT_INTR, "interrupt" }, \
105 { SVM_EXIT_NMI, "nmi" }, \
106 { SVM_EXIT_SMI, "smi" }, \
107 { SVM_EXIT_INIT, "init" }, \
108 { SVM_EXIT_VINTR, "vintr" }, \
109 { SVM_EXIT_CPUID, "cpuid" }, \
110 { SVM_EXIT_INVD, "invd" }, \
111 { SVM_EXIT_HLT, "hlt" }, \
112 { SVM_EXIT_INVLPG, "invlpg" }, \
113 { SVM_EXIT_INVLPGA, "invlpga" }, \
114 { SVM_EXIT_IOIO, "io" }, \
115 { SVM_EXIT_MSR, "msr" }, \
116 { SVM_EXIT_TASK_SWITCH, "task_switch" }, \
117 { SVM_EXIT_SHUTDOWN, "shutdown" }, \
118 { SVM_EXIT_VMRUN, "vmrun" }, \
119 { SVM_EXIT_VMMCALL, "hypercall" }, \
120 { SVM_EXIT_VMLOAD, "vmload" }, \
121 { SVM_EXIT_VMSAVE, "vmsave" }, \
122 { SVM_EXIT_STGI, "stgi" }, \
123 { SVM_EXIT_CLGI, "clgi" }, \
124 { SVM_EXIT_SKINIT, "skinit" }, \
125 { SVM_EXIT_WBINVD, "wbinvd" }, \
126 { SVM_EXIT_MONITOR, "monitor" }, \
127 { SVM_EXIT_MWAIT, "mwait" }, \
128 { SVM_EXIT_XSETBV, "xsetbv" }, \
129 { SVM_EXIT_NPF, "npf" }
130
131#ifdef __KERNEL__
132
4enum { 133enum {
5 INTERCEPT_INTR, 134 INTERCEPT_INTR,
6 INTERCEPT_NMI, 135 INTERCEPT_NMI,
@@ -264,81 +393,6 @@ struct __attribute__ ((__packed__)) vmcb {
264 393
265#define SVM_EXITINFO_REG_MASK 0x0F 394#define SVM_EXITINFO_REG_MASK 0x0F
266 395
267#define SVM_EXIT_READ_CR0 0x000
268#define SVM_EXIT_READ_CR3 0x003
269#define SVM_EXIT_READ_CR4 0x004
270#define SVM_EXIT_READ_CR8 0x008
271#define SVM_EXIT_WRITE_CR0 0x010
272#define SVM_EXIT_WRITE_CR3 0x013
273#define SVM_EXIT_WRITE_CR4 0x014
274#define SVM_EXIT_WRITE_CR8 0x018
275#define SVM_EXIT_READ_DR0 0x020
276#define SVM_EXIT_READ_DR1 0x021
277#define SVM_EXIT_READ_DR2 0x022
278#define SVM_EXIT_READ_DR3 0x023
279#define SVM_EXIT_READ_DR4 0x024
280#define SVM_EXIT_READ_DR5 0x025
281#define SVM_EXIT_READ_DR6 0x026
282#define SVM_EXIT_READ_DR7 0x027
283#define SVM_EXIT_WRITE_DR0 0x030
284#define SVM_EXIT_WRITE_DR1 0x031
285#define SVM_EXIT_WRITE_DR2 0x032
286#define SVM_EXIT_WRITE_DR3 0x033
287#define SVM_EXIT_WRITE_DR4 0x034
288#define SVM_EXIT_WRITE_DR5 0x035
289#define SVM_EXIT_WRITE_DR6 0x036
290#define SVM_EXIT_WRITE_DR7 0x037
291#define SVM_EXIT_EXCP_BASE 0x040
292#define SVM_EXIT_INTR 0x060
293#define SVM_EXIT_NMI 0x061
294#define SVM_EXIT_SMI 0x062
295#define SVM_EXIT_INIT 0x063
296#define SVM_EXIT_VINTR 0x064
297#define SVM_EXIT_CR0_SEL_WRITE 0x065
298#define SVM_EXIT_IDTR_READ 0x066
299#define SVM_EXIT_GDTR_READ 0x067
300#define SVM_EXIT_LDTR_READ 0x068
301#define SVM_EXIT_TR_READ 0x069
302#define SVM_EXIT_IDTR_WRITE 0x06a
303#define SVM_EXIT_GDTR_WRITE 0x06b
304#define SVM_EXIT_LDTR_WRITE 0x06c
305#define SVM_EXIT_TR_WRITE 0x06d
306#define SVM_EXIT_RDTSC 0x06e
307#define SVM_EXIT_RDPMC 0x06f
308#define SVM_EXIT_PUSHF 0x070
309#define SVM_EXIT_POPF 0x071
310#define SVM_EXIT_CPUID 0x072
311#define SVM_EXIT_RSM 0x073
312#define SVM_EXIT_IRET 0x074
313#define SVM_EXIT_SWINT 0x075
314#define SVM_EXIT_INVD 0x076
315#define SVM_EXIT_PAUSE 0x077
316#define SVM_EXIT_HLT 0x078
317#define SVM_EXIT_INVLPG 0x079
318#define SVM_EXIT_INVLPGA 0x07a
319#define SVM_EXIT_IOIO 0x07b
320#define SVM_EXIT_MSR 0x07c
321#define SVM_EXIT_TASK_SWITCH 0x07d
322#define SVM_EXIT_FERR_FREEZE 0x07e
323#define SVM_EXIT_SHUTDOWN 0x07f
324#define SVM_EXIT_VMRUN 0x080
325#define SVM_EXIT_VMMCALL 0x081
326#define SVM_EXIT_VMLOAD 0x082
327#define SVM_EXIT_VMSAVE 0x083
328#define SVM_EXIT_STGI 0x084
329#define SVM_EXIT_CLGI 0x085
330#define SVM_EXIT_SKINIT 0x086
331#define SVM_EXIT_RDTSCP 0x087
332#define SVM_EXIT_ICEBP 0x088
333#define SVM_EXIT_WBINVD 0x089
334#define SVM_EXIT_MONITOR 0x08a
335#define SVM_EXIT_MWAIT 0x08b
336#define SVM_EXIT_MWAIT_COND 0x08c
337#define SVM_EXIT_XSETBV 0x08d
338#define SVM_EXIT_NPF 0x400
339
340#define SVM_EXIT_ERR -1
341
342#define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP) 396#define SVM_CR0_SELECTIVE_MASK (X86_CR0_TS | X86_CR0_MP)
343 397
344#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda" 398#define SVM_VMLOAD ".byte 0x0f, 0x01, 0xda"
@@ -350,3 +404,4 @@ struct __attribute__ ((__packed__)) vmcb {
350 404
351#endif 405#endif
352 406
407#endif
diff --git a/arch/x86/include/asm/sys_ia32.h b/arch/x86/include/asm/sys_ia32.h
index 3fda9db48819..a9a8cf3da49d 100644
--- a/arch/x86/include/asm/sys_ia32.h
+++ b/arch/x86/include/asm/sys_ia32.h
@@ -40,7 +40,7 @@ asmlinkage long sys32_sigaction(int, struct old_sigaction32 __user *,
40 struct old_sigaction32 __user *); 40 struct old_sigaction32 __user *);
41asmlinkage long sys32_alarm(unsigned int); 41asmlinkage long sys32_alarm(unsigned int);
42 42
43asmlinkage long sys32_waitpid(compat_pid_t, unsigned int *, int); 43asmlinkage long sys32_waitpid(compat_pid_t, unsigned int __user *, int);
44asmlinkage long sys32_sysfs(int, u32, u32); 44asmlinkage long sys32_sysfs(int, u32, u32);
45 45
46asmlinkage long sys32_sched_rr_get_interval(compat_pid_t, 46asmlinkage long sys32_sched_rr_get_interval(compat_pid_t,
@@ -54,8 +54,6 @@ asmlinkage long sys32_pwrite(unsigned int, const char __user *, u32, u32, u32);
54asmlinkage long sys32_personality(unsigned long); 54asmlinkage long sys32_personality(unsigned long);
55asmlinkage long sys32_sendfile(int, int, compat_off_t __user *, s32); 55asmlinkage long sys32_sendfile(int, int, compat_off_t __user *, s32);
56 56
57asmlinkage long sys32_execve(const char __user *, compat_uptr_t __user *,
58 compat_uptr_t __user *, struct pt_regs *);
59asmlinkage long sys32_clone(unsigned int, unsigned int, struct pt_regs *); 57asmlinkage long sys32_clone(unsigned int, unsigned int, struct pt_regs *);
60 58
61long sys32_lseek(unsigned int, int, unsigned int); 59long sys32_lseek(unsigned int, int, unsigned int);
diff --git a/arch/x86/include/asm/syscalls.h b/arch/x86/include/asm/syscalls.h
index f1d8b441fc77..2be0b880417e 100644
--- a/arch/x86/include/asm/syscalls.h
+++ b/arch/x86/include/asm/syscalls.h
@@ -25,7 +25,7 @@ int sys_fork(struct pt_regs *);
25int sys_vfork(struct pt_regs *); 25int sys_vfork(struct pt_regs *);
26long sys_execve(const char __user *, 26long sys_execve(const char __user *,
27 const char __user *const __user *, 27 const char __user *const __user *,
28 const char __user *const __user *, struct pt_regs *); 28 const char __user *const __user *);
29long sys_clone(unsigned long, unsigned long, void __user *, 29long sys_clone(unsigned long, unsigned long, void __user *,
30 void __user *, struct pt_regs *); 30 void __user *, struct pt_regs *);
31 31
diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index 89f794f007ec..2d946e63ee82 100644
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -79,7 +79,6 @@ struct thread_info {
79#define TIF_SIGPENDING 2 /* signal pending */ 79#define TIF_SIGPENDING 2 /* signal pending */
80#define TIF_NEED_RESCHED 3 /* rescheduling necessary */ 80#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
81#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/ 81#define TIF_SINGLESTEP 4 /* reenable singlestep on user return*/
82#define TIF_IRET 5 /* force IRET */
83#define TIF_SYSCALL_EMU 6 /* syscall emulation active */ 82#define TIF_SYSCALL_EMU 6 /* syscall emulation active */
84#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */ 83#define TIF_SYSCALL_AUDIT 7 /* syscall auditing active */
85#define TIF_SECCOMP 8 /* secure computing */ 84#define TIF_SECCOMP 8 /* secure computing */
@@ -89,6 +88,7 @@ struct thread_info {
89#define TIF_NOTSC 16 /* TSC is not accessible in userland */ 88#define TIF_NOTSC 16 /* TSC is not accessible in userland */
90#define TIF_IA32 17 /* IA32 compatibility process */ 89#define TIF_IA32 17 /* IA32 compatibility process */
91#define TIF_FORK 18 /* ret_from_fork */ 90#define TIF_FORK 18 /* ret_from_fork */
91#define TIF_NOHZ 19 /* in adaptive nohz mode */
92#define TIF_MEMDIE 20 /* is terminating due to OOM killer */ 92#define TIF_MEMDIE 20 /* is terminating due to OOM killer */
93#define TIF_DEBUG 21 /* uses debug registers */ 93#define TIF_DEBUG 21 /* uses debug registers */
94#define TIF_IO_BITMAP 22 /* uses I/O bitmap */ 94#define TIF_IO_BITMAP 22 /* uses I/O bitmap */
@@ -104,7 +104,6 @@ struct thread_info {
104#define _TIF_SIGPENDING (1 << TIF_SIGPENDING) 104#define _TIF_SIGPENDING (1 << TIF_SIGPENDING)
105#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP) 105#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
106#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) 106#define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED)
107#define _TIF_IRET (1 << TIF_IRET)
108#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU) 107#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
109#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) 108#define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT)
110#define _TIF_SECCOMP (1 << TIF_SECCOMP) 109#define _TIF_SECCOMP (1 << TIF_SECCOMP)
@@ -114,6 +113,7 @@ struct thread_info {
114#define _TIF_NOTSC (1 << TIF_NOTSC) 113#define _TIF_NOTSC (1 << TIF_NOTSC)
115#define _TIF_IA32 (1 << TIF_IA32) 114#define _TIF_IA32 (1 << TIF_IA32)
116#define _TIF_FORK (1 << TIF_FORK) 115#define _TIF_FORK (1 << TIF_FORK)
116#define _TIF_NOHZ (1 << TIF_NOHZ)
117#define _TIF_DEBUG (1 << TIF_DEBUG) 117#define _TIF_DEBUG (1 << TIF_DEBUG)
118#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP) 118#define _TIF_IO_BITMAP (1 << TIF_IO_BITMAP)
119#define _TIF_FORCED_TF (1 << TIF_FORCED_TF) 119#define _TIF_FORCED_TF (1 << TIF_FORCED_TF)
@@ -126,12 +126,13 @@ struct thread_info {
126/* work to do in syscall_trace_enter() */ 126/* work to do in syscall_trace_enter() */
127#define _TIF_WORK_SYSCALL_ENTRY \ 127#define _TIF_WORK_SYSCALL_ENTRY \
128 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_AUDIT | \ 128 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_EMU | _TIF_SYSCALL_AUDIT | \
129 _TIF_SECCOMP | _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT) 129 _TIF_SECCOMP | _TIF_SINGLESTEP | _TIF_SYSCALL_TRACEPOINT | \
130 _TIF_NOHZ)
130 131
131/* work to do in syscall_trace_leave() */ 132/* work to do in syscall_trace_leave() */
132#define _TIF_WORK_SYSCALL_EXIT \ 133#define _TIF_WORK_SYSCALL_EXIT \
133 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP | \ 134 (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SINGLESTEP | \
134 _TIF_SYSCALL_TRACEPOINT) 135 _TIF_SYSCALL_TRACEPOINT | _TIF_NOHZ)
135 136
136/* work to do on interrupt/exception return */ 137/* work to do on interrupt/exception return */
137#define _TIF_WORK_MASK \ 138#define _TIF_WORK_MASK \
@@ -141,7 +142,8 @@ struct thread_info {
141 142
142/* work to do on any return to user space */ 143/* work to do on any return to user space */
143#define _TIF_ALLWORK_MASK \ 144#define _TIF_ALLWORK_MASK \
144 ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_TRACEPOINT) 145 ((0x0000FFFF & ~_TIF_SECCOMP) | _TIF_SYSCALL_TRACEPOINT | \
146 _TIF_NOHZ)
145 147
146/* Only used for 64 bit */ 148/* Only used for 64 bit */
147#define _TIF_DO_NOTIFY_MASK \ 149#define _TIF_DO_NOTIFY_MASK \
diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h
index e1f3a17034fc..7ccf8d131535 100644
--- a/arch/x86/include/asm/uaccess.h
+++ b/arch/x86/include/asm/uaccess.h
@@ -9,6 +9,7 @@
9#include <linux/string.h> 9#include <linux/string.h>
10#include <asm/asm.h> 10#include <asm/asm.h>
11#include <asm/page.h> 11#include <asm/page.h>
12#include <asm/smap.h>
12 13
13#define VERIFY_READ 0 14#define VERIFY_READ 0
14#define VERIFY_WRITE 1 15#define VERIFY_WRITE 1
@@ -192,9 +193,10 @@ extern int __get_user_bad(void);
192 193
193#ifdef CONFIG_X86_32 194#ifdef CONFIG_X86_32
194#define __put_user_asm_u64(x, addr, err, errret) \ 195#define __put_user_asm_u64(x, addr, err, errret) \
195 asm volatile("1: movl %%eax,0(%2)\n" \ 196 asm volatile(ASM_STAC "\n" \
197 "1: movl %%eax,0(%2)\n" \
196 "2: movl %%edx,4(%2)\n" \ 198 "2: movl %%edx,4(%2)\n" \
197 "3:\n" \ 199 "3: " ASM_CLAC "\n" \
198 ".section .fixup,\"ax\"\n" \ 200 ".section .fixup,\"ax\"\n" \
199 "4: movl %3,%0\n" \ 201 "4: movl %3,%0\n" \
200 " jmp 3b\n" \ 202 " jmp 3b\n" \
@@ -205,9 +207,10 @@ extern int __get_user_bad(void);
205 : "A" (x), "r" (addr), "i" (errret), "0" (err)) 207 : "A" (x), "r" (addr), "i" (errret), "0" (err))
206 208
207#define __put_user_asm_ex_u64(x, addr) \ 209#define __put_user_asm_ex_u64(x, addr) \
208 asm volatile("1: movl %%eax,0(%1)\n" \ 210 asm volatile(ASM_STAC "\n" \
211 "1: movl %%eax,0(%1)\n" \
209 "2: movl %%edx,4(%1)\n" \ 212 "2: movl %%edx,4(%1)\n" \
210 "3:\n" \ 213 "3: " ASM_CLAC "\n" \
211 _ASM_EXTABLE_EX(1b, 2b) \ 214 _ASM_EXTABLE_EX(1b, 2b) \
212 _ASM_EXTABLE_EX(2b, 3b) \ 215 _ASM_EXTABLE_EX(2b, 3b) \
213 : : "A" (x), "r" (addr)) 216 : : "A" (x), "r" (addr))
@@ -379,8 +382,9 @@ do { \
379} while (0) 382} while (0)
380 383
381#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 384#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
382 asm volatile("1: mov"itype" %2,%"rtype"1\n" \ 385 asm volatile(ASM_STAC "\n" \
383 "2:\n" \ 386 "1: mov"itype" %2,%"rtype"1\n" \
387 "2: " ASM_CLAC "\n" \
384 ".section .fixup,\"ax\"\n" \ 388 ".section .fixup,\"ax\"\n" \
385 "3: mov %3,%0\n" \ 389 "3: mov %3,%0\n" \
386 " xor"itype" %"rtype"1,%"rtype"1\n" \ 390 " xor"itype" %"rtype"1,%"rtype"1\n" \
@@ -443,8 +447,9 @@ struct __large_struct { unsigned long buf[100]; };
443 * aliasing issues. 447 * aliasing issues.
444 */ 448 */
445#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ 449#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
446 asm volatile("1: mov"itype" %"rtype"1,%2\n" \ 450 asm volatile(ASM_STAC "\n" \
447 "2:\n" \ 451 "1: mov"itype" %"rtype"1,%2\n" \
452 "2: " ASM_CLAC "\n" \
448 ".section .fixup,\"ax\"\n" \ 453 ".section .fixup,\"ax\"\n" \
449 "3: mov %3,%0\n" \ 454 "3: mov %3,%0\n" \
450 " jmp 2b\n" \ 455 " jmp 2b\n" \
@@ -463,13 +468,13 @@ struct __large_struct { unsigned long buf[100]; };
463 * uaccess_try and catch 468 * uaccess_try and catch
464 */ 469 */
465#define uaccess_try do { \ 470#define uaccess_try do { \
466 int prev_err = current_thread_info()->uaccess_err; \
467 current_thread_info()->uaccess_err = 0; \ 471 current_thread_info()->uaccess_err = 0; \
472 stac(); \
468 barrier(); 473 barrier();
469 474
470#define uaccess_catch(err) \ 475#define uaccess_catch(err) \
476 clac(); \
471 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \ 477 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
472 current_thread_info()->uaccess_err = prev_err; \
473} while (0) 478} while (0)
474 479
475/** 480/**
@@ -569,6 +574,9 @@ strncpy_from_user(char *dst, const char __user *src, long count);
569extern __must_check long strlen_user(const char __user *str); 574extern __must_check long strlen_user(const char __user *str);
570extern __must_check long strnlen_user(const char __user *str, long n); 575extern __must_check long strnlen_user(const char __user *str, long n);
571 576
577unsigned long __must_check clear_user(void __user *mem, unsigned long len);
578unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
579
572/* 580/*
573 * movsl can be slow when source and dest are not both 8-byte aligned 581 * movsl can be slow when source and dest are not both 8-byte aligned
574 */ 582 */
@@ -581,9 +589,9 @@ extern struct movsl_mask {
581#define ARCH_HAS_NOCACHE_UACCESS 1 589#define ARCH_HAS_NOCACHE_UACCESS 1
582 590
583#ifdef CONFIG_X86_32 591#ifdef CONFIG_X86_32
584# include "uaccess_32.h" 592# include <asm/uaccess_32.h>
585#else 593#else
586# include "uaccess_64.h" 594# include <asm/uaccess_64.h>
587#endif 595#endif
588 596
589#endif /* _ASM_X86_UACCESS_H */ 597#endif /* _ASM_X86_UACCESS_H */
diff --git a/arch/x86/include/asm/uaccess_32.h b/arch/x86/include/asm/uaccess_32.h
index 576e39bca6ad..7f760a9f1f61 100644
--- a/arch/x86/include/asm/uaccess_32.h
+++ b/arch/x86/include/asm/uaccess_32.h
@@ -213,7 +213,4 @@ static inline unsigned long __must_check copy_from_user(void *to,
213 return n; 213 return n;
214} 214}
215 215
216unsigned long __must_check clear_user(void __user *mem, unsigned long len);
217unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
218
219#endif /* _ASM_X86_UACCESS_32_H */ 216#endif /* _ASM_X86_UACCESS_32_H */
diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
index d8def8b3dba0..142810c457dc 100644
--- a/arch/x86/include/asm/uaccess_64.h
+++ b/arch/x86/include/asm/uaccess_64.h
@@ -217,9 +217,6 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
217 } 217 }
218} 218}
219 219
220__must_check unsigned long clear_user(void __user *mem, unsigned long len);
221__must_check unsigned long __clear_user(void __user *mem, unsigned long len);
222
223static __must_check __always_inline int 220static __must_check __always_inline int
224__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size) 221__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
225{ 222{
diff --git a/arch/x86/include/asm/unistd.h b/arch/x86/include/asm/unistd.h
index 0d9776e9e2dc..16f3fc6ebf2e 100644
--- a/arch/x86/include/asm/unistd.h
+++ b/arch/x86/include/asm/unistd.h
@@ -50,6 +50,7 @@
50# define __ARCH_WANT_SYS_TIME 50# define __ARCH_WANT_SYS_TIME
51# define __ARCH_WANT_SYS_UTIME 51# define __ARCH_WANT_SYS_UTIME
52# define __ARCH_WANT_SYS_WAITPID 52# define __ARCH_WANT_SYS_WAITPID
53# define __ARCH_WANT_SYS_EXECVE
53 54
54/* 55/*
55 * "Conditional" syscalls 56 * "Conditional" syscalls
diff --git a/arch/x86/include/asm/uprobes.h b/arch/x86/include/asm/uprobes.h
index f3971bbcd1de..8ff8be7835ab 100644
--- a/arch/x86/include/asm/uprobes.h
+++ b/arch/x86/include/asm/uprobes.h
@@ -42,10 +42,11 @@ struct arch_uprobe {
42}; 42};
43 43
44struct arch_uprobe_task { 44struct arch_uprobe_task {
45 unsigned long saved_trap_nr;
46#ifdef CONFIG_X86_64 45#ifdef CONFIG_X86_64
47 unsigned long saved_scratch_register; 46 unsigned long saved_scratch_register;
48#endif 47#endif
48 unsigned int saved_trap_nr;
49 unsigned int saved_tf;
49}; 50};
50 51
51extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr); 52extern int arch_uprobe_analyze_insn(struct arch_uprobe *aup, struct mm_struct *mm, unsigned long addr);
diff --git a/arch/x86/include/asm/user.h b/arch/x86/include/asm/user.h
index 24532c7da3d6..ccab4af1646d 100644
--- a/arch/x86/include/asm/user.h
+++ b/arch/x86/include/asm/user.h
@@ -2,9 +2,9 @@
2#define _ASM_X86_USER_H 2#define _ASM_X86_USER_H
3 3
4#ifdef CONFIG_X86_32 4#ifdef CONFIG_X86_32
5# include "user_32.h" 5# include <asm/user_32.h>
6#else 6#else
7# include "user_64.h" 7# include <asm/user_64.h>
8#endif 8#endif
9 9
10#include <asm/types.h> 10#include <asm/types.h>
diff --git a/arch/x86/include/asm/vdso.h b/arch/x86/include/asm/vdso.h
index bb0522850b74..fddb53d63915 100644
--- a/arch/x86/include/asm/vdso.h
+++ b/arch/x86/include/asm/vdso.h
@@ -11,7 +11,8 @@ extern const char VDSO32_PRELINK[];
11#define VDSO32_SYMBOL(base, name) \ 11#define VDSO32_SYMBOL(base, name) \
12({ \ 12({ \
13 extern const char VDSO32_##name[]; \ 13 extern const char VDSO32_##name[]; \
14 (void *)(VDSO32_##name - VDSO32_PRELINK + (unsigned long)(base)); \ 14 (void __user *)(VDSO32_##name - VDSO32_PRELINK + \
15 (unsigned long)(base)); \
15}) 16})
16#endif 17#endif
17 18
diff --git a/arch/x86/include/asm/vgtod.h b/arch/x86/include/asm/vgtod.h
index 8b38be2de9e1..46e24d36b7da 100644
--- a/arch/x86/include/asm/vgtod.h
+++ b/arch/x86/include/asm/vgtod.h
@@ -17,8 +17,8 @@ struct vsyscall_gtod_data {
17 17
18 /* open coded 'struct timespec' */ 18 /* open coded 'struct timespec' */
19 time_t wall_time_sec; 19 time_t wall_time_sec;
20 u32 wall_time_nsec; 20 u64 wall_time_snsec;
21 u32 monotonic_time_nsec; 21 u64 monotonic_time_snsec;
22 time_t monotonic_time_sec; 22 time_t monotonic_time_sec;
23 23
24 struct timezone sys_tz; 24 struct timezone sys_tz;
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h
index 74fcb963595b..36ec21c36d68 100644
--- a/arch/x86/include/asm/vmx.h
+++ b/arch/x86/include/asm/vmx.h
@@ -25,6 +25,88 @@
25 * 25 *
26 */ 26 */
27 27
28#define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000
29
30#define EXIT_REASON_EXCEPTION_NMI 0
31#define EXIT_REASON_EXTERNAL_INTERRUPT 1
32#define EXIT_REASON_TRIPLE_FAULT 2
33
34#define EXIT_REASON_PENDING_INTERRUPT 7
35#define EXIT_REASON_NMI_WINDOW 8
36#define EXIT_REASON_TASK_SWITCH 9
37#define EXIT_REASON_CPUID 10
38#define EXIT_REASON_HLT 12
39#define EXIT_REASON_INVD 13
40#define EXIT_REASON_INVLPG 14
41#define EXIT_REASON_RDPMC 15
42#define EXIT_REASON_RDTSC 16
43#define EXIT_REASON_VMCALL 18
44#define EXIT_REASON_VMCLEAR 19
45#define EXIT_REASON_VMLAUNCH 20
46#define EXIT_REASON_VMPTRLD 21
47#define EXIT_REASON_VMPTRST 22
48#define EXIT_REASON_VMREAD 23
49#define EXIT_REASON_VMRESUME 24
50#define EXIT_REASON_VMWRITE 25
51#define EXIT_REASON_VMOFF 26
52#define EXIT_REASON_VMON 27
53#define EXIT_REASON_CR_ACCESS 28
54#define EXIT_REASON_DR_ACCESS 29
55#define EXIT_REASON_IO_INSTRUCTION 30
56#define EXIT_REASON_MSR_READ 31
57#define EXIT_REASON_MSR_WRITE 32
58#define EXIT_REASON_INVALID_STATE 33
59#define EXIT_REASON_MWAIT_INSTRUCTION 36
60#define EXIT_REASON_MONITOR_INSTRUCTION 39
61#define EXIT_REASON_PAUSE_INSTRUCTION 40
62#define EXIT_REASON_MCE_DURING_VMENTRY 41
63#define EXIT_REASON_TPR_BELOW_THRESHOLD 43
64#define EXIT_REASON_APIC_ACCESS 44
65#define EXIT_REASON_EPT_VIOLATION 48
66#define EXIT_REASON_EPT_MISCONFIG 49
67#define EXIT_REASON_WBINVD 54
68#define EXIT_REASON_XSETBV 55
69#define EXIT_REASON_INVPCID 58
70
71#define VMX_EXIT_REASONS \
72 { EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
73 { EXIT_REASON_EXTERNAL_INTERRUPT, "EXTERNAL_INTERRUPT" }, \
74 { EXIT_REASON_TRIPLE_FAULT, "TRIPLE_FAULT" }, \
75 { EXIT_REASON_PENDING_INTERRUPT, "PENDING_INTERRUPT" }, \
76 { EXIT_REASON_NMI_WINDOW, "NMI_WINDOW" }, \
77 { EXIT_REASON_TASK_SWITCH, "TASK_SWITCH" }, \
78 { EXIT_REASON_CPUID, "CPUID" }, \
79 { EXIT_REASON_HLT, "HLT" }, \
80 { EXIT_REASON_INVLPG, "INVLPG" }, \
81 { EXIT_REASON_RDPMC, "RDPMC" }, \
82 { EXIT_REASON_RDTSC, "RDTSC" }, \
83 { EXIT_REASON_VMCALL, "VMCALL" }, \
84 { EXIT_REASON_VMCLEAR, "VMCLEAR" }, \
85 { EXIT_REASON_VMLAUNCH, "VMLAUNCH" }, \
86 { EXIT_REASON_VMPTRLD, "VMPTRLD" }, \
87 { EXIT_REASON_VMPTRST, "VMPTRST" }, \
88 { EXIT_REASON_VMREAD, "VMREAD" }, \
89 { EXIT_REASON_VMRESUME, "VMRESUME" }, \
90 { EXIT_REASON_VMWRITE, "VMWRITE" }, \
91 { EXIT_REASON_VMOFF, "VMOFF" }, \
92 { EXIT_REASON_VMON, "VMON" }, \
93 { EXIT_REASON_CR_ACCESS, "CR_ACCESS" }, \
94 { EXIT_REASON_DR_ACCESS, "DR_ACCESS" }, \
95 { EXIT_REASON_IO_INSTRUCTION, "IO_INSTRUCTION" }, \
96 { EXIT_REASON_MSR_READ, "MSR_READ" }, \
97 { EXIT_REASON_MSR_WRITE, "MSR_WRITE" }, \
98 { EXIT_REASON_MWAIT_INSTRUCTION, "MWAIT_INSTRUCTION" }, \
99 { EXIT_REASON_MONITOR_INSTRUCTION, "MONITOR_INSTRUCTION" }, \
100 { EXIT_REASON_PAUSE_INSTRUCTION, "PAUSE_INSTRUCTION" }, \
101 { EXIT_REASON_MCE_DURING_VMENTRY, "MCE_DURING_VMENTRY" }, \
102 { EXIT_REASON_TPR_BELOW_THRESHOLD, "TPR_BELOW_THRESHOLD" }, \
103 { EXIT_REASON_APIC_ACCESS, "APIC_ACCESS" }, \
104 { EXIT_REASON_EPT_VIOLATION, "EPT_VIOLATION" }, \
105 { EXIT_REASON_EPT_MISCONFIG, "EPT_MISCONFIG" }, \
106 { EXIT_REASON_WBINVD, "WBINVD" }
107
108#ifdef __KERNEL__
109
28#include <linux/types.h> 110#include <linux/types.h>
29 111
30/* 112/*
@@ -241,49 +323,6 @@ enum vmcs_field {
241 HOST_RIP = 0x00006c16, 323 HOST_RIP = 0x00006c16,
242}; 324};
243 325
244#define VMX_EXIT_REASONS_FAILED_VMENTRY 0x80000000
245
246#define EXIT_REASON_EXCEPTION_NMI 0
247#define EXIT_REASON_EXTERNAL_INTERRUPT 1
248#define EXIT_REASON_TRIPLE_FAULT 2
249
250#define EXIT_REASON_PENDING_INTERRUPT 7
251#define EXIT_REASON_NMI_WINDOW 8
252#define EXIT_REASON_TASK_SWITCH 9
253#define EXIT_REASON_CPUID 10
254#define EXIT_REASON_HLT 12
255#define EXIT_REASON_INVD 13
256#define EXIT_REASON_INVLPG 14
257#define EXIT_REASON_RDPMC 15
258#define EXIT_REASON_RDTSC 16
259#define EXIT_REASON_VMCALL 18
260#define EXIT_REASON_VMCLEAR 19
261#define EXIT_REASON_VMLAUNCH 20
262#define EXIT_REASON_VMPTRLD 21
263#define EXIT_REASON_VMPTRST 22
264#define EXIT_REASON_VMREAD 23
265#define EXIT_REASON_VMRESUME 24
266#define EXIT_REASON_VMWRITE 25
267#define EXIT_REASON_VMOFF 26
268#define EXIT_REASON_VMON 27
269#define EXIT_REASON_CR_ACCESS 28
270#define EXIT_REASON_DR_ACCESS 29
271#define EXIT_REASON_IO_INSTRUCTION 30
272#define EXIT_REASON_MSR_READ 31
273#define EXIT_REASON_MSR_WRITE 32
274#define EXIT_REASON_INVALID_STATE 33
275#define EXIT_REASON_MWAIT_INSTRUCTION 36
276#define EXIT_REASON_MONITOR_INSTRUCTION 39
277#define EXIT_REASON_PAUSE_INSTRUCTION 40
278#define EXIT_REASON_MCE_DURING_VMENTRY 41
279#define EXIT_REASON_TPR_BELOW_THRESHOLD 43
280#define EXIT_REASON_APIC_ACCESS 44
281#define EXIT_REASON_EPT_VIOLATION 48
282#define EXIT_REASON_EPT_MISCONFIG 49
283#define EXIT_REASON_WBINVD 54
284#define EXIT_REASON_XSETBV 55
285#define EXIT_REASON_INVPCID 58
286
287/* 326/*
288 * Interruption-information format 327 * Interruption-information format
289 */ 328 */
@@ -488,3 +527,5 @@ enum vm_instruction_error_number {
488}; 527};
489 528
490#endif 529#endif
530
531#endif
diff --git a/arch/x86/include/asm/xen/interface.h b/arch/x86/include/asm/xen/interface.h
index ca9487d4f176..54d52ff1304a 100644
--- a/arch/x86/include/asm/xen/interface.h
+++ b/arch/x86/include/asm/xen/interface.h
@@ -122,9 +122,9 @@ struct arch_shared_info {
122#endif /* !__ASSEMBLY__ */ 122#endif /* !__ASSEMBLY__ */
123 123
124#ifdef CONFIG_X86_32 124#ifdef CONFIG_X86_32
125#include "interface_32.h" 125#include <asm/xen/interface_32.h>
126#else 126#else
127#include "interface_64.h" 127#include <asm/xen/interface_64.h>
128#endif 128#endif
129 129
130#include <asm/pvclock-abi.h> 130#include <asm/pvclock-abi.h>
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 93971e841dd5..472b9b783019 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -51,7 +51,8 @@ extern unsigned long set_phys_range_identity(unsigned long pfn_s,
51 51
52extern int m2p_add_override(unsigned long mfn, struct page *page, 52extern int m2p_add_override(unsigned long mfn, struct page *page,
53 struct gnttab_map_grant_ref *kmap_op); 53 struct gnttab_map_grant_ref *kmap_op);
54extern int m2p_remove_override(struct page *page, bool clear_pte); 54extern int m2p_remove_override(struct page *page,
55 struct gnttab_map_grant_ref *kmap_op);
55extern struct page *m2p_find_override(unsigned long mfn); 56extern struct page *m2p_find_override(unsigned long mfn);
56extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn); 57extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn);
57 58
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
index 7fcf6f3dbcc3..f8fde90bc45e 100644
--- a/arch/x86/include/asm/xor.h
+++ b/arch/x86/include/asm/xor.h
@@ -3,8 +3,8 @@
3# include <asm-generic/xor.h> 3# include <asm-generic/xor.h>
4#else 4#else
5#ifdef CONFIG_X86_32 5#ifdef CONFIG_X86_32
6# include "xor_32.h" 6# include <asm/xor_32.h>
7#else 7#else
8# include "xor_64.h" 8# include <asm/xor_64.h>
9#endif 9#endif
10#endif 10#endif
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
index 454570891bdc..f79cb7ec0e06 100644
--- a/arch/x86/include/asm/xor_32.h
+++ b/arch/x86/include/asm/xor_32.h
@@ -534,38 +534,6 @@ static struct xor_block_template xor_block_p5_mmx = {
534 * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo) 534 * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
535 */ 535 */
536 536
537#define XMMS_SAVE \
538do { \
539 preempt_disable(); \
540 cr0 = read_cr0(); \
541 clts(); \
542 asm volatile( \
543 "movups %%xmm0,(%0) ;\n\t" \
544 "movups %%xmm1,0x10(%0) ;\n\t" \
545 "movups %%xmm2,0x20(%0) ;\n\t" \
546 "movups %%xmm3,0x30(%0) ;\n\t" \
547 : \
548 : "r" (xmm_save) \
549 : "memory"); \
550} while (0)
551
552#define XMMS_RESTORE \
553do { \
554 asm volatile( \
555 "sfence ;\n\t" \
556 "movups (%0),%%xmm0 ;\n\t" \
557 "movups 0x10(%0),%%xmm1 ;\n\t" \
558 "movups 0x20(%0),%%xmm2 ;\n\t" \
559 "movups 0x30(%0),%%xmm3 ;\n\t" \
560 : \
561 : "r" (xmm_save) \
562 : "memory"); \
563 write_cr0(cr0); \
564 preempt_enable(); \
565} while (0)
566
567#define ALIGN16 __attribute__((aligned(16)))
568
569#define OFFS(x) "16*("#x")" 537#define OFFS(x) "16*("#x")"
570#define PF_OFFS(x) "256+16*("#x")" 538#define PF_OFFS(x) "256+16*("#x")"
571#define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n" 539#define PF0(x) " prefetchnta "PF_OFFS(x)"(%1) ;\n"
@@ -587,10 +555,8 @@ static void
587xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 555xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
588{ 556{
589 unsigned long lines = bytes >> 8; 557 unsigned long lines = bytes >> 8;
590 char xmm_save[16*4] ALIGN16;
591 int cr0;
592 558
593 XMMS_SAVE; 559 kernel_fpu_begin();
594 560
595 asm volatile( 561 asm volatile(
596#undef BLOCK 562#undef BLOCK
@@ -633,7 +599,7 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
633 : 599 :
634 : "memory"); 600 : "memory");
635 601
636 XMMS_RESTORE; 602 kernel_fpu_end();
637} 603}
638 604
639static void 605static void
@@ -641,10 +607,8 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
641 unsigned long *p3) 607 unsigned long *p3)
642{ 608{
643 unsigned long lines = bytes >> 8; 609 unsigned long lines = bytes >> 8;
644 char xmm_save[16*4] ALIGN16;
645 int cr0;
646 610
647 XMMS_SAVE; 611 kernel_fpu_begin();
648 612
649 asm volatile( 613 asm volatile(
650#undef BLOCK 614#undef BLOCK
@@ -694,7 +658,7 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
694 : 658 :
695 : "memory" ); 659 : "memory" );
696 660
697 XMMS_RESTORE; 661 kernel_fpu_end();
698} 662}
699 663
700static void 664static void
@@ -702,10 +666,8 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
702 unsigned long *p3, unsigned long *p4) 666 unsigned long *p3, unsigned long *p4)
703{ 667{
704 unsigned long lines = bytes >> 8; 668 unsigned long lines = bytes >> 8;
705 char xmm_save[16*4] ALIGN16;
706 int cr0;
707 669
708 XMMS_SAVE; 670 kernel_fpu_begin();
709 671
710 asm volatile( 672 asm volatile(
711#undef BLOCK 673#undef BLOCK
@@ -762,7 +724,7 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
762 : 724 :
763 : "memory" ); 725 : "memory" );
764 726
765 XMMS_RESTORE; 727 kernel_fpu_end();
766} 728}
767 729
768static void 730static void
@@ -770,10 +732,8 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
770 unsigned long *p3, unsigned long *p4, unsigned long *p5) 732 unsigned long *p3, unsigned long *p4, unsigned long *p5)
771{ 733{
772 unsigned long lines = bytes >> 8; 734 unsigned long lines = bytes >> 8;
773 char xmm_save[16*4] ALIGN16;
774 int cr0;
775 735
776 XMMS_SAVE; 736 kernel_fpu_begin();
777 737
778 /* Make sure GCC forgets anything it knows about p4 or p5, 738 /* Make sure GCC forgets anything it knows about p4 or p5,
779 such that it won't pass to the asm volatile below a 739 such that it won't pass to the asm volatile below a
@@ -850,7 +810,7 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
850 like assuming they have some legal value. */ 810 like assuming they have some legal value. */
851 asm("" : "=r" (p4), "=r" (p5)); 811 asm("" : "=r" (p4), "=r" (p5));
852 812
853 XMMS_RESTORE; 813 kernel_fpu_end();
854} 814}
855 815
856static struct xor_block_template xor_block_pIII_sse = { 816static struct xor_block_template xor_block_pIII_sse = {
@@ -862,7 +822,7 @@ static struct xor_block_template xor_block_pIII_sse = {
862}; 822};
863 823
864/* Also try the AVX routines */ 824/* Also try the AVX routines */
865#include "xor_avx.h" 825#include <asm/xor_avx.h>
866 826
867/* Also try the generic routines. */ 827/* Also try the generic routines. */
868#include <asm-generic/xor.h> 828#include <asm-generic/xor.h>
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
index b9b2323e90fe..87ac522c4af5 100644
--- a/arch/x86/include/asm/xor_64.h
+++ b/arch/x86/include/asm/xor_64.h
@@ -34,41 +34,7 @@
34 * no advantages to be gotten from x86-64 here anyways. 34 * no advantages to be gotten from x86-64 here anyways.
35 */ 35 */
36 36
37typedef struct { 37#include <asm/i387.h>
38 unsigned long a, b;
39} __attribute__((aligned(16))) xmm_store_t;
40
41/* Doesn't use gcc to save the XMM registers, because there is no easy way to
42 tell it to do a clts before the register saving. */
43#define XMMS_SAVE \
44do { \
45 preempt_disable(); \
46 asm volatile( \
47 "movq %%cr0,%0 ;\n\t" \
48 "clts ;\n\t" \
49 "movups %%xmm0,(%1) ;\n\t" \
50 "movups %%xmm1,0x10(%1) ;\n\t" \
51 "movups %%xmm2,0x20(%1) ;\n\t" \
52 "movups %%xmm3,0x30(%1) ;\n\t" \
53 : "=&r" (cr0) \
54 : "r" (xmm_save) \
55 : "memory"); \
56} while (0)
57
58#define XMMS_RESTORE \
59do { \
60 asm volatile( \
61 "sfence ;\n\t" \
62 "movups (%1),%%xmm0 ;\n\t" \
63 "movups 0x10(%1),%%xmm1 ;\n\t" \
64 "movups 0x20(%1),%%xmm2 ;\n\t" \
65 "movups 0x30(%1),%%xmm3 ;\n\t" \
66 "movq %0,%%cr0 ;\n\t" \
67 : \
68 : "r" (cr0), "r" (xmm_save) \
69 : "memory"); \
70 preempt_enable(); \
71} while (0)
72 38
73#define OFFS(x) "16*("#x")" 39#define OFFS(x) "16*("#x")"
74#define PF_OFFS(x) "256+16*("#x")" 40#define PF_OFFS(x) "256+16*("#x")"
@@ -91,10 +57,8 @@ static void
91xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2) 57xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
92{ 58{
93 unsigned int lines = bytes >> 8; 59 unsigned int lines = bytes >> 8;
94 unsigned long cr0;
95 xmm_store_t xmm_save[4];
96 60
97 XMMS_SAVE; 61 kernel_fpu_begin();
98 62
99 asm volatile( 63 asm volatile(
100#undef BLOCK 64#undef BLOCK
@@ -135,7 +99,7 @@ xor_sse_2(unsigned long bytes, unsigned long *p1, unsigned long *p2)
135 : [inc] "r" (256UL) 99 : [inc] "r" (256UL)
136 : "memory"); 100 : "memory");
137 101
138 XMMS_RESTORE; 102 kernel_fpu_end();
139} 103}
140 104
141static void 105static void
@@ -143,11 +107,8 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
143 unsigned long *p3) 107 unsigned long *p3)
144{ 108{
145 unsigned int lines = bytes >> 8; 109 unsigned int lines = bytes >> 8;
146 xmm_store_t xmm_save[4];
147 unsigned long cr0;
148
149 XMMS_SAVE;
150 110
111 kernel_fpu_begin();
151 asm volatile( 112 asm volatile(
152#undef BLOCK 113#undef BLOCK
153#define BLOCK(i) \ 114#define BLOCK(i) \
@@ -194,7 +155,7 @@ xor_sse_3(unsigned long bytes, unsigned long *p1, unsigned long *p2,
194 [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3) 155 [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
195 : [inc] "r" (256UL) 156 : [inc] "r" (256UL)
196 : "memory"); 157 : "memory");
197 XMMS_RESTORE; 158 kernel_fpu_end();
198} 159}
199 160
200static void 161static void
@@ -202,10 +163,8 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
202 unsigned long *p3, unsigned long *p4) 163 unsigned long *p3, unsigned long *p4)
203{ 164{
204 unsigned int lines = bytes >> 8; 165 unsigned int lines = bytes >> 8;
205 xmm_store_t xmm_save[4];
206 unsigned long cr0;
207 166
208 XMMS_SAVE; 167 kernel_fpu_begin();
209 168
210 asm volatile( 169 asm volatile(
211#undef BLOCK 170#undef BLOCK
@@ -261,7 +220,7 @@ xor_sse_4(unsigned long bytes, unsigned long *p1, unsigned long *p2,
261 : [inc] "r" (256UL) 220 : [inc] "r" (256UL)
262 : "memory" ); 221 : "memory" );
263 222
264 XMMS_RESTORE; 223 kernel_fpu_end();
265} 224}
266 225
267static void 226static void
@@ -269,10 +228,8 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
269 unsigned long *p3, unsigned long *p4, unsigned long *p5) 228 unsigned long *p3, unsigned long *p4, unsigned long *p5)
270{ 229{
271 unsigned int lines = bytes >> 8; 230 unsigned int lines = bytes >> 8;
272 xmm_store_t xmm_save[4];
273 unsigned long cr0;
274 231
275 XMMS_SAVE; 232 kernel_fpu_begin();
276 233
277 asm volatile( 234 asm volatile(
278#undef BLOCK 235#undef BLOCK
@@ -336,7 +293,7 @@ xor_sse_5(unsigned long bytes, unsigned long *p1, unsigned long *p2,
336 : [inc] "r" (256UL) 293 : [inc] "r" (256UL)
337 : "memory"); 294 : "memory");
338 295
339 XMMS_RESTORE; 296 kernel_fpu_end();
340} 297}
341 298
342static struct xor_block_template xor_block_sse = { 299static struct xor_block_template xor_block_sse = {
@@ -349,7 +306,7 @@ static struct xor_block_template xor_block_sse = {
349 306
350 307
351/* Also try the AVX routines */ 308/* Also try the AVX routines */
352#include "xor_avx.h" 309#include <asm/xor_avx.h>
353 310
354#undef XOR_TRY_TEMPLATES 311#undef XOR_TRY_TEMPLATES
355#define XOR_TRY_TEMPLATES \ 312#define XOR_TRY_TEMPLATES \
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h
index 2510d35f480e..7ea79c5fa1f2 100644
--- a/arch/x86/include/asm/xor_avx.h
+++ b/arch/x86/include/asm/xor_avx.h
@@ -20,32 +20,6 @@
20#include <linux/compiler.h> 20#include <linux/compiler.h>
21#include <asm/i387.h> 21#include <asm/i387.h>
22 22
23#define ALIGN32 __aligned(32)
24
25#define YMM_SAVED_REGS 4
26
27#define YMMS_SAVE \
28do { \
29 preempt_disable(); \
30 cr0 = read_cr0(); \
31 clts(); \
32 asm volatile("vmovaps %%ymm0, %0" : "=m" (ymm_save[0]) : : "memory"); \
33 asm volatile("vmovaps %%ymm1, %0" : "=m" (ymm_save[32]) : : "memory"); \
34 asm volatile("vmovaps %%ymm2, %0" : "=m" (ymm_save[64]) : : "memory"); \
35 asm volatile("vmovaps %%ymm3, %0" : "=m" (ymm_save[96]) : : "memory"); \
36} while (0);
37
38#define YMMS_RESTORE \
39do { \
40 asm volatile("sfence" : : : "memory"); \
41 asm volatile("vmovaps %0, %%ymm3" : : "m" (ymm_save[96])); \
42 asm volatile("vmovaps %0, %%ymm2" : : "m" (ymm_save[64])); \
43 asm volatile("vmovaps %0, %%ymm1" : : "m" (ymm_save[32])); \
44 asm volatile("vmovaps %0, %%ymm0" : : "m" (ymm_save[0])); \
45 write_cr0(cr0); \
46 preempt_enable(); \
47} while (0);
48
49#define BLOCK4(i) \ 23#define BLOCK4(i) \
50 BLOCK(32 * i, 0) \ 24 BLOCK(32 * i, 0) \
51 BLOCK(32 * (i + 1), 1) \ 25 BLOCK(32 * (i + 1), 1) \
@@ -60,10 +34,9 @@ do { \
60 34
61static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1) 35static void xor_avx_2(unsigned long bytes, unsigned long *p0, unsigned long *p1)
62{ 36{
63 unsigned long cr0, lines = bytes >> 9; 37 unsigned long lines = bytes >> 9;
64 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
65 38
66 YMMS_SAVE 39 kernel_fpu_begin();
67 40
68 while (lines--) { 41 while (lines--) {
69#undef BLOCK 42#undef BLOCK
@@ -82,16 +55,15 @@ do { \
82 p1 = (unsigned long *)((uintptr_t)p1 + 512); 55 p1 = (unsigned long *)((uintptr_t)p1 + 512);
83 } 56 }
84 57
85 YMMS_RESTORE 58 kernel_fpu_end();
86} 59}
87 60
88static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1, 61static void xor_avx_3(unsigned long bytes, unsigned long *p0, unsigned long *p1,
89 unsigned long *p2) 62 unsigned long *p2)
90{ 63{
91 unsigned long cr0, lines = bytes >> 9; 64 unsigned long lines = bytes >> 9;
92 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
93 65
94 YMMS_SAVE 66 kernel_fpu_begin();
95 67
96 while (lines--) { 68 while (lines--) {
97#undef BLOCK 69#undef BLOCK
@@ -113,16 +85,15 @@ do { \
113 p2 = (unsigned long *)((uintptr_t)p2 + 512); 85 p2 = (unsigned long *)((uintptr_t)p2 + 512);
114 } 86 }
115 87
116 YMMS_RESTORE 88 kernel_fpu_end();
117} 89}
118 90
119static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1, 91static void xor_avx_4(unsigned long bytes, unsigned long *p0, unsigned long *p1,
120 unsigned long *p2, unsigned long *p3) 92 unsigned long *p2, unsigned long *p3)
121{ 93{
122 unsigned long cr0, lines = bytes >> 9; 94 unsigned long lines = bytes >> 9;
123 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
124 95
125 YMMS_SAVE 96 kernel_fpu_begin();
126 97
127 while (lines--) { 98 while (lines--) {
128#undef BLOCK 99#undef BLOCK
@@ -147,16 +118,15 @@ do { \
147 p3 = (unsigned long *)((uintptr_t)p3 + 512); 118 p3 = (unsigned long *)((uintptr_t)p3 + 512);
148 } 119 }
149 120
150 YMMS_RESTORE 121 kernel_fpu_end();
151} 122}
152 123
153static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1, 124static void xor_avx_5(unsigned long bytes, unsigned long *p0, unsigned long *p1,
154 unsigned long *p2, unsigned long *p3, unsigned long *p4) 125 unsigned long *p2, unsigned long *p3, unsigned long *p4)
155{ 126{
156 unsigned long cr0, lines = bytes >> 9; 127 unsigned long lines = bytes >> 9;
157 char ymm_save[32 * YMM_SAVED_REGS] ALIGN32;
158 128
159 YMMS_SAVE 129 kernel_fpu_begin();
160 130
161 while (lines--) { 131 while (lines--) {
162#undef BLOCK 132#undef BLOCK
@@ -184,7 +154,7 @@ do { \
184 p4 = (unsigned long *)((uintptr_t)p4 + 512); 154 p4 = (unsigned long *)((uintptr_t)p4 + 512);
185 } 155 }
186 156
187 YMMS_RESTORE 157 kernel_fpu_end();
188} 158}
189 159
190static struct xor_block_template xor_block_avx = { 160static struct xor_block_template xor_block_avx = {
diff --git a/arch/x86/include/asm/xsave.h b/arch/x86/include/asm/xsave.h
index 8a1b6f9b594a..0415cdabb5a6 100644
--- a/arch/x86/include/asm/xsave.h
+++ b/arch/x86/include/asm/xsave.h
@@ -34,17 +34,14 @@
34extern unsigned int xstate_size; 34extern unsigned int xstate_size;
35extern u64 pcntxt_mask; 35extern u64 pcntxt_mask;
36extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS]; 36extern u64 xstate_fx_sw_bytes[USER_XSTATE_FX_SW_WORDS];
37extern struct xsave_struct *init_xstate_buf;
37 38
38extern void xsave_init(void); 39extern void xsave_init(void);
39extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); 40extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask);
40extern int init_fpu(struct task_struct *child); 41extern int init_fpu(struct task_struct *child);
41extern int check_for_xstate(struct i387_fxsave_struct __user *buf,
42 void __user *fpstate,
43 struct _fpx_sw_bytes *sw);
44 42
45static inline int fpu_xrstor_checking(struct fpu *fpu) 43static inline int fpu_xrstor_checking(struct xsave_struct *fx)
46{ 44{
47 struct xsave_struct *fx = &fpu->state->xsave;
48 int err; 45 int err;
49 46
50 asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t" 47 asm volatile("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n\t"
@@ -69,13 +66,13 @@ static inline int xsave_user(struct xsave_struct __user *buf)
69 * Clear the xsave header first, so that reserved fields are 66 * Clear the xsave header first, so that reserved fields are
70 * initialized to zero. 67 * initialized to zero.
71 */ 68 */
72 err = __clear_user(&buf->xsave_hdr, 69 err = __clear_user(&buf->xsave_hdr, sizeof(buf->xsave_hdr));
73 sizeof(struct xsave_hdr_struct));
74 if (unlikely(err)) 70 if (unlikely(err))
75 return -EFAULT; 71 return -EFAULT;
76 72
77 __asm__ __volatile__("1: .byte " REX_PREFIX "0x0f,0xae,0x27\n" 73 __asm__ __volatile__(ASM_STAC "\n"
78 "2:\n" 74 "1: .byte " REX_PREFIX "0x0f,0xae,0x27\n"
75 "2: " ASM_CLAC "\n"
79 ".section .fixup,\"ax\"\n" 76 ".section .fixup,\"ax\"\n"
80 "3: movl $-1,%[err]\n" 77 "3: movl $-1,%[err]\n"
81 " jmp 2b\n" 78 " jmp 2b\n"
@@ -84,9 +81,6 @@ static inline int xsave_user(struct xsave_struct __user *buf)
84 : [err] "=r" (err) 81 : [err] "=r" (err)
85 : "D" (buf), "a" (-1), "d" (-1), "0" (0) 82 : "D" (buf), "a" (-1), "d" (-1), "0" (0)
86 : "memory"); 83 : "memory");
87 if (unlikely(err) && __clear_user(buf, xstate_size))
88 err = -EFAULT;
89 /* No need to clear here because the caller clears USED_MATH */
90 return err; 84 return err;
91} 85}
92 86
@@ -97,8 +91,9 @@ static inline int xrestore_user(struct xsave_struct __user *buf, u64 mask)
97 u32 lmask = mask; 91 u32 lmask = mask;
98 u32 hmask = mask >> 32; 92 u32 hmask = mask >> 32;
99 93
100 __asm__ __volatile__("1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n" 94 __asm__ __volatile__(ASM_STAC "\n"
101 "2:\n" 95 "1: .byte " REX_PREFIX "0x0f,0xae,0x2f\n"
96 "2: " ASM_CLAC "\n"
102 ".section .fixup,\"ax\"\n" 97 ".section .fixup,\"ax\"\n"
103 "3: movl $-1,%[err]\n" 98 "3: movl $-1,%[err]\n"
104 " jmp 2b\n" 99 " jmp 2b\n"
diff --git a/arch/x86/include/uapi/asm/Kbuild b/arch/x86/include/uapi/asm/Kbuild
new file mode 100644
index 000000000000..83b6e9a0dce4
--- /dev/null
+++ b/arch/x86/include/uapi/asm/Kbuild
@@ -0,0 +1,6 @@
1# UAPI Header export list
2include include/uapi/asm-generic/Kbuild.asm
3
4genhdr-y += unistd_32.h
5genhdr-y += unistd_64.h
6genhdr-y += unistd_x32.h
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
index 8215e5652d97..91ce48f05f9f 100644
--- a/arch/x86/kernel/Makefile
+++ b/arch/x86/kernel/Makefile
@@ -23,7 +23,7 @@ obj-y += time.o ioport.o ldt.o dumpstack.o nmi.o
23obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o 23obj-y += setup.o x86_init.o i8259.o irqinit.o jump_label.o
24obj-$(CONFIG_IRQ_WORK) += irq_work.o 24obj-$(CONFIG_IRQ_WORK) += irq_work.o
25obj-y += probe_roms.o 25obj-y += probe_roms.o
26obj-$(CONFIG_X86_32) += sys_i386_32.o i386_ksyms_32.o 26obj-$(CONFIG_X86_32) += i386_ksyms_32.o
27obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o 27obj-$(CONFIG_X86_64) += sys_x86_64.o x8664_ksyms_64.o
28obj-y += syscall_$(BITS).o 28obj-y += syscall_$(BITS).o
29obj-$(CONFIG_X86_64) += vsyscall_64.o 29obj-$(CONFIG_X86_64) += vsyscall_64.o
@@ -81,8 +81,7 @@ obj-$(CONFIG_DEBUG_RODATA_TEST) += test_rodata.o
81obj-$(CONFIG_DEBUG_NX_TEST) += test_nx.o 81obj-$(CONFIG_DEBUG_NX_TEST) += test_nx.o
82obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o 82obj-$(CONFIG_DEBUG_NMI_SELFTEST) += nmi_selftest.o
83 83
84obj-$(CONFIG_KVM_GUEST) += kvm.o 84obj-$(CONFIG_KVM_GUEST) += kvm.o kvmclock.o
85obj-$(CONFIG_KVM_CLOCK) += kvmclock.o
86obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o 85obj-$(CONFIG_PARAVIRT) += paravirt.o paravirt_patch_$(BITS).o
87obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o 86obj-$(CONFIG_PARAVIRT_SPINLOCKS)+= paravirt-spinlocks.o
88obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o 87obj-$(CONFIG_PARAVIRT_CLOCK) += pvclock.o
@@ -100,6 +99,8 @@ obj-$(CONFIG_SWIOTLB) += pci-swiotlb.o
100obj-$(CONFIG_OF) += devicetree.o 99obj-$(CONFIG_OF) += devicetree.o
101obj-$(CONFIG_UPROBES) += uprobes.o 100obj-$(CONFIG_UPROBES) += uprobes.o
102 101
102obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
103
103### 104###
104# 64 bit specific files 105# 64 bit specific files
105ifeq ($(CONFIG_X86_64),y) 106ifeq ($(CONFIG_X86_64),y)
diff --git a/arch/x86/kernel/acpi/boot.c b/arch/x86/kernel/acpi/boot.c
index b2297e58c6ed..e651f7a589ac 100644
--- a/arch/x86/kernel/acpi/boot.c
+++ b/arch/x86/kernel/acpi/boot.c
@@ -656,7 +656,7 @@ static int __cpuinit _acpi_map_lsapic(acpi_handle handle, int *pcpu)
656 acpi_register_lapic(physid, ACPI_MADT_ENABLED); 656 acpi_register_lapic(physid, ACPI_MADT_ENABLED);
657 657
658 /* 658 /*
659 * If mp_register_lapic successfully generates a new logical cpu 659 * If acpi_register_lapic successfully generates a new logical cpu
660 * number, then the following will get us exactly what was mapped 660 * number, then the following will get us exactly what was mapped
661 */ 661 */
662 cpumask_andnot(new_map, cpu_present_mask, tmp_map); 662 cpumask_andnot(new_map, cpu_present_mask, tmp_map);
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 1b8e5a03d942..11676cf65aee 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -43,17 +43,22 @@ int acpi_suspend_lowlevel(void)
43 43
44 header->video_mode = saved_video_mode; 44 header->video_mode = saved_video_mode;
45 45
46 header->pmode_behavior = 0;
47
46#ifndef CONFIG_64BIT 48#ifndef CONFIG_64BIT
47 store_gdt((struct desc_ptr *)&header->pmode_gdt); 49 store_gdt((struct desc_ptr *)&header->pmode_gdt);
48 50
49 if (rdmsr_safe(MSR_EFER, &header->pmode_efer_low, 51 if (!rdmsr_safe(MSR_EFER,
50 &header->pmode_efer_high)) 52 &header->pmode_efer_low,
51 header->pmode_efer_low = header->pmode_efer_high = 0; 53 &header->pmode_efer_high))
54 header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_EFER);
52#endif /* !CONFIG_64BIT */ 55#endif /* !CONFIG_64BIT */
53 56
54 header->pmode_cr0 = read_cr0(); 57 header->pmode_cr0 = read_cr0();
55 header->pmode_cr4 = read_cr4_safe(); 58 if (__this_cpu_read(cpu_info.cpuid_level) >= 0) {
56 header->pmode_behavior = 0; 59 header->pmode_cr4 = read_cr4();
60 header->pmode_behavior |= (1 << WAKEUP_BEHAVIOR_RESTORE_CR4);
61 }
57 if (!rdmsr_safe(MSR_IA32_MISC_ENABLE, 62 if (!rdmsr_safe(MSR_IA32_MISC_ENABLE,
58 &header->pmode_misc_en_low, 63 &header->pmode_misc_en_low,
59 &header->pmode_misc_en_high)) 64 &header->pmode_misc_en_high))
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index ced4534baed5..ef5ccca79a6c 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -23,19 +23,6 @@
23 23
24#define MAX_PATCH_LEN (255-1) 24#define MAX_PATCH_LEN (255-1)
25 25
26#ifdef CONFIG_HOTPLUG_CPU
27static int smp_alt_once;
28
29static int __init bootonly(char *str)
30{
31 smp_alt_once = 1;
32 return 1;
33}
34__setup("smp-alt-boot", bootonly);
35#else
36#define smp_alt_once 1
37#endif
38
39static int __initdata_or_module debug_alternative; 26static int __initdata_or_module debug_alternative;
40 27
41static int __init debug_alt(char *str) 28static int __init debug_alt(char *str)
@@ -317,7 +304,7 @@ static void alternatives_smp_lock(const s32 *start, const s32 *end,
317 /* turn DS segment override prefix into lock prefix */ 304 /* turn DS segment override prefix into lock prefix */
318 if (*ptr == 0x3e) 305 if (*ptr == 0x3e)
319 text_poke(ptr, ((unsigned char []){0xf0}), 1); 306 text_poke(ptr, ((unsigned char []){0xf0}), 1);
320 }; 307 }
321 mutex_unlock(&text_mutex); 308 mutex_unlock(&text_mutex);
322} 309}
323 310
@@ -326,9 +313,6 @@ static void alternatives_smp_unlock(const s32 *start, const s32 *end,
326{ 313{
327 const s32 *poff; 314 const s32 *poff;
328 315
329 if (noreplace_smp)
330 return;
331
332 mutex_lock(&text_mutex); 316 mutex_lock(&text_mutex);
333 for (poff = start; poff < end; poff++) { 317 for (poff = start; poff < end; poff++) {
334 u8 *ptr = (u8 *)poff + *poff; 318 u8 *ptr = (u8 *)poff + *poff;
@@ -338,7 +322,7 @@ static void alternatives_smp_unlock(const s32 *start, const s32 *end,
338 /* turn lock prefix into DS segment override prefix */ 322 /* turn lock prefix into DS segment override prefix */
339 if (*ptr == 0xf0) 323 if (*ptr == 0xf0)
340 text_poke(ptr, ((unsigned char []){0x3E}), 1); 324 text_poke(ptr, ((unsigned char []){0x3E}), 1);
341 }; 325 }
342 mutex_unlock(&text_mutex); 326 mutex_unlock(&text_mutex);
343} 327}
344 328
@@ -359,7 +343,7 @@ struct smp_alt_module {
359}; 343};
360static LIST_HEAD(smp_alt_modules); 344static LIST_HEAD(smp_alt_modules);
361static DEFINE_MUTEX(smp_alt); 345static DEFINE_MUTEX(smp_alt);
362static int smp_mode = 1; /* protected by smp_alt */ 346static bool uniproc_patched = false; /* protected by smp_alt */
363 347
364void __init_or_module alternatives_smp_module_add(struct module *mod, 348void __init_or_module alternatives_smp_module_add(struct module *mod,
365 char *name, 349 char *name,
@@ -368,19 +352,18 @@ void __init_or_module alternatives_smp_module_add(struct module *mod,
368{ 352{
369 struct smp_alt_module *smp; 353 struct smp_alt_module *smp;
370 354
371 if (noreplace_smp) 355 mutex_lock(&smp_alt);
372 return; 356 if (!uniproc_patched)
357 goto unlock;
373 358
374 if (smp_alt_once) { 359 if (num_possible_cpus() == 1)
375 if (boot_cpu_has(X86_FEATURE_UP)) 360 /* Don't bother remembering, we'll never have to undo it. */
376 alternatives_smp_unlock(locks, locks_end, 361 goto smp_unlock;
377 text, text_end);
378 return;
379 }
380 362
381 smp = kzalloc(sizeof(*smp), GFP_KERNEL); 363 smp = kzalloc(sizeof(*smp), GFP_KERNEL);
382 if (NULL == smp) 364 if (NULL == smp)
383 return; /* we'll run the (safe but slow) SMP code then ... */ 365 /* we'll run the (safe but slow) SMP code then ... */
366 goto unlock;
384 367
385 smp->mod = mod; 368 smp->mod = mod;
386 smp->name = name; 369 smp->name = name;
@@ -392,11 +375,10 @@ void __init_or_module alternatives_smp_module_add(struct module *mod,
392 __func__, smp->locks, smp->locks_end, 375 __func__, smp->locks, smp->locks_end,
393 smp->text, smp->text_end, smp->name); 376 smp->text, smp->text_end, smp->name);
394 377
395 mutex_lock(&smp_alt);
396 list_add_tail(&smp->next, &smp_alt_modules); 378 list_add_tail(&smp->next, &smp_alt_modules);
397 if (boot_cpu_has(X86_FEATURE_UP)) 379smp_unlock:
398 alternatives_smp_unlock(smp->locks, smp->locks_end, 380 alternatives_smp_unlock(locks, locks_end, text, text_end);
399 smp->text, smp->text_end); 381unlock:
400 mutex_unlock(&smp_alt); 382 mutex_unlock(&smp_alt);
401} 383}
402 384
@@ -404,24 +386,18 @@ void __init_or_module alternatives_smp_module_del(struct module *mod)
404{ 386{
405 struct smp_alt_module *item; 387 struct smp_alt_module *item;
406 388
407 if (smp_alt_once || noreplace_smp)
408 return;
409
410 mutex_lock(&smp_alt); 389 mutex_lock(&smp_alt);
411 list_for_each_entry(item, &smp_alt_modules, next) { 390 list_for_each_entry(item, &smp_alt_modules, next) {
412 if (mod != item->mod) 391 if (mod != item->mod)
413 continue; 392 continue;
414 list_del(&item->next); 393 list_del(&item->next);
415 mutex_unlock(&smp_alt);
416 DPRINTK("%s: %s\n", __func__, item->name);
417 kfree(item); 394 kfree(item);
418 return; 395 break;
419 } 396 }
420 mutex_unlock(&smp_alt); 397 mutex_unlock(&smp_alt);
421} 398}
422 399
423bool skip_smp_alternatives; 400void alternatives_enable_smp(void)
424void alternatives_smp_switch(int smp)
425{ 401{
426 struct smp_alt_module *mod; 402 struct smp_alt_module *mod;
427 403
@@ -436,34 +412,21 @@ void alternatives_smp_switch(int smp)
436 pr_info("lockdep: fixing up alternatives\n"); 412 pr_info("lockdep: fixing up alternatives\n");
437#endif 413#endif
438 414
439 if (noreplace_smp || smp_alt_once || skip_smp_alternatives) 415 /* Why bother if there are no other CPUs? */
440 return; 416 BUG_ON(num_possible_cpus() == 1);
441 BUG_ON(!smp && (num_online_cpus() > 1));
442 417
443 mutex_lock(&smp_alt); 418 mutex_lock(&smp_alt);
444 419
445 /* 420 if (uniproc_patched) {
446 * Avoid unnecessary switches because it forces JIT based VMs to
447 * throw away all cached translations, which can be quite costly.
448 */
449 if (smp == smp_mode) {
450 /* nothing */
451 } else if (smp) {
452 pr_info("switching to SMP code\n"); 421 pr_info("switching to SMP code\n");
422 BUG_ON(num_online_cpus() != 1);
453 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP); 423 clear_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
454 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP); 424 clear_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
455 list_for_each_entry(mod, &smp_alt_modules, next) 425 list_for_each_entry(mod, &smp_alt_modules, next)
456 alternatives_smp_lock(mod->locks, mod->locks_end, 426 alternatives_smp_lock(mod->locks, mod->locks_end,
457 mod->text, mod->text_end); 427 mod->text, mod->text_end);
458 } else { 428 uniproc_patched = false;
459 pr_info("switching to UP code\n");
460 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
461 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
462 list_for_each_entry(mod, &smp_alt_modules, next)
463 alternatives_smp_unlock(mod->locks, mod->locks_end,
464 mod->text, mod->text_end);
465 } 429 }
466 smp_mode = smp;
467 mutex_unlock(&smp_alt); 430 mutex_unlock(&smp_alt);
468} 431}
469 432
@@ -540,40 +503,22 @@ void __init alternative_instructions(void)
540 503
541 apply_alternatives(__alt_instructions, __alt_instructions_end); 504 apply_alternatives(__alt_instructions, __alt_instructions_end);
542 505
543 /* switch to patch-once-at-boottime-only mode and free the
544 * tables in case we know the number of CPUs will never ever
545 * change */
546#ifdef CONFIG_HOTPLUG_CPU
547 if (num_possible_cpus() < 2)
548 smp_alt_once = 1;
549#endif
550
551#ifdef CONFIG_SMP 506#ifdef CONFIG_SMP
552 if (smp_alt_once) { 507 /* Patch to UP if other cpus not imminent. */
553 if (1 == num_possible_cpus()) { 508 if (!noreplace_smp && (num_present_cpus() == 1 || setup_max_cpus <= 1)) {
554 pr_info("switching to UP code\n"); 509 uniproc_patched = true;
555 set_cpu_cap(&boot_cpu_data, X86_FEATURE_UP);
556 set_cpu_cap(&cpu_data(0), X86_FEATURE_UP);
557
558 alternatives_smp_unlock(__smp_locks, __smp_locks_end,
559 _text, _etext);
560 }
561 } else {
562 alternatives_smp_module_add(NULL, "core kernel", 510 alternatives_smp_module_add(NULL, "core kernel",
563 __smp_locks, __smp_locks_end, 511 __smp_locks, __smp_locks_end,
564 _text, _etext); 512 _text, _etext);
565
566 /* Only switch to UP mode if we don't immediately boot others */
567 if (num_present_cpus() == 1 || setup_max_cpus <= 1)
568 alternatives_smp_switch(0);
569 } 513 }
570#endif
571 apply_paravirt(__parainstructions, __parainstructions_end);
572 514
573 if (smp_alt_once) 515 if (!uniproc_patched || num_possible_cpus() == 1)
574 free_init_pages("SMP alternatives", 516 free_init_pages("SMP alternatives",
575 (unsigned long)__smp_locks, 517 (unsigned long)__smp_locks,
576 (unsigned long)__smp_locks_end); 518 (unsigned long)__smp_locks_end);
519#endif
520
521 apply_paravirt(__parainstructions, __parainstructions_end);
577 522
578 restart_nmi(); 523 restart_nmi();
579} 524}
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 24deb3082328..b17416e72fbd 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -1934,7 +1934,7 @@ void smp_error_interrupt(struct pt_regs *regs)
1934 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]); 1934 apic_printk(APIC_DEBUG, KERN_CONT " : %s", error_interrupt_reason[i]);
1935 i++; 1935 i++;
1936 v1 >>= 1; 1936 v1 >>= 1;
1937 }; 1937 }
1938 1938
1939 apic_printk(APIC_DEBUG, KERN_CONT "\n"); 1939 apic_printk(APIC_DEBUG, KERN_CONT "\n");
1940 1940
diff --git a/arch/x86/kernel/apic/apic_numachip.c b/arch/x86/kernel/apic/apic_numachip.c
index bc552cff2578..a65829ac2b9a 100644
--- a/arch/x86/kernel/apic/apic_numachip.c
+++ b/arch/x86/kernel/apic/apic_numachip.c
@@ -30,7 +30,7 @@
30 30
31static int numachip_system __read_mostly; 31static int numachip_system __read_mostly;
32 32
33static struct apic apic_numachip __read_mostly; 33static const struct apic apic_numachip __read_mostly;
34 34
35static unsigned int get_apic_id(unsigned long x) 35static unsigned int get_apic_id(unsigned long x)
36{ 36{
@@ -199,7 +199,7 @@ static int numachip_acpi_madt_oem_check(char *oem_id, char *oem_table_id)
199 return 0; 199 return 0;
200} 200}
201 201
202static struct apic apic_numachip __refconst = { 202static const struct apic apic_numachip __refconst = {
203 203
204 .name = "NumaConnect system", 204 .name = "NumaConnect system",
205 .probe = numachip_probe, 205 .probe = numachip_probe,
diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 68de2dc962ec..28610822fb3c 100644
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -69,4 +69,7 @@ void common(void) {
69 OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment); 69 OFFSET(BP_kernel_alignment, boot_params, hdr.kernel_alignment);
70 OFFSET(BP_pref_address, boot_params, hdr.pref_address); 70 OFFSET(BP_pref_address, boot_params, hdr.pref_address);
71 OFFSET(BP_code32_start, boot_params, hdr.code32_start); 71 OFFSET(BP_code32_start, boot_params, hdr.code32_start);
72
73 BLANK();
74 DEFINE(PTREGS_SIZE, sizeof(struct pt_regs));
72} 75}
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index d30a6a9a0121..a0e067d3d96c 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -32,7 +32,7 @@ obj-$(CONFIG_PERF_EVENTS) += perf_event.o
32 32
33ifdef CONFIG_PERF_EVENTS 33ifdef CONFIG_PERF_EVENTS
34obj-$(CONFIG_CPU_SUP_AMD) += perf_event_amd.o 34obj-$(CONFIG_CPU_SUP_AMD) += perf_event_amd.o
35obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_p6.o perf_event_p4.o 35obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_p6.o perf_event_knc.o perf_event_p4.o
36obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o 36obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o
37obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_uncore.o 37obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_uncore.o
38endif 38endif
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 9d92e19039f0..f7e98a2c0d12 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -737,6 +737,72 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
737} 737}
738#endif 738#endif
739 739
740static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
741{
742 if (!cpu_has_invlpg)
743 return;
744
745 tlb_flushall_shift = 5;
746
747 if (c->x86 <= 0x11)
748 tlb_flushall_shift = 4;
749}
750
751static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
752{
753 u32 ebx, eax, ecx, edx;
754 u16 mask = 0xfff;
755
756 if (c->x86 < 0xf)
757 return;
758
759 if (c->extended_cpuid_level < 0x80000006)
760 return;
761
762 cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
763
764 tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
765 tlb_lli_4k[ENTRIES] = ebx & mask;
766
767 /*
768 * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
769 * characteristics from the CPUID function 0x80000005 instead.
770 */
771 if (c->x86 == 0xf) {
772 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
773 mask = 0xff;
774 }
775
776 /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
777 if (!((eax >> 16) & mask)) {
778 u32 a, b, c, d;
779
780 cpuid(0x80000005, &a, &b, &c, &d);
781 tlb_lld_2m[ENTRIES] = (a >> 16) & 0xff;
782 } else {
783 tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
784 }
785
786 /* a 4M entry uses two 2M entries */
787 tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
788
789 /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
790 if (!(eax & mask)) {
791 /* Erratum 658 */
792 if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
793 tlb_lli_2m[ENTRIES] = 1024;
794 } else {
795 cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
796 tlb_lli_2m[ENTRIES] = eax & 0xff;
797 }
798 } else
799 tlb_lli_2m[ENTRIES] = eax & mask;
800
801 tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
802
803 cpu_set_tlb_flushall_shift(c);
804}
805
740static const struct cpu_dev __cpuinitconst amd_cpu_dev = { 806static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
741 .c_vendor = "AMD", 807 .c_vendor = "AMD",
742 .c_ident = { "AuthenticAMD" }, 808 .c_ident = { "AuthenticAMD" },
@@ -756,6 +822,7 @@ static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
756 .c_size_cache = amd_size_cache, 822 .c_size_cache = amd_size_cache,
757#endif 823#endif
758 .c_early_init = early_init_amd, 824 .c_early_init = early_init_amd,
825 .c_detect_tlb = cpu_detect_tlb_amd,
759 .c_bsp_init = bsp_init_amd, 826 .c_bsp_init = bsp_init_amd,
760 .c_init = init_amd, 827 .c_init = init_amd,
761 .c_x86_vendor = X86_VENDOR_AMD, 828 .c_x86_vendor = X86_VENDOR_AMD,
diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c
index c97bb7b5a9f8..d0e910da16c5 100644
--- a/arch/x86/kernel/cpu/bugs.c
+++ b/arch/x86/kernel/cpu/bugs.c
@@ -165,10 +165,15 @@ void __init check_bugs(void)
165 print_cpu_info(&boot_cpu_data); 165 print_cpu_info(&boot_cpu_data);
166#endif 166#endif
167 check_config(); 167 check_config();
168 check_fpu();
169 check_hlt(); 168 check_hlt();
170 check_popad(); 169 check_popad();
171 init_utsname()->machine[1] = 170 init_utsname()->machine[1] =
172 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86); 171 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
173 alternative_instructions(); 172 alternative_instructions();
173
174 /*
175 * kernel_fpu_begin/end() in check_fpu() relies on the patched
176 * alternative instructions.
177 */
178 check_fpu();
174} 179}
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a5fbc3c5fccc..7505f7b13e71 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -259,23 +259,36 @@ static inline void squash_the_stupid_serial_number(struct cpuinfo_x86 *c)
259} 259}
260#endif 260#endif
261 261
262static int disable_smep __cpuinitdata;
263static __init int setup_disable_smep(char *arg) 262static __init int setup_disable_smep(char *arg)
264{ 263{
265 disable_smep = 1; 264 setup_clear_cpu_cap(X86_FEATURE_SMEP);
266 return 1; 265 return 1;
267} 266}
268__setup("nosmep", setup_disable_smep); 267__setup("nosmep", setup_disable_smep);
269 268
270static __cpuinit void setup_smep(struct cpuinfo_x86 *c) 269static __always_inline void setup_smep(struct cpuinfo_x86 *c)
271{ 270{
272 if (cpu_has(c, X86_FEATURE_SMEP)) { 271 if (cpu_has(c, X86_FEATURE_SMEP))
273 if (unlikely(disable_smep)) { 272 set_in_cr4(X86_CR4_SMEP);
274 setup_clear_cpu_cap(X86_FEATURE_SMEP); 273}
275 clear_in_cr4(X86_CR4_SMEP); 274
276 } else 275static __init int setup_disable_smap(char *arg)
277 set_in_cr4(X86_CR4_SMEP); 276{
278 } 277 setup_clear_cpu_cap(X86_FEATURE_SMAP);
278 return 1;
279}
280__setup("nosmap", setup_disable_smap);
281
282static __always_inline void setup_smap(struct cpuinfo_x86 *c)
283{
284 unsigned long eflags;
285
286 /* This should have been cleared long ago */
287 raw_local_save_flags(eflags);
288 BUG_ON(eflags & X86_EFLAGS_AC);
289
290 if (cpu_has(c, X86_FEATURE_SMAP))
291 set_in_cr4(X86_CR4_SMAP);
279} 292}
280 293
281/* 294/*
@@ -476,7 +489,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
476 489
477 printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ 490 printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
478 "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ 491 "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
479 "tlb_flushall_shift is 0x%x\n", 492 "tlb_flushall_shift: %d\n",
480 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES], 493 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
481 tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES], 494 tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
482 tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES], 495 tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
@@ -712,8 +725,6 @@ static void __init early_identify_cpu(struct cpuinfo_x86 *c)
712 c->cpu_index = 0; 725 c->cpu_index = 0;
713 filter_cpuid_features(c, false); 726 filter_cpuid_features(c, false);
714 727
715 setup_smep(c);
716
717 if (this_cpu->c_bsp_init) 728 if (this_cpu->c_bsp_init)
718 this_cpu->c_bsp_init(c); 729 this_cpu->c_bsp_init(c);
719} 730}
@@ -798,8 +809,6 @@ static void __cpuinit generic_identify(struct cpuinfo_x86 *c)
798 c->phys_proc_id = c->initial_apicid; 809 c->phys_proc_id = c->initial_apicid;
799 } 810 }
800 811
801 setup_smep(c);
802
803 get_model_name(c); /* Default name */ 812 get_model_name(c); /* Default name */
804 813
805 detect_nopl(c); 814 detect_nopl(c);
@@ -864,6 +873,10 @@ static void __cpuinit identify_cpu(struct cpuinfo_x86 *c)
864 /* Disable the PN if appropriate */ 873 /* Disable the PN if appropriate */
865 squash_the_stupid_serial_number(c); 874 squash_the_stupid_serial_number(c);
866 875
876 /* Set up SMEP/SMAP */
877 setup_smep(c);
878 setup_smap(c);
879
867 /* 880 /*
868 * The vendor-specific functions might have changed features. 881 * The vendor-specific functions might have changed features.
869 * Now we do "generic changes." 882 * Now we do "generic changes."
@@ -942,8 +955,7 @@ void __init identify_boot_cpu(void)
942#else 955#else
943 vgetcpu_set_mode(); 956 vgetcpu_set_mode();
944#endif 957#endif
945 if (boot_cpu_data.cpuid_level >= 2) 958 cpu_detect_tlb(&boot_cpu_data);
946 cpu_detect_tlb(&boot_cpu_data);
947} 959}
948 960
949void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) 961void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
@@ -1023,14 +1035,16 @@ void __cpuinit print_cpu_info(struct cpuinfo_x86 *c)
1023 printk(KERN_CONT "%s ", vendor); 1035 printk(KERN_CONT "%s ", vendor);
1024 1036
1025 if (c->x86_model_id[0]) 1037 if (c->x86_model_id[0])
1026 printk(KERN_CONT "%s", c->x86_model_id); 1038 printk(KERN_CONT "%s", strim(c->x86_model_id));
1027 else 1039 else
1028 printk(KERN_CONT "%d86", c->x86); 1040 printk(KERN_CONT "%d86", c->x86);
1029 1041
1042 printk(KERN_CONT " (fam: %02x, model: %02x", c->x86, c->x86_model);
1043
1030 if (c->x86_mask || c->cpuid_level >= 0) 1044 if (c->x86_mask || c->cpuid_level >= 0)
1031 printk(KERN_CONT " stepping %02x\n", c->x86_mask); 1045 printk(KERN_CONT ", stepping: %02x)\n", c->x86_mask);
1032 else 1046 else
1033 printk(KERN_CONT "\n"); 1047 printk(KERN_CONT ")\n");
1034 1048
1035 print_cpu_msr(c); 1049 print_cpu_msr(c);
1036} 1050}
@@ -1113,11 +1127,10 @@ void syscall_init(void)
1113 1127
1114 /* Flags to clear on syscall */ 1128 /* Flags to clear on syscall */
1115 wrmsrl(MSR_SYSCALL_MASK, 1129 wrmsrl(MSR_SYSCALL_MASK,
1116 X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|X86_EFLAGS_IOPL); 1130 X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF|
1131 X86_EFLAGS_IOPL|X86_EFLAGS_AC);
1117} 1132}
1118 1133
1119unsigned long kernel_eflags;
1120
1121/* 1134/*
1122 * Copies of the original ist values from the tss are only accessed during 1135 * Copies of the original ist values from the tss are only accessed during
1123 * debugging, no special alignment required. 1136 * debugging, no special alignment required.
@@ -1297,9 +1310,6 @@ void __cpuinit cpu_init(void)
1297 dbg_restore_debug_regs(); 1310 dbg_restore_debug_regs();
1298 1311
1299 fpu_init(); 1312 fpu_init();
1300 xsave_init();
1301
1302 raw_local_save_flags(kernel_eflags);
1303 1313
1304 if (is_uv_system()) 1314 if (is_uv_system())
1305 uv_cpu_init(); 1315 uv_cpu_init();
@@ -1352,6 +1362,5 @@ void __cpuinit cpu_init(void)
1352 dbg_restore_debug_regs(); 1362 dbg_restore_debug_regs();
1353 1363
1354 fpu_init(); 1364 fpu_init();
1355 xsave_init();
1356} 1365}
1357#endif 1366#endif
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 0a4ce2980a5a..198e019a531a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -648,6 +648,10 @@ static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c)
648 int i, j, n; 648 int i, j, n;
649 unsigned int regs[4]; 649 unsigned int regs[4];
650 unsigned char *desc = (unsigned char *)regs; 650 unsigned char *desc = (unsigned char *)regs;
651
652 if (c->cpuid_level < 2)
653 return;
654
651 /* Number of times to iterate */ 655 /* Number of times to iterate */
652 n = cpuid_eax(2) & 0xFF; 656 n = cpuid_eax(2) & 0xFF;
653 657
diff --git a/arch/x86/kernel/cpu/mcheck/mce-inject.c b/arch/x86/kernel/cpu/mcheck/mce-inject.c
index fc4beb393577..ddc72f839332 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-inject.c
+++ b/arch/x86/kernel/cpu/mcheck/mce-inject.c
@@ -78,6 +78,7 @@ static void raise_exception(struct mce *m, struct pt_regs *pregs)
78} 78}
79 79
80static cpumask_var_t mce_inject_cpumask; 80static cpumask_var_t mce_inject_cpumask;
81static DEFINE_MUTEX(mce_inject_mutex);
81 82
82static int mce_raise_notify(unsigned int cmd, struct pt_regs *regs) 83static int mce_raise_notify(unsigned int cmd, struct pt_regs *regs)
83{ 84{
@@ -194,7 +195,11 @@ static void raise_mce(struct mce *m)
194 put_online_cpus(); 195 put_online_cpus();
195 } else 196 } else
196#endif 197#endif
198 {
199 preempt_disable();
197 raise_local(); 200 raise_local();
201 preempt_enable();
202 }
198} 203}
199 204
200/* Error injection interface */ 205/* Error injection interface */
@@ -225,7 +230,10 @@ static ssize_t mce_write(struct file *filp, const char __user *ubuf,
225 * so do it a jiffie or two later everywhere. 230 * so do it a jiffie or two later everywhere.
226 */ 231 */
227 schedule_timeout(2); 232 schedule_timeout(2);
233
234 mutex_lock(&mce_inject_mutex);
228 raise_mce(&m); 235 raise_mce(&m);
236 mutex_unlock(&mce_inject_mutex);
229 return usize; 237 return usize;
230} 238}
231 239
diff --git a/arch/x86/kernel/cpu/mcheck/mce-internal.h b/arch/x86/kernel/cpu/mcheck/mce-internal.h
index ed44c8a65858..6a05c1d327a9 100644
--- a/arch/x86/kernel/cpu/mcheck/mce-internal.h
+++ b/arch/x86/kernel/cpu/mcheck/mce-internal.h
@@ -28,6 +28,18 @@ extern int mce_ser;
28 28
29extern struct mce_bank *mce_banks; 29extern struct mce_bank *mce_banks;
30 30
31#ifdef CONFIG_X86_MCE_INTEL
32unsigned long mce_intel_adjust_timer(unsigned long interval);
33void mce_intel_cmci_poll(void);
34void mce_intel_hcpu_update(unsigned long cpu);
35#else
36# define mce_intel_adjust_timer mce_adjust_timer_default
37static inline void mce_intel_cmci_poll(void) { }
38static inline void mce_intel_hcpu_update(unsigned long cpu) { }
39#endif
40
41void mce_timer_kick(unsigned long interval);
42
31#ifdef CONFIG_ACPI_APEI 43#ifdef CONFIG_ACPI_APEI
32int apei_write_mce(struct mce *m); 44int apei_write_mce(struct mce *m);
33ssize_t apei_read_mce(struct mce *m, u64 *record_id); 45ssize_t apei_read_mce(struct mce *m, u64 *record_id);
diff --git a/arch/x86/kernel/cpu/mcheck/mce.c b/arch/x86/kernel/cpu/mcheck/mce.c
index 292d0258311c..29e87d3b2843 100644
--- a/arch/x86/kernel/cpu/mcheck/mce.c
+++ b/arch/x86/kernel/cpu/mcheck/mce.c
@@ -83,6 +83,7 @@ static int mce_dont_log_ce __read_mostly;
83int mce_cmci_disabled __read_mostly; 83int mce_cmci_disabled __read_mostly;
84int mce_ignore_ce __read_mostly; 84int mce_ignore_ce __read_mostly;
85int mce_ser __read_mostly; 85int mce_ser __read_mostly;
86int mce_bios_cmci_threshold __read_mostly;
86 87
87struct mce_bank *mce_banks __read_mostly; 88struct mce_bank *mce_banks __read_mostly;
88 89
@@ -1266,6 +1267,14 @@ static unsigned long check_interval = 5 * 60; /* 5 minutes */
1266static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */ 1267static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1267static DEFINE_PER_CPU(struct timer_list, mce_timer); 1268static DEFINE_PER_CPU(struct timer_list, mce_timer);
1268 1269
1270static unsigned long mce_adjust_timer_default(unsigned long interval)
1271{
1272 return interval;
1273}
1274
1275static unsigned long (*mce_adjust_timer)(unsigned long interval) =
1276 mce_adjust_timer_default;
1277
1269static void mce_timer_fn(unsigned long data) 1278static void mce_timer_fn(unsigned long data)
1270{ 1279{
1271 struct timer_list *t = &__get_cpu_var(mce_timer); 1280 struct timer_list *t = &__get_cpu_var(mce_timer);
@@ -1276,6 +1285,7 @@ static void mce_timer_fn(unsigned long data)
1276 if (mce_available(__this_cpu_ptr(&cpu_info))) { 1285 if (mce_available(__this_cpu_ptr(&cpu_info))) {
1277 machine_check_poll(MCP_TIMESTAMP, 1286 machine_check_poll(MCP_TIMESTAMP,
1278 &__get_cpu_var(mce_poll_banks)); 1287 &__get_cpu_var(mce_poll_banks));
1288 mce_intel_cmci_poll();
1279 } 1289 }
1280 1290
1281 /* 1291 /*
@@ -1283,14 +1293,38 @@ static void mce_timer_fn(unsigned long data)
1283 * polling interval, otherwise increase the polling interval. 1293 * polling interval, otherwise increase the polling interval.
1284 */ 1294 */
1285 iv = __this_cpu_read(mce_next_interval); 1295 iv = __this_cpu_read(mce_next_interval);
1286 if (mce_notify_irq()) 1296 if (mce_notify_irq()) {
1287 iv = max(iv / 2, (unsigned long) HZ/100); 1297 iv = max(iv / 2, (unsigned long) HZ/100);
1288 else 1298 } else {
1289 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ)); 1299 iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1300 iv = mce_adjust_timer(iv);
1301 }
1290 __this_cpu_write(mce_next_interval, iv); 1302 __this_cpu_write(mce_next_interval, iv);
1303 /* Might have become 0 after CMCI storm subsided */
1304 if (iv) {
1305 t->expires = jiffies + iv;
1306 add_timer_on(t, smp_processor_id());
1307 }
1308}
1291 1309
1292 t->expires = jiffies + iv; 1310/*
1293 add_timer_on(t, smp_processor_id()); 1311 * Ensure that the timer is firing in @interval from now.
1312 */
1313void mce_timer_kick(unsigned long interval)
1314{
1315 struct timer_list *t = &__get_cpu_var(mce_timer);
1316 unsigned long when = jiffies + interval;
1317 unsigned long iv = __this_cpu_read(mce_next_interval);
1318
1319 if (timer_pending(t)) {
1320 if (time_before(when, t->expires))
1321 mod_timer_pinned(t, when);
1322 } else {
1323 t->expires = round_jiffies(when);
1324 add_timer_on(t, smp_processor_id());
1325 }
1326 if (interval < iv)
1327 __this_cpu_write(mce_next_interval, interval);
1294} 1328}
1295 1329
1296/* Must not be called in IRQ context where del_timer_sync() can deadlock */ 1330/* Must not be called in IRQ context where del_timer_sync() can deadlock */
@@ -1585,6 +1619,7 @@ static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
1585 switch (c->x86_vendor) { 1619 switch (c->x86_vendor) {
1586 case X86_VENDOR_INTEL: 1620 case X86_VENDOR_INTEL:
1587 mce_intel_feature_init(c); 1621 mce_intel_feature_init(c);
1622 mce_adjust_timer = mce_intel_adjust_timer;
1588 break; 1623 break;
1589 case X86_VENDOR_AMD: 1624 case X86_VENDOR_AMD:
1590 mce_amd_feature_init(c); 1625 mce_amd_feature_init(c);
@@ -1594,23 +1629,28 @@ static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
1594 } 1629 }
1595} 1630}
1596 1631
1597static void __mcheck_cpu_init_timer(void) 1632static void mce_start_timer(unsigned int cpu, struct timer_list *t)
1598{ 1633{
1599 struct timer_list *t = &__get_cpu_var(mce_timer); 1634 unsigned long iv = mce_adjust_timer(check_interval * HZ);
1600 unsigned long iv = check_interval * HZ;
1601 1635
1602 setup_timer(t, mce_timer_fn, smp_processor_id()); 1636 __this_cpu_write(mce_next_interval, iv);
1603 1637
1604 if (mce_ignore_ce) 1638 if (mce_ignore_ce || !iv)
1605 return; 1639 return;
1606 1640
1607 __this_cpu_write(mce_next_interval, iv);
1608 if (!iv)
1609 return;
1610 t->expires = round_jiffies(jiffies + iv); 1641 t->expires = round_jiffies(jiffies + iv);
1611 add_timer_on(t, smp_processor_id()); 1642 add_timer_on(t, smp_processor_id());
1612} 1643}
1613 1644
1645static void __mcheck_cpu_init_timer(void)
1646{
1647 struct timer_list *t = &__get_cpu_var(mce_timer);
1648 unsigned int cpu = smp_processor_id();
1649
1650 setup_timer(t, mce_timer_fn, cpu);
1651 mce_start_timer(cpu, t);
1652}
1653
1614/* Handle unconfigured int18 (should never happen) */ 1654/* Handle unconfigured int18 (should never happen) */
1615static void unexpected_machine_check(struct pt_regs *regs, long error_code) 1655static void unexpected_machine_check(struct pt_regs *regs, long error_code)
1616{ 1656{
@@ -1907,6 +1947,7 @@ static struct miscdevice mce_chrdev_device = {
1907 * check, or 0 to not wait 1947 * check, or 0 to not wait
1908 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD. 1948 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
1909 * mce=nobootlog Don't log MCEs from before booting. 1949 * mce=nobootlog Don't log MCEs from before booting.
1950 * mce=bios_cmci_threshold Don't program the CMCI threshold
1910 */ 1951 */
1911static int __init mcheck_enable(char *str) 1952static int __init mcheck_enable(char *str)
1912{ 1953{
@@ -1926,6 +1967,8 @@ static int __init mcheck_enable(char *str)
1926 mce_ignore_ce = 1; 1967 mce_ignore_ce = 1;
1927 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) 1968 else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
1928 mce_bootlog = (str[0] == 'b'); 1969 mce_bootlog = (str[0] == 'b');
1970 else if (!strcmp(str, "bios_cmci_threshold"))
1971 mce_bios_cmci_threshold = 1;
1929 else if (isdigit(str[0])) { 1972 else if (isdigit(str[0])) {
1930 get_option(&str, &tolerant); 1973 get_option(&str, &tolerant);
1931 if (*str == ',') { 1974 if (*str == ',') {
@@ -2166,6 +2209,11 @@ static struct dev_ext_attribute dev_attr_cmci_disabled = {
2166 &mce_cmci_disabled 2209 &mce_cmci_disabled
2167}; 2210};
2168 2211
2212static struct dev_ext_attribute dev_attr_bios_cmci_threshold = {
2213 __ATTR(bios_cmci_threshold, 0444, device_show_int, NULL),
2214 &mce_bios_cmci_threshold
2215};
2216
2169static struct device_attribute *mce_device_attrs[] = { 2217static struct device_attribute *mce_device_attrs[] = {
2170 &dev_attr_tolerant.attr, 2218 &dev_attr_tolerant.attr,
2171 &dev_attr_check_interval.attr, 2219 &dev_attr_check_interval.attr,
@@ -2174,6 +2222,7 @@ static struct device_attribute *mce_device_attrs[] = {
2174 &dev_attr_dont_log_ce.attr, 2222 &dev_attr_dont_log_ce.attr,
2175 &dev_attr_ignore_ce.attr, 2223 &dev_attr_ignore_ce.attr,
2176 &dev_attr_cmci_disabled.attr, 2224 &dev_attr_cmci_disabled.attr,
2225 &dev_attr_bios_cmci_threshold.attr,
2177 NULL 2226 NULL
2178}; 2227};
2179 2228
@@ -2294,38 +2343,33 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2294 unsigned int cpu = (unsigned long)hcpu; 2343 unsigned int cpu = (unsigned long)hcpu;
2295 struct timer_list *t = &per_cpu(mce_timer, cpu); 2344 struct timer_list *t = &per_cpu(mce_timer, cpu);
2296 2345
2297 switch (action) { 2346 switch (action & ~CPU_TASKS_FROZEN) {
2298 case CPU_ONLINE: 2347 case CPU_ONLINE:
2299 case CPU_ONLINE_FROZEN:
2300 mce_device_create(cpu); 2348 mce_device_create(cpu);
2301 if (threshold_cpu_callback) 2349 if (threshold_cpu_callback)
2302 threshold_cpu_callback(action, cpu); 2350 threshold_cpu_callback(action, cpu);
2303 break; 2351 break;
2304 case CPU_DEAD: 2352 case CPU_DEAD:
2305 case CPU_DEAD_FROZEN:
2306 if (threshold_cpu_callback) 2353 if (threshold_cpu_callback)
2307 threshold_cpu_callback(action, cpu); 2354 threshold_cpu_callback(action, cpu);
2308 mce_device_remove(cpu); 2355 mce_device_remove(cpu);
2356 mce_intel_hcpu_update(cpu);
2309 break; 2357 break;
2310 case CPU_DOWN_PREPARE: 2358 case CPU_DOWN_PREPARE:
2311 case CPU_DOWN_PREPARE_FROZEN:
2312 del_timer_sync(t);
2313 smp_call_function_single(cpu, mce_disable_cpu, &action, 1); 2359 smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
2360 del_timer_sync(t);
2314 break; 2361 break;
2315 case CPU_DOWN_FAILED: 2362 case CPU_DOWN_FAILED:
2316 case CPU_DOWN_FAILED_FROZEN:
2317 if (!mce_ignore_ce && check_interval) {
2318 t->expires = round_jiffies(jiffies +
2319 per_cpu(mce_next_interval, cpu));
2320 add_timer_on(t, cpu);
2321 }
2322 smp_call_function_single(cpu, mce_reenable_cpu, &action, 1); 2363 smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
2364 mce_start_timer(cpu, t);
2323 break; 2365 break;
2324 case CPU_POST_DEAD: 2366 }
2367
2368 if (action == CPU_POST_DEAD) {
2325 /* intentionally ignoring frozen here */ 2369 /* intentionally ignoring frozen here */
2326 cmci_rediscover(cpu); 2370 cmci_rediscover(cpu);
2327 break;
2328 } 2371 }
2372
2329 return NOTIFY_OK; 2373 return NOTIFY_OK;
2330} 2374}
2331 2375
diff --git a/arch/x86/kernel/cpu/mcheck/mce_intel.c b/arch/x86/kernel/cpu/mcheck/mce_intel.c
index 38e49bc95ffc..5f88abf07e9c 100644
--- a/arch/x86/kernel/cpu/mcheck/mce_intel.c
+++ b/arch/x86/kernel/cpu/mcheck/mce_intel.c
@@ -15,6 +15,8 @@
15#include <asm/msr.h> 15#include <asm/msr.h>
16#include <asm/mce.h> 16#include <asm/mce.h>
17 17
18#include "mce-internal.h"
19
18/* 20/*
19 * Support for Intel Correct Machine Check Interrupts. This allows 21 * Support for Intel Correct Machine Check Interrupts. This allows
20 * the CPU to raise an interrupt when a corrected machine check happened. 22 * the CPU to raise an interrupt when a corrected machine check happened.
@@ -30,7 +32,22 @@ static DEFINE_PER_CPU(mce_banks_t, mce_banks_owned);
30 */ 32 */
31static DEFINE_RAW_SPINLOCK(cmci_discover_lock); 33static DEFINE_RAW_SPINLOCK(cmci_discover_lock);
32 34
33#define CMCI_THRESHOLD 1 35#define CMCI_THRESHOLD 1
36#define CMCI_POLL_INTERVAL (30 * HZ)
37#define CMCI_STORM_INTERVAL (1 * HZ)
38#define CMCI_STORM_THRESHOLD 15
39
40static DEFINE_PER_CPU(unsigned long, cmci_time_stamp);
41static DEFINE_PER_CPU(unsigned int, cmci_storm_cnt);
42static DEFINE_PER_CPU(unsigned int, cmci_storm_state);
43
44enum {
45 CMCI_STORM_NONE,
46 CMCI_STORM_ACTIVE,
47 CMCI_STORM_SUBSIDED,
48};
49
50static atomic_t cmci_storm_on_cpus;
34 51
35static int cmci_supported(int *banks) 52static int cmci_supported(int *banks)
36{ 53{
@@ -53,6 +70,93 @@ static int cmci_supported(int *banks)
53 return !!(cap & MCG_CMCI_P); 70 return !!(cap & MCG_CMCI_P);
54} 71}
55 72
73void mce_intel_cmci_poll(void)
74{
75 if (__this_cpu_read(cmci_storm_state) == CMCI_STORM_NONE)
76 return;
77 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
78}
79
80void mce_intel_hcpu_update(unsigned long cpu)
81{
82 if (per_cpu(cmci_storm_state, cpu) == CMCI_STORM_ACTIVE)
83 atomic_dec(&cmci_storm_on_cpus);
84
85 per_cpu(cmci_storm_state, cpu) = CMCI_STORM_NONE;
86}
87
88unsigned long mce_intel_adjust_timer(unsigned long interval)
89{
90 int r;
91
92 if (interval < CMCI_POLL_INTERVAL)
93 return interval;
94
95 switch (__this_cpu_read(cmci_storm_state)) {
96 case CMCI_STORM_ACTIVE:
97 /*
98 * We switch back to interrupt mode once the poll timer has
99 * silenced itself. That means no events recorded and the
100 * timer interval is back to our poll interval.
101 */
102 __this_cpu_write(cmci_storm_state, CMCI_STORM_SUBSIDED);
103 r = atomic_sub_return(1, &cmci_storm_on_cpus);
104 if (r == 0)
105 pr_notice("CMCI storm subsided: switching to interrupt mode\n");
106 /* FALLTHROUGH */
107
108 case CMCI_STORM_SUBSIDED:
109 /*
110 * We wait for all cpus to go back to SUBSIDED
111 * state. When that happens we switch back to
112 * interrupt mode.
113 */
114 if (!atomic_read(&cmci_storm_on_cpus)) {
115 __this_cpu_write(cmci_storm_state, CMCI_STORM_NONE);
116 cmci_reenable();
117 cmci_recheck();
118 }
119 return CMCI_POLL_INTERVAL;
120 default:
121 /*
122 * We have shiny weather. Let the poll do whatever it
123 * thinks.
124 */
125 return interval;
126 }
127}
128
129static bool cmci_storm_detect(void)
130{
131 unsigned int cnt = __this_cpu_read(cmci_storm_cnt);
132 unsigned long ts = __this_cpu_read(cmci_time_stamp);
133 unsigned long now = jiffies;
134 int r;
135
136 if (__this_cpu_read(cmci_storm_state) != CMCI_STORM_NONE)
137 return true;
138
139 if (time_before_eq(now, ts + CMCI_STORM_INTERVAL)) {
140 cnt++;
141 } else {
142 cnt = 1;
143 __this_cpu_write(cmci_time_stamp, now);
144 }
145 __this_cpu_write(cmci_storm_cnt, cnt);
146
147 if (cnt <= CMCI_STORM_THRESHOLD)
148 return false;
149
150 cmci_clear();
151 __this_cpu_write(cmci_storm_state, CMCI_STORM_ACTIVE);
152 r = atomic_add_return(1, &cmci_storm_on_cpus);
153 mce_timer_kick(CMCI_POLL_INTERVAL);
154
155 if (r == 1)
156 pr_notice("CMCI storm detected: switching to poll mode\n");
157 return true;
158}
159
56/* 160/*
57 * The interrupt handler. This is called on every event. 161 * The interrupt handler. This is called on every event.
58 * Just call the poller directly to log any events. 162 * Just call the poller directly to log any events.
@@ -61,33 +165,28 @@ static int cmci_supported(int *banks)
61 */ 165 */
62static void intel_threshold_interrupt(void) 166static void intel_threshold_interrupt(void)
63{ 167{
168 if (cmci_storm_detect())
169 return;
64 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned)); 170 machine_check_poll(MCP_TIMESTAMP, &__get_cpu_var(mce_banks_owned));
65 mce_notify_irq(); 171 mce_notify_irq();
66} 172}
67 173
68static void print_update(char *type, int *hdr, int num)
69{
70 if (*hdr == 0)
71 printk(KERN_INFO "CPU %d MCA banks", smp_processor_id());
72 *hdr = 1;
73 printk(KERN_CONT " %s:%d", type, num);
74}
75
76/* 174/*
77 * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks 175 * Enable CMCI (Corrected Machine Check Interrupt) for available MCE banks
78 * on this CPU. Use the algorithm recommended in the SDM to discover shared 176 * on this CPU. Use the algorithm recommended in the SDM to discover shared
79 * banks. 177 * banks.
80 */ 178 */
81static void cmci_discover(int banks, int boot) 179static void cmci_discover(int banks)
82{ 180{
83 unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned); 181 unsigned long *owned = (void *)&__get_cpu_var(mce_banks_owned);
84 unsigned long flags; 182 unsigned long flags;
85 int hdr = 0;
86 int i; 183 int i;
184 int bios_wrong_thresh = 0;
87 185
88 raw_spin_lock_irqsave(&cmci_discover_lock, flags); 186 raw_spin_lock_irqsave(&cmci_discover_lock, flags);
89 for (i = 0; i < banks; i++) { 187 for (i = 0; i < banks; i++) {
90 u64 val; 188 u64 val;
189 int bios_zero_thresh = 0;
91 190
92 if (test_bit(i, owned)) 191 if (test_bit(i, owned))
93 continue; 192 continue;
@@ -96,29 +195,52 @@ static void cmci_discover(int banks, int boot)
96 195
97 /* Already owned by someone else? */ 196 /* Already owned by someone else? */
98 if (val & MCI_CTL2_CMCI_EN) { 197 if (val & MCI_CTL2_CMCI_EN) {
99 if (test_and_clear_bit(i, owned) && !boot) 198 clear_bit(i, owned);
100 print_update("SHD", &hdr, i);
101 __clear_bit(i, __get_cpu_var(mce_poll_banks)); 199 __clear_bit(i, __get_cpu_var(mce_poll_banks));
102 continue; 200 continue;
103 } 201 }
104 202
105 val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK; 203 if (!mce_bios_cmci_threshold) {
106 val |= MCI_CTL2_CMCI_EN | CMCI_THRESHOLD; 204 val &= ~MCI_CTL2_CMCI_THRESHOLD_MASK;
205 val |= CMCI_THRESHOLD;
206 } else if (!(val & MCI_CTL2_CMCI_THRESHOLD_MASK)) {
207 /*
208 * If bios_cmci_threshold boot option was specified
209 * but the threshold is zero, we'll try to initialize
210 * it to 1.
211 */
212 bios_zero_thresh = 1;
213 val |= CMCI_THRESHOLD;
214 }
215
216 val |= MCI_CTL2_CMCI_EN;
107 wrmsrl(MSR_IA32_MCx_CTL2(i), val); 217 wrmsrl(MSR_IA32_MCx_CTL2(i), val);
108 rdmsrl(MSR_IA32_MCx_CTL2(i), val); 218 rdmsrl(MSR_IA32_MCx_CTL2(i), val);
109 219
110 /* Did the enable bit stick? -- the bank supports CMCI */ 220 /* Did the enable bit stick? -- the bank supports CMCI */
111 if (val & MCI_CTL2_CMCI_EN) { 221 if (val & MCI_CTL2_CMCI_EN) {
112 if (!test_and_set_bit(i, owned) && !boot) 222 set_bit(i, owned);
113 print_update("CMCI", &hdr, i);
114 __clear_bit(i, __get_cpu_var(mce_poll_banks)); 223 __clear_bit(i, __get_cpu_var(mce_poll_banks));
224 /*
225 * We are able to set thresholds for some banks that
226 * had a threshold of 0. This means the BIOS has not
227 * set the thresholds properly or does not work with
228 * this boot option. Note down now and report later.
229 */
230 if (mce_bios_cmci_threshold && bios_zero_thresh &&
231 (val & MCI_CTL2_CMCI_THRESHOLD_MASK))
232 bios_wrong_thresh = 1;
115 } else { 233 } else {
116 WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks))); 234 WARN_ON(!test_bit(i, __get_cpu_var(mce_poll_banks)));
117 } 235 }
118 } 236 }
119 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags); 237 raw_spin_unlock_irqrestore(&cmci_discover_lock, flags);
120 if (hdr) 238 if (mce_bios_cmci_threshold && bios_wrong_thresh) {
121 printk(KERN_CONT "\n"); 239 pr_info_once(
240 "bios_cmci_threshold: Some banks do not have valid thresholds set\n");
241 pr_info_once(
242 "bios_cmci_threshold: Make sure your BIOS supports this boot option\n");
243 }
122} 244}
123 245
124/* 246/*
@@ -156,7 +278,7 @@ void cmci_clear(void)
156 continue; 278 continue;
157 /* Disable CMCI */ 279 /* Disable CMCI */
158 rdmsrl(MSR_IA32_MCx_CTL2(i), val); 280 rdmsrl(MSR_IA32_MCx_CTL2(i), val);
159 val &= ~(MCI_CTL2_CMCI_EN|MCI_CTL2_CMCI_THRESHOLD_MASK); 281 val &= ~MCI_CTL2_CMCI_EN;
160 wrmsrl(MSR_IA32_MCx_CTL2(i), val); 282 wrmsrl(MSR_IA32_MCx_CTL2(i), val);
161 __clear_bit(i, __get_cpu_var(mce_banks_owned)); 283 __clear_bit(i, __get_cpu_var(mce_banks_owned));
162 } 284 }
@@ -186,7 +308,7 @@ void cmci_rediscover(int dying)
186 continue; 308 continue;
187 /* Recheck banks in case CPUs don't all have the same */ 309 /* Recheck banks in case CPUs don't all have the same */
188 if (cmci_supported(&banks)) 310 if (cmci_supported(&banks))
189 cmci_discover(banks, 0); 311 cmci_discover(banks);
190 } 312 }
191 313
192 set_cpus_allowed_ptr(current, old); 314 set_cpus_allowed_ptr(current, old);
@@ -200,7 +322,7 @@ void cmci_reenable(void)
200{ 322{
201 int banks; 323 int banks;
202 if (cmci_supported(&banks)) 324 if (cmci_supported(&banks))
203 cmci_discover(banks, 0); 325 cmci_discover(banks);
204} 326}
205 327
206static void intel_init_cmci(void) 328static void intel_init_cmci(void)
@@ -211,7 +333,7 @@ static void intel_init_cmci(void)
211 return; 333 return;
212 334
213 mce_threshold_vector = intel_threshold_interrupt; 335 mce_threshold_vector = intel_threshold_interrupt;
214 cmci_discover(banks, 1); 336 cmci_discover(banks);
215 /* 337 /*
216 * For CPU #0 this runs with still disabled APIC, but that's 338 * For CPU #0 this runs with still disabled APIC, but that's
217 * ok because only the vector is set up. We still do another 339 * ok because only the vector is set up. We still do another
diff --git a/arch/x86/kernel/cpu/mkcapflags.pl b/arch/x86/kernel/cpu/mkcapflags.pl
index c7b3fe2d72e0..091972ef49de 100644
--- a/arch/x86/kernel/cpu/mkcapflags.pl
+++ b/arch/x86/kernel/cpu/mkcapflags.pl
@@ -8,7 +8,10 @@
8open(IN, "< $in\0") or die "$0: cannot open: $in: $!\n"; 8open(IN, "< $in\0") or die "$0: cannot open: $in: $!\n";
9open(OUT, "> $out\0") or die "$0: cannot create: $out: $!\n"; 9open(OUT, "> $out\0") or die "$0: cannot create: $out: $!\n";
10 10
11print OUT "#include <asm/cpufeature.h>\n\n"; 11print OUT "#ifndef _ASM_X86_CPUFEATURE_H\n";
12print OUT "#include <asm/cpufeature.h>\n";
13print OUT "#endif\n";
14print OUT "\n";
12print OUT "const char * const x86_cap_flags[NCAPINTS*32] = {\n"; 15print OUT "const char * const x86_cap_flags[NCAPINTS*32] = {\n";
13 16
14%features = (); 17%features = ();
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 6605a81ba339..271d25700297 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -586,6 +586,8 @@ extern struct event_constraint intel_westmere_pebs_event_constraints[];
586 586
587extern struct event_constraint intel_snb_pebs_event_constraints[]; 587extern struct event_constraint intel_snb_pebs_event_constraints[];
588 588
589extern struct event_constraint intel_ivb_pebs_event_constraints[];
590
589struct event_constraint *intel_pebs_constraints(struct perf_event *event); 591struct event_constraint *intel_pebs_constraints(struct perf_event *event);
590 592
591void intel_pmu_pebs_enable(struct perf_event *event); 593void intel_pmu_pebs_enable(struct perf_event *event);
@@ -624,6 +626,8 @@ int p4_pmu_init(void);
624 626
625int p6_pmu_init(void); 627int p6_pmu_init(void);
626 628
629int knc_pmu_init(void);
630
627#else /* CONFIG_CPU_SUP_INTEL */ 631#else /* CONFIG_CPU_SUP_INTEL */
628 632
629static inline void reserve_ds_buffers(void) 633static inline void reserve_ds_buffers(void)
diff --git a/arch/x86/kernel/cpu/perf_event_amd_ibs.c b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
index 7bfb5bec8630..6336bcbd0618 100644
--- a/arch/x86/kernel/cpu/perf_event_amd_ibs.c
+++ b/arch/x86/kernel/cpu/perf_event_amd_ibs.c
@@ -41,17 +41,22 @@ struct cpu_perf_ibs {
41}; 41};
42 42
43struct perf_ibs { 43struct perf_ibs {
44 struct pmu pmu; 44 struct pmu pmu;
45 unsigned int msr; 45 unsigned int msr;
46 u64 config_mask; 46 u64 config_mask;
47 u64 cnt_mask; 47 u64 cnt_mask;
48 u64 enable_mask; 48 u64 enable_mask;
49 u64 valid_mask; 49 u64 valid_mask;
50 u64 max_period; 50 u64 max_period;
51 unsigned long offset_mask[1]; 51 unsigned long offset_mask[1];
52 int offset_max; 52 int offset_max;
53 struct cpu_perf_ibs __percpu *pcpu; 53 struct cpu_perf_ibs __percpu *pcpu;
54 u64 (*get_count)(u64 config); 54
55 struct attribute **format_attrs;
56 struct attribute_group format_group;
57 const struct attribute_group *attr_groups[2];
58
59 u64 (*get_count)(u64 config);
55}; 60};
56 61
57struct perf_ibs_data { 62struct perf_ibs_data {
@@ -209,6 +214,15 @@ static int perf_ibs_precise_event(struct perf_event *event, u64 *config)
209 return -EOPNOTSUPP; 214 return -EOPNOTSUPP;
210} 215}
211 216
217static const struct perf_event_attr ibs_notsupp = {
218 .exclude_user = 1,
219 .exclude_kernel = 1,
220 .exclude_hv = 1,
221 .exclude_idle = 1,
222 .exclude_host = 1,
223 .exclude_guest = 1,
224};
225
212static int perf_ibs_init(struct perf_event *event) 226static int perf_ibs_init(struct perf_event *event)
213{ 227{
214 struct hw_perf_event *hwc = &event->hw; 228 struct hw_perf_event *hwc = &event->hw;
@@ -229,6 +243,9 @@ static int perf_ibs_init(struct perf_event *event)
229 if (event->pmu != &perf_ibs->pmu) 243 if (event->pmu != &perf_ibs->pmu)
230 return -ENOENT; 244 return -ENOENT;
231 245
246 if (perf_flags(&event->attr) & perf_flags(&ibs_notsupp))
247 return -EINVAL;
248
232 if (config & ~perf_ibs->config_mask) 249 if (config & ~perf_ibs->config_mask)
233 return -EINVAL; 250 return -EINVAL;
234 251
@@ -434,6 +451,19 @@ static void perf_ibs_del(struct perf_event *event, int flags)
434 451
435static void perf_ibs_read(struct perf_event *event) { } 452static void perf_ibs_read(struct perf_event *event) { }
436 453
454PMU_FORMAT_ATTR(rand_en, "config:57");
455PMU_FORMAT_ATTR(cnt_ctl, "config:19");
456
457static struct attribute *ibs_fetch_format_attrs[] = {
458 &format_attr_rand_en.attr,
459 NULL,
460};
461
462static struct attribute *ibs_op_format_attrs[] = {
463 NULL, /* &format_attr_cnt_ctl.attr if IBS_CAPS_OPCNT */
464 NULL,
465};
466
437static struct perf_ibs perf_ibs_fetch = { 467static struct perf_ibs perf_ibs_fetch = {
438 .pmu = { 468 .pmu = {
439 .task_ctx_nr = perf_invalid_context, 469 .task_ctx_nr = perf_invalid_context,
@@ -453,6 +483,7 @@ static struct perf_ibs perf_ibs_fetch = {
453 .max_period = IBS_FETCH_MAX_CNT << 4, 483 .max_period = IBS_FETCH_MAX_CNT << 4,
454 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK }, 484 .offset_mask = { MSR_AMD64_IBSFETCH_REG_MASK },
455 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT, 485 .offset_max = MSR_AMD64_IBSFETCH_REG_COUNT,
486 .format_attrs = ibs_fetch_format_attrs,
456 487
457 .get_count = get_ibs_fetch_count, 488 .get_count = get_ibs_fetch_count,
458}; 489};
@@ -476,6 +507,7 @@ static struct perf_ibs perf_ibs_op = {
476 .max_period = IBS_OP_MAX_CNT << 4, 507 .max_period = IBS_OP_MAX_CNT << 4,
477 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK }, 508 .offset_mask = { MSR_AMD64_IBSOP_REG_MASK },
478 .offset_max = MSR_AMD64_IBSOP_REG_COUNT, 509 .offset_max = MSR_AMD64_IBSOP_REG_COUNT,
510 .format_attrs = ibs_op_format_attrs,
479 511
480 .get_count = get_ibs_op_count, 512 .get_count = get_ibs_op_count,
481}; 513};
@@ -585,6 +617,17 @@ static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
585 617
586 perf_ibs->pcpu = pcpu; 618 perf_ibs->pcpu = pcpu;
587 619
620 /* register attributes */
621 if (perf_ibs->format_attrs[0]) {
622 memset(&perf_ibs->format_group, 0, sizeof(perf_ibs->format_group));
623 perf_ibs->format_group.name = "format";
624 perf_ibs->format_group.attrs = perf_ibs->format_attrs;
625
626 memset(&perf_ibs->attr_groups, 0, sizeof(perf_ibs->attr_groups));
627 perf_ibs->attr_groups[0] = &perf_ibs->format_group;
628 perf_ibs->pmu.attr_groups = perf_ibs->attr_groups;
629 }
630
588 ret = perf_pmu_register(&perf_ibs->pmu, name, -1); 631 ret = perf_pmu_register(&perf_ibs->pmu, name, -1);
589 if (ret) { 632 if (ret) {
590 perf_ibs->pcpu = NULL; 633 perf_ibs->pcpu = NULL;
@@ -596,13 +639,19 @@ static __init int perf_ibs_pmu_init(struct perf_ibs *perf_ibs, char *name)
596 639
597static __init int perf_event_ibs_init(void) 640static __init int perf_event_ibs_init(void)
598{ 641{
642 struct attribute **attr = ibs_op_format_attrs;
643
599 if (!ibs_caps) 644 if (!ibs_caps)
600 return -ENODEV; /* ibs not supported by the cpu */ 645 return -ENODEV; /* ibs not supported by the cpu */
601 646
602 perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch"); 647 perf_ibs_pmu_init(&perf_ibs_fetch, "ibs_fetch");
603 if (ibs_caps & IBS_CAPS_OPCNT) 648
649 if (ibs_caps & IBS_CAPS_OPCNT) {
604 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL; 650 perf_ibs_op.config_mask |= IBS_OP_CNT_CTL;
651 *attr++ = &format_attr_cnt_ctl.attr;
652 }
605 perf_ibs_pmu_init(&perf_ibs_op, "ibs_op"); 653 perf_ibs_pmu_init(&perf_ibs_op, "ibs_op");
654
606 register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs"); 655 register_nmi_handler(NMI_LOCAL, perf_ibs_nmi_handler, 0, "perf_ibs");
607 printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps); 656 printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps);
608 657
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index 7f2739e03e79..324bb523d9d9 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -1906,6 +1906,8 @@ __init int intel_pmu_init(void)
1906 switch (boot_cpu_data.x86) { 1906 switch (boot_cpu_data.x86) {
1907 case 0x6: 1907 case 0x6:
1908 return p6_pmu_init(); 1908 return p6_pmu_init();
1909 case 0xb:
1910 return knc_pmu_init();
1909 case 0xf: 1911 case 0xf:
1910 return p4_pmu_init(); 1912 return p4_pmu_init();
1911 } 1913 }
@@ -2008,6 +2010,7 @@ __init int intel_pmu_init(void)
2008 break; 2010 break;
2009 2011
2010 case 28: /* Atom */ 2012 case 28: /* Atom */
2013 case 54: /* Cedariew */
2011 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids, 2014 memcpy(hw_cache_event_ids, atom_hw_cache_event_ids,
2012 sizeof(hw_cache_event_ids)); 2015 sizeof(hw_cache_event_ids));
2013 2016
@@ -2047,7 +2050,6 @@ __init int intel_pmu_init(void)
2047 case 42: /* SandyBridge */ 2050 case 42: /* SandyBridge */
2048 case 45: /* SandyBridge, "Romely-EP" */ 2051 case 45: /* SandyBridge, "Romely-EP" */
2049 x86_add_quirk(intel_sandybridge_quirk); 2052 x86_add_quirk(intel_sandybridge_quirk);
2050 case 58: /* IvyBridge */
2051 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids, 2053 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
2052 sizeof(hw_cache_event_ids)); 2054 sizeof(hw_cache_event_ids));
2053 memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs, 2055 memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
@@ -2072,6 +2074,29 @@ __init int intel_pmu_init(void)
2072 2074
2073 pr_cont("SandyBridge events, "); 2075 pr_cont("SandyBridge events, ");
2074 break; 2076 break;
2077 case 58: /* IvyBridge */
2078 memcpy(hw_cache_event_ids, snb_hw_cache_event_ids,
2079 sizeof(hw_cache_event_ids));
2080 memcpy(hw_cache_extra_regs, snb_hw_cache_extra_regs,
2081 sizeof(hw_cache_extra_regs));
2082
2083 intel_pmu_lbr_init_snb();
2084
2085 x86_pmu.event_constraints = intel_snb_event_constraints;
2086 x86_pmu.pebs_constraints = intel_ivb_pebs_event_constraints;
2087 x86_pmu.pebs_aliases = intel_pebs_aliases_snb;
2088 x86_pmu.extra_regs = intel_snb_extra_regs;
2089 /* all extra regs are per-cpu when HT is on */
2090 x86_pmu.er_flags |= ERF_HAS_RSP_1;
2091 x86_pmu.er_flags |= ERF_NO_HT_SHARING;
2092
2093 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
2094 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
2095 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
2096
2097 pr_cont("IvyBridge events, ");
2098 break;
2099
2075 2100
2076 default: 2101 default:
2077 switch (x86_pmu.version) { 2102 switch (x86_pmu.version) {
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index e38d97bf4259..826054a4f2ee 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -407,6 +407,20 @@ struct event_constraint intel_snb_pebs_event_constraints[] = {
407 EVENT_CONSTRAINT_END 407 EVENT_CONSTRAINT_END
408}; 408};
409 409
410struct event_constraint intel_ivb_pebs_event_constraints[] = {
411 INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PRECDIST */
412 INTEL_UEVENT_CONSTRAINT(0x01c2, 0xf), /* UOPS_RETIRED.ALL */
413 INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
414 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
415 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
416 INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.* */
417 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
418 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
419 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
420 INTEL_EVENT_CONSTRAINT(0xd3, 0xf), /* MEM_LOAD_UOPS_LLC_MISS_RETIRED.* */
421 EVENT_CONSTRAINT_END
422};
423
410struct event_constraint *intel_pebs_constraints(struct perf_event *event) 424struct event_constraint *intel_pebs_constraints(struct perf_event *event)
411{ 425{
412 struct event_constraint *c; 426 struct event_constraint *c;
diff --git a/arch/x86/kernel/cpu/perf_event_intel_lbr.c b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
index 520b4265fcd2..da02e9cc3754 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_lbr.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_lbr.c
@@ -686,7 +686,8 @@ void intel_pmu_lbr_init_atom(void)
686 * to have an operational LBR which can freeze 686 * to have an operational LBR which can freeze
687 * on PMU interrupt 687 * on PMU interrupt
688 */ 688 */
689 if (boot_cpu_data.x86_mask < 10) { 689 if (boot_cpu_data.x86_model == 28
690 && boot_cpu_data.x86_mask < 10) {
690 pr_cont("LBR disabled due to erratum"); 691 pr_cont("LBR disabled due to erratum");
691 return; 692 return;
692 } 693 }
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index 0a5571080e74..99d96a4978b5 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -661,6 +661,11 @@ static void snb_uncore_msr_init_box(struct intel_uncore_box *box)
661 } 661 }
662} 662}
663 663
664static struct uncore_event_desc snb_uncore_events[] = {
665 INTEL_UNCORE_EVENT_DESC(clockticks, "event=0xff,umask=0x00"),
666 { /* end: all zeroes */ },
667};
668
664static struct attribute *snb_uncore_formats_attr[] = { 669static struct attribute *snb_uncore_formats_attr[] = {
665 &format_attr_event.attr, 670 &format_attr_event.attr,
666 &format_attr_umask.attr, 671 &format_attr_umask.attr,
@@ -704,6 +709,7 @@ static struct intel_uncore_type snb_uncore_cbox = {
704 .constraints = snb_uncore_cbox_constraints, 709 .constraints = snb_uncore_cbox_constraints,
705 .ops = &snb_uncore_msr_ops, 710 .ops = &snb_uncore_msr_ops,
706 .format_group = &snb_uncore_format_group, 711 .format_group = &snb_uncore_format_group,
712 .event_descs = snb_uncore_events,
707}; 713};
708 714
709static struct intel_uncore_type *snb_msr_uncores[] = { 715static struct intel_uncore_type *snb_msr_uncores[] = {
@@ -1944,7 +1950,7 @@ struct intel_uncore_box *uncore_alloc_box(struct intel_uncore_type *type, int cp
1944static struct intel_uncore_box * 1950static struct intel_uncore_box *
1945uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu) 1951uncore_pmu_to_box(struct intel_uncore_pmu *pmu, int cpu)
1946{ 1952{
1947 static struct intel_uncore_box *box; 1953 struct intel_uncore_box *box;
1948 1954
1949 box = *per_cpu_ptr(pmu->box, cpu); 1955 box = *per_cpu_ptr(pmu->box, cpu);
1950 if (box) 1956 if (box)
@@ -2341,6 +2347,27 @@ int uncore_pmu_event_init(struct perf_event *event)
2341 return ret; 2347 return ret;
2342} 2348}
2343 2349
2350static ssize_t uncore_get_attr_cpumask(struct device *dev,
2351 struct device_attribute *attr, char *buf)
2352{
2353 int n = cpulist_scnprintf(buf, PAGE_SIZE - 2, &uncore_cpu_mask);
2354
2355 buf[n++] = '\n';
2356 buf[n] = '\0';
2357 return n;
2358}
2359
2360static DEVICE_ATTR(cpumask, S_IRUGO, uncore_get_attr_cpumask, NULL);
2361
2362static struct attribute *uncore_pmu_attrs[] = {
2363 &dev_attr_cpumask.attr,
2364 NULL,
2365};
2366
2367static struct attribute_group uncore_pmu_attr_group = {
2368 .attrs = uncore_pmu_attrs,
2369};
2370
2344static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu) 2371static int __init uncore_pmu_register(struct intel_uncore_pmu *pmu)
2345{ 2372{
2346 int ret; 2373 int ret;
@@ -2378,8 +2405,8 @@ static void __init uncore_type_exit(struct intel_uncore_type *type)
2378 free_percpu(type->pmus[i].box); 2405 free_percpu(type->pmus[i].box);
2379 kfree(type->pmus); 2406 kfree(type->pmus);
2380 type->pmus = NULL; 2407 type->pmus = NULL;
2381 kfree(type->attr_groups[1]); 2408 kfree(type->events_group);
2382 type->attr_groups[1] = NULL; 2409 type->events_group = NULL;
2383} 2410}
2384 2411
2385static void __init uncore_types_exit(struct intel_uncore_type **types) 2412static void __init uncore_types_exit(struct intel_uncore_type **types)
@@ -2431,9 +2458,10 @@ static int __init uncore_type_init(struct intel_uncore_type *type)
2431 for (j = 0; j < i; j++) 2458 for (j = 0; j < i; j++)
2432 attrs[j] = &type->event_descs[j].attr.attr; 2459 attrs[j] = &type->event_descs[j].attr.attr;
2433 2460
2434 type->attr_groups[1] = events_group; 2461 type->events_group = events_group;
2435 } 2462 }
2436 2463
2464 type->pmu_group = &uncore_pmu_attr_group;
2437 type->pmus = pmus; 2465 type->pmus = pmus;
2438 return 0; 2466 return 0;
2439fail: 2467fail:
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.h b/arch/x86/kernel/cpu/perf_event_intel_uncore.h
index 5b81c1856aac..e68a4550e952 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.h
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.h
@@ -369,10 +369,12 @@ struct intel_uncore_type {
369 struct intel_uncore_pmu *pmus; 369 struct intel_uncore_pmu *pmus;
370 struct intel_uncore_ops *ops; 370 struct intel_uncore_ops *ops;
371 struct uncore_event_desc *event_descs; 371 struct uncore_event_desc *event_descs;
372 const struct attribute_group *attr_groups[3]; 372 const struct attribute_group *attr_groups[4];
373}; 373};
374 374
375#define format_group attr_groups[0] 375#define pmu_group attr_groups[0]
376#define format_group attr_groups[1]
377#define events_group attr_groups[2]
376 378
377struct intel_uncore_ops { 379struct intel_uncore_ops {
378 void (*init_box)(struct intel_uncore_box *); 380 void (*init_box)(struct intel_uncore_box *);
diff --git a/arch/x86/kernel/cpu/perf_event_knc.c b/arch/x86/kernel/cpu/perf_event_knc.c
new file mode 100644
index 000000000000..7c46bfdbc373
--- /dev/null
+++ b/arch/x86/kernel/cpu/perf_event_knc.c
@@ -0,0 +1,248 @@
1/* Driver for Intel Xeon Phi "Knights Corner" PMU */
2
3#include <linux/perf_event.h>
4#include <linux/types.h>
5
6#include "perf_event.h"
7
8static const u64 knc_perfmon_event_map[] =
9{
10 [PERF_COUNT_HW_CPU_CYCLES] = 0x002a,
11 [PERF_COUNT_HW_INSTRUCTIONS] = 0x0016,
12 [PERF_COUNT_HW_CACHE_REFERENCES] = 0x0028,
13 [PERF_COUNT_HW_CACHE_MISSES] = 0x0029,
14 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x0012,
15 [PERF_COUNT_HW_BRANCH_MISSES] = 0x002b,
16};
17
18static __initconst u64 knc_hw_cache_event_ids
19 [PERF_COUNT_HW_CACHE_MAX]
20 [PERF_COUNT_HW_CACHE_OP_MAX]
21 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
22{
23 [ C(L1D) ] = {
24 [ C(OP_READ) ] = {
25 /* On Xeon Phi event "0" is a valid DATA_READ */
26 /* (L1 Data Cache Reads) Instruction. */
27 /* We code this as ARCH_PERFMON_EVENTSEL_INT as this */
28 /* bit will always be set in x86_pmu_hw_config(). */
29 [ C(RESULT_ACCESS) ] = ARCH_PERFMON_EVENTSEL_INT,
30 /* DATA_READ */
31 [ C(RESULT_MISS) ] = 0x0003, /* DATA_READ_MISS */
32 },
33 [ C(OP_WRITE) ] = {
34 [ C(RESULT_ACCESS) ] = 0x0001, /* DATA_WRITE */
35 [ C(RESULT_MISS) ] = 0x0004, /* DATA_WRITE_MISS */
36 },
37 [ C(OP_PREFETCH) ] = {
38 [ C(RESULT_ACCESS) ] = 0x0011, /* L1_DATA_PF1 */
39 [ C(RESULT_MISS) ] = 0x001c, /* L1_DATA_PF1_MISS */
40 },
41 },
42 [ C(L1I ) ] = {
43 [ C(OP_READ) ] = {
44 [ C(RESULT_ACCESS) ] = 0x000c, /* CODE_READ */
45 [ C(RESULT_MISS) ] = 0x000e, /* CODE_CACHE_MISS */
46 },
47 [ C(OP_WRITE) ] = {
48 [ C(RESULT_ACCESS) ] = -1,
49 [ C(RESULT_MISS) ] = -1,
50 },
51 [ C(OP_PREFETCH) ] = {
52 [ C(RESULT_ACCESS) ] = 0x0,
53 [ C(RESULT_MISS) ] = 0x0,
54 },
55 },
56 [ C(LL ) ] = {
57 [ C(OP_READ) ] = {
58 [ C(RESULT_ACCESS) ] = 0,
59 [ C(RESULT_MISS) ] = 0x10cb, /* L2_READ_MISS */
60 },
61 [ C(OP_WRITE) ] = {
62 [ C(RESULT_ACCESS) ] = 0x10cc, /* L2_WRITE_HIT */
63 [ C(RESULT_MISS) ] = 0,
64 },
65 [ C(OP_PREFETCH) ] = {
66 [ C(RESULT_ACCESS) ] = 0x10fc, /* L2_DATA_PF2 */
67 [ C(RESULT_MISS) ] = 0x10fe, /* L2_DATA_PF2_MISS */
68 },
69 },
70 [ C(DTLB) ] = {
71 [ C(OP_READ) ] = {
72 [ C(RESULT_ACCESS) ] = ARCH_PERFMON_EVENTSEL_INT,
73 /* DATA_READ */
74 /* see note on L1 OP_READ */
75 [ C(RESULT_MISS) ] = 0x0002, /* DATA_PAGE_WALK */
76 },
77 [ C(OP_WRITE) ] = {
78 [ C(RESULT_ACCESS) ] = 0x0001, /* DATA_WRITE */
79 [ C(RESULT_MISS) ] = 0x0002, /* DATA_PAGE_WALK */
80 },
81 [ C(OP_PREFETCH) ] = {
82 [ C(RESULT_ACCESS) ] = 0x0,
83 [ C(RESULT_MISS) ] = 0x0,
84 },
85 },
86 [ C(ITLB) ] = {
87 [ C(OP_READ) ] = {
88 [ C(RESULT_ACCESS) ] = 0x000c, /* CODE_READ */
89 [ C(RESULT_MISS) ] = 0x000d, /* CODE_PAGE_WALK */
90 },
91 [ C(OP_WRITE) ] = {
92 [ C(RESULT_ACCESS) ] = -1,
93 [ C(RESULT_MISS) ] = -1,
94 },
95 [ C(OP_PREFETCH) ] = {
96 [ C(RESULT_ACCESS) ] = -1,
97 [ C(RESULT_MISS) ] = -1,
98 },
99 },
100 [ C(BPU ) ] = {
101 [ C(OP_READ) ] = {
102 [ C(RESULT_ACCESS) ] = 0x0012, /* BRANCHES */
103 [ C(RESULT_MISS) ] = 0x002b, /* BRANCHES_MISPREDICTED */
104 },
105 [ C(OP_WRITE) ] = {
106 [ C(RESULT_ACCESS) ] = -1,
107 [ C(RESULT_MISS) ] = -1,
108 },
109 [ C(OP_PREFETCH) ] = {
110 [ C(RESULT_ACCESS) ] = -1,
111 [ C(RESULT_MISS) ] = -1,
112 },
113 },
114};
115
116
117static u64 knc_pmu_event_map(int hw_event)
118{
119 return knc_perfmon_event_map[hw_event];
120}
121
122static struct event_constraint knc_event_constraints[] =
123{
124 INTEL_EVENT_CONSTRAINT(0xc3, 0x1), /* HWP_L2HIT */
125 INTEL_EVENT_CONSTRAINT(0xc4, 0x1), /* HWP_L2MISS */
126 INTEL_EVENT_CONSTRAINT(0xc8, 0x1), /* L2_READ_HIT_E */
127 INTEL_EVENT_CONSTRAINT(0xc9, 0x1), /* L2_READ_HIT_M */
128 INTEL_EVENT_CONSTRAINT(0xca, 0x1), /* L2_READ_HIT_S */
129 INTEL_EVENT_CONSTRAINT(0xcb, 0x1), /* L2_READ_MISS */
130 INTEL_EVENT_CONSTRAINT(0xcc, 0x1), /* L2_WRITE_HIT */
131 INTEL_EVENT_CONSTRAINT(0xce, 0x1), /* L2_STRONGLY_ORDERED_STREAMING_VSTORES_MISS */
132 INTEL_EVENT_CONSTRAINT(0xcf, 0x1), /* L2_WEAKLY_ORDERED_STREAMING_VSTORE_MISS */
133 INTEL_EVENT_CONSTRAINT(0xd7, 0x1), /* L2_VICTIM_REQ_WITH_DATA */
134 INTEL_EVENT_CONSTRAINT(0xe3, 0x1), /* SNP_HITM_BUNIT */
135 INTEL_EVENT_CONSTRAINT(0xe6, 0x1), /* SNP_HIT_L2 */
136 INTEL_EVENT_CONSTRAINT(0xe7, 0x1), /* SNP_HITM_L2 */
137 INTEL_EVENT_CONSTRAINT(0xf1, 0x1), /* L2_DATA_READ_MISS_CACHE_FILL */
138 INTEL_EVENT_CONSTRAINT(0xf2, 0x1), /* L2_DATA_WRITE_MISS_CACHE_FILL */
139 INTEL_EVENT_CONSTRAINT(0xf6, 0x1), /* L2_DATA_READ_MISS_MEM_FILL */
140 INTEL_EVENT_CONSTRAINT(0xf7, 0x1), /* L2_DATA_WRITE_MISS_MEM_FILL */
141 INTEL_EVENT_CONSTRAINT(0xfc, 0x1), /* L2_DATA_PF2 */
142 INTEL_EVENT_CONSTRAINT(0xfd, 0x1), /* L2_DATA_PF2_DROP */
143 INTEL_EVENT_CONSTRAINT(0xfe, 0x1), /* L2_DATA_PF2_MISS */
144 INTEL_EVENT_CONSTRAINT(0xff, 0x1), /* L2_DATA_HIT_INFLIGHT_PF2 */
145 EVENT_CONSTRAINT_END
146};
147
148#define MSR_KNC_IA32_PERF_GLOBAL_STATUS 0x0000002d
149#define MSR_KNC_IA32_PERF_GLOBAL_OVF_CONTROL 0x0000002e
150#define MSR_KNC_IA32_PERF_GLOBAL_CTRL 0x0000002f
151
152#define KNC_ENABLE_COUNTER0 0x00000001
153#define KNC_ENABLE_COUNTER1 0x00000002
154
155static void knc_pmu_disable_all(void)
156{
157 u64 val;
158
159 rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
160 val &= ~(KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1);
161 wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
162}
163
164static void knc_pmu_enable_all(int added)
165{
166 u64 val;
167
168 rdmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
169 val |= (KNC_ENABLE_COUNTER0|KNC_ENABLE_COUNTER1);
170 wrmsrl(MSR_KNC_IA32_PERF_GLOBAL_CTRL, val);
171}
172
173static inline void
174knc_pmu_disable_event(struct perf_event *event)
175{
176 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
177 struct hw_perf_event *hwc = &event->hw;
178 u64 val;
179
180 val = hwc->config;
181 if (cpuc->enabled)
182 val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
183
184 (void)wrmsrl_safe(hwc->config_base + hwc->idx, val);
185}
186
187static void knc_pmu_enable_event(struct perf_event *event)
188{
189 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
190 struct hw_perf_event *hwc = &event->hw;
191 u64 val;
192
193 val = hwc->config;
194 if (cpuc->enabled)
195 val |= ARCH_PERFMON_EVENTSEL_ENABLE;
196
197 (void)wrmsrl_safe(hwc->config_base + hwc->idx, val);
198}
199
200PMU_FORMAT_ATTR(event, "config:0-7" );
201PMU_FORMAT_ATTR(umask, "config:8-15" );
202PMU_FORMAT_ATTR(edge, "config:18" );
203PMU_FORMAT_ATTR(inv, "config:23" );
204PMU_FORMAT_ATTR(cmask, "config:24-31" );
205
206static struct attribute *intel_knc_formats_attr[] = {
207 &format_attr_event.attr,
208 &format_attr_umask.attr,
209 &format_attr_edge.attr,
210 &format_attr_inv.attr,
211 &format_attr_cmask.attr,
212 NULL,
213};
214
215static __initconst struct x86_pmu knc_pmu = {
216 .name = "knc",
217 .handle_irq = x86_pmu_handle_irq,
218 .disable_all = knc_pmu_disable_all,
219 .enable_all = knc_pmu_enable_all,
220 .enable = knc_pmu_enable_event,
221 .disable = knc_pmu_disable_event,
222 .hw_config = x86_pmu_hw_config,
223 .schedule_events = x86_schedule_events,
224 .eventsel = MSR_KNC_EVNTSEL0,
225 .perfctr = MSR_KNC_PERFCTR0,
226 .event_map = knc_pmu_event_map,
227 .max_events = ARRAY_SIZE(knc_perfmon_event_map),
228 .apic = 1,
229 .max_period = (1ULL << 31) - 1,
230 .version = 0,
231 .num_counters = 2,
232 /* in theory 40 bits, early silicon is buggy though */
233 .cntval_bits = 32,
234 .cntval_mask = (1ULL << 32) - 1,
235 .get_event_constraints = x86_get_event_constraints,
236 .event_constraints = knc_event_constraints,
237 .format_attrs = intel_knc_formats_attr,
238};
239
240__init int knc_pmu_init(void)
241{
242 x86_pmu = knc_pmu;
243
244 memcpy(hw_cache_event_ids, knc_hw_cache_event_ids,
245 sizeof(hw_cache_event_ids));
246
247 return 0;
248}
diff --git a/arch/x86/kernel/cpu/perfctr-watchdog.c b/arch/x86/kernel/cpu/perfctr-watchdog.c
index 966512b2cacf..2e8caf03f593 100644
--- a/arch/x86/kernel/cpu/perfctr-watchdog.c
+++ b/arch/x86/kernel/cpu/perfctr-watchdog.c
@@ -56,6 +56,8 @@ static inline unsigned int nmi_perfctr_msr_to_bit(unsigned int msr)
56 switch (boot_cpu_data.x86) { 56 switch (boot_cpu_data.x86) {
57 case 6: 57 case 6:
58 return msr - MSR_P6_PERFCTR0; 58 return msr - MSR_P6_PERFCTR0;
59 case 11:
60 return msr - MSR_KNC_PERFCTR0;
59 case 15: 61 case 15:
60 return msr - MSR_P4_BPU_PERFCTR0; 62 return msr - MSR_P4_BPU_PERFCTR0;
61 } 63 }
@@ -82,6 +84,8 @@ static inline unsigned int nmi_evntsel_msr_to_bit(unsigned int msr)
82 switch (boot_cpu_data.x86) { 84 switch (boot_cpu_data.x86) {
83 case 6: 85 case 6:
84 return msr - MSR_P6_EVNTSEL0; 86 return msr - MSR_P6_EVNTSEL0;
87 case 11:
88 return msr - MSR_KNC_EVNTSEL0;
85 case 15: 89 case 15:
86 return msr - MSR_P4_BSU_ESCR0; 90 return msr - MSR_P4_BSU_ESCR0;
87 } 91 }
diff --git a/arch/x86/kernel/cpu/proc.c b/arch/x86/kernel/cpu/proc.c
index 8022c6681485..fbd895562292 100644
--- a/arch/x86/kernel/cpu/proc.c
+++ b/arch/x86/kernel/cpu/proc.c
@@ -140,10 +140,7 @@ static int show_cpuinfo(struct seq_file *m, void *v)
140 140
141static void *c_start(struct seq_file *m, loff_t *pos) 141static void *c_start(struct seq_file *m, loff_t *pos)
142{ 142{
143 if (*pos == 0) /* just in case, cpu 0 is not the first */ 143 *pos = cpumask_next(*pos - 1, cpu_online_mask);
144 *pos = cpumask_first(cpu_online_mask);
145 else
146 *pos = cpumask_next(*pos - 1, cpu_online_mask);
147 if ((*pos) < nr_cpu_ids) 144 if ((*pos) < nr_cpu_ids)
148 return &cpu_data(*pos); 145 return &cpu_data(*pos);
149 return NULL; 146 return NULL;
diff --git a/arch/x86/kernel/cpuid.c b/arch/x86/kernel/cpuid.c
index 39472dd2323f..60c78917190c 100644
--- a/arch/x86/kernel/cpuid.c
+++ b/arch/x86/kernel/cpuid.c
@@ -199,12 +199,14 @@ static int __init cpuid_init(void)
199 goto out_chrdev; 199 goto out_chrdev;
200 } 200 }
201 cpuid_class->devnode = cpuid_devnode; 201 cpuid_class->devnode = cpuid_devnode;
202 get_online_cpus();
202 for_each_online_cpu(i) { 203 for_each_online_cpu(i) {
203 err = cpuid_device_create(i); 204 err = cpuid_device_create(i);
204 if (err != 0) 205 if (err != 0)
205 goto out_class; 206 goto out_class;
206 } 207 }
207 register_hotcpu_notifier(&cpuid_class_cpu_notifier); 208 register_hotcpu_notifier(&cpuid_class_cpu_notifier);
209 put_online_cpus();
208 210
209 err = 0; 211 err = 0;
210 goto out; 212 goto out;
@@ -214,6 +216,7 @@ out_class:
214 for_each_online_cpu(i) { 216 for_each_online_cpu(i) {
215 cpuid_device_destroy(i); 217 cpuid_device_destroy(i);
216 } 218 }
219 put_online_cpus();
217 class_destroy(cpuid_class); 220 class_destroy(cpuid_class);
218out_chrdev: 221out_chrdev:
219 __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); 222 __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
@@ -225,11 +228,13 @@ static void __exit cpuid_exit(void)
225{ 228{
226 int cpu = 0; 229 int cpu = 0;
227 230
231 get_online_cpus();
228 for_each_online_cpu(cpu) 232 for_each_online_cpu(cpu)
229 cpuid_device_destroy(cpu); 233 cpuid_device_destroy(cpu);
230 class_destroy(cpuid_class); 234 class_destroy(cpuid_class);
231 __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid"); 235 __unregister_chrdev(CPUID_MAJOR, 0, NR_CPUS, "cpu/cpuid");
232 unregister_hotcpu_notifier(&cpuid_class_cpu_notifier); 236 unregister_hotcpu_notifier(&cpuid_class_cpu_notifier);
237 put_online_cpus();
233} 238}
234 239
235module_init(cpuid_init); 240module_init(cpuid_init);
diff --git a/arch/x86/kernel/devicetree.c b/arch/x86/kernel/devicetree.c
index 3ae2ced4a874..b1581527a236 100644
--- a/arch/x86/kernel/devicetree.c
+++ b/arch/x86/kernel/devicetree.c
@@ -342,6 +342,47 @@ const struct irq_domain_ops ioapic_irq_domain_ops = {
342 .xlate = ioapic_xlate, 342 .xlate = ioapic_xlate,
343}; 343};
344 344
345static void dt_add_ioapic_domain(unsigned int ioapic_num,
346 struct device_node *np)
347{
348 struct irq_domain *id;
349 struct mp_ioapic_gsi *gsi_cfg;
350 int ret;
351 int num;
352
353 gsi_cfg = mp_ioapic_gsi_routing(ioapic_num);
354 num = gsi_cfg->gsi_end - gsi_cfg->gsi_base + 1;
355
356 id = irq_domain_add_linear(np, num, &ioapic_irq_domain_ops,
357 (void *)ioapic_num);
358 BUG_ON(!id);
359 if (gsi_cfg->gsi_base == 0) {
360 /*
361 * The first NR_IRQS_LEGACY irq descs are allocated in
362 * early_irq_init() and need just a mapping. The
363 * remaining irqs need both. All of them are preallocated
364 * and assigned so we can keep the 1:1 mapping which the ioapic
365 * is having.
366 */
367 ret = irq_domain_associate_many(id, 0, 0, NR_IRQS_LEGACY);
368 if (ret)
369 pr_err("Error mapping legacy IRQs: %d\n", ret);
370
371 if (num > NR_IRQS_LEGACY) {
372 ret = irq_create_strict_mappings(id, NR_IRQS_LEGACY,
373 NR_IRQS_LEGACY, num - NR_IRQS_LEGACY);
374 if (ret)
375 pr_err("Error creating mapping for the "
376 "remaining IRQs: %d\n", ret);
377 }
378 irq_set_default_host(id);
379 } else {
380 ret = irq_create_strict_mappings(id, gsi_cfg->gsi_base, 0, num);
381 if (ret)
382 pr_err("Error creating IRQ mapping: %d\n", ret);
383 }
384}
385
345static void __init ioapic_add_ofnode(struct device_node *np) 386static void __init ioapic_add_ofnode(struct device_node *np)
346{ 387{
347 struct resource r; 388 struct resource r;
@@ -356,15 +397,7 @@ static void __init ioapic_add_ofnode(struct device_node *np)
356 397
357 for (i = 0; i < nr_ioapics; i++) { 398 for (i = 0; i < nr_ioapics; i++) {
358 if (r.start == mpc_ioapic_addr(i)) { 399 if (r.start == mpc_ioapic_addr(i)) {
359 struct irq_domain *id; 400 dt_add_ioapic_domain(i, np);
360 struct mp_ioapic_gsi *gsi_cfg;
361
362 gsi_cfg = mp_ioapic_gsi_routing(i);
363
364 id = irq_domain_add_legacy(np, 32, gsi_cfg->gsi_base, 0,
365 &ioapic_irq_domain_ops,
366 (void*)i);
367 BUG_ON(!id);
368 return; 401 return;
369 } 402 }
370 } 403 }
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index 8f8e8eea9085..88b725aa1d52 100644
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -57,6 +57,7 @@
57#include <asm/cpufeature.h> 57#include <asm/cpufeature.h>
58#include <asm/alternative-asm.h> 58#include <asm/alternative-asm.h>
59#include <asm/asm.h> 59#include <asm/asm.h>
60#include <asm/smap.h>
60 61
61/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 62/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
62#include <linux/elf-em.h> 63#include <linux/elf-em.h>
@@ -298,6 +299,21 @@ ENTRY(ret_from_fork)
298 CFI_ENDPROC 299 CFI_ENDPROC
299END(ret_from_fork) 300END(ret_from_fork)
300 301
302ENTRY(ret_from_kernel_thread)
303 CFI_STARTPROC
304 pushl_cfi %eax
305 call schedule_tail
306 GET_THREAD_INFO(%ebp)
307 popl_cfi %eax
308 pushl_cfi $0x0202 # Reset kernel eflags
309 popfl_cfi
310 movl PT_EBP(%esp),%eax
311 call *PT_EBX(%esp)
312 movl $0,PT_EAX(%esp)
313 jmp syscall_exit
314 CFI_ENDPROC
315ENDPROC(ret_from_kernel_thread)
316
301/* 317/*
302 * Interrupt exit functions should be protected against kprobes 318 * Interrupt exit functions should be protected against kprobes
303 */ 319 */
@@ -322,8 +338,7 @@ ret_from_intr:
322 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax 338 andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax
323#else 339#else
324 /* 340 /*
325 * We can be coming here from a syscall done in the kernel space, 341 * We can be coming here from child spawned by kernel_thread().
326 * e.g. a failed kernel_execve().
327 */ 342 */
328 movl PT_CS(%esp), %eax 343 movl PT_CS(%esp), %eax
329 andl $SEGMENT_RPL_MASK, %eax 344 andl $SEGMENT_RPL_MASK, %eax
@@ -407,7 +422,9 @@ sysenter_past_esp:
407 */ 422 */
408 cmpl $__PAGE_OFFSET-3,%ebp 423 cmpl $__PAGE_OFFSET-3,%ebp
409 jae syscall_fault 424 jae syscall_fault
425 ASM_STAC
4101: movl (%ebp),%ebp 4261: movl (%ebp),%ebp
427 ASM_CLAC
411 movl %ebp,PT_EBP(%esp) 428 movl %ebp,PT_EBP(%esp)
412 _ASM_EXTABLE(1b,syscall_fault) 429 _ASM_EXTABLE(1b,syscall_fault)
413 430
@@ -488,6 +505,7 @@ ENDPROC(ia32_sysenter_target)
488 # system call handler stub 505 # system call handler stub
489ENTRY(system_call) 506ENTRY(system_call)
490 RING0_INT_FRAME # can't unwind into user space anyway 507 RING0_INT_FRAME # can't unwind into user space anyway
508 ASM_CLAC
491 pushl_cfi %eax # save orig_eax 509 pushl_cfi %eax # save orig_eax
492 SAVE_ALL 510 SAVE_ALL
493 GET_THREAD_INFO(%ebp) 511 GET_THREAD_INFO(%ebp)
@@ -612,6 +630,10 @@ work_notifysig: # deal with pending signals and
612 movl %esp, %eax 630 movl %esp, %eax
613 jne work_notifysig_v86 # returning to kernel-space or 631 jne work_notifysig_v86 # returning to kernel-space or
614 # vm86-space 632 # vm86-space
6331:
634#else
635 movl %esp, %eax
636#endif
615 TRACE_IRQS_ON 637 TRACE_IRQS_ON
616 ENABLE_INTERRUPTS(CLBR_NONE) 638 ENABLE_INTERRUPTS(CLBR_NONE)
617 movb PT_CS(%esp), %bl 639 movb PT_CS(%esp), %bl
@@ -622,24 +644,15 @@ work_notifysig: # deal with pending signals and
622 call do_notify_resume 644 call do_notify_resume
623 jmp resume_userspace 645 jmp resume_userspace
624 646
647#ifdef CONFIG_VM86
625 ALIGN 648 ALIGN
626work_notifysig_v86: 649work_notifysig_v86:
627 pushl_cfi %ecx # save ti_flags for do_notify_resume 650 pushl_cfi %ecx # save ti_flags for do_notify_resume
628 call save_v86_state # %eax contains pt_regs pointer 651 call save_v86_state # %eax contains pt_regs pointer
629 popl_cfi %ecx 652 popl_cfi %ecx
630 movl %eax, %esp 653 movl %eax, %esp
631#else 654 jmp 1b
632 movl %esp, %eax
633#endif 655#endif
634 TRACE_IRQS_ON
635 ENABLE_INTERRUPTS(CLBR_NONE)
636 movb PT_CS(%esp), %bl
637 andb $SEGMENT_RPL_MASK, %bl
638 cmpb $USER_RPL, %bl
639 jb resume_kernel
640 xorl %edx, %edx
641 call do_notify_resume
642 jmp resume_userspace
643END(work_pending) 656END(work_pending)
644 657
645 # perform syscall exit tracing 658 # perform syscall exit tracing
@@ -670,6 +683,7 @@ END(syscall_exit_work)
670 683
671 RING0_INT_FRAME # can't unwind into user space anyway 684 RING0_INT_FRAME # can't unwind into user space anyway
672syscall_fault: 685syscall_fault:
686 ASM_CLAC
673 GET_THREAD_INFO(%ebp) 687 GET_THREAD_INFO(%ebp)
674 movl $-EFAULT,PT_EAX(%esp) 688 movl $-EFAULT,PT_EAX(%esp)
675 jmp resume_userspace 689 jmp resume_userspace
@@ -727,7 +741,6 @@ ENDPROC(ptregs_##name)
727PTREGSCALL1(iopl) 741PTREGSCALL1(iopl)
728PTREGSCALL0(fork) 742PTREGSCALL0(fork)
729PTREGSCALL0(vfork) 743PTREGSCALL0(vfork)
730PTREGSCALL3(execve)
731PTREGSCALL2(sigaltstack) 744PTREGSCALL2(sigaltstack)
732PTREGSCALL0(sigreturn) 745PTREGSCALL0(sigreturn)
733PTREGSCALL0(rt_sigreturn) 746PTREGSCALL0(rt_sigreturn)
@@ -825,6 +838,7 @@ END(interrupt)
825 */ 838 */
826 .p2align CONFIG_X86_L1_CACHE_SHIFT 839 .p2align CONFIG_X86_L1_CACHE_SHIFT
827common_interrupt: 840common_interrupt:
841 ASM_CLAC
828 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */ 842 addl $-0x80,(%esp) /* Adjust vector into the [-256,-1] range */
829 SAVE_ALL 843 SAVE_ALL
830 TRACE_IRQS_OFF 844 TRACE_IRQS_OFF
@@ -841,6 +855,7 @@ ENDPROC(common_interrupt)
841#define BUILD_INTERRUPT3(name, nr, fn) \ 855#define BUILD_INTERRUPT3(name, nr, fn) \
842ENTRY(name) \ 856ENTRY(name) \
843 RING0_INT_FRAME; \ 857 RING0_INT_FRAME; \
858 ASM_CLAC; \
844 pushl_cfi $~(nr); \ 859 pushl_cfi $~(nr); \
845 SAVE_ALL; \ 860 SAVE_ALL; \
846 TRACE_IRQS_OFF \ 861 TRACE_IRQS_OFF \
@@ -857,6 +872,7 @@ ENDPROC(name)
857 872
858ENTRY(coprocessor_error) 873ENTRY(coprocessor_error)
859 RING0_INT_FRAME 874 RING0_INT_FRAME
875 ASM_CLAC
860 pushl_cfi $0 876 pushl_cfi $0
861 pushl_cfi $do_coprocessor_error 877 pushl_cfi $do_coprocessor_error
862 jmp error_code 878 jmp error_code
@@ -865,6 +881,7 @@ END(coprocessor_error)
865 881
866ENTRY(simd_coprocessor_error) 882ENTRY(simd_coprocessor_error)
867 RING0_INT_FRAME 883 RING0_INT_FRAME
884 ASM_CLAC
868 pushl_cfi $0 885 pushl_cfi $0
869#ifdef CONFIG_X86_INVD_BUG 886#ifdef CONFIG_X86_INVD_BUG
870 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ 887 /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */
@@ -886,6 +903,7 @@ END(simd_coprocessor_error)
886 903
887ENTRY(device_not_available) 904ENTRY(device_not_available)
888 RING0_INT_FRAME 905 RING0_INT_FRAME
906 ASM_CLAC
889 pushl_cfi $-1 # mark this as an int 907 pushl_cfi $-1 # mark this as an int
890 pushl_cfi $do_device_not_available 908 pushl_cfi $do_device_not_available
891 jmp error_code 909 jmp error_code
@@ -906,6 +924,7 @@ END(native_irq_enable_sysexit)
906 924
907ENTRY(overflow) 925ENTRY(overflow)
908 RING0_INT_FRAME 926 RING0_INT_FRAME
927 ASM_CLAC
909 pushl_cfi $0 928 pushl_cfi $0
910 pushl_cfi $do_overflow 929 pushl_cfi $do_overflow
911 jmp error_code 930 jmp error_code
@@ -914,6 +933,7 @@ END(overflow)
914 933
915ENTRY(bounds) 934ENTRY(bounds)
916 RING0_INT_FRAME 935 RING0_INT_FRAME
936 ASM_CLAC
917 pushl_cfi $0 937 pushl_cfi $0
918 pushl_cfi $do_bounds 938 pushl_cfi $do_bounds
919 jmp error_code 939 jmp error_code
@@ -922,6 +942,7 @@ END(bounds)
922 942
923ENTRY(invalid_op) 943ENTRY(invalid_op)
924 RING0_INT_FRAME 944 RING0_INT_FRAME
945 ASM_CLAC
925 pushl_cfi $0 946 pushl_cfi $0
926 pushl_cfi $do_invalid_op 947 pushl_cfi $do_invalid_op
927 jmp error_code 948 jmp error_code
@@ -930,6 +951,7 @@ END(invalid_op)
930 951
931ENTRY(coprocessor_segment_overrun) 952ENTRY(coprocessor_segment_overrun)
932 RING0_INT_FRAME 953 RING0_INT_FRAME
954 ASM_CLAC
933 pushl_cfi $0 955 pushl_cfi $0
934 pushl_cfi $do_coprocessor_segment_overrun 956 pushl_cfi $do_coprocessor_segment_overrun
935 jmp error_code 957 jmp error_code
@@ -938,6 +960,7 @@ END(coprocessor_segment_overrun)
938 960
939ENTRY(invalid_TSS) 961ENTRY(invalid_TSS)
940 RING0_EC_FRAME 962 RING0_EC_FRAME
963 ASM_CLAC
941 pushl_cfi $do_invalid_TSS 964 pushl_cfi $do_invalid_TSS
942 jmp error_code 965 jmp error_code
943 CFI_ENDPROC 966 CFI_ENDPROC
@@ -945,6 +968,7 @@ END(invalid_TSS)
945 968
946ENTRY(segment_not_present) 969ENTRY(segment_not_present)
947 RING0_EC_FRAME 970 RING0_EC_FRAME
971 ASM_CLAC
948 pushl_cfi $do_segment_not_present 972 pushl_cfi $do_segment_not_present
949 jmp error_code 973 jmp error_code
950 CFI_ENDPROC 974 CFI_ENDPROC
@@ -952,6 +976,7 @@ END(segment_not_present)
952 976
953ENTRY(stack_segment) 977ENTRY(stack_segment)
954 RING0_EC_FRAME 978 RING0_EC_FRAME
979 ASM_CLAC
955 pushl_cfi $do_stack_segment 980 pushl_cfi $do_stack_segment
956 jmp error_code 981 jmp error_code
957 CFI_ENDPROC 982 CFI_ENDPROC
@@ -959,6 +984,7 @@ END(stack_segment)
959 984
960ENTRY(alignment_check) 985ENTRY(alignment_check)
961 RING0_EC_FRAME 986 RING0_EC_FRAME
987 ASM_CLAC
962 pushl_cfi $do_alignment_check 988 pushl_cfi $do_alignment_check
963 jmp error_code 989 jmp error_code
964 CFI_ENDPROC 990 CFI_ENDPROC
@@ -966,6 +992,7 @@ END(alignment_check)
966 992
967ENTRY(divide_error) 993ENTRY(divide_error)
968 RING0_INT_FRAME 994 RING0_INT_FRAME
995 ASM_CLAC
969 pushl_cfi $0 # no error code 996 pushl_cfi $0 # no error code
970 pushl_cfi $do_divide_error 997 pushl_cfi $do_divide_error
971 jmp error_code 998 jmp error_code
@@ -975,6 +1002,7 @@ END(divide_error)
975#ifdef CONFIG_X86_MCE 1002#ifdef CONFIG_X86_MCE
976ENTRY(machine_check) 1003ENTRY(machine_check)
977 RING0_INT_FRAME 1004 RING0_INT_FRAME
1005 ASM_CLAC
978 pushl_cfi $0 1006 pushl_cfi $0
979 pushl_cfi machine_check_vector 1007 pushl_cfi machine_check_vector
980 jmp error_code 1008 jmp error_code
@@ -984,6 +1012,7 @@ END(machine_check)
984 1012
985ENTRY(spurious_interrupt_bug) 1013ENTRY(spurious_interrupt_bug)
986 RING0_INT_FRAME 1014 RING0_INT_FRAME
1015 ASM_CLAC
987 pushl_cfi $0 1016 pushl_cfi $0
988 pushl_cfi $do_spurious_interrupt_bug 1017 pushl_cfi $do_spurious_interrupt_bug
989 jmp error_code 1018 jmp error_code
@@ -994,16 +1023,6 @@ END(spurious_interrupt_bug)
994 */ 1023 */
995 .popsection 1024 .popsection
996 1025
997ENTRY(kernel_thread_helper)
998 pushl $0 # fake return address for unwinder
999 CFI_STARTPROC
1000 movl %edi,%eax
1001 call *%esi
1002 call do_exit
1003 ud2 # padding for call trace
1004 CFI_ENDPROC
1005ENDPROC(kernel_thread_helper)
1006
1007#ifdef CONFIG_XEN 1026#ifdef CONFIG_XEN
1008/* Xen doesn't set %esp to be precisely what the normal sysenter 1027/* Xen doesn't set %esp to be precisely what the normal sysenter
1009 entrypoint expects, so fix it up before using the normal path. */ 1028 entrypoint expects, so fix it up before using the normal path. */
@@ -1111,17 +1130,21 @@ ENTRY(ftrace_caller)
1111 pushl %eax 1130 pushl %eax
1112 pushl %ecx 1131 pushl %ecx
1113 pushl %edx 1132 pushl %edx
1114 movl 0xc(%esp), %eax 1133 pushl $0 /* Pass NULL as regs pointer */
1134 movl 4*4(%esp), %eax
1115 movl 0x4(%ebp), %edx 1135 movl 0x4(%ebp), %edx
1136 leal function_trace_op, %ecx
1116 subl $MCOUNT_INSN_SIZE, %eax 1137 subl $MCOUNT_INSN_SIZE, %eax
1117 1138
1118.globl ftrace_call 1139.globl ftrace_call
1119ftrace_call: 1140ftrace_call:
1120 call ftrace_stub 1141 call ftrace_stub
1121 1142
1143 addl $4,%esp /* skip NULL pointer */
1122 popl %edx 1144 popl %edx
1123 popl %ecx 1145 popl %ecx
1124 popl %eax 1146 popl %eax
1147ftrace_ret:
1125#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1148#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1126.globl ftrace_graph_call 1149.globl ftrace_graph_call
1127ftrace_graph_call: 1150ftrace_graph_call:
@@ -1133,6 +1156,71 @@ ftrace_stub:
1133 ret 1156 ret
1134END(ftrace_caller) 1157END(ftrace_caller)
1135 1158
1159ENTRY(ftrace_regs_caller)
1160 pushf /* push flags before compare (in cs location) */
1161 cmpl $0, function_trace_stop
1162 jne ftrace_restore_flags
1163
1164 /*
1165 * i386 does not save SS and ESP when coming from kernel.
1166 * Instead, to get sp, &regs->sp is used (see ptrace.h).
1167 * Unfortunately, that means eflags must be at the same location
1168 * as the current return ip is. We move the return ip into the
1169 * ip location, and move flags into the return ip location.
1170 */
1171 pushl 4(%esp) /* save return ip into ip slot */
1172
1173 pushl $0 /* Load 0 into orig_ax */
1174 pushl %gs
1175 pushl %fs
1176 pushl %es
1177 pushl %ds
1178 pushl %eax
1179 pushl %ebp
1180 pushl %edi
1181 pushl %esi
1182 pushl %edx
1183 pushl %ecx
1184 pushl %ebx
1185
1186 movl 13*4(%esp), %eax /* Get the saved flags */
1187 movl %eax, 14*4(%esp) /* Move saved flags into regs->flags location */
1188 /* clobbering return ip */
1189 movl $__KERNEL_CS,13*4(%esp)
1190
1191 movl 12*4(%esp), %eax /* Load ip (1st parameter) */
1192 subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
1193 movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
1194 leal function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
1195 pushl %esp /* Save pt_regs as 4th parameter */
1196
1197GLOBAL(ftrace_regs_call)
1198 call ftrace_stub
1199
1200 addl $4, %esp /* Skip pt_regs */
1201 movl 14*4(%esp), %eax /* Move flags back into cs */
1202 movl %eax, 13*4(%esp) /* Needed to keep addl from modifying flags */
1203 movl 12*4(%esp), %eax /* Get return ip from regs->ip */
1204 movl %eax, 14*4(%esp) /* Put return ip back for ret */
1205
1206 popl %ebx
1207 popl %ecx
1208 popl %edx
1209 popl %esi
1210 popl %edi
1211 popl %ebp
1212 popl %eax
1213 popl %ds
1214 popl %es
1215 popl %fs
1216 popl %gs
1217 addl $8, %esp /* Skip orig_ax and ip */
1218 popf /* Pop flags at end (no addl to corrupt flags) */
1219 jmp ftrace_ret
1220
1221ftrace_restore_flags:
1222 popf
1223 jmp ftrace_stub
1136#else /* ! CONFIG_DYNAMIC_FTRACE */ 1224#else /* ! CONFIG_DYNAMIC_FTRACE */
1137 1225
1138ENTRY(mcount) 1226ENTRY(mcount)
@@ -1173,9 +1261,6 @@ END(mcount)
1173 1261
1174#ifdef CONFIG_FUNCTION_GRAPH_TRACER 1262#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1175ENTRY(ftrace_graph_caller) 1263ENTRY(ftrace_graph_caller)
1176 cmpl $0, function_trace_stop
1177 jne ftrace_stub
1178
1179 pushl %eax 1264 pushl %eax
1180 pushl %ecx 1265 pushl %ecx
1181 pushl %edx 1266 pushl %edx
@@ -1209,6 +1294,7 @@ return_to_handler:
1209 1294
1210ENTRY(page_fault) 1295ENTRY(page_fault)
1211 RING0_EC_FRAME 1296 RING0_EC_FRAME
1297 ASM_CLAC
1212 pushl_cfi $do_page_fault 1298 pushl_cfi $do_page_fault
1213 ALIGN 1299 ALIGN
1214error_code: 1300error_code:
@@ -1281,6 +1367,7 @@ END(page_fault)
1281 1367
1282ENTRY(debug) 1368ENTRY(debug)
1283 RING0_INT_FRAME 1369 RING0_INT_FRAME
1370 ASM_CLAC
1284 cmpl $ia32_sysenter_target,(%esp) 1371 cmpl $ia32_sysenter_target,(%esp)
1285 jne debug_stack_correct 1372 jne debug_stack_correct
1286 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn 1373 FIX_STACK 12, debug_stack_correct, debug_esp_fix_insn
@@ -1305,6 +1392,7 @@ END(debug)
1305 */ 1392 */
1306ENTRY(nmi) 1393ENTRY(nmi)
1307 RING0_INT_FRAME 1394 RING0_INT_FRAME
1395 ASM_CLAC
1308 pushl_cfi %eax 1396 pushl_cfi %eax
1309 movl %ss, %eax 1397 movl %ss, %eax
1310 cmpw $__ESPFIX_SS, %ax 1398 cmpw $__ESPFIX_SS, %ax
@@ -1375,6 +1463,7 @@ END(nmi)
1375 1463
1376ENTRY(int3) 1464ENTRY(int3)
1377 RING0_INT_FRAME 1465 RING0_INT_FRAME
1466 ASM_CLAC
1378 pushl_cfi $-1 # mark this as an int 1467 pushl_cfi $-1 # mark this as an int
1379 SAVE_ALL 1468 SAVE_ALL
1380 TRACE_IRQS_OFF 1469 TRACE_IRQS_OFF
@@ -1395,6 +1484,7 @@ END(general_protection)
1395#ifdef CONFIG_KVM_GUEST 1484#ifdef CONFIG_KVM_GUEST
1396ENTRY(async_page_fault) 1485ENTRY(async_page_fault)
1397 RING0_EC_FRAME 1486 RING0_EC_FRAME
1487 ASM_CLAC
1398 pushl_cfi $do_async_page_fault 1488 pushl_cfi $do_async_page_fault
1399 jmp error_code 1489 jmp error_code
1400 CFI_ENDPROC 1490 CFI_ENDPROC
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index dcdd0ea33a32..b51b2c7ee51f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -56,6 +56,8 @@
56#include <asm/ftrace.h> 56#include <asm/ftrace.h>
57#include <asm/percpu.h> 57#include <asm/percpu.h>
58#include <asm/asm.h> 58#include <asm/asm.h>
59#include <asm/rcu.h>
60#include <asm/smap.h>
59#include <linux/err.h> 61#include <linux/err.h>
60 62
61/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */ 63/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
@@ -68,25 +70,51 @@
68 .section .entry.text, "ax" 70 .section .entry.text, "ax"
69 71
70#ifdef CONFIG_FUNCTION_TRACER 72#ifdef CONFIG_FUNCTION_TRACER
73
74#ifdef CC_USING_FENTRY
75# define function_hook __fentry__
76#else
77# define function_hook mcount
78#endif
79
71#ifdef CONFIG_DYNAMIC_FTRACE 80#ifdef CONFIG_DYNAMIC_FTRACE
72ENTRY(mcount) 81
82ENTRY(function_hook)
73 retq 83 retq
74END(mcount) 84END(function_hook)
85
86/* skip is set if stack has been adjusted */
87.macro ftrace_caller_setup skip=0
88 MCOUNT_SAVE_FRAME \skip
89
90 /* Load the ftrace_ops into the 3rd parameter */
91 leaq function_trace_op, %rdx
92
93 /* Load ip into the first parameter */
94 movq RIP(%rsp), %rdi
95 subq $MCOUNT_INSN_SIZE, %rdi
96 /* Load the parent_ip into the second parameter */
97#ifdef CC_USING_FENTRY
98 movq SS+16(%rsp), %rsi
99#else
100 movq 8(%rbp), %rsi
101#endif
102.endm
75 103
76ENTRY(ftrace_caller) 104ENTRY(ftrace_caller)
105 /* Check if tracing was disabled (quick check) */
77 cmpl $0, function_trace_stop 106 cmpl $0, function_trace_stop
78 jne ftrace_stub 107 jne ftrace_stub
79 108
80 MCOUNT_SAVE_FRAME 109 ftrace_caller_setup
81 110 /* regs go into 4th parameter (but make it NULL) */
82 movq 0x38(%rsp), %rdi 111 movq $0, %rcx
83 movq 8(%rbp), %rsi
84 subq $MCOUNT_INSN_SIZE, %rdi
85 112
86GLOBAL(ftrace_call) 113GLOBAL(ftrace_call)
87 call ftrace_stub 114 call ftrace_stub
88 115
89 MCOUNT_RESTORE_FRAME 116 MCOUNT_RESTORE_FRAME
117ftrace_return:
90 118
91#ifdef CONFIG_FUNCTION_GRAPH_TRACER 119#ifdef CONFIG_FUNCTION_GRAPH_TRACER
92GLOBAL(ftrace_graph_call) 120GLOBAL(ftrace_graph_call)
@@ -97,8 +125,78 @@ GLOBAL(ftrace_stub)
97 retq 125 retq
98END(ftrace_caller) 126END(ftrace_caller)
99 127
128ENTRY(ftrace_regs_caller)
129 /* Save the current flags before compare (in SS location)*/
130 pushfq
131
132 /* Check if tracing was disabled (quick check) */
133 cmpl $0, function_trace_stop
134 jne ftrace_restore_flags
135
136 /* skip=8 to skip flags saved in SS */
137 ftrace_caller_setup 8
138
139 /* Save the rest of pt_regs */
140 movq %r15, R15(%rsp)
141 movq %r14, R14(%rsp)
142 movq %r13, R13(%rsp)
143 movq %r12, R12(%rsp)
144 movq %r11, R11(%rsp)
145 movq %r10, R10(%rsp)
146 movq %rbp, RBP(%rsp)
147 movq %rbx, RBX(%rsp)
148 /* Copy saved flags */
149 movq SS(%rsp), %rcx
150 movq %rcx, EFLAGS(%rsp)
151 /* Kernel segments */
152 movq $__KERNEL_DS, %rcx
153 movq %rcx, SS(%rsp)
154 movq $__KERNEL_CS, %rcx
155 movq %rcx, CS(%rsp)
156 /* Stack - skipping return address */
157 leaq SS+16(%rsp), %rcx
158 movq %rcx, RSP(%rsp)
159
160 /* regs go into 4th parameter */
161 leaq (%rsp), %rcx
162
163GLOBAL(ftrace_regs_call)
164 call ftrace_stub
165
166 /* Copy flags back to SS, to restore them */
167 movq EFLAGS(%rsp), %rax
168 movq %rax, SS(%rsp)
169
170 /* Handlers can change the RIP */
171 movq RIP(%rsp), %rax
172 movq %rax, SS+8(%rsp)
173
174 /* restore the rest of pt_regs */
175 movq R15(%rsp), %r15
176 movq R14(%rsp), %r14
177 movq R13(%rsp), %r13
178 movq R12(%rsp), %r12
179 movq R10(%rsp), %r10
180 movq RBP(%rsp), %rbp
181 movq RBX(%rsp), %rbx
182
183 /* skip=8 to skip flags saved in SS */
184 MCOUNT_RESTORE_FRAME 8
185
186 /* Restore flags */
187 popfq
188
189 jmp ftrace_return
190ftrace_restore_flags:
191 popfq
192 jmp ftrace_stub
193
194END(ftrace_regs_caller)
195
196
100#else /* ! CONFIG_DYNAMIC_FTRACE */ 197#else /* ! CONFIG_DYNAMIC_FTRACE */
101ENTRY(mcount) 198
199ENTRY(function_hook)
102 cmpl $0, function_trace_stop 200 cmpl $0, function_trace_stop
103 jne ftrace_stub 201 jne ftrace_stub
104 202
@@ -119,8 +217,12 @@ GLOBAL(ftrace_stub)
119trace: 217trace:
120 MCOUNT_SAVE_FRAME 218 MCOUNT_SAVE_FRAME
121 219
122 movq 0x38(%rsp), %rdi 220 movq RIP(%rsp), %rdi
221#ifdef CC_USING_FENTRY
222 movq SS+16(%rsp), %rsi
223#else
123 movq 8(%rbp), %rsi 224 movq 8(%rbp), %rsi
225#endif
124 subq $MCOUNT_INSN_SIZE, %rdi 226 subq $MCOUNT_INSN_SIZE, %rdi
125 227
126 call *ftrace_trace_function 228 call *ftrace_trace_function
@@ -128,20 +230,22 @@ trace:
128 MCOUNT_RESTORE_FRAME 230 MCOUNT_RESTORE_FRAME
129 231
130 jmp ftrace_stub 232 jmp ftrace_stub
131END(mcount) 233END(function_hook)
132#endif /* CONFIG_DYNAMIC_FTRACE */ 234#endif /* CONFIG_DYNAMIC_FTRACE */
133#endif /* CONFIG_FUNCTION_TRACER */ 235#endif /* CONFIG_FUNCTION_TRACER */
134 236
135#ifdef CONFIG_FUNCTION_GRAPH_TRACER 237#ifdef CONFIG_FUNCTION_GRAPH_TRACER
136ENTRY(ftrace_graph_caller) 238ENTRY(ftrace_graph_caller)
137 cmpl $0, function_trace_stop
138 jne ftrace_stub
139
140 MCOUNT_SAVE_FRAME 239 MCOUNT_SAVE_FRAME
141 240
241#ifdef CC_USING_FENTRY
242 leaq SS+16(%rsp), %rdi
243 movq $0, %rdx /* No framepointers needed */
244#else
142 leaq 8(%rbp), %rdi 245 leaq 8(%rbp), %rdi
143 movq 0x38(%rsp), %rsi
144 movq (%rbp), %rdx 246 movq (%rbp), %rdx
247#endif
248 movq RIP(%rsp), %rsi
145 subq $MCOUNT_INSN_SIZE, %rsi 249 subq $MCOUNT_INSN_SIZE, %rsi
146 250
147 call prepare_ftrace_return 251 call prepare_ftrace_return
@@ -342,15 +446,15 @@ ENDPROC(native_usergs_sysret64)
342 .macro SAVE_ARGS_IRQ 446 .macro SAVE_ARGS_IRQ
343 cld 447 cld
344 /* start from rbp in pt_regs and jump over */ 448 /* start from rbp in pt_regs and jump over */
345 movq_cfi rdi, RDI-RBP 449 movq_cfi rdi, (RDI-RBP)
346 movq_cfi rsi, RSI-RBP 450 movq_cfi rsi, (RSI-RBP)
347 movq_cfi rdx, RDX-RBP 451 movq_cfi rdx, (RDX-RBP)
348 movq_cfi rcx, RCX-RBP 452 movq_cfi rcx, (RCX-RBP)
349 movq_cfi rax, RAX-RBP 453 movq_cfi rax, (RAX-RBP)
350 movq_cfi r8, R8-RBP 454 movq_cfi r8, (R8-RBP)
351 movq_cfi r9, R9-RBP 455 movq_cfi r9, (R9-RBP)
352 movq_cfi r10, R10-RBP 456 movq_cfi r10, (R10-RBP)
353 movq_cfi r11, R11-RBP 457 movq_cfi r11, (R11-RBP)
354 458
355 /* Save rbp so that we can unwind from get_irq_regs() */ 459 /* Save rbp so that we can unwind from get_irq_regs() */
356 movq_cfi rbp, 0 460 movq_cfi rbp, 0
@@ -384,7 +488,7 @@ ENDPROC(native_usergs_sysret64)
384 .endm 488 .endm
385 489
386ENTRY(save_rest) 490ENTRY(save_rest)
387 PARTIAL_FRAME 1 REST_SKIP+8 491 PARTIAL_FRAME 1 (REST_SKIP+8)
388 movq 5*8+16(%rsp), %r11 /* save return address */ 492 movq 5*8+16(%rsp), %r11 /* save return address */
389 movq_cfi rbx, RBX+16 493 movq_cfi rbx, RBX+16
390 movq_cfi rbp, RBP+16 494 movq_cfi rbp, RBP+16
@@ -440,7 +544,7 @@ ENTRY(ret_from_fork)
440 544
441 LOCK ; btr $TIF_FORK,TI_flags(%r8) 545 LOCK ; btr $TIF_FORK,TI_flags(%r8)
442 546
443 pushq_cfi kernel_eflags(%rip) 547 pushq_cfi $0x0002
444 popfq_cfi # reset kernel eflags 548 popfq_cfi # reset kernel eflags
445 549
446 call schedule_tail # rdi: 'prev' task parameter 550 call schedule_tail # rdi: 'prev' task parameter
@@ -450,7 +554,7 @@ ENTRY(ret_from_fork)
450 RESTORE_REST 554 RESTORE_REST
451 555
452 testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread? 556 testl $3, CS-ARGOFFSET(%rsp) # from kernel_thread?
453 jz retint_restore_args 557 jz 1f
454 558
455 testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET 559 testl $_TIF_IA32, TI_flags(%rcx) # 32-bit compat task needs IRET
456 jnz int_ret_from_sys_call 560 jnz int_ret_from_sys_call
@@ -458,6 +562,14 @@ ENTRY(ret_from_fork)
458 RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET 562 RESTORE_TOP_OF_STACK %rdi, -ARGOFFSET
459 jmp ret_from_sys_call # go to the SYSRET fastpath 563 jmp ret_from_sys_call # go to the SYSRET fastpath
460 564
5651:
566 subq $REST_SKIP, %rsp # leave space for volatiles
567 CFI_ADJUST_CFA_OFFSET REST_SKIP
568 movq %rbp, %rdi
569 call *%rbx
570 movl $0, RAX(%rsp)
571 RESTORE_REST
572 jmp int_ret_from_sys_call
461 CFI_ENDPROC 573 CFI_ENDPROC
462END(ret_from_fork) 574END(ret_from_fork)
463 575
@@ -465,7 +577,8 @@ END(ret_from_fork)
465 * System call entry. Up to 6 arguments in registers are supported. 577 * System call entry. Up to 6 arguments in registers are supported.
466 * 578 *
467 * SYSCALL does not save anything on the stack and does not change the 579 * SYSCALL does not save anything on the stack and does not change the
468 * stack pointer. 580 * stack pointer. However, it does mask the flags register for us, so
581 * CLD and CLAC are not needed.
469 */ 582 */
470 583
471/* 584/*
@@ -565,7 +678,7 @@ sysret_careful:
565 TRACE_IRQS_ON 678 TRACE_IRQS_ON
566 ENABLE_INTERRUPTS(CLBR_NONE) 679 ENABLE_INTERRUPTS(CLBR_NONE)
567 pushq_cfi %rdi 680 pushq_cfi %rdi
568 call schedule 681 SCHEDULE_USER
569 popq_cfi %rdi 682 popq_cfi %rdi
570 jmp sysret_check 683 jmp sysret_check
571 684
@@ -678,7 +791,7 @@ int_careful:
678 TRACE_IRQS_ON 791 TRACE_IRQS_ON
679 ENABLE_INTERRUPTS(CLBR_NONE) 792 ENABLE_INTERRUPTS(CLBR_NONE)
680 pushq_cfi %rdi 793 pushq_cfi %rdi
681 call schedule 794 SCHEDULE_USER
682 popq_cfi %rdi 795 popq_cfi %rdi
683 DISABLE_INTERRUPTS(CLBR_NONE) 796 DISABLE_INTERRUPTS(CLBR_NONE)
684 TRACE_IRQS_OFF 797 TRACE_IRQS_OFF
@@ -757,7 +870,6 @@ ENTRY(stub_execve)
757 PARTIAL_FRAME 0 870 PARTIAL_FRAME 0
758 SAVE_REST 871 SAVE_REST
759 FIXUP_TOP_OF_STACK %r11 872 FIXUP_TOP_OF_STACK %r11
760 movq %rsp, %rcx
761 call sys_execve 873 call sys_execve
762 RESTORE_TOP_OF_STACK %r11 874 RESTORE_TOP_OF_STACK %r11
763 movq %rax,RAX(%rsp) 875 movq %rax,RAX(%rsp)
@@ -807,8 +919,7 @@ ENTRY(stub_x32_execve)
807 PARTIAL_FRAME 0 919 PARTIAL_FRAME 0
808 SAVE_REST 920 SAVE_REST
809 FIXUP_TOP_OF_STACK %r11 921 FIXUP_TOP_OF_STACK %r11
810 movq %rsp, %rcx 922 call compat_sys_execve
811 call sys32_execve
812 RESTORE_TOP_OF_STACK %r11 923 RESTORE_TOP_OF_STACK %r11
813 movq %rax,RAX(%rsp) 924 movq %rax,RAX(%rsp)
814 RESTORE_REST 925 RESTORE_REST
@@ -884,6 +995,7 @@ END(interrupt)
884 */ 995 */
885 .p2align CONFIG_X86_L1_CACHE_SHIFT 996 .p2align CONFIG_X86_L1_CACHE_SHIFT
886common_interrupt: 997common_interrupt:
998 ASM_CLAC
887 XCPT_FRAME 999 XCPT_FRAME
888 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */ 1000 addq $-0x80,(%rsp) /* Adjust vector to [-256,-1] range */
889 interrupt do_IRQ 1001 interrupt do_IRQ
@@ -974,7 +1086,7 @@ retint_careful:
974 TRACE_IRQS_ON 1086 TRACE_IRQS_ON
975 ENABLE_INTERRUPTS(CLBR_NONE) 1087 ENABLE_INTERRUPTS(CLBR_NONE)
976 pushq_cfi %rdi 1088 pushq_cfi %rdi
977 call schedule 1089 SCHEDULE_USER
978 popq_cfi %rdi 1090 popq_cfi %rdi
979 GET_THREAD_INFO(%rcx) 1091 GET_THREAD_INFO(%rcx)
980 DISABLE_INTERRUPTS(CLBR_NONE) 1092 DISABLE_INTERRUPTS(CLBR_NONE)
@@ -1023,6 +1135,7 @@ END(common_interrupt)
1023 */ 1135 */
1024.macro apicinterrupt num sym do_sym 1136.macro apicinterrupt num sym do_sym
1025ENTRY(\sym) 1137ENTRY(\sym)
1138 ASM_CLAC
1026 INTR_FRAME 1139 INTR_FRAME
1027 pushq_cfi $~(\num) 1140 pushq_cfi $~(\num)
1028.Lcommon_\sym: 1141.Lcommon_\sym:
@@ -1077,6 +1190,7 @@ apicinterrupt IRQ_WORK_VECTOR \
1077 */ 1190 */
1078.macro zeroentry sym do_sym 1191.macro zeroentry sym do_sym
1079ENTRY(\sym) 1192ENTRY(\sym)
1193 ASM_CLAC
1080 INTR_FRAME 1194 INTR_FRAME
1081 PARAVIRT_ADJUST_EXCEPTION_FRAME 1195 PARAVIRT_ADJUST_EXCEPTION_FRAME
1082 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1196 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
@@ -1094,6 +1208,7 @@ END(\sym)
1094 1208
1095.macro paranoidzeroentry sym do_sym 1209.macro paranoidzeroentry sym do_sym
1096ENTRY(\sym) 1210ENTRY(\sym)
1211 ASM_CLAC
1097 INTR_FRAME 1212 INTR_FRAME
1098 PARAVIRT_ADJUST_EXCEPTION_FRAME 1213 PARAVIRT_ADJUST_EXCEPTION_FRAME
1099 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1214 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
@@ -1112,6 +1227,7 @@ END(\sym)
1112#define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8) 1227#define INIT_TSS_IST(x) PER_CPU_VAR(init_tss) + (TSS_ist + ((x) - 1) * 8)
1113.macro paranoidzeroentry_ist sym do_sym ist 1228.macro paranoidzeroentry_ist sym do_sym ist
1114ENTRY(\sym) 1229ENTRY(\sym)
1230 ASM_CLAC
1115 INTR_FRAME 1231 INTR_FRAME
1116 PARAVIRT_ADJUST_EXCEPTION_FRAME 1232 PARAVIRT_ADJUST_EXCEPTION_FRAME
1117 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */ 1233 pushq_cfi $-1 /* ORIG_RAX: no syscall to restart */
@@ -1131,6 +1247,7 @@ END(\sym)
1131 1247
1132.macro errorentry sym do_sym 1248.macro errorentry sym do_sym
1133ENTRY(\sym) 1249ENTRY(\sym)
1250 ASM_CLAC
1134 XCPT_FRAME 1251 XCPT_FRAME
1135 PARAVIRT_ADJUST_EXCEPTION_FRAME 1252 PARAVIRT_ADJUST_EXCEPTION_FRAME
1136 subq $ORIG_RAX-R15, %rsp 1253 subq $ORIG_RAX-R15, %rsp
@@ -1149,6 +1266,7 @@ END(\sym)
1149 /* error code is on the stack already */ 1266 /* error code is on the stack already */
1150.macro paranoiderrorentry sym do_sym 1267.macro paranoiderrorentry sym do_sym
1151ENTRY(\sym) 1268ENTRY(\sym)
1269 ASM_CLAC
1152 XCPT_FRAME 1270 XCPT_FRAME
1153 PARAVIRT_ADJUST_EXCEPTION_FRAME 1271 PARAVIRT_ADJUST_EXCEPTION_FRAME
1154 subq $ORIG_RAX-R15, %rsp 1272 subq $ORIG_RAX-R15, %rsp
@@ -1206,52 +1324,6 @@ bad_gs:
1206 jmp 2b 1324 jmp 2b
1207 .previous 1325 .previous
1208 1326
1209ENTRY(kernel_thread_helper)
1210 pushq $0 # fake return address
1211 CFI_STARTPROC
1212 /*
1213 * Here we are in the child and the registers are set as they were
1214 * at kernel_thread() invocation in the parent.
1215 */
1216 call *%rsi
1217 # exit
1218 mov %eax, %edi
1219 call do_exit
1220 ud2 # padding for call trace
1221 CFI_ENDPROC
1222END(kernel_thread_helper)
1223
1224/*
1225 * execve(). This function needs to use IRET, not SYSRET, to set up all state properly.
1226 *
1227 * C extern interface:
1228 * extern long execve(const char *name, char **argv, char **envp)
1229 *
1230 * asm input arguments:
1231 * rdi: name, rsi: argv, rdx: envp
1232 *
1233 * We want to fallback into:
1234 * extern long sys_execve(const char *name, char **argv,char **envp, struct pt_regs *regs)
1235 *
1236 * do_sys_execve asm fallback arguments:
1237 * rdi: name, rsi: argv, rdx: envp, rcx: fake frame on the stack
1238 */
1239ENTRY(kernel_execve)
1240 CFI_STARTPROC
1241 FAKE_STACK_FRAME $0
1242 SAVE_ALL
1243 movq %rsp,%rcx
1244 call sys_execve
1245 movq %rax, RAX(%rsp)
1246 RESTORE_REST
1247 testq %rax,%rax
1248 je int_ret_from_sys_call
1249 RESTORE_ARGS
1250 UNFAKE_STACK_FRAME
1251 ret
1252 CFI_ENDPROC
1253END(kernel_execve)
1254
1255/* Call softirq on interrupt stack. Interrupts are off. */ 1327/* Call softirq on interrupt stack. Interrupts are off. */
1256ENTRY(call_softirq) 1328ENTRY(call_softirq)
1257 CFI_STARTPROC 1329 CFI_STARTPROC
@@ -1449,7 +1521,7 @@ paranoid_userspace:
1449paranoid_schedule: 1521paranoid_schedule:
1450 TRACE_IRQS_ON 1522 TRACE_IRQS_ON
1451 ENABLE_INTERRUPTS(CLBR_ANY) 1523 ENABLE_INTERRUPTS(CLBR_ANY)
1452 call schedule 1524 SCHEDULE_USER
1453 DISABLE_INTERRUPTS(CLBR_ANY) 1525 DISABLE_INTERRUPTS(CLBR_ANY)
1454 TRACE_IRQS_OFF 1526 TRACE_IRQS_OFF
1455 jmp paranoid_userspace 1527 jmp paranoid_userspace
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
index c3a7cb4bf6e6..1d414029f1d8 100644
--- a/arch/x86/kernel/ftrace.c
+++ b/arch/x86/kernel/ftrace.c
@@ -206,6 +206,21 @@ static int
206ftrace_modify_code(unsigned long ip, unsigned const char *old_code, 206ftrace_modify_code(unsigned long ip, unsigned const char *old_code,
207 unsigned const char *new_code); 207 unsigned const char *new_code);
208 208
209/*
210 * Should never be called:
211 * As it is only called by __ftrace_replace_code() which is called by
212 * ftrace_replace_code() that x86 overrides, and by ftrace_update_code()
213 * which is called to turn mcount into nops or nops into function calls
214 * but not to convert a function from not using regs to one that uses
215 * regs, which ftrace_modify_call() is for.
216 */
217int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
218 unsigned long addr)
219{
220 WARN_ON(1);
221 return -EINVAL;
222}
223
209int ftrace_update_ftrace_func(ftrace_func_t func) 224int ftrace_update_ftrace_func(ftrace_func_t func)
210{ 225{
211 unsigned long ip = (unsigned long)(&ftrace_call); 226 unsigned long ip = (unsigned long)(&ftrace_call);
@@ -220,6 +235,14 @@ int ftrace_update_ftrace_func(ftrace_func_t func)
220 235
221 ret = ftrace_modify_code(ip, old, new); 236 ret = ftrace_modify_code(ip, old, new);
222 237
238 /* Also update the regs callback function */
239 if (!ret) {
240 ip = (unsigned long)(&ftrace_regs_call);
241 memcpy(old, &ftrace_regs_call, MCOUNT_INSN_SIZE);
242 new = ftrace_call_replace(ip, (unsigned long)func);
243 ret = ftrace_modify_code(ip, old, new);
244 }
245
223 atomic_dec(&modifying_ftrace_code); 246 atomic_dec(&modifying_ftrace_code);
224 247
225 return ret; 248 return ret;
@@ -299,6 +322,32 @@ static int add_brk_on_nop(struct dyn_ftrace *rec)
299 return add_break(rec->ip, old); 322 return add_break(rec->ip, old);
300} 323}
301 324
325/*
326 * If the record has the FTRACE_FL_REGS set, that means that it
327 * wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
328 * is not not set, then it wants to convert to the normal callback.
329 */
330static unsigned long get_ftrace_addr(struct dyn_ftrace *rec)
331{
332 if (rec->flags & FTRACE_FL_REGS)
333 return (unsigned long)FTRACE_REGS_ADDR;
334 else
335 return (unsigned long)FTRACE_ADDR;
336}
337
338/*
339 * The FTRACE_FL_REGS_EN is set when the record already points to
340 * a function that saves all the regs. Basically the '_EN' version
341 * represents the current state of the function.
342 */
343static unsigned long get_ftrace_old_addr(struct dyn_ftrace *rec)
344{
345 if (rec->flags & FTRACE_FL_REGS_EN)
346 return (unsigned long)FTRACE_REGS_ADDR;
347 else
348 return (unsigned long)FTRACE_ADDR;
349}
350
302static int add_breakpoints(struct dyn_ftrace *rec, int enable) 351static int add_breakpoints(struct dyn_ftrace *rec, int enable)
303{ 352{
304 unsigned long ftrace_addr; 353 unsigned long ftrace_addr;
@@ -306,7 +355,7 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable)
306 355
307 ret = ftrace_test_record(rec, enable); 356 ret = ftrace_test_record(rec, enable);
308 357
309 ftrace_addr = (unsigned long)FTRACE_ADDR; 358 ftrace_addr = get_ftrace_addr(rec);
310 359
311 switch (ret) { 360 switch (ret) {
312 case FTRACE_UPDATE_IGNORE: 361 case FTRACE_UPDATE_IGNORE:
@@ -316,6 +365,10 @@ static int add_breakpoints(struct dyn_ftrace *rec, int enable)
316 /* converting nop to call */ 365 /* converting nop to call */
317 return add_brk_on_nop(rec); 366 return add_brk_on_nop(rec);
318 367
368 case FTRACE_UPDATE_MODIFY_CALL_REGS:
369 case FTRACE_UPDATE_MODIFY_CALL:
370 ftrace_addr = get_ftrace_old_addr(rec);
371 /* fall through */
319 case FTRACE_UPDATE_MAKE_NOP: 372 case FTRACE_UPDATE_MAKE_NOP:
320 /* converting a call to a nop */ 373 /* converting a call to a nop */
321 return add_brk_on_call(rec, ftrace_addr); 374 return add_brk_on_call(rec, ftrace_addr);
@@ -360,13 +413,21 @@ static int remove_breakpoint(struct dyn_ftrace *rec)
360 * If not, don't touch the breakpoint, we make just create 413 * If not, don't touch the breakpoint, we make just create
361 * a disaster. 414 * a disaster.
362 */ 415 */
363 ftrace_addr = (unsigned long)FTRACE_ADDR; 416 ftrace_addr = get_ftrace_addr(rec);
417 nop = ftrace_call_replace(ip, ftrace_addr);
418
419 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) == 0)
420 goto update;
421
422 /* Check both ftrace_addr and ftrace_old_addr */
423 ftrace_addr = get_ftrace_old_addr(rec);
364 nop = ftrace_call_replace(ip, ftrace_addr); 424 nop = ftrace_call_replace(ip, ftrace_addr);
365 425
366 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0) 426 if (memcmp(&ins[1], &nop[1], MCOUNT_INSN_SIZE - 1) != 0)
367 return -EINVAL; 427 return -EINVAL;
368 } 428 }
369 429
430 update:
370 return probe_kernel_write((void *)ip, &nop[0], 1); 431 return probe_kernel_write((void *)ip, &nop[0], 1);
371} 432}
372 433
@@ -405,12 +466,14 @@ static int add_update(struct dyn_ftrace *rec, int enable)
405 466
406 ret = ftrace_test_record(rec, enable); 467 ret = ftrace_test_record(rec, enable);
407 468
408 ftrace_addr = (unsigned long)FTRACE_ADDR; 469 ftrace_addr = get_ftrace_addr(rec);
409 470
410 switch (ret) { 471 switch (ret) {
411 case FTRACE_UPDATE_IGNORE: 472 case FTRACE_UPDATE_IGNORE:
412 return 0; 473 return 0;
413 474
475 case FTRACE_UPDATE_MODIFY_CALL_REGS:
476 case FTRACE_UPDATE_MODIFY_CALL:
414 case FTRACE_UPDATE_MAKE_CALL: 477 case FTRACE_UPDATE_MAKE_CALL:
415 /* converting nop to call */ 478 /* converting nop to call */
416 return add_update_call(rec, ftrace_addr); 479 return add_update_call(rec, ftrace_addr);
@@ -455,12 +518,14 @@ static int finish_update(struct dyn_ftrace *rec, int enable)
455 518
456 ret = ftrace_update_record(rec, enable); 519 ret = ftrace_update_record(rec, enable);
457 520
458 ftrace_addr = (unsigned long)FTRACE_ADDR; 521 ftrace_addr = get_ftrace_addr(rec);
459 522
460 switch (ret) { 523 switch (ret) {
461 case FTRACE_UPDATE_IGNORE: 524 case FTRACE_UPDATE_IGNORE:
462 return 0; 525 return 0;
463 526
527 case FTRACE_UPDATE_MODIFY_CALL_REGS:
528 case FTRACE_UPDATE_MODIFY_CALL:
464 case FTRACE_UPDATE_MAKE_CALL: 529 case FTRACE_UPDATE_MAKE_CALL:
465 /* converting nop to call */ 530 /* converting nop to call */
466 return finish_update_call(rec, ftrace_addr); 531 return finish_update_call(rec, ftrace_addr);
diff --git a/arch/x86/kernel/head_32.S b/arch/x86/kernel/head_32.S
index d42ab17b7397..957a47aec64e 100644
--- a/arch/x86/kernel/head_32.S
+++ b/arch/x86/kernel/head_32.S
@@ -287,27 +287,28 @@ ENTRY(startup_32_smp)
287 leal -__PAGE_OFFSET(%ecx),%esp 287 leal -__PAGE_OFFSET(%ecx),%esp
288 288
289default_entry: 289default_entry:
290
291/* 290/*
292 * New page tables may be in 4Mbyte page mode and may 291 * New page tables may be in 4Mbyte page mode and may
293 * be using the global pages. 292 * be using the global pages.
294 * 293 *
295 * NOTE! If we are on a 486 we may have no cr4 at all! 294 * NOTE! If we are on a 486 we may have no cr4 at all!
296 * So we do not try to touch it unless we really have 295 * Specifically, cr4 exists if and only if CPUID exists,
297 * some bits in it to set. This won't work if the BSP 296 * which in turn exists if and only if EFLAGS.ID exists.
298 * implements cr4 but this AP does not -- very unlikely
299 * but be warned! The same applies to the pse feature
300 * if not equally supported. --macro
301 *
302 * NOTE! We have to correct for the fact that we're
303 * not yet offset PAGE_OFFSET..
304 */ 297 */
305#define cr4_bits pa(mmu_cr4_features) 298 movl $X86_EFLAGS_ID,%ecx
306 movl cr4_bits,%edx 299 pushl %ecx
307 andl %edx,%edx 300 popfl
308 jz 6f 301 pushfl
309 movl %cr4,%eax # Turn on paging options (PSE,PAE,..) 302 popl %eax
310 orl %edx,%eax 303 pushl $0
304 popfl
305 pushfl
306 popl %edx
307 xorl %edx,%eax
308 testl %ecx,%eax
309 jz 6f # No ID flag = no CPUID = no CR4
310
311 movl pa(mmu_cr4_features),%eax
311 movl %eax,%cr4 312 movl %eax,%cr4
312 313
313 testb $X86_CR4_PAE, %al # check if PAE is enabled 314 testb $X86_CR4_PAE, %al # check if PAE is enabled
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index f250431fb505..675a05012449 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -19,24 +19,17 @@
19#include <asm/fpu-internal.h> 19#include <asm/fpu-internal.h>
20#include <asm/user.h> 20#include <asm/user.h>
21 21
22#ifdef CONFIG_X86_64
23# include <asm/sigcontext32.h>
24# include <asm/user32.h>
25#else
26# define save_i387_xstate_ia32 save_i387_xstate
27# define restore_i387_xstate_ia32 restore_i387_xstate
28# define _fpstate_ia32 _fpstate
29# define _xstate_ia32 _xstate
30# define sig_xstate_ia32_size sig_xstate_size
31# define fx_sw_reserved_ia32 fx_sw_reserved
32# define user_i387_ia32_struct user_i387_struct
33# define user32_fxsr_struct user_fxsr_struct
34#endif
35
36/* 22/*
37 * Were we in an interrupt that interrupted kernel mode? 23 * Were we in an interrupt that interrupted kernel mode?
38 * 24 *
39 * We can do a kernel_fpu_begin/end() pair *ONLY* if that 25 * For now, with eagerfpu we will return interrupted kernel FPU
26 * state as not-idle. TBD: Ideally we can change the return value
27 * to something like __thread_has_fpu(current). But we need to
28 * be careful of doing __thread_clear_has_fpu() before saving
29 * the FPU etc for supporting nested uses etc. For now, take
30 * the simple route!
31 *
32 * On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
40 * pair does nothing at all: the thread must not have fpu (so 33 * pair does nothing at all: the thread must not have fpu (so
41 * that we don't try to save the FPU state), and TS must 34 * that we don't try to save the FPU state), and TS must
42 * be set (so that the clts/stts pair does nothing that is 35 * be set (so that the clts/stts pair does nothing that is
@@ -44,6 +37,9 @@
44 */ 37 */
45static inline bool interrupted_kernel_fpu_idle(void) 38static inline bool interrupted_kernel_fpu_idle(void)
46{ 39{
40 if (use_eager_fpu())
41 return 0;
42
47 return !__thread_has_fpu(current) && 43 return !__thread_has_fpu(current) &&
48 (read_cr0() & X86_CR0_TS); 44 (read_cr0() & X86_CR0_TS);
49} 45}
@@ -77,29 +73,29 @@ bool irq_fpu_usable(void)
77} 73}
78EXPORT_SYMBOL(irq_fpu_usable); 74EXPORT_SYMBOL(irq_fpu_usable);
79 75
80void kernel_fpu_begin(void) 76void __kernel_fpu_begin(void)
81{ 77{
82 struct task_struct *me = current; 78 struct task_struct *me = current;
83 79
84 WARN_ON_ONCE(!irq_fpu_usable());
85 preempt_disable();
86 if (__thread_has_fpu(me)) { 80 if (__thread_has_fpu(me)) {
87 __save_init_fpu(me); 81 __save_init_fpu(me);
88 __thread_clear_has_fpu(me); 82 __thread_clear_has_fpu(me);
89 /* We do 'stts()' in kernel_fpu_end() */ 83 /* We do 'stts()' in __kernel_fpu_end() */
90 } else { 84 } else if (!use_eager_fpu()) {
91 this_cpu_write(fpu_owner_task, NULL); 85 this_cpu_write(fpu_owner_task, NULL);
92 clts(); 86 clts();
93 } 87 }
94} 88}
95EXPORT_SYMBOL(kernel_fpu_begin); 89EXPORT_SYMBOL(__kernel_fpu_begin);
96 90
97void kernel_fpu_end(void) 91void __kernel_fpu_end(void)
98{ 92{
99 stts(); 93 if (use_eager_fpu())
100 preempt_enable(); 94 math_state_restore();
95 else
96 stts();
101} 97}
102EXPORT_SYMBOL(kernel_fpu_end); 98EXPORT_SYMBOL(__kernel_fpu_end);
103 99
104void unlazy_fpu(struct task_struct *tsk) 100void unlazy_fpu(struct task_struct *tsk)
105{ 101{
@@ -113,23 +109,15 @@ void unlazy_fpu(struct task_struct *tsk)
113} 109}
114EXPORT_SYMBOL(unlazy_fpu); 110EXPORT_SYMBOL(unlazy_fpu);
115 111
116#ifdef CONFIG_MATH_EMULATION 112unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
117# define HAVE_HWFP (boot_cpu_data.hard_math)
118#else
119# define HAVE_HWFP 1
120#endif
121
122static unsigned int mxcsr_feature_mask __read_mostly = 0xffffffffu;
123unsigned int xstate_size; 113unsigned int xstate_size;
124EXPORT_SYMBOL_GPL(xstate_size); 114EXPORT_SYMBOL_GPL(xstate_size);
125unsigned int sig_xstate_ia32_size = sizeof(struct _fpstate_ia32);
126static struct i387_fxsave_struct fx_scratch __cpuinitdata; 115static struct i387_fxsave_struct fx_scratch __cpuinitdata;
127 116
128static void __cpuinit mxcsr_feature_mask_init(void) 117static void __cpuinit mxcsr_feature_mask_init(void)
129{ 118{
130 unsigned long mask = 0; 119 unsigned long mask = 0;
131 120
132 clts();
133 if (cpu_has_fxsr) { 121 if (cpu_has_fxsr) {
134 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct)); 122 memset(&fx_scratch, 0, sizeof(struct i387_fxsave_struct));
135 asm volatile("fxsave %0" : : "m" (fx_scratch)); 123 asm volatile("fxsave %0" : : "m" (fx_scratch));
@@ -138,7 +126,6 @@ static void __cpuinit mxcsr_feature_mask_init(void)
138 mask = 0x0000ffbf; 126 mask = 0x0000ffbf;
139 } 127 }
140 mxcsr_feature_mask &= mask; 128 mxcsr_feature_mask &= mask;
141 stts();
142} 129}
143 130
144static void __cpuinit init_thread_xstate(void) 131static void __cpuinit init_thread_xstate(void)
@@ -192,9 +179,8 @@ void __cpuinit fpu_init(void)
192 init_thread_xstate(); 179 init_thread_xstate();
193 180
194 mxcsr_feature_mask_init(); 181 mxcsr_feature_mask_init();
195 /* clean state in init */ 182 xsave_init();
196 current_thread_info()->status = 0; 183 eager_fpu_init();
197 clear_used_math();
198} 184}
199 185
200void fpu_finit(struct fpu *fpu) 186void fpu_finit(struct fpu *fpu)
@@ -205,12 +191,7 @@ void fpu_finit(struct fpu *fpu)
205 } 191 }
206 192
207 if (cpu_has_fxsr) { 193 if (cpu_has_fxsr) {
208 struct i387_fxsave_struct *fx = &fpu->state->fxsave; 194 fx_finit(&fpu->state->fxsave);
209
210 memset(fx, 0, xstate_size);
211 fx->cwd = 0x37f;
212 if (cpu_has_xmm)
213 fx->mxcsr = MXCSR_DEFAULT;
214 } else { 195 } else {
215 struct i387_fsave_struct *fp = &fpu->state->fsave; 196 struct i387_fsave_struct *fp = &fpu->state->fsave;
216 memset(fp, 0, xstate_size); 197 memset(fp, 0, xstate_size);
@@ -454,7 +435,7 @@ static inline u32 twd_fxsr_to_i387(struct i387_fxsave_struct *fxsave)
454 * FXSR floating point environment conversions. 435 * FXSR floating point environment conversions.
455 */ 436 */
456 437
457static void 438void
458convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk) 439convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
459{ 440{
460 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave; 441 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
@@ -491,8 +472,8 @@ convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
491 memcpy(&to[i], &from[i], sizeof(to[0])); 472 memcpy(&to[i], &from[i], sizeof(to[0]));
492} 473}
493 474
494static void convert_to_fxsr(struct task_struct *tsk, 475void convert_to_fxsr(struct task_struct *tsk,
495 const struct user_i387_ia32_struct *env) 476 const struct user_i387_ia32_struct *env)
496 477
497{ 478{
498 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave; 479 struct i387_fxsave_struct *fxsave = &tsk->thread.fpu.state->fxsave;
@@ -589,223 +570,6 @@ int fpregs_set(struct task_struct *target, const struct user_regset *regset,
589} 570}
590 571
591/* 572/*
592 * Signal frame handlers.
593 */
594
595static inline int save_i387_fsave(struct _fpstate_ia32 __user *buf)
596{
597 struct task_struct *tsk = current;
598 struct i387_fsave_struct *fp = &tsk->thread.fpu.state->fsave;
599
600 fp->status = fp->swd;
601 if (__copy_to_user(buf, fp, sizeof(struct i387_fsave_struct)))
602 return -1;
603 return 1;
604}
605
606static int save_i387_fxsave(struct _fpstate_ia32 __user *buf)
607{
608 struct task_struct *tsk = current;
609 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
610 struct user_i387_ia32_struct env;
611 int err = 0;
612
613 convert_from_fxsr(&env, tsk);
614 if (__copy_to_user(buf, &env, sizeof(env)))
615 return -1;
616
617 err |= __put_user(fx->swd, &buf->status);
618 err |= __put_user(X86_FXSR_MAGIC, &buf->magic);
619 if (err)
620 return -1;
621
622 if (__copy_to_user(&buf->_fxsr_env[0], fx, xstate_size))
623 return -1;
624 return 1;
625}
626
627static int save_i387_xsave(void __user *buf)
628{
629 struct task_struct *tsk = current;
630 struct _fpstate_ia32 __user *fx = buf;
631 int err = 0;
632
633
634 sanitize_i387_state(tsk);
635
636 /*
637 * For legacy compatible, we always set FP/SSE bits in the bit
638 * vector while saving the state to the user context.
639 * This will enable us capturing any changes(during sigreturn) to
640 * the FP/SSE bits by the legacy applications which don't touch
641 * xstate_bv in the xsave header.
642 *
643 * xsave aware applications can change the xstate_bv in the xsave
644 * header as well as change any contents in the memory layout.
645 * xrestore as part of sigreturn will capture all the changes.
646 */
647 tsk->thread.fpu.state->xsave.xsave_hdr.xstate_bv |= XSTATE_FPSSE;
648
649 if (save_i387_fxsave(fx) < 0)
650 return -1;
651
652 err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved_ia32,
653 sizeof(struct _fpx_sw_bytes));
654 err |= __put_user(FP_XSTATE_MAGIC2,
655 (__u32 __user *) (buf + sig_xstate_ia32_size
656 - FP_XSTATE_MAGIC2_SIZE));
657 if (err)
658 return -1;
659
660 return 1;
661}
662
663int save_i387_xstate_ia32(void __user *buf)
664{
665 struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
666 struct task_struct *tsk = current;
667
668 if (!used_math())
669 return 0;
670
671 if (!access_ok(VERIFY_WRITE, buf, sig_xstate_ia32_size))
672 return -EACCES;
673 /*
674 * This will cause a "finit" to be triggered by the next
675 * attempted FPU operation by the 'current' process.
676 */
677 clear_used_math();
678
679 if (!HAVE_HWFP) {
680 return fpregs_soft_get(current, NULL,
681 0, sizeof(struct user_i387_ia32_struct),
682 NULL, fp) ? -1 : 1;
683 }
684
685 unlazy_fpu(tsk);
686
687 if (cpu_has_xsave)
688 return save_i387_xsave(fp);
689 if (cpu_has_fxsr)
690 return save_i387_fxsave(fp);
691 else
692 return save_i387_fsave(fp);
693}
694
695static inline int restore_i387_fsave(struct _fpstate_ia32 __user *buf)
696{
697 struct task_struct *tsk = current;
698
699 return __copy_from_user(&tsk->thread.fpu.state->fsave, buf,
700 sizeof(struct i387_fsave_struct));
701}
702
703static int restore_i387_fxsave(struct _fpstate_ia32 __user *buf,
704 unsigned int size)
705{
706 struct task_struct *tsk = current;
707 struct user_i387_ia32_struct env;
708 int err;
709
710 err = __copy_from_user(&tsk->thread.fpu.state->fxsave, &buf->_fxsr_env[0],
711 size);
712 /* mxcsr reserved bits must be masked to zero for security reasons */
713 tsk->thread.fpu.state->fxsave.mxcsr &= mxcsr_feature_mask;
714 if (err || __copy_from_user(&env, buf, sizeof(env)))
715 return 1;
716 convert_to_fxsr(tsk, &env);
717
718 return 0;
719}
720
721static int restore_i387_xsave(void __user *buf)
722{
723 struct _fpx_sw_bytes fx_sw_user;
724 struct _fpstate_ia32 __user *fx_user =
725 ((struct _fpstate_ia32 __user *) buf);
726 struct i387_fxsave_struct __user *fx =
727 (struct i387_fxsave_struct __user *) &fx_user->_fxsr_env[0];
728 struct xsave_hdr_struct *xsave_hdr =
729 &current->thread.fpu.state->xsave.xsave_hdr;
730 u64 mask;
731 int err;
732
733 if (check_for_xstate(fx, buf, &fx_sw_user))
734 goto fx_only;
735
736 mask = fx_sw_user.xstate_bv;
737
738 err = restore_i387_fxsave(buf, fx_sw_user.xstate_size);
739
740 xsave_hdr->xstate_bv &= pcntxt_mask;
741 /*
742 * These bits must be zero.
743 */
744 xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
745
746 /*
747 * Init the state that is not present in the memory layout
748 * and enabled by the OS.
749 */
750 mask = ~(pcntxt_mask & ~mask);
751 xsave_hdr->xstate_bv &= mask;
752
753 return err;
754fx_only:
755 /*
756 * Couldn't find the extended state information in the memory
757 * layout. Restore the FP/SSE and init the other extended state
758 * enabled by the OS.
759 */
760 xsave_hdr->xstate_bv = XSTATE_FPSSE;
761 return restore_i387_fxsave(buf, sizeof(struct i387_fxsave_struct));
762}
763
764int restore_i387_xstate_ia32(void __user *buf)
765{
766 int err;
767 struct task_struct *tsk = current;
768 struct _fpstate_ia32 __user *fp = (struct _fpstate_ia32 __user *) buf;
769
770 if (HAVE_HWFP)
771 clear_fpu(tsk);
772
773 if (!buf) {
774 if (used_math()) {
775 clear_fpu(tsk);
776 clear_used_math();
777 }
778
779 return 0;
780 } else
781 if (!access_ok(VERIFY_READ, buf, sig_xstate_ia32_size))
782 return -EACCES;
783
784 if (!used_math()) {
785 err = init_fpu(tsk);
786 if (err)
787 return err;
788 }
789
790 if (HAVE_HWFP) {
791 if (cpu_has_xsave)
792 err = restore_i387_xsave(buf);
793 else if (cpu_has_fxsr)
794 err = restore_i387_fxsave(fp, sizeof(struct
795 i387_fxsave_struct));
796 else
797 err = restore_i387_fsave(fp);
798 } else {
799 err = fpregs_soft_set(current, NULL,
800 0, sizeof(struct user_i387_ia32_struct),
801 NULL, fp) != 0;
802 }
803 set_used_math();
804
805 return err;
806}
807
808/*
809 * FPU state for core dumps. 573 * FPU state for core dumps.
810 * This is only used for a.out dumps now. 574 * This is only used for a.out dumps now.
811 * It is declared generically using elf_fpregset_t (which is 575 * It is declared generically using elf_fpregset_t (which is
diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c
index 36d1853e91af..9a5c460404dc 100644
--- a/arch/x86/kernel/i8259.c
+++ b/arch/x86/kernel/i8259.c
@@ -263,7 +263,7 @@ static void i8259A_shutdown(void)
263 * out of. 263 * out of.
264 */ 264 */
265 outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */ 265 outb(0xff, PIC_MASTER_IMR); /* mask all of 8259A-1 */
266 outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-1 */ 266 outb(0xff, PIC_SLAVE_IMR); /* mask all of 8259A-2 */
267} 267}
268 268
269static struct syscore_ops i8259_syscore_ops = { 269static struct syscore_ops i8259_syscore_ops = {
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index d44f7829968e..e4595f105910 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -92,7 +92,8 @@ int arch_show_interrupts(struct seq_file *p, int prec)
92 seq_printf(p, " Rescheduling interrupts\n"); 92 seq_printf(p, " Rescheduling interrupts\n");
93 seq_printf(p, "%*s: ", prec, "CAL"); 93 seq_printf(p, "%*s: ", prec, "CAL");
94 for_each_online_cpu(j) 94 for_each_online_cpu(j)
95 seq_printf(p, "%10u ", irq_stats(j)->irq_call_count); 95 seq_printf(p, "%10u ", irq_stats(j)->irq_call_count -
96 irq_stats(j)->irq_tlb_count);
96 seq_printf(p, " Function call interrupts\n"); 97 seq_printf(p, " Function call interrupts\n");
97 seq_printf(p, "%*s: ", prec, "TLB"); 98 seq_printf(p, "%*s: ", prec, "TLB");
98 for_each_online_cpu(j) 99 for_each_online_cpu(j)
@@ -147,7 +148,6 @@ u64 arch_irq_stat_cpu(unsigned int cpu)
147#ifdef CONFIG_SMP 148#ifdef CONFIG_SMP
148 sum += irq_stats(cpu)->irq_resched_count; 149 sum += irq_stats(cpu)->irq_resched_count;
149 sum += irq_stats(cpu)->irq_call_count; 150 sum += irq_stats(cpu)->irq_call_count;
150 sum += irq_stats(cpu)->irq_tlb_count;
151#endif 151#endif
152#ifdef CONFIG_X86_THERMAL_VECTOR 152#ifdef CONFIG_X86_THERMAL_VECTOR
153 sum += irq_stats(cpu)->irq_thermal_count; 153 sum += irq_stats(cpu)->irq_thermal_count;
diff --git a/arch/x86/kernel/kgdb.c b/arch/x86/kernel/kgdb.c
index 3f61904365cf..836f8322960e 100644
--- a/arch/x86/kernel/kgdb.c
+++ b/arch/x86/kernel/kgdb.c
@@ -746,7 +746,9 @@ void kgdb_arch_set_pc(struct pt_regs *regs, unsigned long ip)
746int kgdb_arch_set_breakpoint(struct kgdb_bkpt *bpt) 746int kgdb_arch_set_breakpoint(struct kgdb_bkpt *bpt)
747{ 747{
748 int err; 748 int err;
749#ifdef CONFIG_DEBUG_RODATA
749 char opc[BREAK_INSTR_SIZE]; 750 char opc[BREAK_INSTR_SIZE];
751#endif /* CONFIG_DEBUG_RODATA */
750 752
751 bpt->type = BP_BREAKPOINT; 753 bpt->type = BP_BREAKPOINT;
752 err = probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr, 754 err = probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr,
diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c
index e2f751efb7b1..57916c0d3cf6 100644
--- a/arch/x86/kernel/kprobes.c
+++ b/arch/x86/kernel/kprobes.c
@@ -541,6 +541,23 @@ reenter_kprobe(struct kprobe *p, struct pt_regs *regs, struct kprobe_ctlblk *kcb
541 return 1; 541 return 1;
542} 542}
543 543
544#ifdef KPROBES_CAN_USE_FTRACE
545static void __kprobes skip_singlestep(struct kprobe *p, struct pt_regs *regs,
546 struct kprobe_ctlblk *kcb)
547{
548 /*
549 * Emulate singlestep (and also recover regs->ip)
550 * as if there is a 5byte nop
551 */
552 regs->ip = (unsigned long)p->addr + MCOUNT_INSN_SIZE;
553 if (unlikely(p->post_handler)) {
554 kcb->kprobe_status = KPROBE_HIT_SSDONE;
555 p->post_handler(p, regs, 0);
556 }
557 __this_cpu_write(current_kprobe, NULL);
558}
559#endif
560
544/* 561/*
545 * Interrupts are disabled on entry as trap3 is an interrupt gate and they 562 * Interrupts are disabled on entry as trap3 is an interrupt gate and they
546 * remain disabled throughout this function. 563 * remain disabled throughout this function.
@@ -599,6 +616,12 @@ static int __kprobes kprobe_handler(struct pt_regs *regs)
599 } else if (kprobe_running()) { 616 } else if (kprobe_running()) {
600 p = __this_cpu_read(current_kprobe); 617 p = __this_cpu_read(current_kprobe);
601 if (p->break_handler && p->break_handler(p, regs)) { 618 if (p->break_handler && p->break_handler(p, regs)) {
619#ifdef KPROBES_CAN_USE_FTRACE
620 if (kprobe_ftrace(p)) {
621 skip_singlestep(p, regs, kcb);
622 return 1;
623 }
624#endif
602 setup_singlestep(p, regs, kcb, 0); 625 setup_singlestep(p, regs, kcb, 0);
603 return 1; 626 return 1;
604 } 627 }
@@ -1052,6 +1075,50 @@ int __kprobes longjmp_break_handler(struct kprobe *p, struct pt_regs *regs)
1052 return 0; 1075 return 0;
1053} 1076}
1054 1077
1078#ifdef KPROBES_CAN_USE_FTRACE
1079/* Ftrace callback handler for kprobes */
1080void __kprobes kprobe_ftrace_handler(unsigned long ip, unsigned long parent_ip,
1081 struct ftrace_ops *ops, struct pt_regs *regs)
1082{
1083 struct kprobe *p;
1084 struct kprobe_ctlblk *kcb;
1085 unsigned long flags;
1086
1087 /* Disable irq for emulating a breakpoint and avoiding preempt */
1088 local_irq_save(flags);
1089
1090 p = get_kprobe((kprobe_opcode_t *)ip);
1091 if (unlikely(!p) || kprobe_disabled(p))
1092 goto end;
1093
1094 kcb = get_kprobe_ctlblk();
1095 if (kprobe_running()) {
1096 kprobes_inc_nmissed_count(p);
1097 } else {
1098 /* Kprobe handler expects regs->ip = ip + 1 as breakpoint hit */
1099 regs->ip = ip + sizeof(kprobe_opcode_t);
1100
1101 __this_cpu_write(current_kprobe, p);
1102 kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1103 if (!p->pre_handler || !p->pre_handler(p, regs))
1104 skip_singlestep(p, regs, kcb);
1105 /*
1106 * If pre_handler returns !0, it sets regs->ip and
1107 * resets current kprobe.
1108 */
1109 }
1110end:
1111 local_irq_restore(flags);
1112}
1113
1114int __kprobes arch_prepare_kprobe_ftrace(struct kprobe *p)
1115{
1116 p->ainsn.insn = NULL;
1117 p->ainsn.boostable = -1;
1118 return 0;
1119}
1120#endif
1121
1055int __init arch_init_kprobes(void) 1122int __init arch_init_kprobes(void)
1056{ 1123{
1057 return arch_init_optprobes(); 1124 return arch_init_optprobes();
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index c1d61ee4b4f1..b3e5e51bc907 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -354,6 +354,7 @@ static void kvm_pv_guest_cpu_reboot(void *unused)
354 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI)) 354 if (kvm_para_has_feature(KVM_FEATURE_PV_EOI))
355 wrmsrl(MSR_KVM_PV_EOI_EN, 0); 355 wrmsrl(MSR_KVM_PV_EOI_EN, 0);
356 kvm_pv_disable_apf(); 356 kvm_pv_disable_apf();
357 kvm_disable_steal_time();
357} 358}
358 359
359static int kvm_pv_reboot_notify(struct notifier_block *nb, 360static int kvm_pv_reboot_notify(struct notifier_block *nb,
@@ -396,9 +397,7 @@ void kvm_disable_steal_time(void)
396#ifdef CONFIG_SMP 397#ifdef CONFIG_SMP
397static void __init kvm_smp_prepare_boot_cpu(void) 398static void __init kvm_smp_prepare_boot_cpu(void)
398{ 399{
399#ifdef CONFIG_KVM_CLOCK
400 WARN_ON(kvm_register_clock("primary cpu clock")); 400 WARN_ON(kvm_register_clock("primary cpu clock"));
401#endif
402 kvm_guest_cpu_init(); 401 kvm_guest_cpu_init();
403 native_smp_prepare_boot_cpu(); 402 native_smp_prepare_boot_cpu();
404} 403}
diff --git a/arch/x86/kernel/microcode_amd.c b/arch/x86/kernel/microcode_amd.c
index 82746f942cd8..7720ff5a9ee2 100644
--- a/arch/x86/kernel/microcode_amd.c
+++ b/arch/x86/kernel/microcode_amd.c
@@ -75,20 +75,113 @@ struct microcode_amd {
75 75
76static struct equiv_cpu_entry *equiv_cpu_table; 76static struct equiv_cpu_entry *equiv_cpu_table;
77 77
78/* page-sized ucode patch buffer */ 78struct ucode_patch {
79void *patch; 79 struct list_head plist;
80 void *data;
81 u32 patch_id;
82 u16 equiv_cpu;
83};
84
85static LIST_HEAD(pcache);
86
87static u16 find_equiv_id(unsigned int cpu)
88{
89 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
90 int i = 0;
91
92 if (!equiv_cpu_table)
93 return 0;
94
95 while (equiv_cpu_table[i].installed_cpu != 0) {
96 if (uci->cpu_sig.sig == equiv_cpu_table[i].installed_cpu)
97 return equiv_cpu_table[i].equiv_cpu;
98
99 i++;
100 }
101 return 0;
102}
103
104static u32 find_cpu_family_by_equiv_cpu(u16 equiv_cpu)
105{
106 int i = 0;
107
108 BUG_ON(!equiv_cpu_table);
109
110 while (equiv_cpu_table[i].equiv_cpu != 0) {
111 if (equiv_cpu == equiv_cpu_table[i].equiv_cpu)
112 return equiv_cpu_table[i].installed_cpu;
113 i++;
114 }
115 return 0;
116}
117
118/*
119 * a small, trivial cache of per-family ucode patches
120 */
121static struct ucode_patch *cache_find_patch(u16 equiv_cpu)
122{
123 struct ucode_patch *p;
124
125 list_for_each_entry(p, &pcache, plist)
126 if (p->equiv_cpu == equiv_cpu)
127 return p;
128 return NULL;
129}
130
131static void update_cache(struct ucode_patch *new_patch)
132{
133 struct ucode_patch *p;
134
135 list_for_each_entry(p, &pcache, plist) {
136 if (p->equiv_cpu == new_patch->equiv_cpu) {
137 if (p->patch_id >= new_patch->patch_id)
138 /* we already have the latest patch */
139 return;
140
141 list_replace(&p->plist, &new_patch->plist);
142 kfree(p->data);
143 kfree(p);
144 return;
145 }
146 }
147 /* no patch found, add it */
148 list_add_tail(&new_patch->plist, &pcache);
149}
150
151static void free_cache(void)
152{
153 struct ucode_patch *p, *tmp;
154
155 list_for_each_entry_safe(p, tmp, &pcache, plist) {
156 __list_del(p->plist.prev, p->plist.next);
157 kfree(p->data);
158 kfree(p);
159 }
160}
161
162static struct ucode_patch *find_patch(unsigned int cpu)
163{
164 u16 equiv_id;
165
166 equiv_id = find_equiv_id(cpu);
167 if (!equiv_id)
168 return NULL;
169
170 return cache_find_patch(equiv_id);
171}
80 172
81static int collect_cpu_info_amd(int cpu, struct cpu_signature *csig) 173static int collect_cpu_info_amd(int cpu, struct cpu_signature *csig)
82{ 174{
83 struct cpuinfo_x86 *c = &cpu_data(cpu); 175 struct cpuinfo_x86 *c = &cpu_data(cpu);
84 176
177 csig->sig = cpuid_eax(0x00000001);
85 csig->rev = c->microcode; 178 csig->rev = c->microcode;
86 pr_info("CPU%d: patch_level=0x%08x\n", cpu, csig->rev); 179 pr_info("CPU%d: patch_level=0x%08x\n", cpu, csig->rev);
87 180
88 return 0; 181 return 0;
89} 182}
90 183
91static unsigned int verify_ucode_size(int cpu, u32 patch_size, 184static unsigned int verify_patch_size(int cpu, u32 patch_size,
92 unsigned int size) 185 unsigned int size)
93{ 186{
94 struct cpuinfo_x86 *c = &cpu_data(cpu); 187 struct cpuinfo_x86 *c = &cpu_data(cpu);
@@ -118,95 +211,37 @@ static unsigned int verify_ucode_size(int cpu, u32 patch_size,
118 return patch_size; 211 return patch_size;
119} 212}
120 213
121static u16 find_equiv_id(void) 214static int apply_microcode_amd(int cpu)
122{ 215{
123 unsigned int current_cpu_id, i = 0; 216 struct cpuinfo_x86 *c = &cpu_data(cpu);
124 217 struct microcode_amd *mc_amd;
125 BUG_ON(equiv_cpu_table == NULL); 218 struct ucode_cpu_info *uci;
126 219 struct ucode_patch *p;
127 current_cpu_id = cpuid_eax(0x00000001); 220 u32 rev, dummy;
128
129 while (equiv_cpu_table[i].installed_cpu != 0) {
130 if (current_cpu_id == equiv_cpu_table[i].installed_cpu)
131 return equiv_cpu_table[i].equiv_cpu;
132
133 i++;
134 }
135 return 0;
136}
137 221
138/* 222 BUG_ON(raw_smp_processor_id() != cpu);
139 * we signal a good patch is found by returning its size > 0
140 */
141static int get_matching_microcode(int cpu, const u8 *ucode_ptr,
142 unsigned int leftover_size, int rev,
143 unsigned int *current_size)
144{
145 struct microcode_header_amd *mc_hdr;
146 unsigned int actual_size, patch_size;
147 u16 equiv_cpu_id;
148 223
149 /* size of the current patch we're staring at */ 224 uci = ucode_cpu_info + cpu;
150 patch_size = *(u32 *)(ucode_ptr + 4);
151 *current_size = patch_size + SECTION_HDR_SIZE;
152 225
153 equiv_cpu_id = find_equiv_id(); 226 p = find_patch(cpu);
154 if (!equiv_cpu_id) 227 if (!p)
155 return 0; 228 return 0;
156 229
157 /* 230 mc_amd = p->data;
158 * let's look at the patch header itself now 231 uci->mc = p->data;
159 */
160 mc_hdr = (struct microcode_header_amd *)(ucode_ptr + SECTION_HDR_SIZE);
161 232
162 if (mc_hdr->processor_rev_id != equiv_cpu_id) 233 rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
163 return 0;
164 234
165 /* ucode might be chipset specific -- currently we don't support this */ 235 /* need to apply patch? */
166 if (mc_hdr->nb_dev_id || mc_hdr->sb_dev_id) { 236 if (rev >= mc_amd->hdr.patch_id) {
167 pr_err("CPU%d: chipset specific code not yet supported\n", 237 c->microcode = rev;
168 cpu);
169 return 0; 238 return 0;
170 } 239 }
171 240
172 if (mc_hdr->patch_id <= rev)
173 return 0;
174
175 /*
176 * now that the header looks sane, verify its size
177 */
178 actual_size = verify_ucode_size(cpu, patch_size, leftover_size);
179 if (!actual_size)
180 return 0;
181
182 /* clear the patch buffer */
183 memset(patch, 0, PAGE_SIZE);
184
185 /* all looks ok, get the binary patch */
186 get_ucode_data(patch, ucode_ptr + SECTION_HDR_SIZE, actual_size);
187
188 return actual_size;
189}
190
191static int apply_microcode_amd(int cpu)
192{
193 u32 rev, dummy;
194 int cpu_num = raw_smp_processor_id();
195 struct ucode_cpu_info *uci = ucode_cpu_info + cpu_num;
196 struct microcode_amd *mc_amd = uci->mc;
197 struct cpuinfo_x86 *c = &cpu_data(cpu);
198
199 /* We should bind the task to the CPU */
200 BUG_ON(cpu_num != cpu);
201
202 if (mc_amd == NULL)
203 return 0;
204
205 wrmsrl(MSR_AMD64_PATCH_LOADER, (u64)(long)&mc_amd->hdr.data_code); 241 wrmsrl(MSR_AMD64_PATCH_LOADER, (u64)(long)&mc_amd->hdr.data_code);
206 /* get patch id after patching */
207 rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
208 242
209 /* check current patch id and patch's id for match */ 243 /* verify patch application was successful */
244 rdmsr(MSR_AMD64_PATCH_LEVEL, rev, dummy);
210 if (rev != mc_amd->hdr.patch_id) { 245 if (rev != mc_amd->hdr.patch_id) {
211 pr_err("CPU%d: update failed for patch_level=0x%08x\n", 246 pr_err("CPU%d: update failed for patch_level=0x%08x\n",
212 cpu, mc_amd->hdr.patch_id); 247 cpu, mc_amd->hdr.patch_id);
@@ -238,7 +273,7 @@ static int install_equiv_cpu_table(const u8 *buf)
238 return -ENOMEM; 273 return -ENOMEM;
239 } 274 }
240 275
241 get_ucode_data(equiv_cpu_table, buf + CONTAINER_HDR_SZ, size); 276 memcpy(equiv_cpu_table, buf + CONTAINER_HDR_SZ, size);
242 277
243 /* add header length */ 278 /* add header length */
244 return size + CONTAINER_HDR_SZ; 279 return size + CONTAINER_HDR_SZ;
@@ -250,61 +285,113 @@ static void free_equiv_cpu_table(void)
250 equiv_cpu_table = NULL; 285 equiv_cpu_table = NULL;
251} 286}
252 287
253static enum ucode_state 288static void cleanup(void)
254generic_load_microcode(int cpu, const u8 *data, size_t size)
255{ 289{
256 struct ucode_cpu_info *uci = ucode_cpu_info + cpu; 290 free_equiv_cpu_table();
257 struct microcode_header_amd *mc_hdr = NULL; 291 free_cache();
258 unsigned int mc_size, leftover, current_size = 0; 292}
293
294/*
295 * We return the current size even if some of the checks failed so that
296 * we can skip over the next patch. If we return a negative value, we
297 * signal a grave error like a memory allocation has failed and the
298 * driver cannot continue functioning normally. In such cases, we tear
299 * down everything we've used up so far and exit.
300 */
301static int verify_and_add_patch(unsigned int cpu, u8 *fw, unsigned int leftover)
302{
303 struct cpuinfo_x86 *c = &cpu_data(cpu);
304 struct microcode_header_amd *mc_hdr;
305 struct ucode_patch *patch;
306 unsigned int patch_size, crnt_size, ret;
307 u32 proc_fam;
308 u16 proc_id;
309
310 patch_size = *(u32 *)(fw + 4);
311 crnt_size = patch_size + SECTION_HDR_SIZE;
312 mc_hdr = (struct microcode_header_amd *)(fw + SECTION_HDR_SIZE);
313 proc_id = mc_hdr->processor_rev_id;
314
315 proc_fam = find_cpu_family_by_equiv_cpu(proc_id);
316 if (!proc_fam) {
317 pr_err("No patch family for equiv ID: 0x%04x\n", proc_id);
318 return crnt_size;
319 }
320
321 /* check if patch is for the current family */
322 proc_fam = ((proc_fam >> 8) & 0xf) + ((proc_fam >> 20) & 0xff);
323 if (proc_fam != c->x86)
324 return crnt_size;
325
326 if (mc_hdr->nb_dev_id || mc_hdr->sb_dev_id) {
327 pr_err("Patch-ID 0x%08x: chipset-specific code unsupported.\n",
328 mc_hdr->patch_id);
329 return crnt_size;
330 }
331
332 ret = verify_patch_size(cpu, patch_size, leftover);
333 if (!ret) {
334 pr_err("Patch-ID 0x%08x: size mismatch.\n", mc_hdr->patch_id);
335 return crnt_size;
336 }
337
338 patch = kzalloc(sizeof(*patch), GFP_KERNEL);
339 if (!patch) {
340 pr_err("Patch allocation failure.\n");
341 return -EINVAL;
342 }
343
344 patch->data = kzalloc(patch_size, GFP_KERNEL);
345 if (!patch->data) {
346 pr_err("Patch data allocation failure.\n");
347 kfree(patch);
348 return -EINVAL;
349 }
350
351 /* All looks ok, copy patch... */
352 memcpy(patch->data, fw + SECTION_HDR_SIZE, patch_size);
353 INIT_LIST_HEAD(&patch->plist);
354 patch->patch_id = mc_hdr->patch_id;
355 patch->equiv_cpu = proc_id;
356
357 /* ... and add to cache. */
358 update_cache(patch);
359
360 return crnt_size;
361}
362
363static enum ucode_state load_microcode_amd(int cpu, const u8 *data, size_t size)
364{
365 enum ucode_state ret = UCODE_ERROR;
366 unsigned int leftover;
367 u8 *fw = (u8 *)data;
368 int crnt_size = 0;
259 int offset; 369 int offset;
260 const u8 *ucode_ptr = data;
261 void *new_mc = NULL;
262 unsigned int new_rev = uci->cpu_sig.rev;
263 enum ucode_state state = UCODE_ERROR;
264 370
265 offset = install_equiv_cpu_table(ucode_ptr); 371 offset = install_equiv_cpu_table(data);
266 if (offset < 0) { 372 if (offset < 0) {
267 pr_err("failed to create equivalent cpu table\n"); 373 pr_err("failed to create equivalent cpu table\n");
268 goto out; 374 return ret;
269 } 375 }
270 ucode_ptr += offset; 376 fw += offset;
271 leftover = size - offset; 377 leftover = size - offset;
272 378
273 if (*(u32 *)ucode_ptr != UCODE_UCODE_TYPE) { 379 if (*(u32 *)fw != UCODE_UCODE_TYPE) {
274 pr_err("invalid type field in container file section header\n"); 380 pr_err("invalid type field in container file section header\n");
275 goto free_table; 381 free_equiv_cpu_table();
382 return ret;
276 } 383 }
277 384
278 while (leftover) { 385 while (leftover) {
279 mc_size = get_matching_microcode(cpu, ucode_ptr, leftover, 386 crnt_size = verify_and_add_patch(cpu, fw, leftover);
280 new_rev, &current_size); 387 if (crnt_size < 0)
281 if (mc_size) { 388 return ret;
282 mc_hdr = patch;
283 new_mc = patch;
284 new_rev = mc_hdr->patch_id;
285 goto out_ok;
286 }
287
288 ucode_ptr += current_size;
289 leftover -= current_size;
290 }
291 389
292 if (!new_mc) { 390 fw += crnt_size;
293 state = UCODE_NFOUND; 391 leftover -= crnt_size;
294 goto free_table;
295 } 392 }
296 393
297out_ok: 394 return UCODE_OK;
298 uci->mc = new_mc;
299 state = UCODE_OK;
300 pr_debug("CPU%d update ucode (0x%08x -> 0x%08x)\n",
301 cpu, uci->cpu_sig.rev, new_rev);
302
303free_table:
304 free_equiv_cpu_table();
305
306out:
307 return state;
308} 395}
309 396
310/* 397/*
@@ -315,7 +402,7 @@ out:
315 * 402 *
316 * This legacy file is always smaller than 2K in size. 403 * This legacy file is always smaller than 2K in size.
317 * 404 *
318 * Starting at family 15h they are in family specific firmware files: 405 * Beginning with family 15h, they are in family-specific firmware files:
319 * 406 *
320 * amd-ucode/microcode_amd_fam15h.bin 407 * amd-ucode/microcode_amd_fam15h.bin
321 * amd-ucode/microcode_amd_fam16h.bin 408 * amd-ucode/microcode_amd_fam16h.bin
@@ -323,12 +410,17 @@ out:
323 * 410 *
324 * These might be larger than 2K. 411 * These might be larger than 2K.
325 */ 412 */
326static enum ucode_state request_microcode_amd(int cpu, struct device *device) 413static enum ucode_state request_microcode_amd(int cpu, struct device *device,
414 bool refresh_fw)
327{ 415{
328 char fw_name[36] = "amd-ucode/microcode_amd.bin"; 416 char fw_name[36] = "amd-ucode/microcode_amd.bin";
329 const struct firmware *fw;
330 enum ucode_state ret = UCODE_NFOUND;
331 struct cpuinfo_x86 *c = &cpu_data(cpu); 417 struct cpuinfo_x86 *c = &cpu_data(cpu);
418 enum ucode_state ret = UCODE_NFOUND;
419 const struct firmware *fw;
420
421 /* reload ucode container only on the boot cpu */
422 if (!refresh_fw || c->cpu_index != boot_cpu_data.cpu_index)
423 return UCODE_OK;
332 424
333 if (c->x86 >= 0x15) 425 if (c->x86 >= 0x15)
334 snprintf(fw_name, sizeof(fw_name), "amd-ucode/microcode_amd_fam%.2xh.bin", c->x86); 426 snprintf(fw_name, sizeof(fw_name), "amd-ucode/microcode_amd_fam%.2xh.bin", c->x86);
@@ -344,12 +436,17 @@ static enum ucode_state request_microcode_amd(int cpu, struct device *device)
344 goto fw_release; 436 goto fw_release;
345 } 437 }
346 438
347 ret = generic_load_microcode(cpu, fw->data, fw->size); 439 /* free old equiv table */
440 free_equiv_cpu_table();
441
442 ret = load_microcode_amd(cpu, fw->data, fw->size);
443 if (ret != UCODE_OK)
444 cleanup();
348 445
349fw_release: 446 fw_release:
350 release_firmware(fw); 447 release_firmware(fw);
351 448
352out: 449 out:
353 return ret; 450 return ret;
354} 451}
355 452
@@ -383,14 +480,10 @@ struct microcode_ops * __init init_amd_microcode(void)
383 return NULL; 480 return NULL;
384 } 481 }
385 482
386 patch = (void *)get_zeroed_page(GFP_KERNEL);
387 if (!patch)
388 return NULL;
389
390 return &microcode_amd_ops; 483 return &microcode_amd_ops;
391} 484}
392 485
393void __exit exit_amd_microcode(void) 486void __exit exit_amd_microcode(void)
394{ 487{
395 free_page((unsigned long)patch); 488 cleanup();
396} 489}
diff --git a/arch/x86/kernel/microcode_core.c b/arch/x86/kernel/microcode_core.c
index 4873e62db6a1..3a04b224d0c0 100644
--- a/arch/x86/kernel/microcode_core.c
+++ b/arch/x86/kernel/microcode_core.c
@@ -225,6 +225,9 @@ static ssize_t microcode_write(struct file *file, const char __user *buf,
225 if (do_microcode_update(buf, len) == 0) 225 if (do_microcode_update(buf, len) == 0)
226 ret = (ssize_t)len; 226 ret = (ssize_t)len;
227 227
228 if (ret > 0)
229 perf_check_microcode();
230
228 mutex_unlock(&microcode_mutex); 231 mutex_unlock(&microcode_mutex);
229 put_online_cpus(); 232 put_online_cpus();
230 233
@@ -276,19 +279,18 @@ static struct platform_device *microcode_pdev;
276static int reload_for_cpu(int cpu) 279static int reload_for_cpu(int cpu)
277{ 280{
278 struct ucode_cpu_info *uci = ucode_cpu_info + cpu; 281 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
282 enum ucode_state ustate;
279 int err = 0; 283 int err = 0;
280 284
281 if (uci->valid) { 285 if (!uci->valid)
282 enum ucode_state ustate; 286 return err;
283
284 ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev);
285 if (ustate == UCODE_OK)
286 apply_microcode_on_target(cpu);
287 else
288 if (ustate == UCODE_ERROR)
289 err = -EINVAL;
290 }
291 287
288 ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev, true);
289 if (ustate == UCODE_OK)
290 apply_microcode_on_target(cpu);
291 else
292 if (ustate == UCODE_ERROR)
293 err = -EINVAL;
292 return err; 294 return err;
293} 295}
294 296
@@ -370,18 +372,15 @@ static void microcode_fini_cpu(int cpu)
370 372
371static enum ucode_state microcode_resume_cpu(int cpu) 373static enum ucode_state microcode_resume_cpu(int cpu)
372{ 374{
373 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
374
375 if (!uci->mc)
376 return UCODE_NFOUND;
377
378 pr_debug("CPU%d updated upon resume\n", cpu); 375 pr_debug("CPU%d updated upon resume\n", cpu);
379 apply_microcode_on_target(cpu); 376
377 if (apply_microcode_on_target(cpu))
378 return UCODE_ERROR;
380 379
381 return UCODE_OK; 380 return UCODE_OK;
382} 381}
383 382
384static enum ucode_state microcode_init_cpu(int cpu) 383static enum ucode_state microcode_init_cpu(int cpu, bool refresh_fw)
385{ 384{
386 enum ucode_state ustate; 385 enum ucode_state ustate;
387 386
@@ -392,7 +391,8 @@ static enum ucode_state microcode_init_cpu(int cpu)
392 if (system_state != SYSTEM_RUNNING) 391 if (system_state != SYSTEM_RUNNING)
393 return UCODE_NFOUND; 392 return UCODE_NFOUND;
394 393
395 ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev); 394 ustate = microcode_ops->request_microcode_fw(cpu, &microcode_pdev->dev,
395 refresh_fw);
396 396
397 if (ustate == UCODE_OK) { 397 if (ustate == UCODE_OK) {
398 pr_debug("CPU%d updated upon init\n", cpu); 398 pr_debug("CPU%d updated upon init\n", cpu);
@@ -405,14 +405,11 @@ static enum ucode_state microcode_init_cpu(int cpu)
405static enum ucode_state microcode_update_cpu(int cpu) 405static enum ucode_state microcode_update_cpu(int cpu)
406{ 406{
407 struct ucode_cpu_info *uci = ucode_cpu_info + cpu; 407 struct ucode_cpu_info *uci = ucode_cpu_info + cpu;
408 enum ucode_state ustate;
409 408
410 if (uci->valid) 409 if (uci->valid)
411 ustate = microcode_resume_cpu(cpu); 410 return microcode_resume_cpu(cpu);
412 else
413 ustate = microcode_init_cpu(cpu);
414 411
415 return ustate; 412 return microcode_init_cpu(cpu, false);
416} 413}
417 414
418static int mc_device_add(struct device *dev, struct subsys_interface *sif) 415static int mc_device_add(struct device *dev, struct subsys_interface *sif)
@@ -428,7 +425,7 @@ static int mc_device_add(struct device *dev, struct subsys_interface *sif)
428 if (err) 425 if (err)
429 return err; 426 return err;
430 427
431 if (microcode_init_cpu(cpu) == UCODE_ERROR) 428 if (microcode_init_cpu(cpu, true) == UCODE_ERROR)
432 return -EINVAL; 429 return -EINVAL;
433 430
434 return err; 431 return err;
@@ -477,34 +474,41 @@ mc_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu)
477 struct device *dev; 474 struct device *dev;
478 475
479 dev = get_cpu_device(cpu); 476 dev = get_cpu_device(cpu);
480 switch (action) { 477
478 switch (action & ~CPU_TASKS_FROZEN) {
481 case CPU_ONLINE: 479 case CPU_ONLINE:
482 case CPU_ONLINE_FROZEN:
483 microcode_update_cpu(cpu); 480 microcode_update_cpu(cpu);
484 case CPU_DOWN_FAILED:
485 case CPU_DOWN_FAILED_FROZEN:
486 pr_debug("CPU%d added\n", cpu); 481 pr_debug("CPU%d added\n", cpu);
482 /*
483 * "break" is missing on purpose here because we want to fall
484 * through in order to create the sysfs group.
485 */
486
487 case CPU_DOWN_FAILED:
487 if (sysfs_create_group(&dev->kobj, &mc_attr_group)) 488 if (sysfs_create_group(&dev->kobj, &mc_attr_group))
488 pr_err("Failed to create group for CPU%d\n", cpu); 489 pr_err("Failed to create group for CPU%d\n", cpu);
489 break; 490 break;
491
490 case CPU_DOWN_PREPARE: 492 case CPU_DOWN_PREPARE:
491 case CPU_DOWN_PREPARE_FROZEN:
492 /* Suspend is in progress, only remove the interface */ 493 /* Suspend is in progress, only remove the interface */
493 sysfs_remove_group(&dev->kobj, &mc_attr_group); 494 sysfs_remove_group(&dev->kobj, &mc_attr_group);
494 pr_debug("CPU%d removed\n", cpu); 495 pr_debug("CPU%d removed\n", cpu);
495 break; 496 break;
496 497
497 /* 498 /*
499 * case CPU_DEAD:
500 *
498 * When a CPU goes offline, don't free up or invalidate the copy of 501 * When a CPU goes offline, don't free up or invalidate the copy of
499 * the microcode in kernel memory, so that we can reuse it when the 502 * the microcode in kernel memory, so that we can reuse it when the
500 * CPU comes back online without unnecessarily requesting the userspace 503 * CPU comes back online without unnecessarily requesting the userspace
501 * for it again. 504 * for it again.
502 */ 505 */
503 case CPU_UP_CANCELED_FROZEN:
504 /* The CPU refused to come up during a system resume */
505 microcode_fini_cpu(cpu);
506 break;
507 } 506 }
507
508 /* The CPU refused to come up during a system resume */
509 if (action == CPU_UP_CANCELED_FROZEN)
510 microcode_fini_cpu(cpu);
511
508 return NOTIFY_OK; 512 return NOTIFY_OK;
509} 513}
510 514
diff --git a/arch/x86/kernel/microcode_intel.c b/arch/x86/kernel/microcode_intel.c
index 0327e2b3c408..3544aed39338 100644
--- a/arch/x86/kernel/microcode_intel.c
+++ b/arch/x86/kernel/microcode_intel.c
@@ -405,7 +405,8 @@ static int get_ucode_fw(void *to, const void *from, size_t n)
405 return 0; 405 return 0;
406} 406}
407 407
408static enum ucode_state request_microcode_fw(int cpu, struct device *device) 408static enum ucode_state request_microcode_fw(int cpu, struct device *device,
409 bool refresh_fw)
409{ 410{
410 char name[30]; 411 char name[30];
411 struct cpuinfo_x86 *c = &cpu_data(cpu); 412 struct cpuinfo_x86 *c = &cpu_data(cpu);
diff --git a/arch/x86/kernel/msr.c b/arch/x86/kernel/msr.c
index eb113693f043..a7c5661f8496 100644
--- a/arch/x86/kernel/msr.c
+++ b/arch/x86/kernel/msr.c
@@ -257,12 +257,14 @@ static int __init msr_init(void)
257 goto out_chrdev; 257 goto out_chrdev;
258 } 258 }
259 msr_class->devnode = msr_devnode; 259 msr_class->devnode = msr_devnode;
260 get_online_cpus();
260 for_each_online_cpu(i) { 261 for_each_online_cpu(i) {
261 err = msr_device_create(i); 262 err = msr_device_create(i);
262 if (err != 0) 263 if (err != 0)
263 goto out_class; 264 goto out_class;
264 } 265 }
265 register_hotcpu_notifier(&msr_class_cpu_notifier); 266 register_hotcpu_notifier(&msr_class_cpu_notifier);
267 put_online_cpus();
266 268
267 err = 0; 269 err = 0;
268 goto out; 270 goto out;
@@ -271,6 +273,7 @@ out_class:
271 i = 0; 273 i = 0;
272 for_each_online_cpu(i) 274 for_each_online_cpu(i)
273 msr_device_destroy(i); 275 msr_device_destroy(i);
276 put_online_cpus();
274 class_destroy(msr_class); 277 class_destroy(msr_class);
275out_chrdev: 278out_chrdev:
276 __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr"); 279 __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
@@ -281,11 +284,13 @@ out:
281static void __exit msr_exit(void) 284static void __exit msr_exit(void)
282{ 285{
283 int cpu = 0; 286 int cpu = 0;
287 get_online_cpus();
284 for_each_online_cpu(cpu) 288 for_each_online_cpu(cpu)
285 msr_device_destroy(cpu); 289 msr_device_destroy(cpu);
286 class_destroy(msr_class); 290 class_destroy(msr_class);
287 __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr"); 291 __unregister_chrdev(MSR_MAJOR, 0, NR_CPUS, "cpu/msr");
288 unregister_hotcpu_notifier(&msr_class_cpu_notifier); 292 unregister_hotcpu_notifier(&msr_class_cpu_notifier);
293 put_online_cpus();
289} 294}
290 295
291module_init(msr_init); 296module_init(msr_init);
diff --git a/arch/x86/kernel/perf_regs.c b/arch/x86/kernel/perf_regs.c
new file mode 100644
index 000000000000..e309cc5c276e
--- /dev/null
+++ b/arch/x86/kernel/perf_regs.c
@@ -0,0 +1,105 @@
1#include <linux/errno.h>
2#include <linux/kernel.h>
3#include <linux/sched.h>
4#include <linux/perf_event.h>
5#include <linux/bug.h>
6#include <linux/stddef.h>
7#include <asm/perf_regs.h>
8#include <asm/ptrace.h>
9
10#ifdef CONFIG_X86_32
11#define PERF_REG_X86_MAX PERF_REG_X86_32_MAX
12#else
13#define PERF_REG_X86_MAX PERF_REG_X86_64_MAX
14#endif
15
16#define PT_REGS_OFFSET(id, r) [id] = offsetof(struct pt_regs, r)
17
18static unsigned int pt_regs_offset[PERF_REG_X86_MAX] = {
19 PT_REGS_OFFSET(PERF_REG_X86_AX, ax),
20 PT_REGS_OFFSET(PERF_REG_X86_BX, bx),
21 PT_REGS_OFFSET(PERF_REG_X86_CX, cx),
22 PT_REGS_OFFSET(PERF_REG_X86_DX, dx),
23 PT_REGS_OFFSET(PERF_REG_X86_SI, si),
24 PT_REGS_OFFSET(PERF_REG_X86_DI, di),
25 PT_REGS_OFFSET(PERF_REG_X86_BP, bp),
26 PT_REGS_OFFSET(PERF_REG_X86_SP, sp),
27 PT_REGS_OFFSET(PERF_REG_X86_IP, ip),
28 PT_REGS_OFFSET(PERF_REG_X86_FLAGS, flags),
29 PT_REGS_OFFSET(PERF_REG_X86_CS, cs),
30 PT_REGS_OFFSET(PERF_REG_X86_SS, ss),
31#ifdef CONFIG_X86_32
32 PT_REGS_OFFSET(PERF_REG_X86_DS, ds),
33 PT_REGS_OFFSET(PERF_REG_X86_ES, es),
34 PT_REGS_OFFSET(PERF_REG_X86_FS, fs),
35 PT_REGS_OFFSET(PERF_REG_X86_GS, gs),
36#else
37 /*
38 * The pt_regs struct does not store
39 * ds, es, fs, gs in 64 bit mode.
40 */
41 (unsigned int) -1,
42 (unsigned int) -1,
43 (unsigned int) -1,
44 (unsigned int) -1,
45#endif
46#ifdef CONFIG_X86_64
47 PT_REGS_OFFSET(PERF_REG_X86_R8, r8),
48 PT_REGS_OFFSET(PERF_REG_X86_R9, r9),
49 PT_REGS_OFFSET(PERF_REG_X86_R10, r10),
50 PT_REGS_OFFSET(PERF_REG_X86_R11, r11),
51 PT_REGS_OFFSET(PERF_REG_X86_R12, r12),
52 PT_REGS_OFFSET(PERF_REG_X86_R13, r13),
53 PT_REGS_OFFSET(PERF_REG_X86_R14, r14),
54 PT_REGS_OFFSET(PERF_REG_X86_R15, r15),
55#endif
56};
57
58u64 perf_reg_value(struct pt_regs *regs, int idx)
59{
60 if (WARN_ON_ONCE(idx >= ARRAY_SIZE(pt_regs_offset)))
61 return 0;
62
63 return regs_get_register(regs, pt_regs_offset[idx]);
64}
65
66#define REG_RESERVED (~((1ULL << PERF_REG_X86_MAX) - 1ULL))
67
68#ifdef CONFIG_X86_32
69int perf_reg_validate(u64 mask)
70{
71 if (!mask || mask & REG_RESERVED)
72 return -EINVAL;
73
74 return 0;
75}
76
77u64 perf_reg_abi(struct task_struct *task)
78{
79 return PERF_SAMPLE_REGS_ABI_32;
80}
81#else /* CONFIG_X86_64 */
82#define REG_NOSUPPORT ((1ULL << PERF_REG_X86_DS) | \
83 (1ULL << PERF_REG_X86_ES) | \
84 (1ULL << PERF_REG_X86_FS) | \
85 (1ULL << PERF_REG_X86_GS))
86
87int perf_reg_validate(u64 mask)
88{
89 if (!mask || mask & REG_RESERVED)
90 return -EINVAL;
91
92 if (mask & REG_NOSUPPORT)
93 return -EINVAL;
94
95 return 0;
96}
97
98u64 perf_reg_abi(struct task_struct *task)
99{
100 if (test_tsk_thread_flag(task, TIF_IA32))
101 return PERF_SAMPLE_REGS_ABI_32;
102 else
103 return PERF_SAMPLE_REGS_ABI_64;
104}
105#endif /* CONFIG_X86_32 */
diff --git a/arch/x86/kernel/probe_roms.c b/arch/x86/kernel/probe_roms.c
index 0bc72e2069e3..d5f15c3f7b25 100644
--- a/arch/x86/kernel/probe_roms.c
+++ b/arch/x86/kernel/probe_roms.c
@@ -150,7 +150,7 @@ static struct resource *find_oprom(struct pci_dev *pdev)
150 return oprom; 150 return oprom;
151} 151}
152 152
153void *pci_map_biosrom(struct pci_dev *pdev) 153void __iomem *pci_map_biosrom(struct pci_dev *pdev)
154{ 154{
155 struct resource *oprom = find_oprom(pdev); 155 struct resource *oprom = find_oprom(pdev);
156 156
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
index ef6a8456f719..b644e1c765dc 100644
--- a/arch/x86/kernel/process.c
+++ b/arch/x86/kernel/process.c
@@ -66,15 +66,13 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
66{ 66{
67 int ret; 67 int ret;
68 68
69 unlazy_fpu(src);
70
71 *dst = *src; 69 *dst = *src;
72 if (fpu_allocated(&src->thread.fpu)) { 70 if (fpu_allocated(&src->thread.fpu)) {
73 memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu)); 71 memset(&dst->thread.fpu, 0, sizeof(dst->thread.fpu));
74 ret = fpu_alloc(&dst->thread.fpu); 72 ret = fpu_alloc(&dst->thread.fpu);
75 if (ret) 73 if (ret)
76 return ret; 74 return ret;
77 fpu_copy(&dst->thread.fpu, &src->thread.fpu); 75 fpu_copy(dst, src);
78 } 76 }
79 return 0; 77 return 0;
80} 78}
@@ -97,16 +95,6 @@ void arch_task_cache_init(void)
97 SLAB_PANIC | SLAB_NOTRACK, NULL); 95 SLAB_PANIC | SLAB_NOTRACK, NULL);
98} 96}
99 97
100static inline void drop_fpu(struct task_struct *tsk)
101{
102 /*
103 * Forget coprocessor state..
104 */
105 tsk->fpu_counter = 0;
106 clear_fpu(tsk);
107 clear_used_math();
108}
109
110/* 98/*
111 * Free current thread data structures etc.. 99 * Free current thread data structures etc..
112 */ 100 */
@@ -163,7 +151,13 @@ void flush_thread(void)
163 151
164 flush_ptrace_hw_breakpoint(tsk); 152 flush_ptrace_hw_breakpoint(tsk);
165 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array)); 153 memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
166 drop_fpu(tsk); 154 drop_init_fpu(tsk);
155 /*
156 * Free the FPU state for non xsave platforms. They get reallocated
157 * lazily at the first use.
158 */
159 if (!use_eager_fpu())
160 free_thread_xstate(tsk);
167} 161}
168 162
169static void hard_disable_TSC(void) 163static void hard_disable_TSC(void)
@@ -299,71 +293,6 @@ sys_clone(unsigned long clone_flags, unsigned long newsp,
299} 293}
300 294
301/* 295/*
302 * This gets run with %si containing the
303 * function to call, and %di containing
304 * the "args".
305 */
306extern void kernel_thread_helper(void);
307
308/*
309 * Create a kernel thread
310 */
311int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
312{
313 struct pt_regs regs;
314
315 memset(&regs, 0, sizeof(regs));
316
317 regs.si = (unsigned long) fn;
318 regs.di = (unsigned long) arg;
319
320#ifdef CONFIG_X86_32
321 regs.ds = __USER_DS;
322 regs.es = __USER_DS;
323 regs.fs = __KERNEL_PERCPU;
324 regs.gs = __KERNEL_STACK_CANARY;
325#else
326 regs.ss = __KERNEL_DS;
327#endif
328
329 regs.orig_ax = -1;
330 regs.ip = (unsigned long) kernel_thread_helper;
331 regs.cs = __KERNEL_CS | get_kernel_rpl();
332 regs.flags = X86_EFLAGS_IF | X86_EFLAGS_BIT1;
333
334 /* Ok, create the new process.. */
335 return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, &regs, 0, NULL, NULL);
336}
337EXPORT_SYMBOL(kernel_thread);
338
339/*
340 * sys_execve() executes a new program.
341 */
342long sys_execve(const char __user *name,
343 const char __user *const __user *argv,
344 const char __user *const __user *envp, struct pt_regs *regs)
345{
346 long error;
347 char *filename;
348
349 filename = getname(name);
350 error = PTR_ERR(filename);
351 if (IS_ERR(filename))
352 return error;
353 error = do_execve(filename, argv, envp, regs);
354
355#ifdef CONFIG_X86_32
356 if (error == 0) {
357 /* Make sure we don't return using sysenter.. */
358 set_thread_flag(TIF_IRET);
359 }
360#endif
361
362 putname(filename);
363 return error;
364}
365
366/*
367 * Idle related variables and functions 296 * Idle related variables and functions
368 */ 297 */
369unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE; 298unsigned long boot_option_idle_override = IDLE_NO_OVERRIDE;
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 516fa186121b..44e0bff38e72 100644
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -57,6 +57,7 @@
57#include <asm/switch_to.h> 57#include <asm/switch_to.h>
58 58
59asmlinkage void ret_from_fork(void) __asm__("ret_from_fork"); 59asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
60asmlinkage void ret_from_kernel_thread(void) __asm__("ret_from_kernel_thread");
60 61
61/* 62/*
62 * Return saved PC of a blocked thread. 63 * Return saved PC of a blocked thread.
@@ -127,23 +128,39 @@ void release_thread(struct task_struct *dead_task)
127} 128}
128 129
129int copy_thread(unsigned long clone_flags, unsigned long sp, 130int copy_thread(unsigned long clone_flags, unsigned long sp,
130 unsigned long unused, 131 unsigned long arg,
131 struct task_struct *p, struct pt_regs *regs) 132 struct task_struct *p, struct pt_regs *regs)
132{ 133{
133 struct pt_regs *childregs; 134 struct pt_regs *childregs = task_pt_regs(p);
134 struct task_struct *tsk; 135 struct task_struct *tsk;
135 int err; 136 int err;
136 137
137 childregs = task_pt_regs(p); 138 p->thread.sp = (unsigned long) childregs;
139 p->thread.sp0 = (unsigned long) (childregs+1);
140
141 if (unlikely(!regs)) {
142 /* kernel thread */
143 memset(childregs, 0, sizeof(struct pt_regs));
144 p->thread.ip = (unsigned long) ret_from_kernel_thread;
145 task_user_gs(p) = __KERNEL_STACK_CANARY;
146 childregs->ds = __USER_DS;
147 childregs->es = __USER_DS;
148 childregs->fs = __KERNEL_PERCPU;
149 childregs->bx = sp; /* function */
150 childregs->bp = arg;
151 childregs->orig_ax = -1;
152 childregs->cs = __KERNEL_CS | get_kernel_rpl();
153 childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_BIT1;
154 p->fpu_counter = 0;
155 p->thread.io_bitmap_ptr = NULL;
156 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
157 return 0;
158 }
138 *childregs = *regs; 159 *childregs = *regs;
139 childregs->ax = 0; 160 childregs->ax = 0;
140 childregs->sp = sp; 161 childregs->sp = sp;
141 162
142 p->thread.sp = (unsigned long) childregs;
143 p->thread.sp0 = (unsigned long) (childregs+1);
144
145 p->thread.ip = (unsigned long) ret_from_fork; 163 p->thread.ip = (unsigned long) ret_from_fork;
146
147 task_user_gs(p) = get_user_gs(regs); 164 task_user_gs(p) = get_user_gs(regs);
148 165
149 p->fpu_counter = 0; 166 p->fpu_counter = 0;
@@ -190,10 +207,12 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
190 regs->cs = __USER_CS; 207 regs->cs = __USER_CS;
191 regs->ip = new_ip; 208 regs->ip = new_ip;
192 regs->sp = new_sp; 209 regs->sp = new_sp;
210 regs->flags = X86_EFLAGS_IF;
193 /* 211 /*
194 * Free the old FP and other extended state 212 * force it to the iret return path by making it look as if there was
213 * some work pending.
195 */ 214 */
196 free_thread_xstate(current); 215 set_thread_flag(TIF_NOTIFY_RESUME);
197} 216}
198EXPORT_SYMBOL_GPL(start_thread); 217EXPORT_SYMBOL_GPL(start_thread);
199 218
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 0a980c9d7cb8..16c6365e2b86 100644
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -146,29 +146,18 @@ static inline u32 read_32bit_tls(struct task_struct *t, int tls)
146} 146}
147 147
148int copy_thread(unsigned long clone_flags, unsigned long sp, 148int copy_thread(unsigned long clone_flags, unsigned long sp,
149 unsigned long unused, 149 unsigned long arg,
150 struct task_struct *p, struct pt_regs *regs) 150 struct task_struct *p, struct pt_regs *regs)
151{ 151{
152 int err; 152 int err;
153 struct pt_regs *childregs; 153 struct pt_regs *childregs;
154 struct task_struct *me = current; 154 struct task_struct *me = current;
155 155
156 childregs = ((struct pt_regs *) 156 p->thread.sp0 = (unsigned long)task_stack_page(p) + THREAD_SIZE;
157 (THREAD_SIZE + task_stack_page(p))) - 1; 157 childregs = task_pt_regs(p);
158 *childregs = *regs;
159
160 childregs->ax = 0;
161 if (user_mode(regs))
162 childregs->sp = sp;
163 else
164 childregs->sp = (unsigned long)childregs;
165
166 p->thread.sp = (unsigned long) childregs; 158 p->thread.sp = (unsigned long) childregs;
167 p->thread.sp0 = (unsigned long) (childregs+1);
168 p->thread.usersp = me->thread.usersp; 159 p->thread.usersp = me->thread.usersp;
169
170 set_tsk_thread_flag(p, TIF_FORK); 160 set_tsk_thread_flag(p, TIF_FORK);
171
172 p->fpu_counter = 0; 161 p->fpu_counter = 0;
173 p->thread.io_bitmap_ptr = NULL; 162 p->thread.io_bitmap_ptr = NULL;
174 163
@@ -178,6 +167,24 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
178 p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs; 167 p->thread.fs = p->thread.fsindex ? 0 : me->thread.fs;
179 savesegment(es, p->thread.es); 168 savesegment(es, p->thread.es);
180 savesegment(ds, p->thread.ds); 169 savesegment(ds, p->thread.ds);
170 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
171
172 if (unlikely(!regs)) {
173 /* kernel thread */
174 memset(childregs, 0, sizeof(struct pt_regs));
175 childregs->sp = (unsigned long)childregs;
176 childregs->ss = __KERNEL_DS;
177 childregs->bx = sp; /* function */
178 childregs->bp = arg;
179 childregs->orig_ax = -1;
180 childregs->cs = __KERNEL_CS | get_kernel_rpl();
181 childregs->flags = X86_EFLAGS_IF | X86_EFLAGS_BIT1;
182 return 0;
183 }
184 *childregs = *regs;
185
186 childregs->ax = 0;
187 childregs->sp = sp;
181 188
182 err = -ENOMEM; 189 err = -ENOMEM;
183 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps)); 190 memset(p->thread.ptrace_bps, 0, sizeof(p->thread.ptrace_bps));
@@ -232,10 +239,6 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip,
232 regs->cs = _cs; 239 regs->cs = _cs;
233 regs->ss = _ss; 240 regs->ss = _ss;
234 regs->flags = X86_EFLAGS_IF; 241 regs->flags = X86_EFLAGS_IF;
235 /*
236 * Free the old FP and other extended state
237 */
238 free_thread_xstate(current);
239} 242}
240 243
241void 244void
diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index c4c6a5c2bf0f..b00b33a18390 100644
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -21,6 +21,7 @@
21#include <linux/signal.h> 21#include <linux/signal.h>
22#include <linux/perf_event.h> 22#include <linux/perf_event.h>
23#include <linux/hw_breakpoint.h> 23#include <linux/hw_breakpoint.h>
24#include <linux/rcupdate.h>
24 25
25#include <asm/uaccess.h> 26#include <asm/uaccess.h>
26#include <asm/pgtable.h> 27#include <asm/pgtable.h>
@@ -1332,9 +1333,6 @@ static const struct user_regset_view user_x86_64_view = {
1332#define genregs32_get genregs_get 1333#define genregs32_get genregs_get
1333#define genregs32_set genregs_set 1334#define genregs32_set genregs_set
1334 1335
1335#define user_i387_ia32_struct user_i387_struct
1336#define user32_fxsr_struct user_fxsr_struct
1337
1338#endif /* CONFIG_X86_64 */ 1336#endif /* CONFIG_X86_64 */
1339 1337
1340#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION 1338#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
@@ -1463,6 +1461,8 @@ long syscall_trace_enter(struct pt_regs *regs)
1463{ 1461{
1464 long ret = 0; 1462 long ret = 0;
1465 1463
1464 rcu_user_exit();
1465
1466 /* 1466 /*
1467 * If we stepped into a sysenter/syscall insn, it trapped in 1467 * If we stepped into a sysenter/syscall insn, it trapped in
1468 * kernel mode; do_debug() cleared TF and set TIF_SINGLESTEP. 1468 * kernel mode; do_debug() cleared TF and set TIF_SINGLESTEP.
@@ -1526,4 +1526,6 @@ void syscall_trace_leave(struct pt_regs *regs)
1526 !test_thread_flag(TIF_SYSCALL_EMU); 1526 !test_thread_flag(TIF_SYSCALL_EMU);
1527 if (step || test_thread_flag(TIF_SYSCALL_TRACE)) 1527 if (step || test_thread_flag(TIF_SYSCALL_TRACE))
1528 tracehook_report_syscall_exit(regs, step); 1528 tracehook_report_syscall_exit(regs, step);
1529
1530 rcu_user_enter();
1529} 1531}
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c
index af6db6ec5b2a..4929c1be0ac0 100644
--- a/arch/x86/kernel/rtc.c
+++ b/arch/x86/kernel/rtc.c
@@ -225,7 +225,7 @@ static struct platform_device rtc_device = {
225static __init int add_rtc_cmos(void) 225static __init int add_rtc_cmos(void)
226{ 226{
227#ifdef CONFIG_PNP 227#ifdef CONFIG_PNP
228 static const char *ids[] __initconst = 228 static const char * const const ids[] __initconst =
229 { "PNP0b00", "PNP0b01", "PNP0b02", }; 229 { "PNP0b00", "PNP0b01", "PNP0b02", };
230 struct pnp_dev *dev; 230 struct pnp_dev *dev;
231 struct pnp_id *id; 231 struct pnp_id *id;
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index 4f165479c453..a2bb18e02839 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -68,6 +68,7 @@
68#include <linux/percpu.h> 68#include <linux/percpu.h>
69#include <linux/crash_dump.h> 69#include <linux/crash_dump.h>
70#include <linux/tboot.h> 70#include <linux/tboot.h>
71#include <linux/jiffies.h>
71 72
72#include <video/edid.h> 73#include <video/edid.h>
73 74
@@ -957,7 +958,7 @@ void __init setup_arch(char **cmdline_p)
957 initmem_init(); 958 initmem_init();
958 memblock_find_dma_reserve(); 959 memblock_find_dma_reserve();
959 960
960#ifdef CONFIG_KVM_CLOCK 961#ifdef CONFIG_KVM_GUEST
961 kvmclock_init(); 962 kvmclock_init();
962#endif 963#endif
963 964
@@ -1032,6 +1033,8 @@ void __init setup_arch(char **cmdline_p)
1032 mcheck_init(); 1033 mcheck_init();
1033 1034
1034 arch_init_ideal_nops(); 1035 arch_init_ideal_nops();
1036
1037 register_refined_jiffies(CLOCK_TICK_RATE);
1035} 1038}
1036 1039
1037#ifdef CONFIG_X86_32 1040#ifdef CONFIG_X86_32
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c
index b280908a376e..29ad351804e9 100644
--- a/arch/x86/kernel/signal.c
+++ b/arch/x86/kernel/signal.c
@@ -114,11 +114,12 @@ int restore_sigcontext(struct pt_regs *regs, struct sigcontext __user *sc,
114 regs->orig_ax = -1; /* disable syscall checks */ 114 regs->orig_ax = -1; /* disable syscall checks */
115 115
116 get_user_ex(buf, &sc->fpstate); 116 get_user_ex(buf, &sc->fpstate);
117 err |= restore_i387_xstate(buf);
118 117
119 get_user_ex(*pax, &sc->ax); 118 get_user_ex(*pax, &sc->ax);
120 } get_user_catch(err); 119 } get_user_catch(err);
121 120
121 err |= restore_xstate_sig(buf, config_enabled(CONFIG_X86_32));
122
122 return err; 123 return err;
123} 124}
124 125
@@ -206,35 +207,32 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
206 void __user **fpstate) 207 void __user **fpstate)
207{ 208{
208 /* Default to using normal stack */ 209 /* Default to using normal stack */
210 unsigned long math_size = 0;
209 unsigned long sp = regs->sp; 211 unsigned long sp = regs->sp;
212 unsigned long buf_fx = 0;
210 int onsigstack = on_sig_stack(sp); 213 int onsigstack = on_sig_stack(sp);
211 214
212#ifdef CONFIG_X86_64
213 /* redzone */ 215 /* redzone */
214 sp -= 128; 216 if (config_enabled(CONFIG_X86_64))
215#endif /* CONFIG_X86_64 */ 217 sp -= 128;
216 218
217 if (!onsigstack) { 219 if (!onsigstack) {
218 /* This is the X/Open sanctioned signal stack switching. */ 220 /* This is the X/Open sanctioned signal stack switching. */
219 if (ka->sa.sa_flags & SA_ONSTACK) { 221 if (ka->sa.sa_flags & SA_ONSTACK) {
220 if (current->sas_ss_size) 222 if (current->sas_ss_size)
221 sp = current->sas_ss_sp + current->sas_ss_size; 223 sp = current->sas_ss_sp + current->sas_ss_size;
222 } else { 224 } else if (config_enabled(CONFIG_X86_32) &&
223#ifdef CONFIG_X86_32 225 (regs->ss & 0xffff) != __USER_DS &&
224 /* This is the legacy signal stack switching. */ 226 !(ka->sa.sa_flags & SA_RESTORER) &&
225 if ((regs->ss & 0xffff) != __USER_DS && 227 ka->sa.sa_restorer) {
226 !(ka->sa.sa_flags & SA_RESTORER) && 228 /* This is the legacy signal stack switching. */
227 ka->sa.sa_restorer)
228 sp = (unsigned long) ka->sa.sa_restorer; 229 sp = (unsigned long) ka->sa.sa_restorer;
229#endif /* CONFIG_X86_32 */
230 } 230 }
231 } 231 }
232 232
233 if (used_math()) { 233 if (used_math()) {
234 sp -= sig_xstate_size; 234 sp = alloc_mathframe(sp, config_enabled(CONFIG_X86_32),
235#ifdef CONFIG_X86_64 235 &buf_fx, &math_size);
236 sp = round_down(sp, 64);
237#endif /* CONFIG_X86_64 */
238 *fpstate = (void __user *)sp; 236 *fpstate = (void __user *)sp;
239 } 237 }
240 238
@@ -247,8 +245,9 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size,
247 if (onsigstack && !likely(on_sig_stack(sp))) 245 if (onsigstack && !likely(on_sig_stack(sp)))
248 return (void __user *)-1L; 246 return (void __user *)-1L;
249 247
250 /* save i387 state */ 248 /* save i387 and extended state */
251 if (used_math() && save_i387_xstate(*fpstate) < 0) 249 if (used_math() &&
250 save_xstate_sig(*fpstate, (void __user *)buf_fx, math_size) < 0)
252 return (void __user *)-1L; 251 return (void __user *)-1L;
253 252
254 return (void __user *)sp; 253 return (void __user *)sp;
@@ -357,7 +356,6 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
357 put_user_ex(sig, &frame->sig); 356 put_user_ex(sig, &frame->sig);
358 put_user_ex(&frame->info, &frame->pinfo); 357 put_user_ex(&frame->info, &frame->pinfo);
359 put_user_ex(&frame->uc, &frame->puc); 358 put_user_ex(&frame->uc, &frame->puc);
360 err |= copy_siginfo_to_user(&frame->info, info);
361 359
362 /* Create the ucontext. */ 360 /* Create the ucontext. */
363 if (cpu_has_xsave) 361 if (cpu_has_xsave)
@@ -369,9 +367,6 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
369 put_user_ex(sas_ss_flags(regs->sp), 367 put_user_ex(sas_ss_flags(regs->sp),
370 &frame->uc.uc_stack.ss_flags); 368 &frame->uc.uc_stack.ss_flags);
371 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size); 369 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
372 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
373 regs, set->sig[0]);
374 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
375 370
376 /* Set up to return from userspace. */ 371 /* Set up to return from userspace. */
377 restorer = VDSO32_SYMBOL(current->mm->context.vdso, rt_sigreturn); 372 restorer = VDSO32_SYMBOL(current->mm->context.vdso, rt_sigreturn);
@@ -388,6 +383,11 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
388 */ 383 */
389 put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode); 384 put_user_ex(*((u64 *)&rt_retcode), (u64 *)frame->retcode);
390 } put_user_catch(err); 385 } put_user_catch(err);
386
387 err |= copy_siginfo_to_user(&frame->info, info);
388 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
389 regs, set->sig[0]);
390 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
391 391
392 if (err) 392 if (err)
393 return -EFAULT; 393 return -EFAULT;
@@ -436,8 +436,6 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
436 put_user_ex(sas_ss_flags(regs->sp), 436 put_user_ex(sas_ss_flags(regs->sp),
437 &frame->uc.uc_stack.ss_flags); 437 &frame->uc.uc_stack.ss_flags);
438 put_user_ex(me->sas_ss_size, &frame->uc.uc_stack.ss_size); 438 put_user_ex(me->sas_ss_size, &frame->uc.uc_stack.ss_size);
439 err |= setup_sigcontext(&frame->uc.uc_mcontext, fp, regs, set->sig[0]);
440 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
441 439
442 /* Set up to return from userspace. If provided, use a stub 440 /* Set up to return from userspace. If provided, use a stub
443 already in userspace. */ 441 already in userspace. */
@@ -450,6 +448,9 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
450 } 448 }
451 } put_user_catch(err); 449 } put_user_catch(err);
452 450
451 err |= setup_sigcontext(&frame->uc.uc_mcontext, fp, regs, set->sig[0]);
452 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
453
453 if (err) 454 if (err)
454 return -EFAULT; 455 return -EFAULT;
455 456
@@ -474,6 +475,75 @@ static int __setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
474} 475}
475#endif /* CONFIG_X86_32 */ 476#endif /* CONFIG_X86_32 */
476 477
478static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
479 siginfo_t *info, compat_sigset_t *set,
480 struct pt_regs *regs)
481{
482#ifdef CONFIG_X86_X32_ABI
483 struct rt_sigframe_x32 __user *frame;
484 void __user *restorer;
485 int err = 0;
486 void __user *fpstate = NULL;
487
488 frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
489
490 if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
491 return -EFAULT;
492
493 if (ka->sa.sa_flags & SA_SIGINFO) {
494 if (copy_siginfo_to_user32(&frame->info, info))
495 return -EFAULT;
496 }
497
498 put_user_try {
499 /* Create the ucontext. */
500 if (cpu_has_xsave)
501 put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
502 else
503 put_user_ex(0, &frame->uc.uc_flags);
504 put_user_ex(0, &frame->uc.uc_link);
505 put_user_ex(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
506 put_user_ex(sas_ss_flags(regs->sp),
507 &frame->uc.uc_stack.ss_flags);
508 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
509 put_user_ex(0, &frame->uc.uc__pad0);
510
511 if (ka->sa.sa_flags & SA_RESTORER) {
512 restorer = ka->sa.sa_restorer;
513 } else {
514 /* could use a vstub here */
515 restorer = NULL;
516 err |= -EFAULT;
517 }
518 put_user_ex(restorer, &frame->pretcode);
519 } put_user_catch(err);
520
521 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
522 regs, set->sig[0]);
523 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
524
525 if (err)
526 return -EFAULT;
527
528 /* Set up registers for signal handler */
529 regs->sp = (unsigned long) frame;
530 regs->ip = (unsigned long) ka->sa.sa_handler;
531
532 /* We use the x32 calling convention here... */
533 regs->di = sig;
534 regs->si = (unsigned long) &frame->info;
535 regs->dx = (unsigned long) &frame->uc;
536
537 loadsegment(ds, __USER_DS);
538 loadsegment(es, __USER_DS);
539
540 regs->cs = __USER_CS;
541 regs->ss = __USER_DS;
542#endif /* CONFIG_X86_X32_ABI */
543
544 return 0;
545}
546
477#ifdef CONFIG_X86_32 547#ifdef CONFIG_X86_32
478/* 548/*
479 * Atomically swap in the new signal mask, and wait for a signal. 549 * Atomically swap in the new signal mask, and wait for a signal.
@@ -612,55 +682,22 @@ static int signr_convert(int sig)
612 return sig; 682 return sig;
613} 683}
614 684
615#ifdef CONFIG_X86_32
616
617#define is_ia32 1
618#define ia32_setup_frame __setup_frame
619#define ia32_setup_rt_frame __setup_rt_frame
620
621#else /* !CONFIG_X86_32 */
622
623#ifdef CONFIG_IA32_EMULATION
624#define is_ia32 test_thread_flag(TIF_IA32)
625#else /* !CONFIG_IA32_EMULATION */
626#define is_ia32 0
627#endif /* CONFIG_IA32_EMULATION */
628
629#ifdef CONFIG_X86_X32_ABI
630#define is_x32 test_thread_flag(TIF_X32)
631
632static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
633 siginfo_t *info, compat_sigset_t *set,
634 struct pt_regs *regs);
635#else /* !CONFIG_X86_X32_ABI */
636#define is_x32 0
637#endif /* CONFIG_X86_X32_ABI */
638
639int ia32_setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
640 sigset_t *set, struct pt_regs *regs);
641int ia32_setup_frame(int sig, struct k_sigaction *ka,
642 sigset_t *set, struct pt_regs *regs);
643
644#endif /* CONFIG_X86_32 */
645
646static int 685static int
647setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info, 686setup_rt_frame(int sig, struct k_sigaction *ka, siginfo_t *info,
648 struct pt_regs *regs) 687 struct pt_regs *regs)
649{ 688{
650 int usig = signr_convert(sig); 689 int usig = signr_convert(sig);
651 sigset_t *set = sigmask_to_save(); 690 sigset_t *set = sigmask_to_save();
691 compat_sigset_t *cset = (compat_sigset_t *) set;
652 692
653 /* Set up the stack frame */ 693 /* Set up the stack frame */
654 if (is_ia32) { 694 if (is_ia32_frame()) {
655 if (ka->sa.sa_flags & SA_SIGINFO) 695 if (ka->sa.sa_flags & SA_SIGINFO)
656 return ia32_setup_rt_frame(usig, ka, info, set, regs); 696 return ia32_setup_rt_frame(usig, ka, info, cset, regs);
657 else 697 else
658 return ia32_setup_frame(usig, ka, set, regs); 698 return ia32_setup_frame(usig, ka, cset, regs);
659#ifdef CONFIG_X86_X32_ABI 699 } else if (is_x32_frame()) {
660 } else if (is_x32) { 700 return x32_setup_rt_frame(usig, ka, info, cset, regs);
661 return x32_setup_rt_frame(usig, ka, info,
662 (compat_sigset_t *)set, regs);
663#endif
664 } else { 701 } else {
665 return __setup_rt_frame(sig, ka, info, set, regs); 702 return __setup_rt_frame(sig, ka, info, set, regs);
666 } 703 }
@@ -779,6 +816,8 @@ static void do_signal(struct pt_regs *regs)
779void 816void
780do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags) 817do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
781{ 818{
819 rcu_user_exit();
820
782#ifdef CONFIG_X86_MCE 821#ifdef CONFIG_X86_MCE
783 /* notify userspace of pending MCEs */ 822 /* notify userspace of pending MCEs */
784 if (thread_info_flags & _TIF_MCE_NOTIFY) 823 if (thread_info_flags & _TIF_MCE_NOTIFY)
@@ -801,9 +840,7 @@ do_notify_resume(struct pt_regs *regs, void *unused, __u32 thread_info_flags)
801 if (thread_info_flags & _TIF_USER_RETURN_NOTIFY) 840 if (thread_info_flags & _TIF_USER_RETURN_NOTIFY)
802 fire_user_return_notifiers(); 841 fire_user_return_notifiers();
803 842
804#ifdef CONFIG_X86_32 843 rcu_user_enter();
805 clear_thread_flag(TIF_IRET);
806#endif /* CONFIG_X86_32 */
807} 844}
808 845
809void signal_fault(struct pt_regs *regs, void __user *frame, char *where) 846void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
@@ -824,72 +861,6 @@ void signal_fault(struct pt_regs *regs, void __user *frame, char *where)
824} 861}
825 862
826#ifdef CONFIG_X86_X32_ABI 863#ifdef CONFIG_X86_X32_ABI
827static int x32_setup_rt_frame(int sig, struct k_sigaction *ka,
828 siginfo_t *info, compat_sigset_t *set,
829 struct pt_regs *regs)
830{
831 struct rt_sigframe_x32 __user *frame;
832 void __user *restorer;
833 int err = 0;
834 void __user *fpstate = NULL;
835
836 frame = get_sigframe(ka, regs, sizeof(*frame), &fpstate);
837
838 if (!access_ok(VERIFY_WRITE, frame, sizeof(*frame)))
839 return -EFAULT;
840
841 if (ka->sa.sa_flags & SA_SIGINFO) {
842 if (copy_siginfo_to_user32(&frame->info, info))
843 return -EFAULT;
844 }
845
846 put_user_try {
847 /* Create the ucontext. */
848 if (cpu_has_xsave)
849 put_user_ex(UC_FP_XSTATE, &frame->uc.uc_flags);
850 else
851 put_user_ex(0, &frame->uc.uc_flags);
852 put_user_ex(0, &frame->uc.uc_link);
853 put_user_ex(current->sas_ss_sp, &frame->uc.uc_stack.ss_sp);
854 put_user_ex(sas_ss_flags(regs->sp),
855 &frame->uc.uc_stack.ss_flags);
856 put_user_ex(current->sas_ss_size, &frame->uc.uc_stack.ss_size);
857 put_user_ex(0, &frame->uc.uc__pad0);
858 err |= setup_sigcontext(&frame->uc.uc_mcontext, fpstate,
859 regs, set->sig[0]);
860 err |= __copy_to_user(&frame->uc.uc_sigmask, set, sizeof(*set));
861
862 if (ka->sa.sa_flags & SA_RESTORER) {
863 restorer = ka->sa.sa_restorer;
864 } else {
865 /* could use a vstub here */
866 restorer = NULL;
867 err |= -EFAULT;
868 }
869 put_user_ex(restorer, &frame->pretcode);
870 } put_user_catch(err);
871
872 if (err)
873 return -EFAULT;
874
875 /* Set up registers for signal handler */
876 regs->sp = (unsigned long) frame;
877 regs->ip = (unsigned long) ka->sa.sa_handler;
878
879 /* We use the x32 calling convention here... */
880 regs->di = sig;
881 regs->si = (unsigned long) &frame->info;
882 regs->dx = (unsigned long) &frame->uc;
883
884 loadsegment(ds, __USER_DS);
885 loadsegment(es, __USER_DS);
886
887 regs->cs = __USER_CS;
888 regs->ss = __USER_DS;
889
890 return 0;
891}
892
893asmlinkage long sys32_x32_rt_sigreturn(struct pt_regs *regs) 864asmlinkage long sys32_x32_rt_sigreturn(struct pt_regs *regs)
894{ 865{
895 struct rt_sigframe_x32 __user *frame; 866 struct rt_sigframe_x32 __user *frame;
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 7c5a8c314c02..c80a33bc528b 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -665,7 +665,8 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
665 unsigned long boot_error = 0; 665 unsigned long boot_error = 0;
666 int timeout; 666 int timeout;
667 667
668 alternatives_smp_switch(1); 668 /* Just in case we booted with a single CPU. */
669 alternatives_enable_smp();
669 670
670 idle->thread.sp = (unsigned long) (((struct pt_regs *) 671 idle->thread.sp = (unsigned long) (((struct pt_regs *)
671 (THREAD_SIZE + task_stack_page(idle))) - 1); 672 (THREAD_SIZE + task_stack_page(idle))) - 1);
@@ -1053,20 +1054,6 @@ out:
1053 preempt_enable(); 1054 preempt_enable();
1054} 1055}
1055 1056
1056void arch_disable_nonboot_cpus_begin(void)
1057{
1058 /*
1059 * Avoid the smp alternatives switch during the disable_nonboot_cpus().
1060 * In the suspend path, we will be back in the SMP mode shortly anyways.
1061 */
1062 skip_smp_alternatives = true;
1063}
1064
1065void arch_disable_nonboot_cpus_end(void)
1066{
1067 skip_smp_alternatives = false;
1068}
1069
1070void arch_enable_nonboot_cpus_begin(void) 1057void arch_enable_nonboot_cpus_begin(void)
1071{ 1058{
1072 set_mtrr_aps_delayed_init(); 1059 set_mtrr_aps_delayed_init();
@@ -1256,9 +1243,6 @@ void native_cpu_die(unsigned int cpu)
1256 if (per_cpu(cpu_state, cpu) == CPU_DEAD) { 1243 if (per_cpu(cpu_state, cpu) == CPU_DEAD) {
1257 if (system_state == SYSTEM_RUNNING) 1244 if (system_state == SYSTEM_RUNNING)
1258 pr_info("CPU %u is now offline\n", cpu); 1245 pr_info("CPU %u is now offline\n", cpu);
1259
1260 if (1 == num_online_cpus())
1261 alternatives_smp_switch(0);
1262 return; 1246 return;
1263 } 1247 }
1264 msleep(100); 1248 msleep(100);
diff --git a/arch/x86/kernel/step.c b/arch/x86/kernel/step.c
index c346d1161488..cd3b2438a980 100644
--- a/arch/x86/kernel/step.c
+++ b/arch/x86/kernel/step.c
@@ -157,6 +157,33 @@ static int enable_single_step(struct task_struct *child)
157 return 1; 157 return 1;
158} 158}
159 159
160void set_task_blockstep(struct task_struct *task, bool on)
161{
162 unsigned long debugctl;
163
164 /*
165 * Ensure irq/preemption can't change debugctl in between.
166 * Note also that both TIF_BLOCKSTEP and debugctl should
167 * be changed atomically wrt preemption.
168 * FIXME: this means that set/clear TIF_BLOCKSTEP is simply
169 * wrong if task != current, SIGKILL can wakeup the stopped
170 * tracee and set/clear can play with the running task, this
171 * can confuse the next __switch_to_xtra().
172 */
173 local_irq_disable();
174 debugctl = get_debugctlmsr();
175 if (on) {
176 debugctl |= DEBUGCTLMSR_BTF;
177 set_tsk_thread_flag(task, TIF_BLOCKSTEP);
178 } else {
179 debugctl &= ~DEBUGCTLMSR_BTF;
180 clear_tsk_thread_flag(task, TIF_BLOCKSTEP);
181 }
182 if (task == current)
183 update_debugctlmsr(debugctl);
184 local_irq_enable();
185}
186
160/* 187/*
161 * Enable single or block step. 188 * Enable single or block step.
162 */ 189 */
@@ -169,19 +196,10 @@ static void enable_step(struct task_struct *child, bool block)
169 * So no one should try to use debugger block stepping in a program 196 * So no one should try to use debugger block stepping in a program
170 * that uses user-mode single stepping itself. 197 * that uses user-mode single stepping itself.
171 */ 198 */
172 if (enable_single_step(child) && block) { 199 if (enable_single_step(child) && block)
173 unsigned long debugctl = get_debugctlmsr(); 200 set_task_blockstep(child, true);
174 201 else if (test_tsk_thread_flag(child, TIF_BLOCKSTEP))
175 debugctl |= DEBUGCTLMSR_BTF; 202 set_task_blockstep(child, false);
176 update_debugctlmsr(debugctl);
177 set_tsk_thread_flag(child, TIF_BLOCKSTEP);
178 } else if (test_tsk_thread_flag(child, TIF_BLOCKSTEP)) {
179 unsigned long debugctl = get_debugctlmsr();
180
181 debugctl &= ~DEBUGCTLMSR_BTF;
182 update_debugctlmsr(debugctl);
183 clear_tsk_thread_flag(child, TIF_BLOCKSTEP);
184 }
185} 203}
186 204
187void user_enable_single_step(struct task_struct *child) 205void user_enable_single_step(struct task_struct *child)
@@ -199,13 +217,8 @@ void user_disable_single_step(struct task_struct *child)
199 /* 217 /*
200 * Make sure block stepping (BTF) is disabled. 218 * Make sure block stepping (BTF) is disabled.
201 */ 219 */
202 if (test_tsk_thread_flag(child, TIF_BLOCKSTEP)) { 220 if (test_tsk_thread_flag(child, TIF_BLOCKSTEP))
203 unsigned long debugctl = get_debugctlmsr(); 221 set_task_blockstep(child, false);
204
205 debugctl &= ~DEBUGCTLMSR_BTF;
206 update_debugctlmsr(debugctl);
207 clear_tsk_thread_flag(child, TIF_BLOCKSTEP);
208 }
209 222
210 /* Always clear TIF_SINGLESTEP... */ 223 /* Always clear TIF_SINGLESTEP... */
211 clear_tsk_thread_flag(child, TIF_SINGLESTEP); 224 clear_tsk_thread_flag(child, TIF_SINGLESTEP);
diff --git a/arch/x86/kernel/sys_i386_32.c b/arch/x86/kernel/sys_i386_32.c
deleted file mode 100644
index 0b0cb5fede19..000000000000
--- a/arch/x86/kernel/sys_i386_32.c
+++ /dev/null
@@ -1,40 +0,0 @@
1/*
2 * This file contains various random system calls that
3 * have a non-standard calling sequence on the Linux/i386
4 * platform.
5 */
6
7#include <linux/errno.h>
8#include <linux/sched.h>
9#include <linux/mm.h>
10#include <linux/fs.h>
11#include <linux/smp.h>
12#include <linux/sem.h>
13#include <linux/msg.h>
14#include <linux/shm.h>
15#include <linux/stat.h>
16#include <linux/syscalls.h>
17#include <linux/mman.h>
18#include <linux/file.h>
19#include <linux/utsname.h>
20#include <linux/ipc.h>
21
22#include <linux/uaccess.h>
23#include <linux/unistd.h>
24
25#include <asm/syscalls.h>
26
27/*
28 * Do a system call from kernel instead of calling sys_execve so we
29 * end up with proper pt_regs.
30 */
31int kernel_execve(const char *filename,
32 const char *const argv[],
33 const char *const envp[])
34{
35 long __res;
36 asm volatile ("int $0x80"
37 : "=a" (__res)
38 : "0" (__NR_execve), "b" (filename), "c" (argv), "d" (envp) : "memory");
39 return __res;
40}
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
index b481341c9369..8276dc6794cc 100644
--- a/arch/x86/kernel/traps.c
+++ b/arch/x86/kernel/traps.c
@@ -55,6 +55,7 @@
55#include <asm/i387.h> 55#include <asm/i387.h>
56#include <asm/fpu-internal.h> 56#include <asm/fpu-internal.h>
57#include <asm/mce.h> 57#include <asm/mce.h>
58#include <asm/rcu.h>
58 59
59#include <asm/mach_traps.h> 60#include <asm/mach_traps.h>
60 61
@@ -107,30 +108,45 @@ static inline void preempt_conditional_cli(struct pt_regs *regs)
107 dec_preempt_count(); 108 dec_preempt_count();
108} 109}
109 110
110static void __kprobes 111static int __kprobes
111do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, 112do_trap_no_signal(struct task_struct *tsk, int trapnr, char *str,
112 long error_code, siginfo_t *info) 113 struct pt_regs *regs, long error_code)
113{ 114{
114 struct task_struct *tsk = current;
115
116#ifdef CONFIG_X86_32 115#ifdef CONFIG_X86_32
117 if (regs->flags & X86_VM_MASK) { 116 if (regs->flags & X86_VM_MASK) {
118 /* 117 /*
119 * traps 0, 1, 3, 4, and 5 should be forwarded to vm86. 118 * Traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
120 * On nmi (interrupt 2), do_trap should not be called. 119 * On nmi (interrupt 2), do_trap should not be called.
121 */ 120 */
122 if (trapnr < X86_TRAP_UD) 121 if (trapnr < X86_TRAP_UD) {
123 goto vm86_trap; 122 if (!handle_vm86_trap((struct kernel_vm86_regs *) regs,
124 goto trap_signal; 123 error_code, trapnr))
124 return 0;
125 }
126 return -1;
125 } 127 }
126#endif 128#endif
129 if (!user_mode(regs)) {
130 if (!fixup_exception(regs)) {
131 tsk->thread.error_code = error_code;
132 tsk->thread.trap_nr = trapnr;
133 die(str, regs, error_code);
134 }
135 return 0;
136 }
127 137
128 if (!user_mode(regs)) 138 return -1;
129 goto kernel_trap; 139}
130 140
131#ifdef CONFIG_X86_32 141static void __kprobes
132trap_signal: 142do_trap(int trapnr, int signr, char *str, struct pt_regs *regs,
133#endif 143 long error_code, siginfo_t *info)
144{
145 struct task_struct *tsk = current;
146
147
148 if (!do_trap_no_signal(tsk, trapnr, str, regs, error_code))
149 return;
134 /* 150 /*
135 * We want error_code and trap_nr set for userspace faults and 151 * We want error_code and trap_nr set for userspace faults and
136 * kernelspace faults which result in die(), but not 152 * kernelspace faults which result in die(), but not
@@ -158,33 +174,20 @@ trap_signal:
158 force_sig_info(signr, info, tsk); 174 force_sig_info(signr, info, tsk);
159 else 175 else
160 force_sig(signr, tsk); 176 force_sig(signr, tsk);
161 return;
162
163kernel_trap:
164 if (!fixup_exception(regs)) {
165 tsk->thread.error_code = error_code;
166 tsk->thread.trap_nr = trapnr;
167 die(str, regs, error_code);
168 }
169 return;
170
171#ifdef CONFIG_X86_32
172vm86_trap:
173 if (handle_vm86_trap((struct kernel_vm86_regs *) regs,
174 error_code, trapnr))
175 goto trap_signal;
176 return;
177#endif
178} 177}
179 178
180#define DO_ERROR(trapnr, signr, str, name) \ 179#define DO_ERROR(trapnr, signr, str, name) \
181dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ 180dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
182{ \ 181{ \
183 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 182 exception_enter(regs); \
184 == NOTIFY_STOP) \ 183 if (notify_die(DIE_TRAP, str, regs, error_code, \
184 trapnr, signr) == NOTIFY_STOP) { \
185 exception_exit(regs); \
185 return; \ 186 return; \
187 } \
186 conditional_sti(regs); \ 188 conditional_sti(regs); \
187 do_trap(trapnr, signr, str, regs, error_code, NULL); \ 189 do_trap(trapnr, signr, str, regs, error_code, NULL); \
190 exception_exit(regs); \
188} 191}
189 192
190#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \ 193#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
@@ -195,11 +198,15 @@ dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
195 info.si_errno = 0; \ 198 info.si_errno = 0; \
196 info.si_code = sicode; \ 199 info.si_code = sicode; \
197 info.si_addr = (void __user *)siaddr; \ 200 info.si_addr = (void __user *)siaddr; \
198 if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \ 201 exception_enter(regs); \
199 == NOTIFY_STOP) \ 202 if (notify_die(DIE_TRAP, str, regs, error_code, \
203 trapnr, signr) == NOTIFY_STOP) { \
204 exception_exit(regs); \
200 return; \ 205 return; \
206 } \
201 conditional_sti(regs); \ 207 conditional_sti(regs); \
202 do_trap(trapnr, signr, str, regs, error_code, &info); \ 208 do_trap(trapnr, signr, str, regs, error_code, &info); \
209 exception_exit(regs); \
203} 210}
204 211
205DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV, 212DO_ERROR_INFO(X86_TRAP_DE, SIGFPE, "divide error", divide_error, FPE_INTDIV,
@@ -222,12 +229,14 @@ DO_ERROR_INFO(X86_TRAP_AC, SIGBUS, "alignment check", alignment_check,
222/* Runs on IST stack */ 229/* Runs on IST stack */
223dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code) 230dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code)
224{ 231{
232 exception_enter(regs);
225 if (notify_die(DIE_TRAP, "stack segment", regs, error_code, 233 if (notify_die(DIE_TRAP, "stack segment", regs, error_code,
226 X86_TRAP_SS, SIGBUS) == NOTIFY_STOP) 234 X86_TRAP_SS, SIGBUS) != NOTIFY_STOP) {
227 return; 235 preempt_conditional_sti(regs);
228 preempt_conditional_sti(regs); 236 do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL);
229 do_trap(X86_TRAP_SS, SIGBUS, "stack segment", regs, error_code, NULL); 237 preempt_conditional_cli(regs);
230 preempt_conditional_cli(regs); 238 }
239 exception_exit(regs);
231} 240}
232 241
233dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code) 242dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
@@ -235,6 +244,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code)
235 static const char str[] = "double fault"; 244 static const char str[] = "double fault";
236 struct task_struct *tsk = current; 245 struct task_struct *tsk = current;
237 246
247 exception_enter(regs);
238 /* Return not checked because double check cannot be ignored */ 248 /* Return not checked because double check cannot be ignored */
239 notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV); 249 notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV);
240 250
@@ -255,16 +265,29 @@ do_general_protection(struct pt_regs *regs, long error_code)
255{ 265{
256 struct task_struct *tsk; 266 struct task_struct *tsk;
257 267
268 exception_enter(regs);
258 conditional_sti(regs); 269 conditional_sti(regs);
259 270
260#ifdef CONFIG_X86_32 271#ifdef CONFIG_X86_32
261 if (regs->flags & X86_VM_MASK) 272 if (regs->flags & X86_VM_MASK) {
262 goto gp_in_vm86; 273 local_irq_enable();
274 handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
275 goto exit;
276 }
263#endif 277#endif
264 278
265 tsk = current; 279 tsk = current;
266 if (!user_mode(regs)) 280 if (!user_mode(regs)) {
267 goto gp_in_kernel; 281 if (fixup_exception(regs))
282 goto exit;
283
284 tsk->thread.error_code = error_code;
285 tsk->thread.trap_nr = X86_TRAP_GP;
286 if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
287 X86_TRAP_GP, SIGSEGV) != NOTIFY_STOP)
288 die("general protection fault", regs, error_code);
289 goto exit;
290 }
268 291
269 tsk->thread.error_code = error_code; 292 tsk->thread.error_code = error_code;
270 tsk->thread.trap_nr = X86_TRAP_GP; 293 tsk->thread.trap_nr = X86_TRAP_GP;
@@ -279,25 +302,8 @@ do_general_protection(struct pt_regs *regs, long error_code)
279 } 302 }
280 303
281 force_sig(SIGSEGV, tsk); 304 force_sig(SIGSEGV, tsk);
282 return; 305exit:
283 306 exception_exit(regs);
284#ifdef CONFIG_X86_32
285gp_in_vm86:
286 local_irq_enable();
287 handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code);
288 return;
289#endif
290
291gp_in_kernel:
292 if (fixup_exception(regs))
293 return;
294
295 tsk->thread.error_code = error_code;
296 tsk->thread.trap_nr = X86_TRAP_GP;
297 if (notify_die(DIE_GPF, "general protection fault", regs, error_code,
298 X86_TRAP_GP, SIGSEGV) == NOTIFY_STOP)
299 return;
300 die("general protection fault", regs, error_code);
301} 307}
302 308
303/* May run on IST stack. */ 309/* May run on IST stack. */
@@ -312,15 +318,16 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
312 ftrace_int3_handler(regs)) 318 ftrace_int3_handler(regs))
313 return; 319 return;
314#endif 320#endif
321 exception_enter(regs);
315#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP 322#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
316 if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, 323 if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
317 SIGTRAP) == NOTIFY_STOP) 324 SIGTRAP) == NOTIFY_STOP)
318 return; 325 goto exit;
319#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ 326#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */
320 327
321 if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, 328 if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP,
322 SIGTRAP) == NOTIFY_STOP) 329 SIGTRAP) == NOTIFY_STOP)
323 return; 330 goto exit;
324 331
325 /* 332 /*
326 * Let others (NMI) know that the debug stack is in use 333 * Let others (NMI) know that the debug stack is in use
@@ -331,6 +338,8 @@ dotraplinkage void __kprobes notrace do_int3(struct pt_regs *regs, long error_co
331 do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL); 338 do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, NULL);
332 preempt_conditional_cli(regs); 339 preempt_conditional_cli(regs);
333 debug_stack_usage_dec(); 340 debug_stack_usage_dec();
341exit:
342 exception_exit(regs);
334} 343}
335 344
336#ifdef CONFIG_X86_64 345#ifdef CONFIG_X86_64
@@ -391,6 +400,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
391 unsigned long dr6; 400 unsigned long dr6;
392 int si_code; 401 int si_code;
393 402
403 exception_enter(regs);
404
394 get_debugreg(dr6, 6); 405 get_debugreg(dr6, 6);
395 406
396 /* Filter out all the reserved bits which are preset to 1 */ 407 /* Filter out all the reserved bits which are preset to 1 */
@@ -406,7 +417,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
406 417
407 /* Catch kmemcheck conditions first of all! */ 418 /* Catch kmemcheck conditions first of all! */
408 if ((dr6 & DR_STEP) && kmemcheck_trap(regs)) 419 if ((dr6 & DR_STEP) && kmemcheck_trap(regs))
409 return; 420 goto exit;
410 421
411 /* DR6 may or may not be cleared by the CPU */ 422 /* DR6 may or may not be cleared by the CPU */
412 set_debugreg(0, 6); 423 set_debugreg(0, 6);
@@ -421,7 +432,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
421 432
422 if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code, 433 if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code,
423 SIGTRAP) == NOTIFY_STOP) 434 SIGTRAP) == NOTIFY_STOP)
424 return; 435 goto exit;
425 436
426 /* 437 /*
427 * Let others (NMI) know that the debug stack is in use 438 * Let others (NMI) know that the debug stack is in use
@@ -437,7 +448,7 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
437 X86_TRAP_DB); 448 X86_TRAP_DB);
438 preempt_conditional_cli(regs); 449 preempt_conditional_cli(regs);
439 debug_stack_usage_dec(); 450 debug_stack_usage_dec();
440 return; 451 goto exit;
441 } 452 }
442 453
443 /* 454 /*
@@ -458,7 +469,8 @@ dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code)
458 preempt_conditional_cli(regs); 469 preempt_conditional_cli(regs);
459 debug_stack_usage_dec(); 470 debug_stack_usage_dec();
460 471
461 return; 472exit:
473 exception_exit(regs);
462} 474}
463 475
464/* 476/*
@@ -555,14 +567,17 @@ dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code)
555#ifdef CONFIG_X86_32 567#ifdef CONFIG_X86_32
556 ignore_fpu_irq = 1; 568 ignore_fpu_irq = 1;
557#endif 569#endif
558 570 exception_enter(regs);
559 math_error(regs, error_code, X86_TRAP_MF); 571 math_error(regs, error_code, X86_TRAP_MF);
572 exception_exit(regs);
560} 573}
561 574
562dotraplinkage void 575dotraplinkage void
563do_simd_coprocessor_error(struct pt_regs *regs, long error_code) 576do_simd_coprocessor_error(struct pt_regs *regs, long error_code)
564{ 577{
578 exception_enter(regs);
565 math_error(regs, error_code, X86_TRAP_XF); 579 math_error(regs, error_code, X86_TRAP_XF);
580 exception_exit(regs);
566} 581}
567 582
568dotraplinkage void 583dotraplinkage void
@@ -613,11 +628,12 @@ void math_state_restore(void)
613 } 628 }
614 629
615 __thread_fpu_begin(tsk); 630 __thread_fpu_begin(tsk);
631
616 /* 632 /*
617 * Paranoid restore. send a SIGSEGV if we fail to restore the state. 633 * Paranoid restore. send a SIGSEGV if we fail to restore the state.
618 */ 634 */
619 if (unlikely(restore_fpu_checking(tsk))) { 635 if (unlikely(restore_fpu_checking(tsk))) {
620 __thread_fpu_end(tsk); 636 drop_init_fpu(tsk);
621 force_sig(SIGSEGV, tsk); 637 force_sig(SIGSEGV, tsk);
622 return; 638 return;
623 } 639 }
@@ -629,6 +645,9 @@ EXPORT_SYMBOL_GPL(math_state_restore);
629dotraplinkage void __kprobes 645dotraplinkage void __kprobes
630do_device_not_available(struct pt_regs *regs, long error_code) 646do_device_not_available(struct pt_regs *regs, long error_code)
631{ 647{
648 exception_enter(regs);
649 BUG_ON(use_eager_fpu());
650
632#ifdef CONFIG_MATH_EMULATION 651#ifdef CONFIG_MATH_EMULATION
633 if (read_cr0() & X86_CR0_EM) { 652 if (read_cr0() & X86_CR0_EM) {
634 struct math_emu_info info = { }; 653 struct math_emu_info info = { };
@@ -637,6 +656,7 @@ do_device_not_available(struct pt_regs *regs, long error_code)
637 656
638 info.regs = regs; 657 info.regs = regs;
639 math_emulate(&info); 658 math_emulate(&info);
659 exception_exit(regs);
640 return; 660 return;
641 } 661 }
642#endif 662#endif
@@ -644,12 +664,15 @@ do_device_not_available(struct pt_regs *regs, long error_code)
644#ifdef CONFIG_X86_32 664#ifdef CONFIG_X86_32
645 conditional_sti(regs); 665 conditional_sti(regs);
646#endif 666#endif
667 exception_exit(regs);
647} 668}
648 669
649#ifdef CONFIG_X86_32 670#ifdef CONFIG_X86_32
650dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) 671dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
651{ 672{
652 siginfo_t info; 673 siginfo_t info;
674
675 exception_enter(regs);
653 local_irq_enable(); 676 local_irq_enable();
654 677
655 info.si_signo = SIGILL; 678 info.si_signo = SIGILL;
@@ -657,10 +680,11 @@ dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code)
657 info.si_code = ILL_BADSTK; 680 info.si_code = ILL_BADSTK;
658 info.si_addr = NULL; 681 info.si_addr = NULL;
659 if (notify_die(DIE_TRAP, "iret exception", regs, error_code, 682 if (notify_die(DIE_TRAP, "iret exception", regs, error_code,
660 X86_TRAP_IRET, SIGILL) == NOTIFY_STOP) 683 X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) {
661 return; 684 do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code,
662 do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code, 685 &info);
663 &info); 686 }
687 exception_exit(regs);
664} 688}
665#endif 689#endif
666 690
diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c
index 36fd42091fa7..9538f00827a9 100644
--- a/arch/x86/kernel/uprobes.c
+++ b/arch/x86/kernel/uprobes.c
@@ -41,6 +41,9 @@
41/* Adjust the return address of a call insn */ 41/* Adjust the return address of a call insn */
42#define UPROBE_FIX_CALL 0x2 42#define UPROBE_FIX_CALL 0x2
43 43
44/* Instruction will modify TF, don't change it */
45#define UPROBE_FIX_SETF 0x4
46
44#define UPROBE_FIX_RIP_AX 0x8000 47#define UPROBE_FIX_RIP_AX 0x8000
45#define UPROBE_FIX_RIP_CX 0x4000 48#define UPROBE_FIX_RIP_CX 0x4000
46 49
@@ -239,6 +242,10 @@ static void prepare_fixups(struct arch_uprobe *auprobe, struct insn *insn)
239 insn_get_opcode(insn); /* should be a nop */ 242 insn_get_opcode(insn); /* should be a nop */
240 243
241 switch (OPCODE1(insn)) { 244 switch (OPCODE1(insn)) {
245 case 0x9d:
246 /* popf */
247 auprobe->fixups |= UPROBE_FIX_SETF;
248 break;
242 case 0xc3: /* ret/lret */ 249 case 0xc3: /* ret/lret */
243 case 0xcb: 250 case 0xcb:
244 case 0xc2: 251 case 0xc2:
@@ -646,7 +653,7 @@ void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
646 * Skip these instructions as per the currently known x86 ISA. 653 * Skip these instructions as per the currently known x86 ISA.
647 * 0x66* { 0x90 | 0x0f 0x1f | 0x0f 0x19 | 0x87 0xc0 } 654 * 0x66* { 0x90 | 0x0f 0x1f | 0x0f 0x19 | 0x87 0xc0 }
648 */ 655 */
649bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs) 656static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
650{ 657{
651 int i; 658 int i;
652 659
@@ -673,3 +680,46 @@ bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
673 } 680 }
674 return false; 681 return false;
675} 682}
683
684bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
685{
686 bool ret = __skip_sstep(auprobe, regs);
687 if (ret && (regs->flags & X86_EFLAGS_TF))
688 send_sig(SIGTRAP, current, 0);
689 return ret;
690}
691
692void arch_uprobe_enable_step(struct arch_uprobe *auprobe)
693{
694 struct task_struct *task = current;
695 struct arch_uprobe_task *autask = &task->utask->autask;
696 struct pt_regs *regs = task_pt_regs(task);
697
698 autask->saved_tf = !!(regs->flags & X86_EFLAGS_TF);
699
700 regs->flags |= X86_EFLAGS_TF;
701 if (test_tsk_thread_flag(task, TIF_BLOCKSTEP))
702 set_task_blockstep(task, false);
703}
704
705void arch_uprobe_disable_step(struct arch_uprobe *auprobe)
706{
707 struct task_struct *task = current;
708 struct arch_uprobe_task *autask = &task->utask->autask;
709 bool trapped = (task->utask->state == UTASK_SSTEP_TRAPPED);
710 struct pt_regs *regs = task_pt_regs(task);
711 /*
712 * The state of TIF_BLOCKSTEP was not saved so we can get an extra
713 * SIGTRAP if we do not clear TF. We need to examine the opcode to
714 * make it right.
715 */
716 if (unlikely(trapped)) {
717 if (!autask->saved_tf)
718 regs->flags &= ~X86_EFLAGS_TF;
719 } else {
720 if (autask->saved_tf)
721 send_sig(SIGTRAP, task, 0);
722 else if (!(auprobe->fixups & UPROBE_FIX_SETF))
723 regs->flags &= ~X86_EFLAGS_TF;
724 }
725}
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
index 54abcc0baf23..5c9687b1bde6 100644
--- a/arch/x86/kernel/vm86_32.c
+++ b/arch/x86/kernel/vm86_32.c
@@ -561,9 +561,9 @@ int handle_vm86_trap(struct kernel_vm86_regs *regs, long error_code, int trapno)
561 if ((trapno == 3) || (trapno == 1)) { 561 if ((trapno == 3) || (trapno == 1)) {
562 KVM86->regs32->ax = VM86_TRAP + (trapno << 8); 562 KVM86->regs32->ax = VM86_TRAP + (trapno << 8);
563 /* setting this flag forces the code in entry_32.S to 563 /* setting this flag forces the code in entry_32.S to
564 call save_v86_state() and change the stack pointer 564 the path where we call save_v86_state() and change
565 to KVM86->regs32 */ 565 the stack pointer to KVM86->regs32 */
566 set_thread_flag(TIF_IRET); 566 set_thread_flag(TIF_NOTIFY_RESUME);
567 return 0; 567 return 0;
568 } 568 }
569 do_int(regs, trapno, (unsigned char __user *) (regs->pt.ss << 4), SP(regs)); 569 do_int(regs, trapno, (unsigned char __user *) (regs->pt.ss << 4), SP(regs));
diff --git a/arch/x86/kernel/vsyscall_64.c b/arch/x86/kernel/vsyscall_64.c
index 8d141b309046..3a3e8c9e280d 100644
--- a/arch/x86/kernel/vsyscall_64.c
+++ b/arch/x86/kernel/vsyscall_64.c
@@ -28,7 +28,7 @@
28#include <linux/jiffies.h> 28#include <linux/jiffies.h>
29#include <linux/sysctl.h> 29#include <linux/sysctl.h>
30#include <linux/topology.h> 30#include <linux/topology.h>
31#include <linux/clocksource.h> 31#include <linux/timekeeper_internal.h>
32#include <linux/getcpu.h> 32#include <linux/getcpu.h>
33#include <linux/cpu.h> 33#include <linux/cpu.h>
34#include <linux/smp.h> 34#include <linux/smp.h>
@@ -82,32 +82,41 @@ void update_vsyscall_tz(void)
82 vsyscall_gtod_data.sys_tz = sys_tz; 82 vsyscall_gtod_data.sys_tz = sys_tz;
83} 83}
84 84
85void update_vsyscall(struct timespec *wall_time, struct timespec *wtm, 85void update_vsyscall(struct timekeeper *tk)
86 struct clocksource *clock, u32 mult)
87{ 86{
88 struct timespec monotonic; 87 struct vsyscall_gtod_data *vdata = &vsyscall_gtod_data;
89 88
90 write_seqcount_begin(&vsyscall_gtod_data.seq); 89 write_seqcount_begin(&vdata->seq);
91 90
92 /* copy vsyscall data */ 91 /* copy vsyscall data */
93 vsyscall_gtod_data.clock.vclock_mode = clock->archdata.vclock_mode; 92 vdata->clock.vclock_mode = tk->clock->archdata.vclock_mode;
94 vsyscall_gtod_data.clock.cycle_last = clock->cycle_last; 93 vdata->clock.cycle_last = tk->clock->cycle_last;
95 vsyscall_gtod_data.clock.mask = clock->mask; 94 vdata->clock.mask = tk->clock->mask;
96 vsyscall_gtod_data.clock.mult = mult; 95 vdata->clock.mult = tk->mult;
97 vsyscall_gtod_data.clock.shift = clock->shift; 96 vdata->clock.shift = tk->shift;
98 97
99 vsyscall_gtod_data.wall_time_sec = wall_time->tv_sec; 98 vdata->wall_time_sec = tk->xtime_sec;
100 vsyscall_gtod_data.wall_time_nsec = wall_time->tv_nsec; 99 vdata->wall_time_snsec = tk->xtime_nsec;
100
101 vdata->monotonic_time_sec = tk->xtime_sec
102 + tk->wall_to_monotonic.tv_sec;
103 vdata->monotonic_time_snsec = tk->xtime_nsec
104 + (tk->wall_to_monotonic.tv_nsec
105 << tk->shift);
106 while (vdata->monotonic_time_snsec >=
107 (((u64)NSEC_PER_SEC) << tk->shift)) {
108 vdata->monotonic_time_snsec -=
109 ((u64)NSEC_PER_SEC) << tk->shift;
110 vdata->monotonic_time_sec++;
111 }
101 112
102 monotonic = timespec_add(*wall_time, *wtm); 113 vdata->wall_time_coarse.tv_sec = tk->xtime_sec;
103 vsyscall_gtod_data.monotonic_time_sec = monotonic.tv_sec; 114 vdata->wall_time_coarse.tv_nsec = (long)(tk->xtime_nsec >> tk->shift);
104 vsyscall_gtod_data.monotonic_time_nsec = monotonic.tv_nsec;
105 115
106 vsyscall_gtod_data.wall_time_coarse = __current_kernel_time(); 116 vdata->monotonic_time_coarse = timespec_add(vdata->wall_time_coarse,
107 vsyscall_gtod_data.monotonic_time_coarse = 117 tk->wall_to_monotonic);
108 timespec_add(vsyscall_gtod_data.wall_time_coarse, *wtm);
109 118
110 write_seqcount_end(&vsyscall_gtod_data.seq); 119 write_seqcount_end(&vdata->seq);
111} 120}
112 121
113static void warn_bad_vsyscall(const char *level, struct pt_regs *regs, 122static void warn_bad_vsyscall(const char *level, struct pt_regs *regs,
diff --git a/arch/x86/kernel/x8664_ksyms_64.c b/arch/x86/kernel/x8664_ksyms_64.c
index 6020f6f5927c..1330dd102950 100644
--- a/arch/x86/kernel/x8664_ksyms_64.c
+++ b/arch/x86/kernel/x8664_ksyms_64.c
@@ -13,9 +13,13 @@
13#include <asm/ftrace.h> 13#include <asm/ftrace.h>
14 14
15#ifdef CONFIG_FUNCTION_TRACER 15#ifdef CONFIG_FUNCTION_TRACER
16/* mcount is defined in assembly */ 16/* mcount and __fentry__ are defined in assembly */
17#ifdef CC_USING_FENTRY
18EXPORT_SYMBOL(__fentry__);
19#else
17EXPORT_SYMBOL(mcount); 20EXPORT_SYMBOL(mcount);
18#endif 21#endif
22#endif
19 23
20EXPORT_SYMBOL(__get_user_1); 24EXPORT_SYMBOL(__get_user_1);
21EXPORT_SYMBOL(__get_user_2); 25EXPORT_SYMBOL(__get_user_2);
diff --git a/arch/x86/kernel/xsave.c b/arch/x86/kernel/xsave.c
index 3d3e20709119..ada87a329edc 100644
--- a/arch/x86/kernel/xsave.c
+++ b/arch/x86/kernel/xsave.c
@@ -10,9 +10,7 @@
10#include <linux/compat.h> 10#include <linux/compat.h>
11#include <asm/i387.h> 11#include <asm/i387.h>
12#include <asm/fpu-internal.h> 12#include <asm/fpu-internal.h>
13#ifdef CONFIG_IA32_EMULATION 13#include <asm/sigframe.h>
14#include <asm/sigcontext32.h>
15#endif
16#include <asm/xcr.h> 14#include <asm/xcr.h>
17 15
18/* 16/*
@@ -23,13 +21,9 @@ u64 pcntxt_mask;
23/* 21/*
24 * Represents init state for the supported extended state. 22 * Represents init state for the supported extended state.
25 */ 23 */
26static struct xsave_struct *init_xstate_buf; 24struct xsave_struct *init_xstate_buf;
27
28struct _fpx_sw_bytes fx_sw_reserved;
29#ifdef CONFIG_IA32_EMULATION
30struct _fpx_sw_bytes fx_sw_reserved_ia32;
31#endif
32 25
26static struct _fpx_sw_bytes fx_sw_reserved, fx_sw_reserved_ia32;
33static unsigned int *xstate_offsets, *xstate_sizes, xstate_features; 27static unsigned int *xstate_offsets, *xstate_sizes, xstate_features;
34 28
35/* 29/*
@@ -44,9 +38,9 @@ static unsigned int *xstate_offsets, *xstate_sizes, xstate_features;
44 */ 38 */
45void __sanitize_i387_state(struct task_struct *tsk) 39void __sanitize_i387_state(struct task_struct *tsk)
46{ 40{
47 u64 xstate_bv;
48 int feature_bit = 0x2;
49 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave; 41 struct i387_fxsave_struct *fx = &tsk->thread.fpu.state->fxsave;
42 int feature_bit = 0x2;
43 u64 xstate_bv;
50 44
51 if (!fx) 45 if (!fx)
52 return; 46 return;
@@ -104,213 +98,326 @@ void __sanitize_i387_state(struct task_struct *tsk)
104 * Check for the presence of extended state information in the 98 * Check for the presence of extended state information in the
105 * user fpstate pointer in the sigcontext. 99 * user fpstate pointer in the sigcontext.
106 */ 100 */
107int check_for_xstate(struct i387_fxsave_struct __user *buf, 101static inline int check_for_xstate(struct i387_fxsave_struct __user *buf,
108 void __user *fpstate, 102 void __user *fpstate,
109 struct _fpx_sw_bytes *fx_sw_user) 103 struct _fpx_sw_bytes *fx_sw)
110{ 104{
111 int min_xstate_size = sizeof(struct i387_fxsave_struct) + 105 int min_xstate_size = sizeof(struct i387_fxsave_struct) +
112 sizeof(struct xsave_hdr_struct); 106 sizeof(struct xsave_hdr_struct);
113 unsigned int magic2; 107 unsigned int magic2;
114 int err;
115 108
116 err = __copy_from_user(fx_sw_user, &buf->sw_reserved[0], 109 if (__copy_from_user(fx_sw, &buf->sw_reserved[0], sizeof(*fx_sw)))
117 sizeof(struct _fpx_sw_bytes)); 110 return -1;
118 if (err)
119 return -EFAULT;
120 111
121 /* 112 /* Check for the first magic field and other error scenarios. */
122 * First Magic check failed. 113 if (fx_sw->magic1 != FP_XSTATE_MAGIC1 ||
123 */ 114 fx_sw->xstate_size < min_xstate_size ||
124 if (fx_sw_user->magic1 != FP_XSTATE_MAGIC1) 115 fx_sw->xstate_size > xstate_size ||
125 return -EINVAL; 116 fx_sw->xstate_size > fx_sw->extended_size)
117 return -1;
126 118
127 /* 119 /*
128 * Check for error scenarios.
129 */
130 if (fx_sw_user->xstate_size < min_xstate_size ||
131 fx_sw_user->xstate_size > xstate_size ||
132 fx_sw_user->xstate_size > fx_sw_user->extended_size)
133 return -EINVAL;
134
135 err = __get_user(magic2, (__u32 *) (((void *)fpstate) +
136 fx_sw_user->extended_size -
137 FP_XSTATE_MAGIC2_SIZE));
138 if (err)
139 return err;
140 /*
141 * Check for the presence of second magic word at the end of memory 120 * Check for the presence of second magic word at the end of memory
142 * layout. This detects the case where the user just copied the legacy 121 * layout. This detects the case where the user just copied the legacy
143 * fpstate layout with out copying the extended state information 122 * fpstate layout with out copying the extended state information
144 * in the memory layout. 123 * in the memory layout.
145 */ 124 */
146 if (magic2 != FP_XSTATE_MAGIC2) 125 if (__get_user(magic2, (__u32 __user *)(fpstate + fx_sw->xstate_size))
147 return -EFAULT; 126 || magic2 != FP_XSTATE_MAGIC2)
127 return -1;
148 128
149 return 0; 129 return 0;
150} 130}
151 131
152#ifdef CONFIG_X86_64
153/* 132/*
154 * Signal frame handlers. 133 * Signal frame handlers.
155 */ 134 */
156 135static inline int save_fsave_header(struct task_struct *tsk, void __user *buf)
157int save_i387_xstate(void __user *buf)
158{ 136{
159 struct task_struct *tsk = current; 137 if (use_fxsr()) {
160 int err = 0; 138 struct xsave_struct *xsave = &tsk->thread.fpu.state->xsave;
161 139 struct user_i387_ia32_struct env;
162 if (!access_ok(VERIFY_WRITE, buf, sig_xstate_size)) 140 struct _fpstate_ia32 __user *fp = buf;
163 return -EACCES;
164 141
165 BUG_ON(sig_xstate_size < xstate_size); 142 convert_from_fxsr(&env, tsk);
166 143
167 if ((unsigned long)buf % 64) 144 if (__copy_to_user(buf, &env, sizeof(env)) ||
168 pr_err("%s: bad fpstate %p\n", __func__, buf); 145 __put_user(xsave->i387.swd, &fp->status) ||
169 146 __put_user(X86_FXSR_MAGIC, &fp->magic))
170 if (!used_math()) 147 return -1;
171 return 0;
172
173 if (user_has_fpu()) {
174 if (use_xsave())
175 err = xsave_user(buf);
176 else
177 err = fxsave_user(buf);
178
179 if (err)
180 return err;
181 user_fpu_end();
182 } else { 148 } else {
183 sanitize_i387_state(tsk); 149 struct i387_fsave_struct __user *fp = buf;
184 if (__copy_to_user(buf, &tsk->thread.fpu.state->fxsave, 150 u32 swd;
185 xstate_size)) 151 if (__get_user(swd, &fp->swd) || __put_user(swd, &fp->status))
186 return -1; 152 return -1;
187 } 153 }
188 154
189 clear_used_math(); /* trigger finit */ 155 return 0;
156}
190 157
191 if (use_xsave()) { 158static inline int save_xstate_epilog(void __user *buf, int ia32_frame)
192 struct _fpstate __user *fx = buf; 159{
193 struct _xstate __user *x = buf; 160 struct xsave_struct __user *x = buf;
194 u64 xstate_bv; 161 struct _fpx_sw_bytes *sw_bytes;
162 u32 xstate_bv;
163 int err;
195 164
196 err = __copy_to_user(&fx->sw_reserved, &fx_sw_reserved, 165 /* Setup the bytes not touched by the [f]xsave and reserved for SW. */
197 sizeof(struct _fpx_sw_bytes)); 166 sw_bytes = ia32_frame ? &fx_sw_reserved_ia32 : &fx_sw_reserved;
167 err = __copy_to_user(&x->i387.sw_reserved, sw_bytes, sizeof(*sw_bytes));
198 168
199 err |= __put_user(FP_XSTATE_MAGIC2, 169 if (!use_xsave())
200 (__u32 __user *) (buf + sig_xstate_size 170 return err;
201 - FP_XSTATE_MAGIC2_SIZE));
202 171
203 /* 172 err |= __put_user(FP_XSTATE_MAGIC2, (__u32 *)(buf + xstate_size));
204 * Read the xstate_bv which we copied (directly from the cpu or
205 * from the state in task struct) to the user buffers and
206 * set the FP/SSE bits.
207 */
208 err |= __get_user(xstate_bv, &x->xstate_hdr.xstate_bv);
209 173
210 /* 174 /*
211 * For legacy compatible, we always set FP/SSE bits in the bit 175 * Read the xstate_bv which we copied (directly from the cpu or
212 * vector while saving the state to the user context. This will 176 * from the state in task struct) to the user buffers.
213 * enable us capturing any changes(during sigreturn) to 177 */
214 * the FP/SSE bits by the legacy applications which don't touch 178 err |= __get_user(xstate_bv, (__u32 *)&x->xsave_hdr.xstate_bv);
215 * xstate_bv in the xsave header.
216 *
217 * xsave aware apps can change the xstate_bv in the xsave
218 * header as well as change any contents in the memory layout.
219 * xrestore as part of sigreturn will capture all the changes.
220 */
221 xstate_bv |= XSTATE_FPSSE;
222 179
223 err |= __put_user(xstate_bv, &x->xstate_hdr.xstate_bv); 180 /*
181 * For legacy compatible, we always set FP/SSE bits in the bit
182 * vector while saving the state to the user context. This will
183 * enable us capturing any changes(during sigreturn) to
184 * the FP/SSE bits by the legacy applications which don't touch
185 * xstate_bv in the xsave header.
186 *
187 * xsave aware apps can change the xstate_bv in the xsave
188 * header as well as change any contents in the memory layout.
189 * xrestore as part of sigreturn will capture all the changes.
190 */
191 xstate_bv |= XSTATE_FPSSE;
224 192
225 if (err) 193 err |= __put_user(xstate_bv, (__u32 *)&x->xsave_hdr.xstate_bv);
226 return err;
227 }
228 194
229 return 1; 195 return err;
196}
197
198static inline int save_user_xstate(struct xsave_struct __user *buf)
199{
200 int err;
201
202 if (use_xsave())
203 err = xsave_user(buf);
204 else if (use_fxsr())
205 err = fxsave_user((struct i387_fxsave_struct __user *) buf);
206 else
207 err = fsave_user((struct i387_fsave_struct __user *) buf);
208
209 if (unlikely(err) && __clear_user(buf, xstate_size))
210 err = -EFAULT;
211 return err;
230} 212}
231 213
232/* 214/*
233 * Restore the extended state if present. Otherwise, restore the FP/SSE 215 * Save the fpu, extended register state to the user signal frame.
234 * state. 216 *
217 * 'buf_fx' is the 64-byte aligned pointer at which the [f|fx|x]save
218 * state is copied.
219 * 'buf' points to the 'buf_fx' or to the fsave header followed by 'buf_fx'.
220 *
221 * buf == buf_fx for 64-bit frames and 32-bit fsave frame.
222 * buf != buf_fx for 32-bit frames with fxstate.
223 *
224 * If the fpu, extended register state is live, save the state directly
225 * to the user frame pointed by the aligned pointer 'buf_fx'. Otherwise,
226 * copy the thread's fpu state to the user frame starting at 'buf_fx'.
227 *
228 * If this is a 32-bit frame with fxstate, put a fsave header before
229 * the aligned state at 'buf_fx'.
230 *
231 * For [f]xsave state, update the SW reserved fields in the [f]xsave frame
232 * indicating the absence/presence of the extended state to the user.
235 */ 233 */
236static int restore_user_xstate(void __user *buf) 234int save_xstate_sig(void __user *buf, void __user *buf_fx, int size)
237{ 235{
238 struct _fpx_sw_bytes fx_sw_user; 236 struct xsave_struct *xsave = &current->thread.fpu.state->xsave;
239 u64 mask; 237 struct task_struct *tsk = current;
240 int err; 238 int ia32_fxstate = (buf != buf_fx);
241 239
242 if (((unsigned long)buf % 64) || 240 ia32_fxstate &= (config_enabled(CONFIG_X86_32) ||
243 check_for_xstate(buf, buf, &fx_sw_user)) 241 config_enabled(CONFIG_IA32_EMULATION));
244 goto fx_only;
245 242
246 mask = fx_sw_user.xstate_bv; 243 if (!access_ok(VERIFY_WRITE, buf, size))
244 return -EACCES;
247 245
248 /* 246 if (!HAVE_HWFP)
249 * restore the state passed by the user. 247 return fpregs_soft_get(current, NULL, 0,
250 */ 248 sizeof(struct user_i387_ia32_struct), NULL,
251 err = xrestore_user(buf, mask); 249 (struct _fpstate_ia32 __user *) buf) ? -1 : 1;
252 if (err)
253 return err;
254 250
255 /* 251 if (user_has_fpu()) {
256 * init the state skipped by the user. 252 /* Save the live register state to the user directly. */
257 */ 253 if (save_user_xstate(buf_fx))
258 mask = pcntxt_mask & ~mask; 254 return -1;
259 if (unlikely(mask)) 255 /* Update the thread's fxstate to save the fsave header. */
260 xrstor_state(init_xstate_buf, mask); 256 if (ia32_fxstate)
257 fpu_fxsave(&tsk->thread.fpu);
258 } else {
259 sanitize_i387_state(tsk);
260 if (__copy_to_user(buf_fx, xsave, xstate_size))
261 return -1;
262 }
263
264 /* Save the fsave header for the 32-bit frames. */
265 if ((ia32_fxstate || !use_fxsr()) && save_fsave_header(tsk, buf))
266 return -1;
267
268 if (use_fxsr() && save_xstate_epilog(buf_fx, ia32_fxstate))
269 return -1;
270
271 drop_init_fpu(tsk); /* trigger finit */
261 272
262 return 0; 273 return 0;
274}
263 275
264fx_only: 276static inline void
265 /* 277sanitize_restored_xstate(struct task_struct *tsk,
266 * couldn't find the extended state information in the 278 struct user_i387_ia32_struct *ia32_env,
267 * memory layout. Restore just the FP/SSE and init all 279 u64 xstate_bv, int fx_only)
268 * the other extended state. 280{
269 */ 281 struct xsave_struct *xsave = &tsk->thread.fpu.state->xsave;
270 xrstor_state(init_xstate_buf, pcntxt_mask & ~XSTATE_FPSSE); 282 struct xsave_hdr_struct *xsave_hdr = &xsave->xsave_hdr;
271 return fxrstor_checking((__force struct i387_fxsave_struct *)buf); 283
284 if (use_xsave()) {
285 /* These bits must be zero. */
286 xsave_hdr->reserved1[0] = xsave_hdr->reserved1[1] = 0;
287
288 /*
289 * Init the state that is not present in the memory
290 * layout and not enabled by the OS.
291 */
292 if (fx_only)
293 xsave_hdr->xstate_bv = XSTATE_FPSSE;
294 else
295 xsave_hdr->xstate_bv &= (pcntxt_mask & xstate_bv);
296 }
297
298 if (use_fxsr()) {
299 /*
300 * mscsr reserved bits must be masked to zero for security
301 * reasons.
302 */
303 xsave->i387.mxcsr &= mxcsr_feature_mask;
304
305 convert_to_fxsr(tsk, ia32_env);
306 }
272} 307}
273 308
274/* 309/*
275 * This restores directly out of user space. Exceptions are handled. 310 * Restore the extended state if present. Otherwise, restore the FP/SSE state.
276 */ 311 */
277int restore_i387_xstate(void __user *buf) 312static inline int restore_user_xstate(void __user *buf, u64 xbv, int fx_only)
278{ 313{
314 if (use_xsave()) {
315 if ((unsigned long)buf % 64 || fx_only) {
316 u64 init_bv = pcntxt_mask & ~XSTATE_FPSSE;
317 xrstor_state(init_xstate_buf, init_bv);
318 return fxrstor_user(buf);
319 } else {
320 u64 init_bv = pcntxt_mask & ~xbv;
321 if (unlikely(init_bv))
322 xrstor_state(init_xstate_buf, init_bv);
323 return xrestore_user(buf, xbv);
324 }
325 } else if (use_fxsr()) {
326 return fxrstor_user(buf);
327 } else
328 return frstor_user(buf);
329}
330
331int __restore_xstate_sig(void __user *buf, void __user *buf_fx, int size)
332{
333 int ia32_fxstate = (buf != buf_fx);
279 struct task_struct *tsk = current; 334 struct task_struct *tsk = current;
280 int err = 0; 335 int state_size = xstate_size;
336 u64 xstate_bv = 0;
337 int fx_only = 0;
338
339 ia32_fxstate &= (config_enabled(CONFIG_X86_32) ||
340 config_enabled(CONFIG_IA32_EMULATION));
281 341
282 if (!buf) { 342 if (!buf) {
283 if (used_math()) 343 drop_init_fpu(tsk);
284 goto clear;
285 return 0; 344 return 0;
286 } else 345 }
287 if (!access_ok(VERIFY_READ, buf, sig_xstate_size))
288 return -EACCES;
289 346
290 if (!used_math()) { 347 if (!access_ok(VERIFY_READ, buf, size))
291 err = init_fpu(tsk); 348 return -EACCES;
292 if (err) 349
293 return err; 350 if (!used_math() && init_fpu(tsk))
351 return -1;
352
353 if (!HAVE_HWFP) {
354 return fpregs_soft_set(current, NULL,
355 0, sizeof(struct user_i387_ia32_struct),
356 NULL, buf) != 0;
294 } 357 }
295 358
296 user_fpu_begin(); 359 if (use_xsave()) {
297 if (use_xsave()) 360 struct _fpx_sw_bytes fx_sw_user;
298 err = restore_user_xstate(buf); 361 if (unlikely(check_for_xstate(buf_fx, buf_fx, &fx_sw_user))) {
299 else 362 /*
300 err = fxrstor_checking((__force struct i387_fxsave_struct *) 363 * Couldn't find the extended state information in the
301 buf); 364 * memory layout. Restore just the FP/SSE and init all
302 if (unlikely(err)) { 365 * the other extended state.
366 */
367 state_size = sizeof(struct i387_fxsave_struct);
368 fx_only = 1;
369 } else {
370 state_size = fx_sw_user.xstate_size;
371 xstate_bv = fx_sw_user.xstate_bv;
372 }
373 }
374
375 if (ia32_fxstate) {
376 /*
377 * For 32-bit frames with fxstate, copy the user state to the
378 * thread's fpu state, reconstruct fxstate from the fsave
379 * header. Sanitize the copied state etc.
380 */
381 struct xsave_struct *xsave = &tsk->thread.fpu.state->xsave;
382 struct user_i387_ia32_struct env;
383 int err = 0;
384
385 /*
386 * Drop the current fpu which clears used_math(). This ensures
387 * that any context-switch during the copy of the new state,
388 * avoids the intermediate state from getting restored/saved.
389 * Thus avoiding the new restored state from getting corrupted.
390 * We will be ready to restore/save the state only after
391 * set_used_math() is again set.
392 */
393 drop_fpu(tsk);
394
395 if (__copy_from_user(xsave, buf_fx, state_size) ||
396 __copy_from_user(&env, buf, sizeof(env))) {
397 err = -1;
398 } else {
399 sanitize_restored_xstate(tsk, &env, xstate_bv, fx_only);
400 set_used_math();
401 }
402
403 if (use_eager_fpu())
404 math_state_restore();
405
406 return err;
407 } else {
303 /* 408 /*
304 * Encountered an error while doing the restore from the 409 * For 64-bit frames and 32-bit fsave frames, restore the user
305 * user buffer, clear the fpu state. 410 * state to the registers directly (with exceptions handled).
306 */ 411 */
307clear: 412 user_fpu_begin();
308 clear_fpu(tsk); 413 if (restore_user_xstate(buf_fx, xstate_bv, fx_only)) {
309 clear_used_math(); 414 drop_init_fpu(tsk);
415 return -1;
416 }
310 } 417 }
311 return err; 418
419 return 0;
312} 420}
313#endif
314 421
315/* 422/*
316 * Prepare the SW reserved portion of the fxsave memory layout, indicating 423 * Prepare the SW reserved portion of the fxsave memory layout, indicating
@@ -321,31 +428,22 @@ clear:
321 */ 428 */
322static void prepare_fx_sw_frame(void) 429static void prepare_fx_sw_frame(void)
323{ 430{
324 int size_extended = (xstate_size - sizeof(struct i387_fxsave_struct)) + 431 int fsave_header_size = sizeof(struct i387_fsave_struct);
325 FP_XSTATE_MAGIC2_SIZE; 432 int size = xstate_size + FP_XSTATE_MAGIC2_SIZE;
326 433
327 sig_xstate_size = sizeof(struct _fpstate) + size_extended; 434 if (config_enabled(CONFIG_X86_32))
328 435 size += fsave_header_size;
329#ifdef CONFIG_IA32_EMULATION
330 sig_xstate_ia32_size = sizeof(struct _fpstate_ia32) + size_extended;
331#endif
332
333 memset(&fx_sw_reserved, 0, sizeof(fx_sw_reserved));
334 436
335 fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1; 437 fx_sw_reserved.magic1 = FP_XSTATE_MAGIC1;
336 fx_sw_reserved.extended_size = sig_xstate_size; 438 fx_sw_reserved.extended_size = size;
337 fx_sw_reserved.xstate_bv = pcntxt_mask; 439 fx_sw_reserved.xstate_bv = pcntxt_mask;
338 fx_sw_reserved.xstate_size = xstate_size; 440 fx_sw_reserved.xstate_size = xstate_size;
339#ifdef CONFIG_IA32_EMULATION
340 memcpy(&fx_sw_reserved_ia32, &fx_sw_reserved,
341 sizeof(struct _fpx_sw_bytes));
342 fx_sw_reserved_ia32.extended_size = sig_xstate_ia32_size;
343#endif
344}
345 441
346#ifdef CONFIG_X86_64 442 if (config_enabled(CONFIG_IA32_EMULATION)) {
347unsigned int sig_xstate_size = sizeof(struct _fpstate); 443 fx_sw_reserved_ia32 = fx_sw_reserved;
348#endif 444 fx_sw_reserved_ia32.extended_size += fsave_header_size;
445 }
446}
349 447
350/* 448/*
351 * Enable the extended processor state save/restore feature 449 * Enable the extended processor state save/restore feature
@@ -384,19 +482,21 @@ static void __init setup_xstate_features(void)
384/* 482/*
385 * setup the xstate image representing the init state 483 * setup the xstate image representing the init state
386 */ 484 */
387static void __init setup_xstate_init(void) 485static void __init setup_init_fpu_buf(void)
388{ 486{
389 setup_xstate_features();
390
391 /* 487 /*
392 * Setup init_xstate_buf to represent the init state of 488 * Setup init_xstate_buf to represent the init state of
393 * all the features managed by the xsave 489 * all the features managed by the xsave
394 */ 490 */
395 init_xstate_buf = alloc_bootmem_align(xstate_size, 491 init_xstate_buf = alloc_bootmem_align(xstate_size,
396 __alignof__(struct xsave_struct)); 492 __alignof__(struct xsave_struct));
397 init_xstate_buf->i387.mxcsr = MXCSR_DEFAULT; 493 fx_finit(&init_xstate_buf->i387);
494
495 if (!cpu_has_xsave)
496 return;
497
498 setup_xstate_features();
398 499
399 clts();
400 /* 500 /*
401 * Init all the features state with header_bv being 0x0 501 * Init all the features state with header_bv being 0x0
402 */ 502 */
@@ -406,9 +506,21 @@ static void __init setup_xstate_init(void)
406 * of any feature which is not represented by all zero's. 506 * of any feature which is not represented by all zero's.
407 */ 507 */
408 xsave_state(init_xstate_buf, -1); 508 xsave_state(init_xstate_buf, -1);
409 stts();
410} 509}
411 510
511static enum { AUTO, ENABLE, DISABLE } eagerfpu = AUTO;
512static int __init eager_fpu_setup(char *s)
513{
514 if (!strcmp(s, "on"))
515 eagerfpu = ENABLE;
516 else if (!strcmp(s, "off"))
517 eagerfpu = DISABLE;
518 else if (!strcmp(s, "auto"))
519 eagerfpu = AUTO;
520 return 1;
521}
522__setup("eagerfpu=", eager_fpu_setup);
523
412/* 524/*
413 * Enable and initialize the xsave feature. 525 * Enable and initialize the xsave feature.
414 */ 526 */
@@ -445,8 +557,11 @@ static void __init xstate_enable_boot_cpu(void)
445 557
446 update_regset_xstate_info(xstate_size, pcntxt_mask); 558 update_regset_xstate_info(xstate_size, pcntxt_mask);
447 prepare_fx_sw_frame(); 559 prepare_fx_sw_frame();
560 setup_init_fpu_buf();
448 561
449 setup_xstate_init(); 562 /* Auto enable eagerfpu for xsaveopt */
563 if (cpu_has_xsaveopt && eagerfpu != DISABLE)
564 eagerfpu = ENABLE;
450 565
451 pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x\n", 566 pr_info("enabled xstate_bv 0x%llx, cntxt size 0x%x\n",
452 pcntxt_mask, xstate_size); 567 pcntxt_mask, xstate_size);
@@ -471,3 +586,43 @@ void __cpuinit xsave_init(void)
471 next_func = xstate_enable; 586 next_func = xstate_enable;
472 this_func(); 587 this_func();
473} 588}
589
590static inline void __init eager_fpu_init_bp(void)
591{
592 current->thread.fpu.state =
593 alloc_bootmem_align(xstate_size, __alignof__(struct xsave_struct));
594 if (!init_xstate_buf)
595 setup_init_fpu_buf();
596}
597
598void __cpuinit eager_fpu_init(void)
599{
600 static __refdata void (*boot_func)(void) = eager_fpu_init_bp;
601
602 clear_used_math();
603 current_thread_info()->status = 0;
604
605 if (eagerfpu == ENABLE)
606 setup_force_cpu_cap(X86_FEATURE_EAGER_FPU);
607
608 if (!cpu_has_eager_fpu) {
609 stts();
610 return;
611 }
612
613 if (boot_func) {
614 boot_func();
615 boot_func = NULL;
616 }
617
618 /*
619 * This is same as math_state_restore(). But use_xsave() is
620 * not yet patched to use math_state_restore().
621 */
622 init_fpu(current);
623 __thread_fpu_begin(current);
624 if (cpu_has_xsave)
625 xrstor_state(init_xstate_buf, -1);
626 else
627 fxrstor_checking(&init_xstate_buf->i387);
628}
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig
index a28f338843ea..586f00059805 100644
--- a/arch/x86/kvm/Kconfig
+++ b/arch/x86/kvm/Kconfig
@@ -20,6 +20,7 @@ if VIRTUALIZATION
20config KVM 20config KVM
21 tristate "Kernel-based Virtual Machine (KVM) support" 21 tristate "Kernel-based Virtual Machine (KVM) support"
22 depends on HAVE_KVM 22 depends on HAVE_KVM
23 depends on HIGH_RES_TIMERS
23 # for device assignment: 24 # for device assignment:
24 depends on PCI 25 depends on PCI
25 # for TASKSTATS/TASK_DELAY_ACCT: 26 # for TASKSTATS/TASK_DELAY_ACCT:
@@ -37,6 +38,7 @@ config KVM
37 select TASK_DELAY_ACCT 38 select TASK_DELAY_ACCT
38 select PERF_EVENTS 39 select PERF_EVENTS
39 select HAVE_KVM_MSI 40 select HAVE_KVM_MSI
41 select HAVE_KVM_CPU_RELAX_INTERCEPT
40 ---help--- 42 ---help---
41 Support hosting fully virtualized guest machines using hardware 43 Support hosting fully virtualized guest machines using hardware
42 virtualization extensions. You will need a fairly recent 44 virtualization extensions. You will need a fairly recent
diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile
index 4f579e8dcacf..04d30401c5cb 100644
--- a/arch/x86/kvm/Makefile
+++ b/arch/x86/kvm/Makefile
@@ -12,7 +12,7 @@ kvm-$(CONFIG_IOMMU_API) += $(addprefix ../../../virt/kvm/, iommu.o)
12kvm-$(CONFIG_KVM_ASYNC_PF) += $(addprefix ../../../virt/kvm/, async_pf.o) 12kvm-$(CONFIG_KVM_ASYNC_PF) += $(addprefix ../../../virt/kvm/, async_pf.o)
13 13
14kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \ 14kvm-y += x86.o mmu.o emulate.o i8259.o irq.o lapic.o \
15 i8254.o timer.o cpuid.o pmu.o 15 i8254.o cpuid.o pmu.o
16kvm-intel-y += vmx.o 16kvm-intel-y += vmx.o
17kvm-amd-y += svm.o 17kvm-amd-y += svm.o
18 18
diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c
index 0595f1397b7c..ec79e773342e 100644
--- a/arch/x86/kvm/cpuid.c
+++ b/arch/x86/kvm/cpuid.c
@@ -316,7 +316,7 @@ static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
316 } 316 }
317 case 7: { 317 case 7: {
318 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX; 318 entry->flags |= KVM_CPUID_FLAG_SIGNIFCANT_INDEX;
319 /* Mask ebx against host capbability word 9 */ 319 /* Mask ebx against host capability word 9 */
320 if (index == 0) { 320 if (index == 0) {
321 entry->ebx &= kvm_supported_word9_x86_features; 321 entry->ebx &= kvm_supported_word9_x86_features;
322 cpuid_mask(&entry->ebx, 9); 322 cpuid_mask(&entry->ebx, 9);
@@ -397,8 +397,8 @@ static int do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function,
397 break; 397 break;
398 } 398 }
399 case KVM_CPUID_SIGNATURE: { 399 case KVM_CPUID_SIGNATURE: {
400 char signature[12] = "KVMKVMKVM\0\0"; 400 static const char signature[12] = "KVMKVMKVM\0\0";
401 u32 *sigptr = (u32 *)signature; 401 const u32 *sigptr = (const u32 *)signature;
402 entry->eax = KVM_CPUID_FEATURES; 402 entry->eax = KVM_CPUID_FEATURES;
403 entry->ebx = sigptr[0]; 403 entry->ebx = sigptr[0];
404 entry->ecx = sigptr[1]; 404 entry->ecx = sigptr[1];
@@ -484,10 +484,10 @@ struct kvm_cpuid_param {
484 u32 func; 484 u32 func;
485 u32 idx; 485 u32 idx;
486 bool has_leaf_count; 486 bool has_leaf_count;
487 bool (*qualifier)(struct kvm_cpuid_param *param); 487 bool (*qualifier)(const struct kvm_cpuid_param *param);
488}; 488};
489 489
490static bool is_centaur_cpu(struct kvm_cpuid_param *param) 490static bool is_centaur_cpu(const struct kvm_cpuid_param *param)
491{ 491{
492 return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR; 492 return boot_cpu_data.x86_vendor == X86_VENDOR_CENTAUR;
493} 493}
@@ -498,7 +498,7 @@ int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
498 struct kvm_cpuid_entry2 *cpuid_entries; 498 struct kvm_cpuid_entry2 *cpuid_entries;
499 int limit, nent = 0, r = -E2BIG, i; 499 int limit, nent = 0, r = -E2BIG, i;
500 u32 func; 500 u32 func;
501 static struct kvm_cpuid_param param[] = { 501 static const struct kvm_cpuid_param param[] = {
502 { .func = 0, .has_leaf_count = true }, 502 { .func = 0, .has_leaf_count = true },
503 { .func = 0x80000000, .has_leaf_count = true }, 503 { .func = 0x80000000, .has_leaf_count = true },
504 { .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true }, 504 { .func = 0xC0000000, .qualifier = is_centaur_cpu, .has_leaf_count = true },
@@ -517,7 +517,7 @@ int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid,
517 517
518 r = 0; 518 r = 0;
519 for (i = 0; i < ARRAY_SIZE(param); i++) { 519 for (i = 0; i < ARRAY_SIZE(param); i++) {
520 struct kvm_cpuid_param *ent = &param[i]; 520 const struct kvm_cpuid_param *ent = &param[i];
521 521
522 if (ent->qualifier && !ent->qualifier(ent)) 522 if (ent->qualifier && !ent->qualifier(ent))
523 continue; 523 continue;
diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
index a3b57a27be88..39171cb307ea 100644
--- a/arch/x86/kvm/emulate.c
+++ b/arch/x86/kvm/emulate.c
@@ -161,9 +161,9 @@ struct opcode {
161 u64 intercept : 8; 161 u64 intercept : 8;
162 union { 162 union {
163 int (*execute)(struct x86_emulate_ctxt *ctxt); 163 int (*execute)(struct x86_emulate_ctxt *ctxt);
164 struct opcode *group; 164 const struct opcode *group;
165 struct group_dual *gdual; 165 const struct group_dual *gdual;
166 struct gprefix *gprefix; 166 const struct gprefix *gprefix;
167 } u; 167 } u;
168 int (*check_perm)(struct x86_emulate_ctxt *ctxt); 168 int (*check_perm)(struct x86_emulate_ctxt *ctxt);
169}; 169};
@@ -202,6 +202,42 @@ struct gprefix {
202#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a 202#define EFLG_RESERVED_ZEROS_MASK 0xffc0802a
203#define EFLG_RESERVED_ONE_MASK 2 203#define EFLG_RESERVED_ONE_MASK 2
204 204
205static ulong reg_read(struct x86_emulate_ctxt *ctxt, unsigned nr)
206{
207 if (!(ctxt->regs_valid & (1 << nr))) {
208 ctxt->regs_valid |= 1 << nr;
209 ctxt->_regs[nr] = ctxt->ops->read_gpr(ctxt, nr);
210 }
211 return ctxt->_regs[nr];
212}
213
214static ulong *reg_write(struct x86_emulate_ctxt *ctxt, unsigned nr)
215{
216 ctxt->regs_valid |= 1 << nr;
217 ctxt->regs_dirty |= 1 << nr;
218 return &ctxt->_regs[nr];
219}
220
221static ulong *reg_rmw(struct x86_emulate_ctxt *ctxt, unsigned nr)
222{
223 reg_read(ctxt, nr);
224 return reg_write(ctxt, nr);
225}
226
227static void writeback_registers(struct x86_emulate_ctxt *ctxt)
228{
229 unsigned reg;
230
231 for_each_set_bit(reg, (ulong *)&ctxt->regs_dirty, 16)
232 ctxt->ops->write_gpr(ctxt, reg, ctxt->_regs[reg]);
233}
234
235static void invalidate_registers(struct x86_emulate_ctxt *ctxt)
236{
237 ctxt->regs_dirty = 0;
238 ctxt->regs_valid = 0;
239}
240
205/* 241/*
206 * Instruction emulation: 242 * Instruction emulation:
207 * Most instructions are emulated directly via a fragment of inline assembly 243 * Most instructions are emulated directly via a fragment of inline assembly
@@ -374,8 +410,8 @@ struct gprefix {
374#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \ 410#define __emulate_1op_rax_rdx(ctxt, _op, _suffix, _ex) \
375 do { \ 411 do { \
376 unsigned long _tmp; \ 412 unsigned long _tmp; \
377 ulong *rax = &(ctxt)->regs[VCPU_REGS_RAX]; \ 413 ulong *rax = reg_rmw((ctxt), VCPU_REGS_RAX); \
378 ulong *rdx = &(ctxt)->regs[VCPU_REGS_RDX]; \ 414 ulong *rdx = reg_rmw((ctxt), VCPU_REGS_RDX); \
379 \ 415 \
380 __asm__ __volatile__ ( \ 416 __asm__ __volatile__ ( \
381 _PRE_EFLAGS("0", "5", "1") \ 417 _PRE_EFLAGS("0", "5", "1") \
@@ -494,7 +530,7 @@ register_address_increment(struct x86_emulate_ctxt *ctxt, unsigned long *reg, in
494 530
495static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc) 531static void rsp_increment(struct x86_emulate_ctxt *ctxt, int inc)
496{ 532{
497 masked_increment(&ctxt->regs[VCPU_REGS_RSP], stack_mask(ctxt), inc); 533 masked_increment(reg_rmw(ctxt, VCPU_REGS_RSP), stack_mask(ctxt), inc);
498} 534}
499 535
500static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel) 536static inline void jmp_rel(struct x86_emulate_ctxt *ctxt, int rel)
@@ -632,8 +668,6 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
632 668
633 la = seg_base(ctxt, addr.seg) + addr.ea; 669 la = seg_base(ctxt, addr.seg) + addr.ea;
634 switch (ctxt->mode) { 670 switch (ctxt->mode) {
635 case X86EMUL_MODE_REAL:
636 break;
637 case X86EMUL_MODE_PROT64: 671 case X86EMUL_MODE_PROT64:
638 if (((signed long)la << 16) >> 16 != la) 672 if (((signed long)la << 16) >> 16 != la)
639 return emulate_gp(ctxt, 0); 673 return emulate_gp(ctxt, 0);
@@ -655,7 +689,7 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
655 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim) 689 if (addr.ea > lim || (u32)(addr.ea + size - 1) > lim)
656 goto bad; 690 goto bad;
657 } else { 691 } else {
658 /* exapand-down segment */ 692 /* expand-down segment */
659 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim) 693 if (addr.ea <= lim || (u32)(addr.ea + size - 1) <= lim)
660 goto bad; 694 goto bad;
661 lim = desc.d ? 0xffffffff : 0xffff; 695 lim = desc.d ? 0xffffffff : 0xffff;
@@ -663,7 +697,10 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
663 goto bad; 697 goto bad;
664 } 698 }
665 cpl = ctxt->ops->cpl(ctxt); 699 cpl = ctxt->ops->cpl(ctxt);
666 rpl = sel & 3; 700 if (ctxt->mode == X86EMUL_MODE_REAL)
701 rpl = 0;
702 else
703 rpl = sel & 3;
667 cpl = max(cpl, rpl); 704 cpl = max(cpl, rpl);
668 if (!(desc.type & 8)) { 705 if (!(desc.type & 8)) {
669 /* data segment */ 706 /* data segment */
@@ -688,9 +725,9 @@ static int __linearize(struct x86_emulate_ctxt *ctxt,
688 return X86EMUL_CONTINUE; 725 return X86EMUL_CONTINUE;
689bad: 726bad:
690 if (addr.seg == VCPU_SREG_SS) 727 if (addr.seg == VCPU_SREG_SS)
691 return emulate_ss(ctxt, addr.seg); 728 return emulate_ss(ctxt, sel);
692 else 729 else
693 return emulate_gp(ctxt, addr.seg); 730 return emulate_gp(ctxt, sel);
694} 731}
695 732
696static int linearize(struct x86_emulate_ctxt *ctxt, 733static int linearize(struct x86_emulate_ctxt *ctxt,
@@ -786,14 +823,15 @@ static int do_insn_fetch(struct x86_emulate_ctxt *ctxt,
786 * pointer into the block that addresses the relevant register. 823 * pointer into the block that addresses the relevant register.
787 * @highbyte_regs specifies whether to decode AH,CH,DH,BH. 824 * @highbyte_regs specifies whether to decode AH,CH,DH,BH.
788 */ 825 */
789static void *decode_register(u8 modrm_reg, unsigned long *regs, 826static void *decode_register(struct x86_emulate_ctxt *ctxt, u8 modrm_reg,
790 int highbyte_regs) 827 int highbyte_regs)
791{ 828{
792 void *p; 829 void *p;
793 830
794 p = &regs[modrm_reg];
795 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8) 831 if (highbyte_regs && modrm_reg >= 4 && modrm_reg < 8)
796 p = (unsigned char *)&regs[modrm_reg & 3] + 1; 832 p = (unsigned char *)reg_rmw(ctxt, modrm_reg & 3) + 1;
833 else
834 p = reg_rmw(ctxt, modrm_reg);
797 return p; 835 return p;
798} 836}
799 837
@@ -871,23 +909,23 @@ static void read_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data, int reg)
871{ 909{
872 ctxt->ops->get_fpu(ctxt); 910 ctxt->ops->get_fpu(ctxt);
873 switch (reg) { 911 switch (reg) {
874 case 0: asm("movdqu %%xmm0, %0" : "=m"(*data)); break; 912 case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break;
875 case 1: asm("movdqu %%xmm1, %0" : "=m"(*data)); break; 913 case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break;
876 case 2: asm("movdqu %%xmm2, %0" : "=m"(*data)); break; 914 case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break;
877 case 3: asm("movdqu %%xmm3, %0" : "=m"(*data)); break; 915 case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break;
878 case 4: asm("movdqu %%xmm4, %0" : "=m"(*data)); break; 916 case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break;
879 case 5: asm("movdqu %%xmm5, %0" : "=m"(*data)); break; 917 case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break;
880 case 6: asm("movdqu %%xmm6, %0" : "=m"(*data)); break; 918 case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break;
881 case 7: asm("movdqu %%xmm7, %0" : "=m"(*data)); break; 919 case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break;
882#ifdef CONFIG_X86_64 920#ifdef CONFIG_X86_64
883 case 8: asm("movdqu %%xmm8, %0" : "=m"(*data)); break; 921 case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break;
884 case 9: asm("movdqu %%xmm9, %0" : "=m"(*data)); break; 922 case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break;
885 case 10: asm("movdqu %%xmm10, %0" : "=m"(*data)); break; 923 case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break;
886 case 11: asm("movdqu %%xmm11, %0" : "=m"(*data)); break; 924 case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break;
887 case 12: asm("movdqu %%xmm12, %0" : "=m"(*data)); break; 925 case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break;
888 case 13: asm("movdqu %%xmm13, %0" : "=m"(*data)); break; 926 case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break;
889 case 14: asm("movdqu %%xmm14, %0" : "=m"(*data)); break; 927 case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break;
890 case 15: asm("movdqu %%xmm15, %0" : "=m"(*data)); break; 928 case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break;
891#endif 929#endif
892 default: BUG(); 930 default: BUG();
893 } 931 }
@@ -899,23 +937,23 @@ static void write_sse_reg(struct x86_emulate_ctxt *ctxt, sse128_t *data,
899{ 937{
900 ctxt->ops->get_fpu(ctxt); 938 ctxt->ops->get_fpu(ctxt);
901 switch (reg) { 939 switch (reg) {
902 case 0: asm("movdqu %0, %%xmm0" : : "m"(*data)); break; 940 case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break;
903 case 1: asm("movdqu %0, %%xmm1" : : "m"(*data)); break; 941 case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break;
904 case 2: asm("movdqu %0, %%xmm2" : : "m"(*data)); break; 942 case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break;
905 case 3: asm("movdqu %0, %%xmm3" : : "m"(*data)); break; 943 case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break;
906 case 4: asm("movdqu %0, %%xmm4" : : "m"(*data)); break; 944 case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break;
907 case 5: asm("movdqu %0, %%xmm5" : : "m"(*data)); break; 945 case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break;
908 case 6: asm("movdqu %0, %%xmm6" : : "m"(*data)); break; 946 case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break;
909 case 7: asm("movdqu %0, %%xmm7" : : "m"(*data)); break; 947 case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break;
910#ifdef CONFIG_X86_64 948#ifdef CONFIG_X86_64
911 case 8: asm("movdqu %0, %%xmm8" : : "m"(*data)); break; 949 case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break;
912 case 9: asm("movdqu %0, %%xmm9" : : "m"(*data)); break; 950 case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break;
913 case 10: asm("movdqu %0, %%xmm10" : : "m"(*data)); break; 951 case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break;
914 case 11: asm("movdqu %0, %%xmm11" : : "m"(*data)); break; 952 case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break;
915 case 12: asm("movdqu %0, %%xmm12" : : "m"(*data)); break; 953 case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break;
916 case 13: asm("movdqu %0, %%xmm13" : : "m"(*data)); break; 954 case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break;
917 case 14: asm("movdqu %0, %%xmm14" : : "m"(*data)); break; 955 case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break;
918 case 15: asm("movdqu %0, %%xmm15" : : "m"(*data)); break; 956 case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break;
919#endif 957#endif
920 default: BUG(); 958 default: BUG();
921 } 959 }
@@ -982,10 +1020,10 @@ static void decode_register_operand(struct x86_emulate_ctxt *ctxt,
982 1020
983 op->type = OP_REG; 1021 op->type = OP_REG;
984 if (ctxt->d & ByteOp) { 1022 if (ctxt->d & ByteOp) {
985 op->addr.reg = decode_register(reg, ctxt->regs, highbyte_regs); 1023 op->addr.reg = decode_register(ctxt, reg, highbyte_regs);
986 op->bytes = 1; 1024 op->bytes = 1;
987 } else { 1025 } else {
988 op->addr.reg = decode_register(reg, ctxt->regs, 0); 1026 op->addr.reg = decode_register(ctxt, reg, 0);
989 op->bytes = ctxt->op_bytes; 1027 op->bytes = ctxt->op_bytes;
990 } 1028 }
991 fetch_register_operand(op); 1029 fetch_register_operand(op);
@@ -1020,8 +1058,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1020 if (ctxt->modrm_mod == 3) { 1058 if (ctxt->modrm_mod == 3) {
1021 op->type = OP_REG; 1059 op->type = OP_REG;
1022 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; 1060 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
1023 op->addr.reg = decode_register(ctxt->modrm_rm, 1061 op->addr.reg = decode_register(ctxt, ctxt->modrm_rm, ctxt->d & ByteOp);
1024 ctxt->regs, ctxt->d & ByteOp);
1025 if (ctxt->d & Sse) { 1062 if (ctxt->d & Sse) {
1026 op->type = OP_XMM; 1063 op->type = OP_XMM;
1027 op->bytes = 16; 1064 op->bytes = 16;
@@ -1042,10 +1079,10 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1042 op->type = OP_MEM; 1079 op->type = OP_MEM;
1043 1080
1044 if (ctxt->ad_bytes == 2) { 1081 if (ctxt->ad_bytes == 2) {
1045 unsigned bx = ctxt->regs[VCPU_REGS_RBX]; 1082 unsigned bx = reg_read(ctxt, VCPU_REGS_RBX);
1046 unsigned bp = ctxt->regs[VCPU_REGS_RBP]; 1083 unsigned bp = reg_read(ctxt, VCPU_REGS_RBP);
1047 unsigned si = ctxt->regs[VCPU_REGS_RSI]; 1084 unsigned si = reg_read(ctxt, VCPU_REGS_RSI);
1048 unsigned di = ctxt->regs[VCPU_REGS_RDI]; 1085 unsigned di = reg_read(ctxt, VCPU_REGS_RDI);
1049 1086
1050 /* 16-bit ModR/M decode. */ 1087 /* 16-bit ModR/M decode. */
1051 switch (ctxt->modrm_mod) { 1088 switch (ctxt->modrm_mod) {
@@ -1102,17 +1139,17 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt,
1102 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0) 1139 if ((base_reg & 7) == 5 && ctxt->modrm_mod == 0)
1103 modrm_ea += insn_fetch(s32, ctxt); 1140 modrm_ea += insn_fetch(s32, ctxt);
1104 else { 1141 else {
1105 modrm_ea += ctxt->regs[base_reg]; 1142 modrm_ea += reg_read(ctxt, base_reg);
1106 adjust_modrm_seg(ctxt, base_reg); 1143 adjust_modrm_seg(ctxt, base_reg);
1107 } 1144 }
1108 if (index_reg != 4) 1145 if (index_reg != 4)
1109 modrm_ea += ctxt->regs[index_reg] << scale; 1146 modrm_ea += reg_read(ctxt, index_reg) << scale;
1110 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) { 1147 } else if ((ctxt->modrm_rm & 7) == 5 && ctxt->modrm_mod == 0) {
1111 if (ctxt->mode == X86EMUL_MODE_PROT64) 1148 if (ctxt->mode == X86EMUL_MODE_PROT64)
1112 ctxt->rip_relative = 1; 1149 ctxt->rip_relative = 1;
1113 } else { 1150 } else {
1114 base_reg = ctxt->modrm_rm; 1151 base_reg = ctxt->modrm_rm;
1115 modrm_ea += ctxt->regs[base_reg]; 1152 modrm_ea += reg_read(ctxt, base_reg);
1116 adjust_modrm_seg(ctxt, base_reg); 1153 adjust_modrm_seg(ctxt, base_reg);
1117 } 1154 }
1118 switch (ctxt->modrm_mod) { 1155 switch (ctxt->modrm_mod) {
@@ -1179,24 +1216,21 @@ static int read_emulated(struct x86_emulate_ctxt *ctxt,
1179 int rc; 1216 int rc;
1180 struct read_cache *mc = &ctxt->mem_read; 1217 struct read_cache *mc = &ctxt->mem_read;
1181 1218
1182 while (size) { 1219 if (mc->pos < mc->end)
1183 int n = min(size, 8u); 1220 goto read_cached;
1184 size -= n;
1185 if (mc->pos < mc->end)
1186 goto read_cached;
1187 1221
1188 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, n, 1222 WARN_ON((mc->end + size) >= sizeof(mc->data));
1189 &ctxt->exception);
1190 if (rc != X86EMUL_CONTINUE)
1191 return rc;
1192 mc->end += n;
1193 1223
1194 read_cached: 1224 rc = ctxt->ops->read_emulated(ctxt, addr, mc->data + mc->end, size,
1195 memcpy(dest, mc->data + mc->pos, n); 1225 &ctxt->exception);
1196 mc->pos += n; 1226 if (rc != X86EMUL_CONTINUE)
1197 dest += n; 1227 return rc;
1198 addr += n; 1228
1199 } 1229 mc->end += size;
1230
1231read_cached:
1232 memcpy(dest, mc->data + mc->pos, size);
1233 mc->pos += size;
1200 return X86EMUL_CONTINUE; 1234 return X86EMUL_CONTINUE;
1201} 1235}
1202 1236
@@ -1253,10 +1287,10 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1253 if (rc->pos == rc->end) { /* refill pio read ahead */ 1287 if (rc->pos == rc->end) { /* refill pio read ahead */
1254 unsigned int in_page, n; 1288 unsigned int in_page, n;
1255 unsigned int count = ctxt->rep_prefix ? 1289 unsigned int count = ctxt->rep_prefix ?
1256 address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) : 1; 1290 address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) : 1;
1257 in_page = (ctxt->eflags & EFLG_DF) ? 1291 in_page = (ctxt->eflags & EFLG_DF) ?
1258 offset_in_page(ctxt->regs[VCPU_REGS_RDI]) : 1292 offset_in_page(reg_read(ctxt, VCPU_REGS_RDI)) :
1259 PAGE_SIZE - offset_in_page(ctxt->regs[VCPU_REGS_RDI]); 1293 PAGE_SIZE - offset_in_page(reg_read(ctxt, VCPU_REGS_RDI));
1260 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size, 1294 n = min(min(in_page, (unsigned int)sizeof(rc->data)) / size,
1261 count); 1295 count);
1262 if (n == 0) 1296 if (n == 0)
@@ -1267,8 +1301,15 @@ static int pio_in_emulated(struct x86_emulate_ctxt *ctxt,
1267 rc->end = n * size; 1301 rc->end = n * size;
1268 } 1302 }
1269 1303
1270 memcpy(dest, rc->data + rc->pos, size); 1304 if (ctxt->rep_prefix && !(ctxt->eflags & EFLG_DF)) {
1271 rc->pos += size; 1305 ctxt->dst.data = rc->data + rc->pos;
1306 ctxt->dst.type = OP_MEM_STR;
1307 ctxt->dst.count = (rc->end - rc->pos) / size;
1308 rc->pos = rc->end;
1309 } else {
1310 memcpy(dest, rc->data + rc->pos, size);
1311 rc->pos += size;
1312 }
1272 return 1; 1313 return 1;
1273} 1314}
1274 1315
@@ -1291,7 +1332,7 @@ static int read_interrupt_descriptor(struct x86_emulate_ctxt *ctxt,
1291static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt, 1332static void get_descriptor_table_ptr(struct x86_emulate_ctxt *ctxt,
1292 u16 selector, struct desc_ptr *dt) 1333 u16 selector, struct desc_ptr *dt)
1293{ 1334{
1294 struct x86_emulate_ops *ops = ctxt->ops; 1335 const struct x86_emulate_ops *ops = ctxt->ops;
1295 1336
1296 if (selector & 1 << 2) { 1337 if (selector & 1 << 2) {
1297 struct desc_struct desc; 1338 struct desc_struct desc;
@@ -1355,19 +1396,15 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1355 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */ 1396 bool null_selector = !(selector & ~0x3); /* 0000-0003 are null */
1356 ulong desc_addr; 1397 ulong desc_addr;
1357 int ret; 1398 int ret;
1399 u16 dummy;
1358 1400
1359 memset(&seg_desc, 0, sizeof seg_desc); 1401 memset(&seg_desc, 0, sizeof seg_desc);
1360 1402
1361 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86) 1403 if ((seg <= VCPU_SREG_GS && ctxt->mode == X86EMUL_MODE_VM86)
1362 || ctxt->mode == X86EMUL_MODE_REAL) { 1404 || ctxt->mode == X86EMUL_MODE_REAL) {
1363 /* set real mode segment descriptor */ 1405 /* set real mode segment descriptor */
1406 ctxt->ops->get_segment(ctxt, &dummy, &seg_desc, NULL, seg);
1364 set_desc_base(&seg_desc, selector << 4); 1407 set_desc_base(&seg_desc, selector << 4);
1365 set_desc_limit(&seg_desc, 0xffff);
1366 seg_desc.type = 3;
1367 seg_desc.p = 1;
1368 seg_desc.s = 1;
1369 if (ctxt->mode == X86EMUL_MODE_VM86)
1370 seg_desc.dpl = 3;
1371 goto load; 1408 goto load;
1372 } 1409 }
1373 1410
@@ -1396,7 +1433,7 @@ static int load_segment_descriptor(struct x86_emulate_ctxt *ctxt,
1396 err_code = selector & 0xfffc; 1433 err_code = selector & 0xfffc;
1397 err_vec = GP_VECTOR; 1434 err_vec = GP_VECTOR;
1398 1435
1399 /* can't load system descriptor into segment selecor */ 1436 /* can't load system descriptor into segment selector */
1400 if (seg <= VCPU_SREG_GS && !seg_desc.s) 1437 if (seg <= VCPU_SREG_GS && !seg_desc.s)
1401 goto exception; 1438 goto exception;
1402 1439
@@ -1516,6 +1553,14 @@ static int writeback(struct x86_emulate_ctxt *ctxt)
1516 if (rc != X86EMUL_CONTINUE) 1553 if (rc != X86EMUL_CONTINUE)
1517 return rc; 1554 return rc;
1518 break; 1555 break;
1556 case OP_MEM_STR:
1557 rc = segmented_write(ctxt,
1558 ctxt->dst.addr.mem,
1559 ctxt->dst.data,
1560 ctxt->dst.bytes * ctxt->dst.count);
1561 if (rc != X86EMUL_CONTINUE)
1562 return rc;
1563 break;
1519 case OP_XMM: 1564 case OP_XMM:
1520 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm); 1565 write_sse_reg(ctxt, &ctxt->dst.vec_val, ctxt->dst.addr.xmm);
1521 break; 1566 break;
@@ -1536,7 +1581,7 @@ static int push(struct x86_emulate_ctxt *ctxt, void *data, int bytes)
1536 struct segmented_address addr; 1581 struct segmented_address addr;
1537 1582
1538 rsp_increment(ctxt, -bytes); 1583 rsp_increment(ctxt, -bytes);
1539 addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt); 1584 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1540 addr.seg = VCPU_SREG_SS; 1585 addr.seg = VCPU_SREG_SS;
1541 1586
1542 return segmented_write(ctxt, addr, data, bytes); 1587 return segmented_write(ctxt, addr, data, bytes);
@@ -1555,7 +1600,7 @@ static int emulate_pop(struct x86_emulate_ctxt *ctxt,
1555 int rc; 1600 int rc;
1556 struct segmented_address addr; 1601 struct segmented_address addr;
1557 1602
1558 addr.ea = ctxt->regs[VCPU_REGS_RSP] & stack_mask(ctxt); 1603 addr.ea = reg_read(ctxt, VCPU_REGS_RSP) & stack_mask(ctxt);
1559 addr.seg = VCPU_SREG_SS; 1604 addr.seg = VCPU_SREG_SS;
1560 rc = segmented_read(ctxt, addr, dest, len); 1605 rc = segmented_read(ctxt, addr, dest, len);
1561 if (rc != X86EMUL_CONTINUE) 1606 if (rc != X86EMUL_CONTINUE)
@@ -1623,26 +1668,28 @@ static int em_enter(struct x86_emulate_ctxt *ctxt)
1623 int rc; 1668 int rc;
1624 unsigned frame_size = ctxt->src.val; 1669 unsigned frame_size = ctxt->src.val;
1625 unsigned nesting_level = ctxt->src2.val & 31; 1670 unsigned nesting_level = ctxt->src2.val & 31;
1671 ulong rbp;
1626 1672
1627 if (nesting_level) 1673 if (nesting_level)
1628 return X86EMUL_UNHANDLEABLE; 1674 return X86EMUL_UNHANDLEABLE;
1629 1675
1630 rc = push(ctxt, &ctxt->regs[VCPU_REGS_RBP], stack_size(ctxt)); 1676 rbp = reg_read(ctxt, VCPU_REGS_RBP);
1677 rc = push(ctxt, &rbp, stack_size(ctxt));
1631 if (rc != X86EMUL_CONTINUE) 1678 if (rc != X86EMUL_CONTINUE)
1632 return rc; 1679 return rc;
1633 assign_masked(&ctxt->regs[VCPU_REGS_RBP], ctxt->regs[VCPU_REGS_RSP], 1680 assign_masked(reg_rmw(ctxt, VCPU_REGS_RBP), reg_read(ctxt, VCPU_REGS_RSP),
1634 stack_mask(ctxt)); 1681 stack_mask(ctxt));
1635 assign_masked(&ctxt->regs[VCPU_REGS_RSP], 1682 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP),
1636 ctxt->regs[VCPU_REGS_RSP] - frame_size, 1683 reg_read(ctxt, VCPU_REGS_RSP) - frame_size,
1637 stack_mask(ctxt)); 1684 stack_mask(ctxt));
1638 return X86EMUL_CONTINUE; 1685 return X86EMUL_CONTINUE;
1639} 1686}
1640 1687
1641static int em_leave(struct x86_emulate_ctxt *ctxt) 1688static int em_leave(struct x86_emulate_ctxt *ctxt)
1642{ 1689{
1643 assign_masked(&ctxt->regs[VCPU_REGS_RSP], ctxt->regs[VCPU_REGS_RBP], 1690 assign_masked(reg_rmw(ctxt, VCPU_REGS_RSP), reg_read(ctxt, VCPU_REGS_RBP),
1644 stack_mask(ctxt)); 1691 stack_mask(ctxt));
1645 return emulate_pop(ctxt, &ctxt->regs[VCPU_REGS_RBP], ctxt->op_bytes); 1692 return emulate_pop(ctxt, reg_rmw(ctxt, VCPU_REGS_RBP), ctxt->op_bytes);
1646} 1693}
1647 1694
1648static int em_push_sreg(struct x86_emulate_ctxt *ctxt) 1695static int em_push_sreg(struct x86_emulate_ctxt *ctxt)
@@ -1670,13 +1717,13 @@ static int em_pop_sreg(struct x86_emulate_ctxt *ctxt)
1670 1717
1671static int em_pusha(struct x86_emulate_ctxt *ctxt) 1718static int em_pusha(struct x86_emulate_ctxt *ctxt)
1672{ 1719{
1673 unsigned long old_esp = ctxt->regs[VCPU_REGS_RSP]; 1720 unsigned long old_esp = reg_read(ctxt, VCPU_REGS_RSP);
1674 int rc = X86EMUL_CONTINUE; 1721 int rc = X86EMUL_CONTINUE;
1675 int reg = VCPU_REGS_RAX; 1722 int reg = VCPU_REGS_RAX;
1676 1723
1677 while (reg <= VCPU_REGS_RDI) { 1724 while (reg <= VCPU_REGS_RDI) {
1678 (reg == VCPU_REGS_RSP) ? 1725 (reg == VCPU_REGS_RSP) ?
1679 (ctxt->src.val = old_esp) : (ctxt->src.val = ctxt->regs[reg]); 1726 (ctxt->src.val = old_esp) : (ctxt->src.val = reg_read(ctxt, reg));
1680 1727
1681 rc = em_push(ctxt); 1728 rc = em_push(ctxt);
1682 if (rc != X86EMUL_CONTINUE) 1729 if (rc != X86EMUL_CONTINUE)
@@ -1705,7 +1752,7 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
1705 --reg; 1752 --reg;
1706 } 1753 }
1707 1754
1708 rc = emulate_pop(ctxt, &ctxt->regs[reg], ctxt->op_bytes); 1755 rc = emulate_pop(ctxt, reg_rmw(ctxt, reg), ctxt->op_bytes);
1709 if (rc != X86EMUL_CONTINUE) 1756 if (rc != X86EMUL_CONTINUE)
1710 break; 1757 break;
1711 --reg; 1758 --reg;
@@ -1713,9 +1760,9 @@ static int em_popa(struct x86_emulate_ctxt *ctxt)
1713 return rc; 1760 return rc;
1714} 1761}
1715 1762
1716int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq) 1763static int __emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1717{ 1764{
1718 struct x86_emulate_ops *ops = ctxt->ops; 1765 const struct x86_emulate_ops *ops = ctxt->ops;
1719 int rc; 1766 int rc;
1720 struct desc_ptr dt; 1767 struct desc_ptr dt;
1721 gva_t cs_addr; 1768 gva_t cs_addr;
@@ -1762,11 +1809,22 @@ int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1762 return rc; 1809 return rc;
1763} 1810}
1764 1811
1812int emulate_int_real(struct x86_emulate_ctxt *ctxt, int irq)
1813{
1814 int rc;
1815
1816 invalidate_registers(ctxt);
1817 rc = __emulate_int_real(ctxt, irq);
1818 if (rc == X86EMUL_CONTINUE)
1819 writeback_registers(ctxt);
1820 return rc;
1821}
1822
1765static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq) 1823static int emulate_int(struct x86_emulate_ctxt *ctxt, int irq)
1766{ 1824{
1767 switch(ctxt->mode) { 1825 switch(ctxt->mode) {
1768 case X86EMUL_MODE_REAL: 1826 case X86EMUL_MODE_REAL:
1769 return emulate_int_real(ctxt, irq); 1827 return __emulate_int_real(ctxt, irq);
1770 case X86EMUL_MODE_VM86: 1828 case X86EMUL_MODE_VM86:
1771 case X86EMUL_MODE_PROT16: 1829 case X86EMUL_MODE_PROT16:
1772 case X86EMUL_MODE_PROT32: 1830 case X86EMUL_MODE_PROT32:
@@ -1973,14 +2031,14 @@ static int em_cmpxchg8b(struct x86_emulate_ctxt *ctxt)
1973{ 2031{
1974 u64 old = ctxt->dst.orig_val64; 2032 u64 old = ctxt->dst.orig_val64;
1975 2033
1976 if (((u32) (old >> 0) != (u32) ctxt->regs[VCPU_REGS_RAX]) || 2034 if (((u32) (old >> 0) != (u32) reg_read(ctxt, VCPU_REGS_RAX)) ||
1977 ((u32) (old >> 32) != (u32) ctxt->regs[VCPU_REGS_RDX])) { 2035 ((u32) (old >> 32) != (u32) reg_read(ctxt, VCPU_REGS_RDX))) {
1978 ctxt->regs[VCPU_REGS_RAX] = (u32) (old >> 0); 2036 *reg_write(ctxt, VCPU_REGS_RAX) = (u32) (old >> 0);
1979 ctxt->regs[VCPU_REGS_RDX] = (u32) (old >> 32); 2037 *reg_write(ctxt, VCPU_REGS_RDX) = (u32) (old >> 32);
1980 ctxt->eflags &= ~EFLG_ZF; 2038 ctxt->eflags &= ~EFLG_ZF;
1981 } else { 2039 } else {
1982 ctxt->dst.val64 = ((u64)ctxt->regs[VCPU_REGS_RCX] << 32) | 2040 ctxt->dst.val64 = ((u64)reg_read(ctxt, VCPU_REGS_RCX) << 32) |
1983 (u32) ctxt->regs[VCPU_REGS_RBX]; 2041 (u32) reg_read(ctxt, VCPU_REGS_RBX);
1984 2042
1985 ctxt->eflags |= EFLG_ZF; 2043 ctxt->eflags |= EFLG_ZF;
1986 } 2044 }
@@ -2016,7 +2074,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2016{ 2074{
2017 /* Save real source value, then compare EAX against destination. */ 2075 /* Save real source value, then compare EAX against destination. */
2018 ctxt->src.orig_val = ctxt->src.val; 2076 ctxt->src.orig_val = ctxt->src.val;
2019 ctxt->src.val = ctxt->regs[VCPU_REGS_RAX]; 2077 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RAX);
2020 emulate_2op_SrcV(ctxt, "cmp"); 2078 emulate_2op_SrcV(ctxt, "cmp");
2021 2079
2022 if (ctxt->eflags & EFLG_ZF) { 2080 if (ctxt->eflags & EFLG_ZF) {
@@ -2025,7 +2083,7 @@ static int em_cmpxchg(struct x86_emulate_ctxt *ctxt)
2025 } else { 2083 } else {
2026 /* Failure: write the value we saw to EAX. */ 2084 /* Failure: write the value we saw to EAX. */
2027 ctxt->dst.type = OP_REG; 2085 ctxt->dst.type = OP_REG;
2028 ctxt->dst.addr.reg = (unsigned long *)&ctxt->regs[VCPU_REGS_RAX]; 2086 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
2029 } 2087 }
2030 return X86EMUL_CONTINUE; 2088 return X86EMUL_CONTINUE;
2031} 2089}
@@ -2050,12 +2108,6 @@ static void
2050setup_syscalls_segments(struct x86_emulate_ctxt *ctxt, 2108setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2051 struct desc_struct *cs, struct desc_struct *ss) 2109 struct desc_struct *cs, struct desc_struct *ss)
2052{ 2110{
2053 u16 selector;
2054
2055 memset(cs, 0, sizeof(struct desc_struct));
2056 ctxt->ops->get_segment(ctxt, &selector, cs, NULL, VCPU_SREG_CS);
2057 memset(ss, 0, sizeof(struct desc_struct));
2058
2059 cs->l = 0; /* will be adjusted later */ 2111 cs->l = 0; /* will be adjusted later */
2060 set_desc_base(cs, 0); /* flat segment */ 2112 set_desc_base(cs, 0); /* flat segment */
2061 cs->g = 1; /* 4kb granularity */ 2113 cs->g = 1; /* 4kb granularity */
@@ -2065,6 +2117,7 @@ setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2065 cs->dpl = 0; /* will be adjusted later */ 2117 cs->dpl = 0; /* will be adjusted later */
2066 cs->p = 1; 2118 cs->p = 1;
2067 cs->d = 1; 2119 cs->d = 1;
2120 cs->avl = 0;
2068 2121
2069 set_desc_base(ss, 0); /* flat segment */ 2122 set_desc_base(ss, 0); /* flat segment */
2070 set_desc_limit(ss, 0xfffff); /* 4GB limit */ 2123 set_desc_limit(ss, 0xfffff); /* 4GB limit */
@@ -2074,6 +2127,8 @@ setup_syscalls_segments(struct x86_emulate_ctxt *ctxt,
2074 ss->d = 1; /* 32bit stack segment */ 2127 ss->d = 1; /* 32bit stack segment */
2075 ss->dpl = 0; 2128 ss->dpl = 0;
2076 ss->p = 1; 2129 ss->p = 1;
2130 ss->l = 0;
2131 ss->avl = 0;
2077} 2132}
2078 2133
2079static bool vendor_intel(struct x86_emulate_ctxt *ctxt) 2134static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
@@ -2089,7 +2144,7 @@ static bool vendor_intel(struct x86_emulate_ctxt *ctxt)
2089 2144
2090static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt) 2145static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2091{ 2146{
2092 struct x86_emulate_ops *ops = ctxt->ops; 2147 const struct x86_emulate_ops *ops = ctxt->ops;
2093 u32 eax, ebx, ecx, edx; 2148 u32 eax, ebx, ecx, edx;
2094 2149
2095 /* 2150 /*
@@ -2133,7 +2188,7 @@ static bool em_syscall_is_enabled(struct x86_emulate_ctxt *ctxt)
2133 2188
2134static int em_syscall(struct x86_emulate_ctxt *ctxt) 2189static int em_syscall(struct x86_emulate_ctxt *ctxt)
2135{ 2190{
2136 struct x86_emulate_ops *ops = ctxt->ops; 2191 const struct x86_emulate_ops *ops = ctxt->ops;
2137 struct desc_struct cs, ss; 2192 struct desc_struct cs, ss;
2138 u64 msr_data; 2193 u64 msr_data;
2139 u16 cs_sel, ss_sel; 2194 u16 cs_sel, ss_sel;
@@ -2165,10 +2220,10 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
2165 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); 2220 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2166 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); 2221 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2167 2222
2168 ctxt->regs[VCPU_REGS_RCX] = ctxt->_eip; 2223 *reg_write(ctxt, VCPU_REGS_RCX) = ctxt->_eip;
2169 if (efer & EFER_LMA) { 2224 if (efer & EFER_LMA) {
2170#ifdef CONFIG_X86_64 2225#ifdef CONFIG_X86_64
2171 ctxt->regs[VCPU_REGS_R11] = ctxt->eflags & ~EFLG_RF; 2226 *reg_write(ctxt, VCPU_REGS_R11) = ctxt->eflags & ~EFLG_RF;
2172 2227
2173 ops->get_msr(ctxt, 2228 ops->get_msr(ctxt,
2174 ctxt->mode == X86EMUL_MODE_PROT64 ? 2229 ctxt->mode == X86EMUL_MODE_PROT64 ?
@@ -2191,7 +2246,7 @@ static int em_syscall(struct x86_emulate_ctxt *ctxt)
2191 2246
2192static int em_sysenter(struct x86_emulate_ctxt *ctxt) 2247static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2193{ 2248{
2194 struct x86_emulate_ops *ops = ctxt->ops; 2249 const struct x86_emulate_ops *ops = ctxt->ops;
2195 struct desc_struct cs, ss; 2250 struct desc_struct cs, ss;
2196 u64 msr_data; 2251 u64 msr_data;
2197 u16 cs_sel, ss_sel; 2252 u16 cs_sel, ss_sel;
@@ -2228,6 +2283,8 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2228 if (msr_data == 0x0) 2283 if (msr_data == 0x0)
2229 return emulate_gp(ctxt, 0); 2284 return emulate_gp(ctxt, 0);
2230 break; 2285 break;
2286 default:
2287 break;
2231 } 2288 }
2232 2289
2233 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF); 2290 ctxt->eflags &= ~(EFLG_VM | EFLG_IF | EFLG_RF);
@@ -2247,14 +2304,14 @@ static int em_sysenter(struct x86_emulate_ctxt *ctxt)
2247 ctxt->_eip = msr_data; 2304 ctxt->_eip = msr_data;
2248 2305
2249 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data); 2306 ops->get_msr(ctxt, MSR_IA32_SYSENTER_ESP, &msr_data);
2250 ctxt->regs[VCPU_REGS_RSP] = msr_data; 2307 *reg_write(ctxt, VCPU_REGS_RSP) = msr_data;
2251 2308
2252 return X86EMUL_CONTINUE; 2309 return X86EMUL_CONTINUE;
2253} 2310}
2254 2311
2255static int em_sysexit(struct x86_emulate_ctxt *ctxt) 2312static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2256{ 2313{
2257 struct x86_emulate_ops *ops = ctxt->ops; 2314 const struct x86_emulate_ops *ops = ctxt->ops;
2258 struct desc_struct cs, ss; 2315 struct desc_struct cs, ss;
2259 u64 msr_data; 2316 u64 msr_data;
2260 int usermode; 2317 int usermode;
@@ -2297,8 +2354,8 @@ static int em_sysexit(struct x86_emulate_ctxt *ctxt)
2297 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS); 2354 ops->set_segment(ctxt, cs_sel, &cs, 0, VCPU_SREG_CS);
2298 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS); 2355 ops->set_segment(ctxt, ss_sel, &ss, 0, VCPU_SREG_SS);
2299 2356
2300 ctxt->_eip = ctxt->regs[VCPU_REGS_RDX]; 2357 ctxt->_eip = reg_read(ctxt, VCPU_REGS_RDX);
2301 ctxt->regs[VCPU_REGS_RSP] = ctxt->regs[VCPU_REGS_RCX]; 2358 *reg_write(ctxt, VCPU_REGS_RSP) = reg_read(ctxt, VCPU_REGS_RCX);
2302 2359
2303 return X86EMUL_CONTINUE; 2360 return X86EMUL_CONTINUE;
2304} 2361}
@@ -2317,7 +2374,7 @@ static bool emulator_bad_iopl(struct x86_emulate_ctxt *ctxt)
2317static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt, 2374static bool emulator_io_port_access_allowed(struct x86_emulate_ctxt *ctxt,
2318 u16 port, u16 len) 2375 u16 port, u16 len)
2319{ 2376{
2320 struct x86_emulate_ops *ops = ctxt->ops; 2377 const struct x86_emulate_ops *ops = ctxt->ops;
2321 struct desc_struct tr_seg; 2378 struct desc_struct tr_seg;
2322 u32 base3; 2379 u32 base3;
2323 int r; 2380 int r;
@@ -2367,14 +2424,14 @@ static void save_state_to_tss16(struct x86_emulate_ctxt *ctxt,
2367{ 2424{
2368 tss->ip = ctxt->_eip; 2425 tss->ip = ctxt->_eip;
2369 tss->flag = ctxt->eflags; 2426 tss->flag = ctxt->eflags;
2370 tss->ax = ctxt->regs[VCPU_REGS_RAX]; 2427 tss->ax = reg_read(ctxt, VCPU_REGS_RAX);
2371 tss->cx = ctxt->regs[VCPU_REGS_RCX]; 2428 tss->cx = reg_read(ctxt, VCPU_REGS_RCX);
2372 tss->dx = ctxt->regs[VCPU_REGS_RDX]; 2429 tss->dx = reg_read(ctxt, VCPU_REGS_RDX);
2373 tss->bx = ctxt->regs[VCPU_REGS_RBX]; 2430 tss->bx = reg_read(ctxt, VCPU_REGS_RBX);
2374 tss->sp = ctxt->regs[VCPU_REGS_RSP]; 2431 tss->sp = reg_read(ctxt, VCPU_REGS_RSP);
2375 tss->bp = ctxt->regs[VCPU_REGS_RBP]; 2432 tss->bp = reg_read(ctxt, VCPU_REGS_RBP);
2376 tss->si = ctxt->regs[VCPU_REGS_RSI]; 2433 tss->si = reg_read(ctxt, VCPU_REGS_RSI);
2377 tss->di = ctxt->regs[VCPU_REGS_RDI]; 2434 tss->di = reg_read(ctxt, VCPU_REGS_RDI);
2378 2435
2379 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); 2436 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2380 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); 2437 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
@@ -2390,14 +2447,14 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2390 2447
2391 ctxt->_eip = tss->ip; 2448 ctxt->_eip = tss->ip;
2392 ctxt->eflags = tss->flag | 2; 2449 ctxt->eflags = tss->flag | 2;
2393 ctxt->regs[VCPU_REGS_RAX] = tss->ax; 2450 *reg_write(ctxt, VCPU_REGS_RAX) = tss->ax;
2394 ctxt->regs[VCPU_REGS_RCX] = tss->cx; 2451 *reg_write(ctxt, VCPU_REGS_RCX) = tss->cx;
2395 ctxt->regs[VCPU_REGS_RDX] = tss->dx; 2452 *reg_write(ctxt, VCPU_REGS_RDX) = tss->dx;
2396 ctxt->regs[VCPU_REGS_RBX] = tss->bx; 2453 *reg_write(ctxt, VCPU_REGS_RBX) = tss->bx;
2397 ctxt->regs[VCPU_REGS_RSP] = tss->sp; 2454 *reg_write(ctxt, VCPU_REGS_RSP) = tss->sp;
2398 ctxt->regs[VCPU_REGS_RBP] = tss->bp; 2455 *reg_write(ctxt, VCPU_REGS_RBP) = tss->bp;
2399 ctxt->regs[VCPU_REGS_RSI] = tss->si; 2456 *reg_write(ctxt, VCPU_REGS_RSI) = tss->si;
2400 ctxt->regs[VCPU_REGS_RDI] = tss->di; 2457 *reg_write(ctxt, VCPU_REGS_RDI) = tss->di;
2401 2458
2402 /* 2459 /*
2403 * SDM says that segment selectors are loaded before segment 2460 * SDM says that segment selectors are loaded before segment
@@ -2410,7 +2467,7 @@ static int load_state_from_tss16(struct x86_emulate_ctxt *ctxt,
2410 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS); 2467 set_segment_selector(ctxt, tss->ds, VCPU_SREG_DS);
2411 2468
2412 /* 2469 /*
2413 * Now load segment descriptors. If fault happenes at this stage 2470 * Now load segment descriptors. If fault happens at this stage
2414 * it is handled in a context of new task 2471 * it is handled in a context of new task
2415 */ 2472 */
2416 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR); 2473 ret = load_segment_descriptor(ctxt, tss->ldt, VCPU_SREG_LDTR);
@@ -2436,7 +2493,7 @@ static int task_switch_16(struct x86_emulate_ctxt *ctxt,
2436 u16 tss_selector, u16 old_tss_sel, 2493 u16 tss_selector, u16 old_tss_sel,
2437 ulong old_tss_base, struct desc_struct *new_desc) 2494 ulong old_tss_base, struct desc_struct *new_desc)
2438{ 2495{
2439 struct x86_emulate_ops *ops = ctxt->ops; 2496 const struct x86_emulate_ops *ops = ctxt->ops;
2440 struct tss_segment_16 tss_seg; 2497 struct tss_segment_16 tss_seg;
2441 int ret; 2498 int ret;
2442 u32 new_tss_base = get_desc_base(new_desc); 2499 u32 new_tss_base = get_desc_base(new_desc);
@@ -2482,14 +2539,14 @@ static void save_state_to_tss32(struct x86_emulate_ctxt *ctxt,
2482 tss->cr3 = ctxt->ops->get_cr(ctxt, 3); 2539 tss->cr3 = ctxt->ops->get_cr(ctxt, 3);
2483 tss->eip = ctxt->_eip; 2540 tss->eip = ctxt->_eip;
2484 tss->eflags = ctxt->eflags; 2541 tss->eflags = ctxt->eflags;
2485 tss->eax = ctxt->regs[VCPU_REGS_RAX]; 2542 tss->eax = reg_read(ctxt, VCPU_REGS_RAX);
2486 tss->ecx = ctxt->regs[VCPU_REGS_RCX]; 2543 tss->ecx = reg_read(ctxt, VCPU_REGS_RCX);
2487 tss->edx = ctxt->regs[VCPU_REGS_RDX]; 2544 tss->edx = reg_read(ctxt, VCPU_REGS_RDX);
2488 tss->ebx = ctxt->regs[VCPU_REGS_RBX]; 2545 tss->ebx = reg_read(ctxt, VCPU_REGS_RBX);
2489 tss->esp = ctxt->regs[VCPU_REGS_RSP]; 2546 tss->esp = reg_read(ctxt, VCPU_REGS_RSP);
2490 tss->ebp = ctxt->regs[VCPU_REGS_RBP]; 2547 tss->ebp = reg_read(ctxt, VCPU_REGS_RBP);
2491 tss->esi = ctxt->regs[VCPU_REGS_RSI]; 2548 tss->esi = reg_read(ctxt, VCPU_REGS_RSI);
2492 tss->edi = ctxt->regs[VCPU_REGS_RDI]; 2549 tss->edi = reg_read(ctxt, VCPU_REGS_RDI);
2493 2550
2494 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES); 2551 tss->es = get_segment_selector(ctxt, VCPU_SREG_ES);
2495 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS); 2552 tss->cs = get_segment_selector(ctxt, VCPU_SREG_CS);
@@ -2511,14 +2568,14 @@ static int load_state_from_tss32(struct x86_emulate_ctxt *ctxt,
2511 ctxt->eflags = tss->eflags | 2; 2568 ctxt->eflags = tss->eflags | 2;
2512 2569
2513 /* General purpose registers */ 2570 /* General purpose registers */
2514 ctxt->regs[VCPU_REGS_RAX] = tss->eax; 2571 *reg_write(ctxt, VCPU_REGS_RAX) = tss->eax;
2515 ctxt->regs[VCPU_REGS_RCX] = tss->ecx; 2572 *reg_write(ctxt, VCPU_REGS_RCX) = tss->ecx;
2516 ctxt->regs[VCPU_REGS_RDX] = tss->edx; 2573 *reg_write(ctxt, VCPU_REGS_RDX) = tss->edx;
2517 ctxt->regs[VCPU_REGS_RBX] = tss->ebx; 2574 *reg_write(ctxt, VCPU_REGS_RBX) = tss->ebx;
2518 ctxt->regs[VCPU_REGS_RSP] = tss->esp; 2575 *reg_write(ctxt, VCPU_REGS_RSP) = tss->esp;
2519 ctxt->regs[VCPU_REGS_RBP] = tss->ebp; 2576 *reg_write(ctxt, VCPU_REGS_RBP) = tss->ebp;
2520 ctxt->regs[VCPU_REGS_RSI] = tss->esi; 2577 *reg_write(ctxt, VCPU_REGS_RSI) = tss->esi;
2521 ctxt->regs[VCPU_REGS_RDI] = tss->edi; 2578 *reg_write(ctxt, VCPU_REGS_RDI) = tss->edi;
2522 2579
2523 /* 2580 /*
2524 * SDM says that segment selectors are loaded before segment 2581 * SDM says that segment selectors are loaded before segment
@@ -2583,7 +2640,7 @@ static int task_switch_32(struct x86_emulate_ctxt *ctxt,
2583 u16 tss_selector, u16 old_tss_sel, 2640 u16 tss_selector, u16 old_tss_sel,
2584 ulong old_tss_base, struct desc_struct *new_desc) 2641 ulong old_tss_base, struct desc_struct *new_desc)
2585{ 2642{
2586 struct x86_emulate_ops *ops = ctxt->ops; 2643 const struct x86_emulate_ops *ops = ctxt->ops;
2587 struct tss_segment_32 tss_seg; 2644 struct tss_segment_32 tss_seg;
2588 int ret; 2645 int ret;
2589 u32 new_tss_base = get_desc_base(new_desc); 2646 u32 new_tss_base = get_desc_base(new_desc);
@@ -2627,7 +2684,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2627 u16 tss_selector, int idt_index, int reason, 2684 u16 tss_selector, int idt_index, int reason,
2628 bool has_error_code, u32 error_code) 2685 bool has_error_code, u32 error_code)
2629{ 2686{
2630 struct x86_emulate_ops *ops = ctxt->ops; 2687 const struct x86_emulate_ops *ops = ctxt->ops;
2631 struct desc_struct curr_tss_desc, next_tss_desc; 2688 struct desc_struct curr_tss_desc, next_tss_desc;
2632 int ret; 2689 int ret;
2633 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR); 2690 u16 old_tss_sel = get_segment_selector(ctxt, VCPU_SREG_TR);
@@ -2652,7 +2709,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2652 * 2709 *
2653 * 1. jmp/call/int to task gate: Check against DPL of the task gate 2710 * 1. jmp/call/int to task gate: Check against DPL of the task gate
2654 * 2. Exception/IRQ/iret: No check is performed 2711 * 2. Exception/IRQ/iret: No check is performed
2655 * 3. jmp/call to TSS: Check agains DPL of the TSS 2712 * 3. jmp/call to TSS: Check against DPL of the TSS
2656 */ 2713 */
2657 if (reason == TASK_SWITCH_GATE) { 2714 if (reason == TASK_SWITCH_GATE) {
2658 if (idt_index != -1) { 2715 if (idt_index != -1) {
@@ -2693,7 +2750,7 @@ static int emulator_do_task_switch(struct x86_emulate_ctxt *ctxt,
2693 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT; 2750 ctxt->eflags = ctxt->eflags & ~X86_EFLAGS_NT;
2694 2751
2695 /* set back link to prev task only if NT bit is set in eflags 2752 /* set back link to prev task only if NT bit is set in eflags
2696 note that old_tss_sel is not used afetr this point */ 2753 note that old_tss_sel is not used after this point */
2697 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE) 2754 if (reason != TASK_SWITCH_CALL && reason != TASK_SWITCH_GATE)
2698 old_tss_sel = 0xffff; 2755 old_tss_sel = 0xffff;
2699 2756
@@ -2733,26 +2790,28 @@ int emulator_task_switch(struct x86_emulate_ctxt *ctxt,
2733{ 2790{
2734 int rc; 2791 int rc;
2735 2792
2793 invalidate_registers(ctxt);
2736 ctxt->_eip = ctxt->eip; 2794 ctxt->_eip = ctxt->eip;
2737 ctxt->dst.type = OP_NONE; 2795 ctxt->dst.type = OP_NONE;
2738 2796
2739 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason, 2797 rc = emulator_do_task_switch(ctxt, tss_selector, idt_index, reason,
2740 has_error_code, error_code); 2798 has_error_code, error_code);
2741 2799
2742 if (rc == X86EMUL_CONTINUE) 2800 if (rc == X86EMUL_CONTINUE) {
2743 ctxt->eip = ctxt->_eip; 2801 ctxt->eip = ctxt->_eip;
2802 writeback_registers(ctxt);
2803 }
2744 2804
2745 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; 2805 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
2746} 2806}
2747 2807
2748static void string_addr_inc(struct x86_emulate_ctxt *ctxt, unsigned seg, 2808static void string_addr_inc(struct x86_emulate_ctxt *ctxt, int reg,
2749 int reg, struct operand *op) 2809 struct operand *op)
2750{ 2810{
2751 int df = (ctxt->eflags & EFLG_DF) ? -1 : 1; 2811 int df = (ctxt->eflags & EFLG_DF) ? -op->count : op->count;
2752 2812
2753 register_address_increment(ctxt, &ctxt->regs[reg], df * op->bytes); 2813 register_address_increment(ctxt, reg_rmw(ctxt, reg), df * op->bytes);
2754 op->addr.mem.ea = register_address(ctxt, ctxt->regs[reg]); 2814 op->addr.mem.ea = register_address(ctxt, reg_read(ctxt, reg));
2755 op->addr.mem.seg = seg;
2756} 2815}
2757 2816
2758static int em_das(struct x86_emulate_ctxt *ctxt) 2817static int em_das(struct x86_emulate_ctxt *ctxt)
@@ -2927,7 +2986,7 @@ static int em_cwd(struct x86_emulate_ctxt *ctxt)
2927{ 2986{
2928 ctxt->dst.type = OP_REG; 2987 ctxt->dst.type = OP_REG;
2929 ctxt->dst.bytes = ctxt->src.bytes; 2988 ctxt->dst.bytes = ctxt->src.bytes;
2930 ctxt->dst.addr.reg = &ctxt->regs[VCPU_REGS_RDX]; 2989 ctxt->dst.addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
2931 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1); 2990 ctxt->dst.val = ~((ctxt->src.val >> (ctxt->src.bytes * 8 - 1)) - 1);
2932 2991
2933 return X86EMUL_CONTINUE; 2992 return X86EMUL_CONTINUE;
@@ -2938,8 +2997,8 @@ static int em_rdtsc(struct x86_emulate_ctxt *ctxt)
2938 u64 tsc = 0; 2997 u64 tsc = 0;
2939 2998
2940 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc); 2999 ctxt->ops->get_msr(ctxt, MSR_IA32_TSC, &tsc);
2941 ctxt->regs[VCPU_REGS_RAX] = (u32)tsc; 3000 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)tsc;
2942 ctxt->regs[VCPU_REGS_RDX] = tsc >> 32; 3001 *reg_write(ctxt, VCPU_REGS_RDX) = tsc >> 32;
2943 return X86EMUL_CONTINUE; 3002 return X86EMUL_CONTINUE;
2944} 3003}
2945 3004
@@ -2947,10 +3006,10 @@ static int em_rdpmc(struct x86_emulate_ctxt *ctxt)
2947{ 3006{
2948 u64 pmc; 3007 u64 pmc;
2949 3008
2950 if (ctxt->ops->read_pmc(ctxt, ctxt->regs[VCPU_REGS_RCX], &pmc)) 3009 if (ctxt->ops->read_pmc(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &pmc))
2951 return emulate_gp(ctxt, 0); 3010 return emulate_gp(ctxt, 0);
2952 ctxt->regs[VCPU_REGS_RAX] = (u32)pmc; 3011 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)pmc;
2953 ctxt->regs[VCPU_REGS_RDX] = pmc >> 32; 3012 *reg_write(ctxt, VCPU_REGS_RDX) = pmc >> 32;
2954 return X86EMUL_CONTINUE; 3013 return X86EMUL_CONTINUE;
2955} 3014}
2956 3015
@@ -2992,9 +3051,9 @@ static int em_wrmsr(struct x86_emulate_ctxt *ctxt)
2992{ 3051{
2993 u64 msr_data; 3052 u64 msr_data;
2994 3053
2995 msr_data = (u32)ctxt->regs[VCPU_REGS_RAX] 3054 msr_data = (u32)reg_read(ctxt, VCPU_REGS_RAX)
2996 | ((u64)ctxt->regs[VCPU_REGS_RDX] << 32); 3055 | ((u64)reg_read(ctxt, VCPU_REGS_RDX) << 32);
2997 if (ctxt->ops->set_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], msr_data)) 3056 if (ctxt->ops->set_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), msr_data))
2998 return emulate_gp(ctxt, 0); 3057 return emulate_gp(ctxt, 0);
2999 3058
3000 return X86EMUL_CONTINUE; 3059 return X86EMUL_CONTINUE;
@@ -3004,11 +3063,11 @@ static int em_rdmsr(struct x86_emulate_ctxt *ctxt)
3004{ 3063{
3005 u64 msr_data; 3064 u64 msr_data;
3006 3065
3007 if (ctxt->ops->get_msr(ctxt, ctxt->regs[VCPU_REGS_RCX], &msr_data)) 3066 if (ctxt->ops->get_msr(ctxt, reg_read(ctxt, VCPU_REGS_RCX), &msr_data))
3008 return emulate_gp(ctxt, 0); 3067 return emulate_gp(ctxt, 0);
3009 3068
3010 ctxt->regs[VCPU_REGS_RAX] = (u32)msr_data; 3069 *reg_write(ctxt, VCPU_REGS_RAX) = (u32)msr_data;
3011 ctxt->regs[VCPU_REGS_RDX] = msr_data >> 32; 3070 *reg_write(ctxt, VCPU_REGS_RDX) = msr_data >> 32;
3012 return X86EMUL_CONTINUE; 3071 return X86EMUL_CONTINUE;
3013} 3072}
3014 3073
@@ -3188,8 +3247,8 @@ static int em_lmsw(struct x86_emulate_ctxt *ctxt)
3188 3247
3189static int em_loop(struct x86_emulate_ctxt *ctxt) 3248static int em_loop(struct x86_emulate_ctxt *ctxt)
3190{ 3249{
3191 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1); 3250 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX), -1);
3192 if ((address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) != 0) && 3251 if ((address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) != 0) &&
3193 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags))) 3252 (ctxt->b == 0xe2 || test_cc(ctxt->b ^ 0x5, ctxt->eflags)))
3194 jmp_rel(ctxt, ctxt->src.val); 3253 jmp_rel(ctxt, ctxt->src.val);
3195 3254
@@ -3198,7 +3257,7 @@ static int em_loop(struct x86_emulate_ctxt *ctxt)
3198 3257
3199static int em_jcxz(struct x86_emulate_ctxt *ctxt) 3258static int em_jcxz(struct x86_emulate_ctxt *ctxt)
3200{ 3259{
3201 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) 3260 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0)
3202 jmp_rel(ctxt, ctxt->src.val); 3261 jmp_rel(ctxt, ctxt->src.val);
3203 3262
3204 return X86EMUL_CONTINUE; 3263 return X86EMUL_CONTINUE;
@@ -3286,20 +3345,20 @@ static int em_cpuid(struct x86_emulate_ctxt *ctxt)
3286{ 3345{
3287 u32 eax, ebx, ecx, edx; 3346 u32 eax, ebx, ecx, edx;
3288 3347
3289 eax = ctxt->regs[VCPU_REGS_RAX]; 3348 eax = reg_read(ctxt, VCPU_REGS_RAX);
3290 ecx = ctxt->regs[VCPU_REGS_RCX]; 3349 ecx = reg_read(ctxt, VCPU_REGS_RCX);
3291 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx); 3350 ctxt->ops->get_cpuid(ctxt, &eax, &ebx, &ecx, &edx);
3292 ctxt->regs[VCPU_REGS_RAX] = eax; 3351 *reg_write(ctxt, VCPU_REGS_RAX) = eax;
3293 ctxt->regs[VCPU_REGS_RBX] = ebx; 3352 *reg_write(ctxt, VCPU_REGS_RBX) = ebx;
3294 ctxt->regs[VCPU_REGS_RCX] = ecx; 3353 *reg_write(ctxt, VCPU_REGS_RCX) = ecx;
3295 ctxt->regs[VCPU_REGS_RDX] = edx; 3354 *reg_write(ctxt, VCPU_REGS_RDX) = edx;
3296 return X86EMUL_CONTINUE; 3355 return X86EMUL_CONTINUE;
3297} 3356}
3298 3357
3299static int em_lahf(struct x86_emulate_ctxt *ctxt) 3358static int em_lahf(struct x86_emulate_ctxt *ctxt)
3300{ 3359{
3301 ctxt->regs[VCPU_REGS_RAX] &= ~0xff00UL; 3360 *reg_rmw(ctxt, VCPU_REGS_RAX) &= ~0xff00UL;
3302 ctxt->regs[VCPU_REGS_RAX] |= (ctxt->eflags & 0xff) << 8; 3361 *reg_rmw(ctxt, VCPU_REGS_RAX) |= (ctxt->eflags & 0xff) << 8;
3303 return X86EMUL_CONTINUE; 3362 return X86EMUL_CONTINUE;
3304} 3363}
3305 3364
@@ -3456,7 +3515,7 @@ static int check_svme(struct x86_emulate_ctxt *ctxt)
3456 3515
3457static int check_svme_pa(struct x86_emulate_ctxt *ctxt) 3516static int check_svme_pa(struct x86_emulate_ctxt *ctxt)
3458{ 3517{
3459 u64 rax = ctxt->regs[VCPU_REGS_RAX]; 3518 u64 rax = reg_read(ctxt, VCPU_REGS_RAX);
3460 3519
3461 /* Valid physical address? */ 3520 /* Valid physical address? */
3462 if (rax & 0xffff000000000000ULL) 3521 if (rax & 0xffff000000000000ULL)
@@ -3478,7 +3537,7 @@ static int check_rdtsc(struct x86_emulate_ctxt *ctxt)
3478static int check_rdpmc(struct x86_emulate_ctxt *ctxt) 3537static int check_rdpmc(struct x86_emulate_ctxt *ctxt)
3479{ 3538{
3480 u64 cr4 = ctxt->ops->get_cr(ctxt, 4); 3539 u64 cr4 = ctxt->ops->get_cr(ctxt, 4);
3481 u64 rcx = ctxt->regs[VCPU_REGS_RCX]; 3540 u64 rcx = reg_read(ctxt, VCPU_REGS_RCX);
3482 3541
3483 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) || 3542 if ((!(cr4 & X86_CR4_PCE) && ctxt->ops->cpl(ctxt)) ||
3484 (rcx > 3)) 3543 (rcx > 3))
@@ -3531,13 +3590,13 @@ static int check_perm_out(struct x86_emulate_ctxt *ctxt)
3531 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \ 3590 I2bv(((_f) | DstReg | SrcMem | ModRM) & ~Lock, _e), \
3532 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e) 3591 I2bv(((_f) & ~Lock) | DstAcc | SrcImm, _e)
3533 3592
3534static struct opcode group7_rm1[] = { 3593static const struct opcode group7_rm1[] = {
3535 DI(SrcNone | Priv, monitor), 3594 DI(SrcNone | Priv, monitor),
3536 DI(SrcNone | Priv, mwait), 3595 DI(SrcNone | Priv, mwait),
3537 N, N, N, N, N, N, 3596 N, N, N, N, N, N,
3538}; 3597};
3539 3598
3540static struct opcode group7_rm3[] = { 3599static const struct opcode group7_rm3[] = {
3541 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa), 3600 DIP(SrcNone | Prot | Priv, vmrun, check_svme_pa),
3542 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall), 3601 II(SrcNone | Prot | VendorSpecific, em_vmmcall, vmmcall),
3543 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa), 3602 DIP(SrcNone | Prot | Priv, vmload, check_svme_pa),
@@ -3548,13 +3607,13 @@ static struct opcode group7_rm3[] = {
3548 DIP(SrcNone | Prot | Priv, invlpga, check_svme), 3607 DIP(SrcNone | Prot | Priv, invlpga, check_svme),
3549}; 3608};
3550 3609
3551static struct opcode group7_rm7[] = { 3610static const struct opcode group7_rm7[] = {
3552 N, 3611 N,
3553 DIP(SrcNone, rdtscp, check_rdtsc), 3612 DIP(SrcNone, rdtscp, check_rdtsc),
3554 N, N, N, N, N, N, 3613 N, N, N, N, N, N,
3555}; 3614};
3556 3615
3557static struct opcode group1[] = { 3616static const struct opcode group1[] = {
3558 I(Lock, em_add), 3617 I(Lock, em_add),
3559 I(Lock | PageTable, em_or), 3618 I(Lock | PageTable, em_or),
3560 I(Lock, em_adc), 3619 I(Lock, em_adc),
@@ -3565,11 +3624,11 @@ static struct opcode group1[] = {
3565 I(0, em_cmp), 3624 I(0, em_cmp),
3566}; 3625};
3567 3626
3568static struct opcode group1A[] = { 3627static const struct opcode group1A[] = {
3569 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N, 3628 I(DstMem | SrcNone | Mov | Stack, em_pop), N, N, N, N, N, N, N,
3570}; 3629};
3571 3630
3572static struct opcode group3[] = { 3631static const struct opcode group3[] = {
3573 I(DstMem | SrcImm, em_test), 3632 I(DstMem | SrcImm, em_test),
3574 I(DstMem | SrcImm, em_test), 3633 I(DstMem | SrcImm, em_test),
3575 I(DstMem | SrcNone | Lock, em_not), 3634 I(DstMem | SrcNone | Lock, em_not),
@@ -3580,13 +3639,13 @@ static struct opcode group3[] = {
3580 I(SrcMem, em_idiv_ex), 3639 I(SrcMem, em_idiv_ex),
3581}; 3640};
3582 3641
3583static struct opcode group4[] = { 3642static const struct opcode group4[] = {
3584 I(ByteOp | DstMem | SrcNone | Lock, em_grp45), 3643 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3585 I(ByteOp | DstMem | SrcNone | Lock, em_grp45), 3644 I(ByteOp | DstMem | SrcNone | Lock, em_grp45),
3586 N, N, N, N, N, N, 3645 N, N, N, N, N, N,
3587}; 3646};
3588 3647
3589static struct opcode group5[] = { 3648static const struct opcode group5[] = {
3590 I(DstMem | SrcNone | Lock, em_grp45), 3649 I(DstMem | SrcNone | Lock, em_grp45),
3591 I(DstMem | SrcNone | Lock, em_grp45), 3650 I(DstMem | SrcNone | Lock, em_grp45),
3592 I(SrcMem | Stack, em_grp45), 3651 I(SrcMem | Stack, em_grp45),
@@ -3596,7 +3655,7 @@ static struct opcode group5[] = {
3596 I(SrcMem | Stack, em_grp45), N, 3655 I(SrcMem | Stack, em_grp45), N,
3597}; 3656};
3598 3657
3599static struct opcode group6[] = { 3658static const struct opcode group6[] = {
3600 DI(Prot, sldt), 3659 DI(Prot, sldt),
3601 DI(Prot, str), 3660 DI(Prot, str),
3602 II(Prot | Priv | SrcMem16, em_lldt, lldt), 3661 II(Prot | Priv | SrcMem16, em_lldt, lldt),
@@ -3604,7 +3663,7 @@ static struct opcode group6[] = {
3604 N, N, N, N, 3663 N, N, N, N,
3605}; 3664};
3606 3665
3607static struct group_dual group7 = { { 3666static const struct group_dual group7 = { {
3608 II(Mov | DstMem | Priv, em_sgdt, sgdt), 3667 II(Mov | DstMem | Priv, em_sgdt, sgdt),
3609 II(Mov | DstMem | Priv, em_sidt, sidt), 3668 II(Mov | DstMem | Priv, em_sidt, sidt),
3610 II(SrcMem | Priv, em_lgdt, lgdt), 3669 II(SrcMem | Priv, em_lgdt, lgdt),
@@ -3621,7 +3680,7 @@ static struct group_dual group7 = { {
3621 EXT(0, group7_rm7), 3680 EXT(0, group7_rm7),
3622} }; 3681} };
3623 3682
3624static struct opcode group8[] = { 3683static const struct opcode group8[] = {
3625 N, N, N, N, 3684 N, N, N, N,
3626 I(DstMem | SrcImmByte, em_bt), 3685 I(DstMem | SrcImmByte, em_bt),
3627 I(DstMem | SrcImmByte | Lock | PageTable, em_bts), 3686 I(DstMem | SrcImmByte | Lock | PageTable, em_bts),
@@ -3629,26 +3688,26 @@ static struct opcode group8[] = {
3629 I(DstMem | SrcImmByte | Lock | PageTable, em_btc), 3688 I(DstMem | SrcImmByte | Lock | PageTable, em_btc),
3630}; 3689};
3631 3690
3632static struct group_dual group9 = { { 3691static const struct group_dual group9 = { {
3633 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N, 3692 N, I(DstMem64 | Lock | PageTable, em_cmpxchg8b), N, N, N, N, N, N,
3634}, { 3693}, {
3635 N, N, N, N, N, N, N, N, 3694 N, N, N, N, N, N, N, N,
3636} }; 3695} };
3637 3696
3638static struct opcode group11[] = { 3697static const struct opcode group11[] = {
3639 I(DstMem | SrcImm | Mov | PageTable, em_mov), 3698 I(DstMem | SrcImm | Mov | PageTable, em_mov),
3640 X7(D(Undefined)), 3699 X7(D(Undefined)),
3641}; 3700};
3642 3701
3643static struct gprefix pfx_0f_6f_0f_7f = { 3702static const struct gprefix pfx_0f_6f_0f_7f = {
3644 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov), 3703 I(Mmx, em_mov), I(Sse | Aligned, em_mov), N, I(Sse | Unaligned, em_mov),
3645}; 3704};
3646 3705
3647static struct gprefix pfx_vmovntpx = { 3706static const struct gprefix pfx_vmovntpx = {
3648 I(0, em_mov), N, N, N, 3707 I(0, em_mov), N, N, N,
3649}; 3708};
3650 3709
3651static struct opcode opcode_table[256] = { 3710static const struct opcode opcode_table[256] = {
3652 /* 0x00 - 0x07 */ 3711 /* 0x00 - 0x07 */
3653 I6ALU(Lock, em_add), 3712 I6ALU(Lock, em_add),
3654 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg), 3713 I(ImplicitOps | Stack | No64 | Src2ES, em_push_sreg),
@@ -3689,7 +3748,7 @@ static struct opcode opcode_table[256] = {
3689 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op), 3748 I(DstReg | SrcMem | ModRM | Src2Imm, em_imul_3op),
3690 I(SrcImmByte | Mov | Stack, em_push), 3749 I(SrcImmByte | Mov | Stack, em_push),
3691 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op), 3750 I(DstReg | SrcMem | ModRM | Src2ImmByte, em_imul_3op),
3692 I2bvIP(DstDI | SrcDX | Mov | String, em_in, ins, check_perm_in), /* insb, insw/insd */ 3751 I2bvIP(DstDI | SrcDX | Mov | String | Unaligned, em_in, ins, check_perm_in), /* insb, insw/insd */
3693 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */ 3752 I2bvIP(SrcSI | DstDX | String, em_out, outs, check_perm_out), /* outsb, outsw/outsd */
3694 /* 0x70 - 0x7F */ 3753 /* 0x70 - 0x7F */
3695 X16(D(SrcImmByte)), 3754 X16(D(SrcImmByte)),
@@ -3765,7 +3824,7 @@ static struct opcode opcode_table[256] = {
3765 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5), 3824 D(ImplicitOps), D(ImplicitOps), G(0, group4), G(0, group5),
3766}; 3825};
3767 3826
3768static struct opcode twobyte_table[256] = { 3827static const struct opcode twobyte_table[256] = {
3769 /* 0x00 - 0x0F */ 3828 /* 0x00 - 0x0F */
3770 G(0, group6), GD(0, &group7), N, N, 3829 G(0, group6), GD(0, &group7), N, N,
3771 N, I(ImplicitOps | VendorSpecific, em_syscall), 3830 N, I(ImplicitOps | VendorSpecific, em_syscall),
@@ -3936,7 +3995,7 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3936 case OpAcc: 3995 case OpAcc:
3937 op->type = OP_REG; 3996 op->type = OP_REG;
3938 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; 3997 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3939 op->addr.reg = &ctxt->regs[VCPU_REGS_RAX]; 3998 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RAX);
3940 fetch_register_operand(op); 3999 fetch_register_operand(op);
3941 op->orig_val = op->val; 4000 op->orig_val = op->val;
3942 break; 4001 break;
@@ -3944,19 +4003,20 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3944 op->type = OP_MEM; 4003 op->type = OP_MEM;
3945 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; 4004 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3946 op->addr.mem.ea = 4005 op->addr.mem.ea =
3947 register_address(ctxt, ctxt->regs[VCPU_REGS_RDI]); 4006 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RDI));
3948 op->addr.mem.seg = VCPU_SREG_ES; 4007 op->addr.mem.seg = VCPU_SREG_ES;
3949 op->val = 0; 4008 op->val = 0;
4009 op->count = 1;
3950 break; 4010 break;
3951 case OpDX: 4011 case OpDX:
3952 op->type = OP_REG; 4012 op->type = OP_REG;
3953 op->bytes = 2; 4013 op->bytes = 2;
3954 op->addr.reg = &ctxt->regs[VCPU_REGS_RDX]; 4014 op->addr.reg = reg_rmw(ctxt, VCPU_REGS_RDX);
3955 fetch_register_operand(op); 4015 fetch_register_operand(op);
3956 break; 4016 break;
3957 case OpCL: 4017 case OpCL:
3958 op->bytes = 1; 4018 op->bytes = 1;
3959 op->val = ctxt->regs[VCPU_REGS_RCX] & 0xff; 4019 op->val = reg_read(ctxt, VCPU_REGS_RCX) & 0xff;
3960 break; 4020 break;
3961 case OpImmByte: 4021 case OpImmByte:
3962 rc = decode_imm(ctxt, op, 1, true); 4022 rc = decode_imm(ctxt, op, 1, true);
@@ -3987,9 +4047,10 @@ static int decode_operand(struct x86_emulate_ctxt *ctxt, struct operand *op,
3987 op->type = OP_MEM; 4047 op->type = OP_MEM;
3988 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes; 4048 op->bytes = (ctxt->d & ByteOp) ? 1 : ctxt->op_bytes;
3989 op->addr.mem.ea = 4049 op->addr.mem.ea =
3990 register_address(ctxt, ctxt->regs[VCPU_REGS_RSI]); 4050 register_address(ctxt, reg_read(ctxt, VCPU_REGS_RSI));
3991 op->addr.mem.seg = seg_override(ctxt); 4051 op->addr.mem.seg = seg_override(ctxt);
3992 op->val = 0; 4052 op->val = 0;
4053 op->count = 1;
3993 break; 4054 break;
3994 case OpImmFAddr: 4055 case OpImmFAddr:
3995 op->type = OP_IMM; 4056 op->type = OP_IMM;
@@ -4293,9 +4354,10 @@ static void fetch_possible_mmx_operand(struct x86_emulate_ctxt *ctxt,
4293 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm); 4354 read_mmx_reg(ctxt, &op->mm_val, op->addr.mm);
4294} 4355}
4295 4356
4357
4296int x86_emulate_insn(struct x86_emulate_ctxt *ctxt) 4358int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4297{ 4359{
4298 struct x86_emulate_ops *ops = ctxt->ops; 4360 const struct x86_emulate_ops *ops = ctxt->ops;
4299 int rc = X86EMUL_CONTINUE; 4361 int rc = X86EMUL_CONTINUE;
4300 int saved_dst_type = ctxt->dst.type; 4362 int saved_dst_type = ctxt->dst.type;
4301 4363
@@ -4356,7 +4418,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4356 } 4418 }
4357 4419
4358 /* Instruction can only be executed in protected mode */ 4420 /* Instruction can only be executed in protected mode */
4359 if ((ctxt->d & Prot) && !(ctxt->mode & X86EMUL_MODE_PROT)) { 4421 if ((ctxt->d & Prot) && ctxt->mode < X86EMUL_MODE_PROT16) {
4360 rc = emulate_ud(ctxt); 4422 rc = emulate_ud(ctxt);
4361 goto done; 4423 goto done;
4362 } 4424 }
@@ -4377,7 +4439,7 @@ int x86_emulate_insn(struct x86_emulate_ctxt *ctxt)
4377 4439
4378 if (ctxt->rep_prefix && (ctxt->d & String)) { 4440 if (ctxt->rep_prefix && (ctxt->d & String)) {
4379 /* All REP prefixes have the same first termination condition */ 4441 /* All REP prefixes have the same first termination condition */
4380 if (address_mask(ctxt, ctxt->regs[VCPU_REGS_RCX]) == 0) { 4442 if (address_mask(ctxt, reg_read(ctxt, VCPU_REGS_RCX)) == 0) {
4381 ctxt->eip = ctxt->_eip; 4443 ctxt->eip = ctxt->_eip;
4382 goto done; 4444 goto done;
4383 } 4445 }
@@ -4450,7 +4512,7 @@ special_insn:
4450 ctxt->dst.val = ctxt->src.addr.mem.ea; 4512 ctxt->dst.val = ctxt->src.addr.mem.ea;
4451 break; 4513 break;
4452 case 0x90 ... 0x97: /* nop / xchg reg, rax */ 4514 case 0x90 ... 0x97: /* nop / xchg reg, rax */
4453 if (ctxt->dst.addr.reg == &ctxt->regs[VCPU_REGS_RAX]) 4515 if (ctxt->dst.addr.reg == reg_rmw(ctxt, VCPU_REGS_RAX))
4454 break; 4516 break;
4455 rc = em_xchg(ctxt); 4517 rc = em_xchg(ctxt);
4456 break; 4518 break;
@@ -4478,7 +4540,7 @@ special_insn:
4478 rc = em_grp2(ctxt); 4540 rc = em_grp2(ctxt);
4479 break; 4541 break;
4480 case 0xd2 ... 0xd3: /* Grp2 */ 4542 case 0xd2 ... 0xd3: /* Grp2 */
4481 ctxt->src.val = ctxt->regs[VCPU_REGS_RCX]; 4543 ctxt->src.val = reg_read(ctxt, VCPU_REGS_RCX);
4482 rc = em_grp2(ctxt); 4544 rc = em_grp2(ctxt);
4483 break; 4545 break;
4484 case 0xe9: /* jmp rel */ 4546 case 0xe9: /* jmp rel */
@@ -4524,23 +4586,27 @@ writeback:
4524 ctxt->dst.type = saved_dst_type; 4586 ctxt->dst.type = saved_dst_type;
4525 4587
4526 if ((ctxt->d & SrcMask) == SrcSI) 4588 if ((ctxt->d & SrcMask) == SrcSI)
4527 string_addr_inc(ctxt, seg_override(ctxt), 4589 string_addr_inc(ctxt, VCPU_REGS_RSI, &ctxt->src);
4528 VCPU_REGS_RSI, &ctxt->src);
4529 4590
4530 if ((ctxt->d & DstMask) == DstDI) 4591 if ((ctxt->d & DstMask) == DstDI)
4531 string_addr_inc(ctxt, VCPU_SREG_ES, VCPU_REGS_RDI, 4592 string_addr_inc(ctxt, VCPU_REGS_RDI, &ctxt->dst);
4532 &ctxt->dst);
4533 4593
4534 if (ctxt->rep_prefix && (ctxt->d & String)) { 4594 if (ctxt->rep_prefix && (ctxt->d & String)) {
4595 unsigned int count;
4535 struct read_cache *r = &ctxt->io_read; 4596 struct read_cache *r = &ctxt->io_read;
4536 register_address_increment(ctxt, &ctxt->regs[VCPU_REGS_RCX], -1); 4597 if ((ctxt->d & SrcMask) == SrcSI)
4598 count = ctxt->src.count;
4599 else
4600 count = ctxt->dst.count;
4601 register_address_increment(ctxt, reg_rmw(ctxt, VCPU_REGS_RCX),
4602 -count);
4537 4603
4538 if (!string_insn_completed(ctxt)) { 4604 if (!string_insn_completed(ctxt)) {
4539 /* 4605 /*
4540 * Re-enter guest when pio read ahead buffer is empty 4606 * Re-enter guest when pio read ahead buffer is empty
4541 * or, if it is not used, after each 1024 iteration. 4607 * or, if it is not used, after each 1024 iteration.
4542 */ 4608 */
4543 if ((r->end != 0 || ctxt->regs[VCPU_REGS_RCX] & 0x3ff) && 4609 if ((r->end != 0 || reg_read(ctxt, VCPU_REGS_RCX) & 0x3ff) &&
4544 (r->end == 0 || r->end != r->pos)) { 4610 (r->end == 0 || r->end != r->pos)) {
4545 /* 4611 /*
4546 * Reset read cache. Usually happens before 4612 * Reset read cache. Usually happens before
@@ -4548,6 +4614,7 @@ writeback:
4548 * we have to do it here. 4614 * we have to do it here.
4549 */ 4615 */
4550 ctxt->mem_read.end = 0; 4616 ctxt->mem_read.end = 0;
4617 writeback_registers(ctxt);
4551 return EMULATION_RESTART; 4618 return EMULATION_RESTART;
4552 } 4619 }
4553 goto done; /* skip rip writeback */ 4620 goto done; /* skip rip writeback */
@@ -4562,6 +4629,9 @@ done:
4562 if (rc == X86EMUL_INTERCEPTED) 4629 if (rc == X86EMUL_INTERCEPTED)
4563 return EMULATION_INTERCEPTED; 4630 return EMULATION_INTERCEPTED;
4564 4631
4632 if (rc == X86EMUL_CONTINUE)
4633 writeback_registers(ctxt);
4634
4565 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK; 4635 return (rc == X86EMUL_UNHANDLEABLE) ? EMULATION_FAILED : EMULATION_OK;
4566 4636
4567twobyte_insn: 4637twobyte_insn:
@@ -4634,3 +4704,13 @@ twobyte_insn:
4634cannot_emulate: 4704cannot_emulate:
4635 return EMULATION_FAILED; 4705 return EMULATION_FAILED;
4636} 4706}
4707
4708void emulator_invalidate_register_cache(struct x86_emulate_ctxt *ctxt)
4709{
4710 invalidate_registers(ctxt);
4711}
4712
4713void emulator_writeback_register_cache(struct x86_emulate_ctxt *ctxt)
4714{
4715 writeback_registers(ctxt);
4716}
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c
index adba28f88d1a..11300d2fa714 100644
--- a/arch/x86/kvm/i8254.c
+++ b/arch/x86/kvm/i8254.c
@@ -108,7 +108,7 @@ static s64 __kpit_elapsed(struct kvm *kvm)
108 ktime_t remaining; 108 ktime_t remaining;
109 struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state; 109 struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state;
110 110
111 if (!ps->pit_timer.period) 111 if (!ps->period)
112 return 0; 112 return 0;
113 113
114 /* 114 /*
@@ -120,9 +120,9 @@ static s64 __kpit_elapsed(struct kvm *kvm)
120 * itself with the initial count and continues counting 120 * itself with the initial count and continues counting
121 * from there. 121 * from there.
122 */ 122 */
123 remaining = hrtimer_get_remaining(&ps->pit_timer.timer); 123 remaining = hrtimer_get_remaining(&ps->timer);
124 elapsed = ps->pit_timer.period - ktime_to_ns(remaining); 124 elapsed = ps->period - ktime_to_ns(remaining);
125 elapsed = mod_64(elapsed, ps->pit_timer.period); 125 elapsed = mod_64(elapsed, ps->period);
126 126
127 return elapsed; 127 return elapsed;
128} 128}
@@ -238,12 +238,12 @@ static void kvm_pit_ack_irq(struct kvm_irq_ack_notifier *kian)
238 int value; 238 int value;
239 239
240 spin_lock(&ps->inject_lock); 240 spin_lock(&ps->inject_lock);
241 value = atomic_dec_return(&ps->pit_timer.pending); 241 value = atomic_dec_return(&ps->pending);
242 if (value < 0) 242 if (value < 0)
243 /* spurious acks can be generated if, for example, the 243 /* spurious acks can be generated if, for example, the
244 * PIC is being reset. Handle it gracefully here 244 * PIC is being reset. Handle it gracefully here
245 */ 245 */
246 atomic_inc(&ps->pit_timer.pending); 246 atomic_inc(&ps->pending);
247 else if (value > 0) 247 else if (value > 0)
248 /* in this case, we had multiple outstanding pit interrupts 248 /* in this case, we had multiple outstanding pit interrupts
249 * that we needed to inject. Reinject 249 * that we needed to inject. Reinject
@@ -261,28 +261,17 @@ void __kvm_migrate_pit_timer(struct kvm_vcpu *vcpu)
261 if (!kvm_vcpu_is_bsp(vcpu) || !pit) 261 if (!kvm_vcpu_is_bsp(vcpu) || !pit)
262 return; 262 return;
263 263
264 timer = &pit->pit_state.pit_timer.timer; 264 timer = &pit->pit_state.timer;
265 if (hrtimer_cancel(timer)) 265 if (hrtimer_cancel(timer))
266 hrtimer_start_expires(timer, HRTIMER_MODE_ABS); 266 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
267} 267}
268 268
269static void destroy_pit_timer(struct kvm_pit *pit) 269static void destroy_pit_timer(struct kvm_pit *pit)
270{ 270{
271 hrtimer_cancel(&pit->pit_state.pit_timer.timer); 271 hrtimer_cancel(&pit->pit_state.timer);
272 flush_kthread_work(&pit->expired); 272 flush_kthread_work(&pit->expired);
273} 273}
274 274
275static bool kpit_is_periodic(struct kvm_timer *ktimer)
276{
277 struct kvm_kpit_state *ps = container_of(ktimer, struct kvm_kpit_state,
278 pit_timer);
279 return ps->is_periodic;
280}
281
282static struct kvm_timer_ops kpit_ops = {
283 .is_periodic = kpit_is_periodic,
284};
285
286static void pit_do_work(struct kthread_work *work) 275static void pit_do_work(struct kthread_work *work)
287{ 276{
288 struct kvm_pit *pit = container_of(work, struct kvm_pit, expired); 277 struct kvm_pit *pit = container_of(work, struct kvm_pit, expired);
@@ -322,16 +311,16 @@ static void pit_do_work(struct kthread_work *work)
322 311
323static enum hrtimer_restart pit_timer_fn(struct hrtimer *data) 312static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
324{ 313{
325 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer); 314 struct kvm_kpit_state *ps = container_of(data, struct kvm_kpit_state, timer);
326 struct kvm_pit *pt = ktimer->kvm->arch.vpit; 315 struct kvm_pit *pt = ps->kvm->arch.vpit;
327 316
328 if (ktimer->reinject || !atomic_read(&ktimer->pending)) { 317 if (ps->reinject || !atomic_read(&ps->pending)) {
329 atomic_inc(&ktimer->pending); 318 atomic_inc(&ps->pending);
330 queue_kthread_work(&pt->worker, &pt->expired); 319 queue_kthread_work(&pt->worker, &pt->expired);
331 } 320 }
332 321
333 if (ktimer->t_ops->is_periodic(ktimer)) { 322 if (ps->is_periodic) {
334 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period); 323 hrtimer_add_expires_ns(&ps->timer, ps->period);
335 return HRTIMER_RESTART; 324 return HRTIMER_RESTART;
336 } else 325 } else
337 return HRTIMER_NORESTART; 326 return HRTIMER_NORESTART;
@@ -340,7 +329,6 @@ static enum hrtimer_restart pit_timer_fn(struct hrtimer *data)
340static void create_pit_timer(struct kvm *kvm, u32 val, int is_period) 329static void create_pit_timer(struct kvm *kvm, u32 val, int is_period)
341{ 330{
342 struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state; 331 struct kvm_kpit_state *ps = &kvm->arch.vpit->pit_state;
343 struct kvm_timer *pt = &ps->pit_timer;
344 s64 interval; 332 s64 interval;
345 333
346 if (!irqchip_in_kernel(kvm) || ps->flags & KVM_PIT_FLAGS_HPET_LEGACY) 334 if (!irqchip_in_kernel(kvm) || ps->flags & KVM_PIT_FLAGS_HPET_LEGACY)
@@ -351,19 +339,18 @@ static void create_pit_timer(struct kvm *kvm, u32 val, int is_period)
351 pr_debug("create pit timer, interval is %llu nsec\n", interval); 339 pr_debug("create pit timer, interval is %llu nsec\n", interval);
352 340
353 /* TODO The new value only affected after the retriggered */ 341 /* TODO The new value only affected after the retriggered */
354 hrtimer_cancel(&pt->timer); 342 hrtimer_cancel(&ps->timer);
355 flush_kthread_work(&ps->pit->expired); 343 flush_kthread_work(&ps->pit->expired);
356 pt->period = interval; 344 ps->period = interval;
357 ps->is_periodic = is_period; 345 ps->is_periodic = is_period;
358 346
359 pt->timer.function = pit_timer_fn; 347 ps->timer.function = pit_timer_fn;
360 pt->t_ops = &kpit_ops; 348 ps->kvm = ps->pit->kvm;
361 pt->kvm = ps->pit->kvm;
362 349
363 atomic_set(&pt->pending, 0); 350 atomic_set(&ps->pending, 0);
364 ps->irq_ack = 1; 351 ps->irq_ack = 1;
365 352
366 hrtimer_start(&pt->timer, ktime_add_ns(ktime_get(), interval), 353 hrtimer_start(&ps->timer, ktime_add_ns(ktime_get(), interval),
367 HRTIMER_MODE_ABS); 354 HRTIMER_MODE_ABS);
368} 355}
369 356
@@ -639,7 +626,7 @@ void kvm_pit_reset(struct kvm_pit *pit)
639 } 626 }
640 mutex_unlock(&pit->pit_state.lock); 627 mutex_unlock(&pit->pit_state.lock);
641 628
642 atomic_set(&pit->pit_state.pit_timer.pending, 0); 629 atomic_set(&pit->pit_state.pending, 0);
643 pit->pit_state.irq_ack = 1; 630 pit->pit_state.irq_ack = 1;
644} 631}
645 632
@@ -648,7 +635,7 @@ static void pit_mask_notifer(struct kvm_irq_mask_notifier *kimn, bool mask)
648 struct kvm_pit *pit = container_of(kimn, struct kvm_pit, mask_notifier); 635 struct kvm_pit *pit = container_of(kimn, struct kvm_pit, mask_notifier);
649 636
650 if (!mask) { 637 if (!mask) {
651 atomic_set(&pit->pit_state.pit_timer.pending, 0); 638 atomic_set(&pit->pit_state.pending, 0);
652 pit->pit_state.irq_ack = 1; 639 pit->pit_state.irq_ack = 1;
653 } 640 }
654} 641}
@@ -706,12 +693,11 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm, u32 flags)
706 693
707 pit_state = &pit->pit_state; 694 pit_state = &pit->pit_state;
708 pit_state->pit = pit; 695 pit_state->pit = pit;
709 hrtimer_init(&pit_state->pit_timer.timer, 696 hrtimer_init(&pit_state->timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
710 CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
711 pit_state->irq_ack_notifier.gsi = 0; 697 pit_state->irq_ack_notifier.gsi = 0;
712 pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq; 698 pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq;
713 kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier); 699 kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier);
714 pit_state->pit_timer.reinject = true; 700 pit_state->reinject = true;
715 mutex_unlock(&pit->pit_state.lock); 701 mutex_unlock(&pit->pit_state.lock);
716 702
717 kvm_pit_reset(pit); 703 kvm_pit_reset(pit);
@@ -761,7 +747,7 @@ void kvm_free_pit(struct kvm *kvm)
761 kvm_unregister_irq_ack_notifier(kvm, 747 kvm_unregister_irq_ack_notifier(kvm,
762 &kvm->arch.vpit->pit_state.irq_ack_notifier); 748 &kvm->arch.vpit->pit_state.irq_ack_notifier);
763 mutex_lock(&kvm->arch.vpit->pit_state.lock); 749 mutex_lock(&kvm->arch.vpit->pit_state.lock);
764 timer = &kvm->arch.vpit->pit_state.pit_timer.timer; 750 timer = &kvm->arch.vpit->pit_state.timer;
765 hrtimer_cancel(timer); 751 hrtimer_cancel(timer);
766 flush_kthread_work(&kvm->arch.vpit->expired); 752 flush_kthread_work(&kvm->arch.vpit->expired);
767 kthread_stop(kvm->arch.vpit->worker_task); 753 kthread_stop(kvm->arch.vpit->worker_task);
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h
index fdf40425ea1d..dd1b16b611b0 100644
--- a/arch/x86/kvm/i8254.h
+++ b/arch/x86/kvm/i8254.h
@@ -24,8 +24,12 @@ struct kvm_kpit_channel_state {
24struct kvm_kpit_state { 24struct kvm_kpit_state {
25 struct kvm_kpit_channel_state channels[3]; 25 struct kvm_kpit_channel_state channels[3];
26 u32 flags; 26 u32 flags;
27 struct kvm_timer pit_timer;
28 bool is_periodic; 27 bool is_periodic;
28 s64 period; /* unit: ns */
29 struct hrtimer timer;
30 atomic_t pending; /* accumulated triggered timers */
31 bool reinject;
32 struct kvm *kvm;
29 u32 speaker_data_on; 33 u32 speaker_data_on;
30 struct mutex lock; 34 struct mutex lock;
31 struct kvm_pit *pit; 35 struct kvm_pit *pit;
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c
index e498b18f010c..848206df0967 100644
--- a/arch/x86/kvm/i8259.c
+++ b/arch/x86/kvm/i8259.c
@@ -190,17 +190,17 @@ void kvm_pic_update_irq(struct kvm_pic *s)
190 190
191int kvm_pic_set_irq(struct kvm_pic *s, int irq, int irq_source_id, int level) 191int kvm_pic_set_irq(struct kvm_pic *s, int irq, int irq_source_id, int level)
192{ 192{
193 int ret = -1; 193 int ret, irq_level;
194
195 BUG_ON(irq < 0 || irq >= PIC_NUM_PINS);
194 196
195 pic_lock(s); 197 pic_lock(s);
196 if (irq >= 0 && irq < PIC_NUM_PINS) { 198 irq_level = __kvm_irq_line_state(&s->irq_states[irq],
197 int irq_level = __kvm_irq_line_state(&s->irq_states[irq], 199 irq_source_id, level);
198 irq_source_id, level); 200 ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, irq_level);
199 ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, irq_level); 201 pic_update_irq(s);
200 pic_update_irq(s); 202 trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr,
201 trace_kvm_pic_set_irq(irq >> 3, irq & 7, s->pics[irq >> 3].elcr, 203 s->pics[irq >> 3].imr, ret == 0);
202 s->pics[irq >> 3].imr, ret == 0);
203 }
204 pic_unlock(s); 204 pic_unlock(s);
205 205
206 return ret; 206 return ret;
@@ -275,23 +275,20 @@ void kvm_pic_reset(struct kvm_kpic_state *s)
275{ 275{
276 int irq, i; 276 int irq, i;
277 struct kvm_vcpu *vcpu; 277 struct kvm_vcpu *vcpu;
278 u8 irr = s->irr, isr = s->imr; 278 u8 edge_irr = s->irr & ~s->elcr;
279 bool found = false; 279 bool found = false;
280 280
281 s->last_irr = 0; 281 s->last_irr = 0;
282 s->irr = 0; 282 s->irr &= s->elcr;
283 s->imr = 0; 283 s->imr = 0;
284 s->isr = 0;
285 s->priority_add = 0; 284 s->priority_add = 0;
286 s->irq_base = 0;
287 s->read_reg_select = 0;
288 s->poll = 0;
289 s->special_mask = 0; 285 s->special_mask = 0;
290 s->init_state = 0; 286 s->read_reg_select = 0;
291 s->auto_eoi = 0; 287 if (!s->init4) {
292 s->rotate_on_auto_eoi = 0; 288 s->special_fully_nested_mode = 0;
293 s->special_fully_nested_mode = 0; 289 s->auto_eoi = 0;
294 s->init4 = 0; 290 }
291 s->init_state = 1;
295 292
296 kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm) 293 kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm)
297 if (kvm_apic_accept_pic_intr(vcpu)) { 294 if (kvm_apic_accept_pic_intr(vcpu)) {
@@ -304,7 +301,7 @@ void kvm_pic_reset(struct kvm_kpic_state *s)
304 return; 301 return;
305 302
306 for (irq = 0; irq < PIC_NUM_PINS/2; irq++) 303 for (irq = 0; irq < PIC_NUM_PINS/2; irq++)
307 if (irr & (1 << irq) || isr & (1 << irq)) 304 if (edge_irr & (1 << irq))
308 pic_clear_isr(s, irq); 305 pic_clear_isr(s, irq);
309} 306}
310 307
@@ -316,40 +313,13 @@ static void pic_ioport_write(void *opaque, u32 addr, u32 val)
316 addr &= 1; 313 addr &= 1;
317 if (addr == 0) { 314 if (addr == 0) {
318 if (val & 0x10) { 315 if (val & 0x10) {
319 u8 edge_irr = s->irr & ~s->elcr;
320 int i;
321 bool found;
322 struct kvm_vcpu *vcpu;
323
324 s->init4 = val & 1; 316 s->init4 = val & 1;
325 s->last_irr = 0;
326 s->irr &= s->elcr;
327 s->imr = 0;
328 s->priority_add = 0;
329 s->special_mask = 0;
330 s->read_reg_select = 0;
331 if (!s->init4) {
332 s->special_fully_nested_mode = 0;
333 s->auto_eoi = 0;
334 }
335 s->init_state = 1;
336 if (val & 0x02) 317 if (val & 0x02)
337 pr_pic_unimpl("single mode not supported"); 318 pr_pic_unimpl("single mode not supported");
338 if (val & 0x08) 319 if (val & 0x08)
339 pr_pic_unimpl( 320 pr_pic_unimpl(
340 "level sensitive irq not supported"); 321 "level sensitive irq not supported");
341 322 kvm_pic_reset(s);
342 kvm_for_each_vcpu(i, vcpu, s->pics_state->kvm)
343 if (kvm_apic_accept_pic_intr(vcpu)) {
344 found = true;
345 break;
346 }
347
348
349 if (found)
350 for (irq = 0; irq < PIC_NUM_PINS/2; irq++)
351 if (edge_irr & (1 << irq))
352 pic_clear_isr(s, irq);
353 } else if (val & 0x08) { 323 } else if (val & 0x08) {
354 if (val & 0x04) 324 if (val & 0x04)
355 s->poll = 1; 325 s->poll = 1;
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h
index 2086f2bfba33..2d03568e9498 100644
--- a/arch/x86/kvm/irq.h
+++ b/arch/x86/kvm/irq.h
@@ -70,7 +70,7 @@ struct kvm_pic {
70 struct kvm_io_device dev_slave; 70 struct kvm_io_device dev_slave;
71 struct kvm_io_device dev_eclr; 71 struct kvm_io_device dev_eclr;
72 void (*ack_notifier)(void *opaque, int irq); 72 void (*ack_notifier)(void *opaque, int irq);
73 unsigned long irq_states[16]; 73 unsigned long irq_states[PIC_NUM_PINS];
74}; 74};
75 75
76struct kvm_pic *kvm_create_pic(struct kvm *kvm); 76struct kvm_pic *kvm_create_pic(struct kvm *kvm);
diff --git a/arch/x86/kvm/kvm_timer.h b/arch/x86/kvm/kvm_timer.h
deleted file mode 100644
index 497dbaa366d4..000000000000
--- a/arch/x86/kvm/kvm_timer.h
+++ /dev/null
@@ -1,18 +0,0 @@
1
2struct kvm_timer {
3 struct hrtimer timer;
4 s64 period; /* unit: ns */
5 u32 timer_mode_mask;
6 u64 tscdeadline;
7 atomic_t pending; /* accumulated triggered timers */
8 bool reinject;
9 struct kvm_timer_ops *t_ops;
10 struct kvm *kvm;
11 struct kvm_vcpu *vcpu;
12};
13
14struct kvm_timer_ops {
15 bool (*is_periodic)(struct kvm_timer *);
16};
17
18enum hrtimer_restart kvm_timer_fn(struct hrtimer *data);
diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c
index ce878788a39f..c6e6b721b6ee 100644
--- a/arch/x86/kvm/lapic.c
+++ b/arch/x86/kvm/lapic.c
@@ -34,6 +34,7 @@
34#include <asm/current.h> 34#include <asm/current.h>
35#include <asm/apicdef.h> 35#include <asm/apicdef.h>
36#include <linux/atomic.h> 36#include <linux/atomic.h>
37#include <linux/jump_label.h>
37#include "kvm_cache_regs.h" 38#include "kvm_cache_regs.h"
38#include "irq.h" 39#include "irq.h"
39#include "trace.h" 40#include "trace.h"
@@ -65,6 +66,7 @@
65#define APIC_DEST_NOSHORT 0x0 66#define APIC_DEST_NOSHORT 0x0
66#define APIC_DEST_MASK 0x800 67#define APIC_DEST_MASK 0x800
67#define MAX_APIC_VECTOR 256 68#define MAX_APIC_VECTOR 256
69#define APIC_VECTORS_PER_REG 32
68 70
69#define VEC_POS(v) ((v) & (32 - 1)) 71#define VEC_POS(v) ((v) & (32 - 1))
70#define REG_POS(v) (((v) >> 5) << 4) 72#define REG_POS(v) (((v) >> 5) << 4)
@@ -72,11 +74,6 @@
72static unsigned int min_timer_period_us = 500; 74static unsigned int min_timer_period_us = 500;
73module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR); 75module_param(min_timer_period_us, uint, S_IRUGO | S_IWUSR);
74 76
75static inline u32 apic_get_reg(struct kvm_lapic *apic, int reg_off)
76{
77 return *((u32 *) (apic->regs + reg_off));
78}
79
80static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val) 77static inline void apic_set_reg(struct kvm_lapic *apic, int reg_off, u32 val)
81{ 78{
82 *((u32 *) (apic->regs + reg_off)) = val; 79 *((u32 *) (apic->regs + reg_off)) = val;
@@ -117,19 +114,23 @@ static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
117 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec)); 114 return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
118} 115}
119 116
120static inline int apic_hw_enabled(struct kvm_lapic *apic) 117struct static_key_deferred apic_hw_disabled __read_mostly;
121{ 118struct static_key_deferred apic_sw_disabled __read_mostly;
122 return (apic)->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
123}
124 119
125static inline int apic_sw_enabled(struct kvm_lapic *apic) 120static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
126{ 121{
127 return apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED; 122 if ((kvm_apic_get_reg(apic, APIC_SPIV) ^ val) & APIC_SPIV_APIC_ENABLED) {
123 if (val & APIC_SPIV_APIC_ENABLED)
124 static_key_slow_dec_deferred(&apic_sw_disabled);
125 else
126 static_key_slow_inc(&apic_sw_disabled.key);
127 }
128 apic_set_reg(apic, APIC_SPIV, val);
128} 129}
129 130
130static inline int apic_enabled(struct kvm_lapic *apic) 131static inline int apic_enabled(struct kvm_lapic *apic)
131{ 132{
132 return apic_sw_enabled(apic) && apic_hw_enabled(apic); 133 return kvm_apic_sw_enabled(apic) && kvm_apic_hw_enabled(apic);
133} 134}
134 135
135#define LVT_MASK \ 136#define LVT_MASK \
@@ -139,36 +140,135 @@ static inline int apic_enabled(struct kvm_lapic *apic)
139 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \ 140 (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
140 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER) 141 APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
141 142
143static inline int apic_x2apic_mode(struct kvm_lapic *apic)
144{
145 return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
146}
147
142static inline int kvm_apic_id(struct kvm_lapic *apic) 148static inline int kvm_apic_id(struct kvm_lapic *apic)
143{ 149{
144 return (apic_get_reg(apic, APIC_ID) >> 24) & 0xff; 150 return (kvm_apic_get_reg(apic, APIC_ID) >> 24) & 0xff;
151}
152
153static inline u16 apic_cluster_id(struct kvm_apic_map *map, u32 ldr)
154{
155 u16 cid;
156 ldr >>= 32 - map->ldr_bits;
157 cid = (ldr >> map->cid_shift) & map->cid_mask;
158
159 BUG_ON(cid >= ARRAY_SIZE(map->logical_map));
160
161 return cid;
162}
163
164static inline u16 apic_logical_id(struct kvm_apic_map *map, u32 ldr)
165{
166 ldr >>= (32 - map->ldr_bits);
167 return ldr & map->lid_mask;
168}
169
170static void recalculate_apic_map(struct kvm *kvm)
171{
172 struct kvm_apic_map *new, *old = NULL;
173 struct kvm_vcpu *vcpu;
174 int i;
175
176 new = kzalloc(sizeof(struct kvm_apic_map), GFP_KERNEL);
177
178 mutex_lock(&kvm->arch.apic_map_lock);
179
180 if (!new)
181 goto out;
182
183 new->ldr_bits = 8;
184 /* flat mode is default */
185 new->cid_shift = 8;
186 new->cid_mask = 0;
187 new->lid_mask = 0xff;
188
189 kvm_for_each_vcpu(i, vcpu, kvm) {
190 struct kvm_lapic *apic = vcpu->arch.apic;
191 u16 cid, lid;
192 u32 ldr;
193
194 if (!kvm_apic_present(vcpu))
195 continue;
196
197 /*
198 * All APICs have to be configured in the same mode by an OS.
199 * We take advatage of this while building logical id loockup
200 * table. After reset APICs are in xapic/flat mode, so if we
201 * find apic with different setting we assume this is the mode
202 * OS wants all apics to be in; build lookup table accordingly.
203 */
204 if (apic_x2apic_mode(apic)) {
205 new->ldr_bits = 32;
206 new->cid_shift = 16;
207 new->cid_mask = new->lid_mask = 0xffff;
208 } else if (kvm_apic_sw_enabled(apic) &&
209 !new->cid_mask /* flat mode */ &&
210 kvm_apic_get_reg(apic, APIC_DFR) == APIC_DFR_CLUSTER) {
211 new->cid_shift = 4;
212 new->cid_mask = 0xf;
213 new->lid_mask = 0xf;
214 }
215
216 new->phys_map[kvm_apic_id(apic)] = apic;
217
218 ldr = kvm_apic_get_reg(apic, APIC_LDR);
219 cid = apic_cluster_id(new, ldr);
220 lid = apic_logical_id(new, ldr);
221
222 if (lid)
223 new->logical_map[cid][ffs(lid) - 1] = apic;
224 }
225out:
226 old = rcu_dereference_protected(kvm->arch.apic_map,
227 lockdep_is_held(&kvm->arch.apic_map_lock));
228 rcu_assign_pointer(kvm->arch.apic_map, new);
229 mutex_unlock(&kvm->arch.apic_map_lock);
230
231 if (old)
232 kfree_rcu(old, rcu);
233}
234
235static inline void kvm_apic_set_id(struct kvm_lapic *apic, u8 id)
236{
237 apic_set_reg(apic, APIC_ID, id << 24);
238 recalculate_apic_map(apic->vcpu->kvm);
239}
240
241static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
242{
243 apic_set_reg(apic, APIC_LDR, id);
244 recalculate_apic_map(apic->vcpu->kvm);
145} 245}
146 246
147static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type) 247static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
148{ 248{
149 return !(apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED); 249 return !(kvm_apic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
150} 250}
151 251
152static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type) 252static inline int apic_lvt_vector(struct kvm_lapic *apic, int lvt_type)
153{ 253{
154 return apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK; 254 return kvm_apic_get_reg(apic, lvt_type) & APIC_VECTOR_MASK;
155} 255}
156 256
157static inline int apic_lvtt_oneshot(struct kvm_lapic *apic) 257static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
158{ 258{
159 return ((apic_get_reg(apic, APIC_LVTT) & 259 return ((kvm_apic_get_reg(apic, APIC_LVTT) &
160 apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT); 260 apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_ONESHOT);
161} 261}
162 262
163static inline int apic_lvtt_period(struct kvm_lapic *apic) 263static inline int apic_lvtt_period(struct kvm_lapic *apic)
164{ 264{
165 return ((apic_get_reg(apic, APIC_LVTT) & 265 return ((kvm_apic_get_reg(apic, APIC_LVTT) &
166 apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC); 266 apic->lapic_timer.timer_mode_mask) == APIC_LVT_TIMER_PERIODIC);
167} 267}
168 268
169static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic) 269static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
170{ 270{
171 return ((apic_get_reg(apic, APIC_LVTT) & 271 return ((kvm_apic_get_reg(apic, APIC_LVTT) &
172 apic->lapic_timer.timer_mode_mask) == 272 apic->lapic_timer.timer_mode_mask) ==
173 APIC_LVT_TIMER_TSCDEADLINE); 273 APIC_LVT_TIMER_TSCDEADLINE);
174} 274}
@@ -184,7 +284,7 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu)
184 struct kvm_cpuid_entry2 *feat; 284 struct kvm_cpuid_entry2 *feat;
185 u32 v = APIC_VERSION; 285 u32 v = APIC_VERSION;
186 286
187 if (!irqchip_in_kernel(vcpu->kvm)) 287 if (!kvm_vcpu_has_lapic(vcpu))
188 return; 288 return;
189 289
190 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0); 290 feat = kvm_find_cpuid_entry(apic->vcpu, 0x1, 0);
@@ -193,12 +293,7 @@ void kvm_apic_set_version(struct kvm_vcpu *vcpu)
193 apic_set_reg(apic, APIC_LVR, v); 293 apic_set_reg(apic, APIC_LVR, v);
194} 294}
195 295
196static inline int apic_x2apic_mode(struct kvm_lapic *apic) 296static const unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
197{
198 return apic->vcpu->arch.apic_base & X2APIC_ENABLE;
199}
200
201static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
202 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */ 297 LVT_MASK , /* part LVTT mask, timer mode mask added at runtime */
203 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */ 298 LVT_MASK | APIC_MODE_MASK, /* LVTTHMR */
204 LVT_MASK | APIC_MODE_MASK, /* LVTPC */ 299 LVT_MASK | APIC_MODE_MASK, /* LVTPC */
@@ -208,25 +303,30 @@ static unsigned int apic_lvt_mask[APIC_LVT_NUM] = {
208 303
209static int find_highest_vector(void *bitmap) 304static int find_highest_vector(void *bitmap)
210{ 305{
211 u32 *word = bitmap; 306 int vec;
212 int word_offset = MAX_APIC_VECTOR >> 5; 307 u32 *reg;
213 308
214 while ((word_offset != 0) && (word[(--word_offset) << 2] == 0)) 309 for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
215 continue; 310 vec >= 0; vec -= APIC_VECTORS_PER_REG) {
311 reg = bitmap + REG_POS(vec);
312 if (*reg)
313 return fls(*reg) - 1 + vec;
314 }
216 315
217 if (likely(!word_offset && !word[0])) 316 return -1;
218 return -1;
219 else
220 return fls(word[word_offset << 2]) - 1 + (word_offset << 5);
221} 317}
222 318
223static u8 count_vectors(void *bitmap) 319static u8 count_vectors(void *bitmap)
224{ 320{
225 u32 *word = bitmap; 321 int vec;
226 int word_offset; 322 u32 *reg;
227 u8 count = 0; 323 u8 count = 0;
228 for (word_offset = 0; word_offset < MAX_APIC_VECTOR >> 5; ++word_offset) 324
229 count += hweight32(word[word_offset << 2]); 325 for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
326 reg = bitmap + REG_POS(vec);
327 count += hweight32(*reg);
328 }
329
230 return count; 330 return count;
231} 331}
232 332
@@ -285,7 +385,6 @@ static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
285 385
286int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu) 386int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
287{ 387{
288 struct kvm_lapic *apic = vcpu->arch.apic;
289 int highest_irr; 388 int highest_irr;
290 389
291 /* This may race with setting of irr in __apic_accept_irq() and 390 /* This may race with setting of irr in __apic_accept_irq() and
@@ -293,9 +392,9 @@ int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
293 * will cause vmexit immediately and the value will be recalculated 392 * will cause vmexit immediately and the value will be recalculated
294 * on the next vmentry. 393 * on the next vmentry.
295 */ 394 */
296 if (!apic) 395 if (!kvm_vcpu_has_lapic(vcpu))
297 return 0; 396 return 0;
298 highest_irr = apic_find_highest_irr(apic); 397 highest_irr = apic_find_highest_irr(vcpu->arch.apic);
299 398
300 return highest_irr; 399 return highest_irr;
301} 400}
@@ -378,8 +477,8 @@ static void apic_update_ppr(struct kvm_lapic *apic)
378 u32 tpr, isrv, ppr, old_ppr; 477 u32 tpr, isrv, ppr, old_ppr;
379 int isr; 478 int isr;
380 479
381 old_ppr = apic_get_reg(apic, APIC_PROCPRI); 480 old_ppr = kvm_apic_get_reg(apic, APIC_PROCPRI);
382 tpr = apic_get_reg(apic, APIC_TASKPRI); 481 tpr = kvm_apic_get_reg(apic, APIC_TASKPRI);
383 isr = apic_find_highest_isr(apic); 482 isr = apic_find_highest_isr(apic);
384 isrv = (isr != -1) ? isr : 0; 483 isrv = (isr != -1) ? isr : 0;
385 484
@@ -415,13 +514,13 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
415 u32 logical_id; 514 u32 logical_id;
416 515
417 if (apic_x2apic_mode(apic)) { 516 if (apic_x2apic_mode(apic)) {
418 logical_id = apic_get_reg(apic, APIC_LDR); 517 logical_id = kvm_apic_get_reg(apic, APIC_LDR);
419 return logical_id & mda; 518 return logical_id & mda;
420 } 519 }
421 520
422 logical_id = GET_APIC_LOGICAL_ID(apic_get_reg(apic, APIC_LDR)); 521 logical_id = GET_APIC_LOGICAL_ID(kvm_apic_get_reg(apic, APIC_LDR));
423 522
424 switch (apic_get_reg(apic, APIC_DFR)) { 523 switch (kvm_apic_get_reg(apic, APIC_DFR)) {
425 case APIC_DFR_FLAT: 524 case APIC_DFR_FLAT:
426 if (logical_id & mda) 525 if (logical_id & mda)
427 result = 1; 526 result = 1;
@@ -433,7 +532,7 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda)
433 break; 532 break;
434 default: 533 default:
435 apic_debug("Bad DFR vcpu %d: %08x\n", 534 apic_debug("Bad DFR vcpu %d: %08x\n",
436 apic->vcpu->vcpu_id, apic_get_reg(apic, APIC_DFR)); 535 apic->vcpu->vcpu_id, kvm_apic_get_reg(apic, APIC_DFR));
437 break; 536 break;
438 } 537 }
439 538
@@ -478,6 +577,72 @@ int kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
478 return result; 577 return result;
479} 578}
480 579
580bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
581 struct kvm_lapic_irq *irq, int *r)
582{
583 struct kvm_apic_map *map;
584 unsigned long bitmap = 1;
585 struct kvm_lapic **dst;
586 int i;
587 bool ret = false;
588
589 *r = -1;
590
591 if (irq->shorthand == APIC_DEST_SELF) {
592 *r = kvm_apic_set_irq(src->vcpu, irq);
593 return true;
594 }
595
596 if (irq->shorthand)
597 return false;
598
599 rcu_read_lock();
600 map = rcu_dereference(kvm->arch.apic_map);
601
602 if (!map)
603 goto out;
604
605 if (irq->dest_mode == 0) { /* physical mode */
606 if (irq->delivery_mode == APIC_DM_LOWEST ||
607 irq->dest_id == 0xff)
608 goto out;
609 dst = &map->phys_map[irq->dest_id & 0xff];
610 } else {
611 u32 mda = irq->dest_id << (32 - map->ldr_bits);
612
613 dst = map->logical_map[apic_cluster_id(map, mda)];
614
615 bitmap = apic_logical_id(map, mda);
616
617 if (irq->delivery_mode == APIC_DM_LOWEST) {
618 int l = -1;
619 for_each_set_bit(i, &bitmap, 16) {
620 if (!dst[i])
621 continue;
622 if (l < 0)
623 l = i;
624 else if (kvm_apic_compare_prio(dst[i]->vcpu, dst[l]->vcpu) < 0)
625 l = i;
626 }
627
628 bitmap = (l >= 0) ? 1 << l : 0;
629 }
630 }
631
632 for_each_set_bit(i, &bitmap, 16) {
633 if (!dst[i])
634 continue;
635 if (*r < 0)
636 *r = 0;
637 *r += kvm_apic_set_irq(dst[i]->vcpu, irq);
638 }
639
640 ret = true;
641out:
642 rcu_read_unlock();
643 return ret;
644}
645
481/* 646/*
482 * Add a pending IRQ into lapic. 647 * Add a pending IRQ into lapic.
483 * Return 1 if successfully added and 0 if discarded. 648 * Return 1 if successfully added and 0 if discarded.
@@ -591,7 +756,7 @@ static int apic_set_eoi(struct kvm_lapic *apic)
591 apic_clear_isr(vector, apic); 756 apic_clear_isr(vector, apic);
592 apic_update_ppr(apic); 757 apic_update_ppr(apic);
593 758
594 if (!(apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) && 759 if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_DIRECTED_EOI) &&
595 kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) { 760 kvm_ioapic_handles_vector(apic->vcpu->kvm, vector)) {
596 int trigger_mode; 761 int trigger_mode;
597 if (apic_test_vector(vector, apic->regs + APIC_TMR)) 762 if (apic_test_vector(vector, apic->regs + APIC_TMR))
@@ -606,8 +771,8 @@ static int apic_set_eoi(struct kvm_lapic *apic)
606 771
607static void apic_send_ipi(struct kvm_lapic *apic) 772static void apic_send_ipi(struct kvm_lapic *apic)
608{ 773{
609 u32 icr_low = apic_get_reg(apic, APIC_ICR); 774 u32 icr_low = kvm_apic_get_reg(apic, APIC_ICR);
610 u32 icr_high = apic_get_reg(apic, APIC_ICR2); 775 u32 icr_high = kvm_apic_get_reg(apic, APIC_ICR2);
611 struct kvm_lapic_irq irq; 776 struct kvm_lapic_irq irq;
612 777
613 irq.vector = icr_low & APIC_VECTOR_MASK; 778 irq.vector = icr_low & APIC_VECTOR_MASK;
@@ -642,7 +807,7 @@ static u32 apic_get_tmcct(struct kvm_lapic *apic)
642 ASSERT(apic != NULL); 807 ASSERT(apic != NULL);
643 808
644 /* if initial count is 0, current count should also be 0 */ 809 /* if initial count is 0, current count should also be 0 */
645 if (apic_get_reg(apic, APIC_TMICT) == 0) 810 if (kvm_apic_get_reg(apic, APIC_TMICT) == 0)
646 return 0; 811 return 0;
647 812
648 remaining = hrtimer_get_remaining(&apic->lapic_timer.timer); 813 remaining = hrtimer_get_remaining(&apic->lapic_timer.timer);
@@ -696,13 +861,15 @@ static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
696 861
697 val = apic_get_tmcct(apic); 862 val = apic_get_tmcct(apic);
698 break; 863 break;
699 864 case APIC_PROCPRI:
865 apic_update_ppr(apic);
866 val = kvm_apic_get_reg(apic, offset);
867 break;
700 case APIC_TASKPRI: 868 case APIC_TASKPRI:
701 report_tpr_access(apic, false); 869 report_tpr_access(apic, false);
702 /* fall thru */ 870 /* fall thru */
703 default: 871 default:
704 apic_update_ppr(apic); 872 val = kvm_apic_get_reg(apic, offset);
705 val = apic_get_reg(apic, offset);
706 break; 873 break;
707 } 874 }
708 875
@@ -719,7 +886,7 @@ static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
719{ 886{
720 unsigned char alignment = offset & 0xf; 887 unsigned char alignment = offset & 0xf;
721 u32 result; 888 u32 result;
722 /* this bitmask has a bit cleared for each reserver register */ 889 /* this bitmask has a bit cleared for each reserved register */
723 static const u64 rmask = 0x43ff01ffffffe70cULL; 890 static const u64 rmask = 0x43ff01ffffffe70cULL;
724 891
725 if ((alignment + len) > 4) { 892 if ((alignment + len) > 4) {
@@ -754,7 +921,7 @@ static int apic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
754 921
755static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr) 922static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
756{ 923{
757 return apic_hw_enabled(apic) && 924 return kvm_apic_hw_enabled(apic) &&
758 addr >= apic->base_address && 925 addr >= apic->base_address &&
759 addr < apic->base_address + LAPIC_MMIO_LENGTH; 926 addr < apic->base_address + LAPIC_MMIO_LENGTH;
760} 927}
@@ -777,7 +944,7 @@ static void update_divide_count(struct kvm_lapic *apic)
777{ 944{
778 u32 tmp1, tmp2, tdcr; 945 u32 tmp1, tmp2, tdcr;
779 946
780 tdcr = apic_get_reg(apic, APIC_TDCR); 947 tdcr = kvm_apic_get_reg(apic, APIC_TDCR);
781 tmp1 = tdcr & 0xf; 948 tmp1 = tdcr & 0xf;
782 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1; 949 tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
783 apic->divide_count = 0x1 << (tmp2 & 0x7); 950 apic->divide_count = 0x1 << (tmp2 & 0x7);
@@ -792,9 +959,9 @@ static void start_apic_timer(struct kvm_lapic *apic)
792 atomic_set(&apic->lapic_timer.pending, 0); 959 atomic_set(&apic->lapic_timer.pending, 0);
793 960
794 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) { 961 if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
795 /* lapic timer in oneshot or peroidic mode */ 962 /* lapic timer in oneshot or periodic mode */
796 now = apic->lapic_timer.timer.base->get_time(); 963 now = apic->lapic_timer.timer.base->get_time();
797 apic->lapic_timer.period = (u64)apic_get_reg(apic, APIC_TMICT) 964 apic->lapic_timer.period = (u64)kvm_apic_get_reg(apic, APIC_TMICT)
798 * APIC_BUS_CYCLE_NS * apic->divide_count; 965 * APIC_BUS_CYCLE_NS * apic->divide_count;
799 966
800 if (!apic->lapic_timer.period) 967 if (!apic->lapic_timer.period)
@@ -826,7 +993,7 @@ static void start_apic_timer(struct kvm_lapic *apic)
826 "timer initial count 0x%x, period %lldns, " 993 "timer initial count 0x%x, period %lldns, "
827 "expire @ 0x%016" PRIx64 ".\n", __func__, 994 "expire @ 0x%016" PRIx64 ".\n", __func__,
828 APIC_BUS_CYCLE_NS, ktime_to_ns(now), 995 APIC_BUS_CYCLE_NS, ktime_to_ns(now),
829 apic_get_reg(apic, APIC_TMICT), 996 kvm_apic_get_reg(apic, APIC_TMICT),
830 apic->lapic_timer.period, 997 apic->lapic_timer.period,
831 ktime_to_ns(ktime_add_ns(now, 998 ktime_to_ns(ktime_add_ns(now,
832 apic->lapic_timer.period))); 999 apic->lapic_timer.period)));
@@ -858,7 +1025,7 @@ static void start_apic_timer(struct kvm_lapic *apic)
858 1025
859static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val) 1026static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
860{ 1027{
861 int nmi_wd_enabled = apic_lvt_nmi_mode(apic_get_reg(apic, APIC_LVT0)); 1028 int nmi_wd_enabled = apic_lvt_nmi_mode(kvm_apic_get_reg(apic, APIC_LVT0));
862 1029
863 if (apic_lvt_nmi_mode(lvt0_val)) { 1030 if (apic_lvt_nmi_mode(lvt0_val)) {
864 if (!nmi_wd_enabled) { 1031 if (!nmi_wd_enabled) {
@@ -879,7 +1046,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
879 switch (reg) { 1046 switch (reg) {
880 case APIC_ID: /* Local APIC ID */ 1047 case APIC_ID: /* Local APIC ID */
881 if (!apic_x2apic_mode(apic)) 1048 if (!apic_x2apic_mode(apic))
882 apic_set_reg(apic, APIC_ID, val); 1049 kvm_apic_set_id(apic, val >> 24);
883 else 1050 else
884 ret = 1; 1051 ret = 1;
885 break; 1052 break;
@@ -895,29 +1062,30 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
895 1062
896 case APIC_LDR: 1063 case APIC_LDR:
897 if (!apic_x2apic_mode(apic)) 1064 if (!apic_x2apic_mode(apic))
898 apic_set_reg(apic, APIC_LDR, val & APIC_LDR_MASK); 1065 kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
899 else 1066 else
900 ret = 1; 1067 ret = 1;
901 break; 1068 break;
902 1069
903 case APIC_DFR: 1070 case APIC_DFR:
904 if (!apic_x2apic_mode(apic)) 1071 if (!apic_x2apic_mode(apic)) {
905 apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF); 1072 apic_set_reg(apic, APIC_DFR, val | 0x0FFFFFFF);
906 else 1073 recalculate_apic_map(apic->vcpu->kvm);
1074 } else
907 ret = 1; 1075 ret = 1;
908 break; 1076 break;
909 1077
910 case APIC_SPIV: { 1078 case APIC_SPIV: {
911 u32 mask = 0x3ff; 1079 u32 mask = 0x3ff;
912 if (apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI) 1080 if (kvm_apic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
913 mask |= APIC_SPIV_DIRECTED_EOI; 1081 mask |= APIC_SPIV_DIRECTED_EOI;
914 apic_set_reg(apic, APIC_SPIV, val & mask); 1082 apic_set_spiv(apic, val & mask);
915 if (!(val & APIC_SPIV_APIC_ENABLED)) { 1083 if (!(val & APIC_SPIV_APIC_ENABLED)) {
916 int i; 1084 int i;
917 u32 lvt_val; 1085 u32 lvt_val;
918 1086
919 for (i = 0; i < APIC_LVT_NUM; i++) { 1087 for (i = 0; i < APIC_LVT_NUM; i++) {
920 lvt_val = apic_get_reg(apic, 1088 lvt_val = kvm_apic_get_reg(apic,
921 APIC_LVTT + 0x10 * i); 1089 APIC_LVTT + 0x10 * i);
922 apic_set_reg(apic, APIC_LVTT + 0x10 * i, 1090 apic_set_reg(apic, APIC_LVTT + 0x10 * i,
923 lvt_val | APIC_LVT_MASKED); 1091 lvt_val | APIC_LVT_MASKED);
@@ -946,7 +1114,7 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
946 case APIC_LVT1: 1114 case APIC_LVT1:
947 case APIC_LVTERR: 1115 case APIC_LVTERR:
948 /* TODO: Check vector */ 1116 /* TODO: Check vector */
949 if (!apic_sw_enabled(apic)) 1117 if (!kvm_apic_sw_enabled(apic))
950 val |= APIC_LVT_MASKED; 1118 val |= APIC_LVT_MASKED;
951 1119
952 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4]; 1120 val &= apic_lvt_mask[(reg - APIC_LVTT) >> 4];
@@ -955,12 +1123,12 @@ static int apic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
955 break; 1123 break;
956 1124
957 case APIC_LVTT: 1125 case APIC_LVTT:
958 if ((apic_get_reg(apic, APIC_LVTT) & 1126 if ((kvm_apic_get_reg(apic, APIC_LVTT) &
959 apic->lapic_timer.timer_mode_mask) != 1127 apic->lapic_timer.timer_mode_mask) !=
960 (val & apic->lapic_timer.timer_mode_mask)) 1128 (val & apic->lapic_timer.timer_mode_mask))
961 hrtimer_cancel(&apic->lapic_timer.timer); 1129 hrtimer_cancel(&apic->lapic_timer.timer);
962 1130
963 if (!apic_sw_enabled(apic)) 1131 if (!kvm_apic_sw_enabled(apic))
964 val |= APIC_LVT_MASKED; 1132 val |= APIC_LVT_MASKED;
965 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask); 1133 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
966 apic_set_reg(apic, APIC_LVTT, val); 1134 apic_set_reg(apic, APIC_LVTT, val);
@@ -1039,24 +1207,30 @@ static int apic_mmio_write(struct kvm_io_device *this,
1039 1207
1040void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu) 1208void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
1041{ 1209{
1042 struct kvm_lapic *apic = vcpu->arch.apic; 1210 if (kvm_vcpu_has_lapic(vcpu))
1043
1044 if (apic)
1045 apic_reg_write(vcpu->arch.apic, APIC_EOI, 0); 1211 apic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
1046} 1212}
1047EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi); 1213EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
1048 1214
1049void kvm_free_lapic(struct kvm_vcpu *vcpu) 1215void kvm_free_lapic(struct kvm_vcpu *vcpu)
1050{ 1216{
1217 struct kvm_lapic *apic = vcpu->arch.apic;
1218
1051 if (!vcpu->arch.apic) 1219 if (!vcpu->arch.apic)
1052 return; 1220 return;
1053 1221
1054 hrtimer_cancel(&vcpu->arch.apic->lapic_timer.timer); 1222 hrtimer_cancel(&apic->lapic_timer.timer);
1223
1224 if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
1225 static_key_slow_dec_deferred(&apic_hw_disabled);
1226
1227 if (!(kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED))
1228 static_key_slow_dec_deferred(&apic_sw_disabled);
1055 1229
1056 if (vcpu->arch.apic->regs) 1230 if (apic->regs)
1057 free_page((unsigned long)vcpu->arch.apic->regs); 1231 free_page((unsigned long)apic->regs);
1058 1232
1059 kfree(vcpu->arch.apic); 1233 kfree(apic);
1060} 1234}
1061 1235
1062/* 1236/*
@@ -1068,10 +1242,9 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu)
1068u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu) 1242u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
1069{ 1243{
1070 struct kvm_lapic *apic = vcpu->arch.apic; 1244 struct kvm_lapic *apic = vcpu->arch.apic;
1071 if (!apic)
1072 return 0;
1073 1245
1074 if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic)) 1246 if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) ||
1247 apic_lvtt_period(apic))
1075 return 0; 1248 return 0;
1076 1249
1077 return apic->lapic_timer.tscdeadline; 1250 return apic->lapic_timer.tscdeadline;
@@ -1080,10 +1253,9 @@ u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
1080void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data) 1253void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
1081{ 1254{
1082 struct kvm_lapic *apic = vcpu->arch.apic; 1255 struct kvm_lapic *apic = vcpu->arch.apic;
1083 if (!apic)
1084 return;
1085 1256
1086 if (apic_lvtt_oneshot(apic) || apic_lvtt_period(apic)) 1257 if (!kvm_vcpu_has_lapic(vcpu) || apic_lvtt_oneshot(apic) ||
1258 apic_lvtt_period(apic))
1087 return; 1259 return;
1088 1260
1089 hrtimer_cancel(&apic->lapic_timer.timer); 1261 hrtimer_cancel(&apic->lapic_timer.timer);
@@ -1095,20 +1267,21 @@ void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
1095{ 1267{
1096 struct kvm_lapic *apic = vcpu->arch.apic; 1268 struct kvm_lapic *apic = vcpu->arch.apic;
1097 1269
1098 if (!apic) 1270 if (!kvm_vcpu_has_lapic(vcpu))
1099 return; 1271 return;
1272
1100 apic_set_tpr(apic, ((cr8 & 0x0f) << 4) 1273 apic_set_tpr(apic, ((cr8 & 0x0f) << 4)
1101 | (apic_get_reg(apic, APIC_TASKPRI) & 4)); 1274 | (kvm_apic_get_reg(apic, APIC_TASKPRI) & 4));
1102} 1275}
1103 1276
1104u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu) 1277u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
1105{ 1278{
1106 struct kvm_lapic *apic = vcpu->arch.apic;
1107 u64 tpr; 1279 u64 tpr;
1108 1280
1109 if (!apic) 1281 if (!kvm_vcpu_has_lapic(vcpu))
1110 return 0; 1282 return 0;
1111 tpr = (u64) apic_get_reg(apic, APIC_TASKPRI); 1283
1284 tpr = (u64) kvm_apic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
1112 1285
1113 return (tpr & 0xf0) >> 4; 1286 return (tpr & 0xf0) >> 4;
1114} 1287}
@@ -1123,6 +1296,15 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1123 return; 1296 return;
1124 } 1297 }
1125 1298
1299 /* update jump label if enable bit changes */
1300 if ((vcpu->arch.apic_base ^ value) & MSR_IA32_APICBASE_ENABLE) {
1301 if (value & MSR_IA32_APICBASE_ENABLE)
1302 static_key_slow_dec_deferred(&apic_hw_disabled);
1303 else
1304 static_key_slow_inc(&apic_hw_disabled.key);
1305 recalculate_apic_map(vcpu->kvm);
1306 }
1307
1126 if (!kvm_vcpu_is_bsp(apic->vcpu)) 1308 if (!kvm_vcpu_is_bsp(apic->vcpu))
1127 value &= ~MSR_IA32_APICBASE_BSP; 1309 value &= ~MSR_IA32_APICBASE_BSP;
1128 1310
@@ -1130,7 +1312,7 @@ void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
1130 if (apic_x2apic_mode(apic)) { 1312 if (apic_x2apic_mode(apic)) {
1131 u32 id = kvm_apic_id(apic); 1313 u32 id = kvm_apic_id(apic);
1132 u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf)); 1314 u32 ldr = ((id & ~0xf) << 16) | (1 << (id & 0xf));
1133 apic_set_reg(apic, APIC_LDR, ldr); 1315 kvm_apic_set_ldr(apic, ldr);
1134 } 1316 }
1135 apic->base_address = apic->vcpu->arch.apic_base & 1317 apic->base_address = apic->vcpu->arch.apic_base &
1136 MSR_IA32_APICBASE_BASE; 1318 MSR_IA32_APICBASE_BASE;
@@ -1155,7 +1337,7 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
1155 /* Stop the timer in case it's a reset to an active apic */ 1337 /* Stop the timer in case it's a reset to an active apic */
1156 hrtimer_cancel(&apic->lapic_timer.timer); 1338 hrtimer_cancel(&apic->lapic_timer.timer);
1157 1339
1158 apic_set_reg(apic, APIC_ID, vcpu->vcpu_id << 24); 1340 kvm_apic_set_id(apic, vcpu->vcpu_id);
1159 kvm_apic_set_version(apic->vcpu); 1341 kvm_apic_set_version(apic->vcpu);
1160 1342
1161 for (i = 0; i < APIC_LVT_NUM; i++) 1343 for (i = 0; i < APIC_LVT_NUM; i++)
@@ -1164,9 +1346,9 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
1164 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT)); 1346 SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
1165 1347
1166 apic_set_reg(apic, APIC_DFR, 0xffffffffU); 1348 apic_set_reg(apic, APIC_DFR, 0xffffffffU);
1167 apic_set_reg(apic, APIC_SPIV, 0xff); 1349 apic_set_spiv(apic, 0xff);
1168 apic_set_reg(apic, APIC_TASKPRI, 0); 1350 apic_set_reg(apic, APIC_TASKPRI, 0);
1169 apic_set_reg(apic, APIC_LDR, 0); 1351 kvm_apic_set_ldr(apic, 0);
1170 apic_set_reg(apic, APIC_ESR, 0); 1352 apic_set_reg(apic, APIC_ESR, 0);
1171 apic_set_reg(apic, APIC_ICR, 0); 1353 apic_set_reg(apic, APIC_ICR, 0);
1172 apic_set_reg(apic, APIC_ICR2, 0); 1354 apic_set_reg(apic, APIC_ICR2, 0);
@@ -1183,7 +1365,8 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
1183 update_divide_count(apic); 1365 update_divide_count(apic);
1184 atomic_set(&apic->lapic_timer.pending, 0); 1366 atomic_set(&apic->lapic_timer.pending, 0);
1185 if (kvm_vcpu_is_bsp(vcpu)) 1367 if (kvm_vcpu_is_bsp(vcpu))
1186 vcpu->arch.apic_base |= MSR_IA32_APICBASE_BSP; 1368 kvm_lapic_set_base(vcpu,
1369 vcpu->arch.apic_base | MSR_IA32_APICBASE_BSP);
1187 vcpu->arch.pv_eoi.msr_val = 0; 1370 vcpu->arch.pv_eoi.msr_val = 0;
1188 apic_update_ppr(apic); 1371 apic_update_ppr(apic);
1189 1372
@@ -1196,45 +1379,34 @@ void kvm_lapic_reset(struct kvm_vcpu *vcpu)
1196 vcpu->arch.apic_base, apic->base_address); 1379 vcpu->arch.apic_base, apic->base_address);
1197} 1380}
1198 1381
1199bool kvm_apic_present(struct kvm_vcpu *vcpu)
1200{
1201 return vcpu->arch.apic && apic_hw_enabled(vcpu->arch.apic);
1202}
1203
1204int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
1205{
1206 return kvm_apic_present(vcpu) && apic_sw_enabled(vcpu->arch.apic);
1207}
1208
1209/* 1382/*
1210 *---------------------------------------------------------------------- 1383 *----------------------------------------------------------------------
1211 * timer interface 1384 * timer interface
1212 *---------------------------------------------------------------------- 1385 *----------------------------------------------------------------------
1213 */ 1386 */
1214 1387
1215static bool lapic_is_periodic(struct kvm_timer *ktimer) 1388static bool lapic_is_periodic(struct kvm_lapic *apic)
1216{ 1389{
1217 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic,
1218 lapic_timer);
1219 return apic_lvtt_period(apic); 1390 return apic_lvtt_period(apic);
1220} 1391}
1221 1392
1222int apic_has_pending_timer(struct kvm_vcpu *vcpu) 1393int apic_has_pending_timer(struct kvm_vcpu *vcpu)
1223{ 1394{
1224 struct kvm_lapic *lapic = vcpu->arch.apic; 1395 struct kvm_lapic *apic = vcpu->arch.apic;
1225 1396
1226 if (lapic && apic_enabled(lapic) && apic_lvt_enabled(lapic, APIC_LVTT)) 1397 if (kvm_vcpu_has_lapic(vcpu) && apic_enabled(apic) &&
1227 return atomic_read(&lapic->lapic_timer.pending); 1398 apic_lvt_enabled(apic, APIC_LVTT))
1399 return atomic_read(&apic->lapic_timer.pending);
1228 1400
1229 return 0; 1401 return 0;
1230} 1402}
1231 1403
1232int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type) 1404int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
1233{ 1405{
1234 u32 reg = apic_get_reg(apic, lvt_type); 1406 u32 reg = kvm_apic_get_reg(apic, lvt_type);
1235 int vector, mode, trig_mode; 1407 int vector, mode, trig_mode;
1236 1408
1237 if (apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) { 1409 if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
1238 vector = reg & APIC_VECTOR_MASK; 1410 vector = reg & APIC_VECTOR_MASK;
1239 mode = reg & APIC_MODE_MASK; 1411 mode = reg & APIC_MODE_MASK;
1240 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER; 1412 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
@@ -1251,15 +1423,40 @@ void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
1251 kvm_apic_local_deliver(apic, APIC_LVT0); 1423 kvm_apic_local_deliver(apic, APIC_LVT0);
1252} 1424}
1253 1425
1254static struct kvm_timer_ops lapic_timer_ops = {
1255 .is_periodic = lapic_is_periodic,
1256};
1257
1258static const struct kvm_io_device_ops apic_mmio_ops = { 1426static const struct kvm_io_device_ops apic_mmio_ops = {
1259 .read = apic_mmio_read, 1427 .read = apic_mmio_read,
1260 .write = apic_mmio_write, 1428 .write = apic_mmio_write,
1261}; 1429};
1262 1430
1431static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
1432{
1433 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
1434 struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
1435 struct kvm_vcpu *vcpu = apic->vcpu;
1436 wait_queue_head_t *q = &vcpu->wq;
1437
1438 /*
1439 * There is a race window between reading and incrementing, but we do
1440 * not care about potentially losing timer events in the !reinject
1441 * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked
1442 * in vcpu_enter_guest.
1443 */
1444 if (!atomic_read(&ktimer->pending)) {
1445 atomic_inc(&ktimer->pending);
1446 /* FIXME: this code should not know anything about vcpus */
1447 kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
1448 }
1449
1450 if (waitqueue_active(q))
1451 wake_up_interruptible(q);
1452
1453 if (lapic_is_periodic(apic)) {
1454 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
1455 return HRTIMER_RESTART;
1456 } else
1457 return HRTIMER_NORESTART;
1458}
1459
1263int kvm_create_lapic(struct kvm_vcpu *vcpu) 1460int kvm_create_lapic(struct kvm_vcpu *vcpu)
1264{ 1461{
1265 struct kvm_lapic *apic; 1462 struct kvm_lapic *apic;
@@ -1283,14 +1480,17 @@ int kvm_create_lapic(struct kvm_vcpu *vcpu)
1283 1480
1284 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC, 1481 hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
1285 HRTIMER_MODE_ABS); 1482 HRTIMER_MODE_ABS);
1286 apic->lapic_timer.timer.function = kvm_timer_fn; 1483 apic->lapic_timer.timer.function = apic_timer_fn;
1287 apic->lapic_timer.t_ops = &lapic_timer_ops;
1288 apic->lapic_timer.kvm = vcpu->kvm;
1289 apic->lapic_timer.vcpu = vcpu;
1290 1484
1291 apic->base_address = APIC_DEFAULT_PHYS_BASE; 1485 /*
1292 vcpu->arch.apic_base = APIC_DEFAULT_PHYS_BASE; 1486 * APIC is created enabled. This will prevent kvm_lapic_set_base from
1487 * thinking that APIC satet has changed.
1488 */
1489 vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
1490 kvm_lapic_set_base(vcpu,
1491 APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE);
1293 1492
1493 static_key_slow_inc(&apic_sw_disabled.key); /* sw disabled at reset */
1294 kvm_lapic_reset(vcpu); 1494 kvm_lapic_reset(vcpu);
1295 kvm_iodevice_init(&apic->dev, &apic_mmio_ops); 1495 kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
1296 1496
@@ -1306,23 +1506,23 @@ int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
1306 struct kvm_lapic *apic = vcpu->arch.apic; 1506 struct kvm_lapic *apic = vcpu->arch.apic;
1307 int highest_irr; 1507 int highest_irr;
1308 1508
1309 if (!apic || !apic_enabled(apic)) 1509 if (!kvm_vcpu_has_lapic(vcpu) || !apic_enabled(apic))
1310 return -1; 1510 return -1;
1311 1511
1312 apic_update_ppr(apic); 1512 apic_update_ppr(apic);
1313 highest_irr = apic_find_highest_irr(apic); 1513 highest_irr = apic_find_highest_irr(apic);
1314 if ((highest_irr == -1) || 1514 if ((highest_irr == -1) ||
1315 ((highest_irr & 0xF0) <= apic_get_reg(apic, APIC_PROCPRI))) 1515 ((highest_irr & 0xF0) <= kvm_apic_get_reg(apic, APIC_PROCPRI)))
1316 return -1; 1516 return -1;
1317 return highest_irr; 1517 return highest_irr;
1318} 1518}
1319 1519
1320int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu) 1520int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
1321{ 1521{
1322 u32 lvt0 = apic_get_reg(vcpu->arch.apic, APIC_LVT0); 1522 u32 lvt0 = kvm_apic_get_reg(vcpu->arch.apic, APIC_LVT0);
1323 int r = 0; 1523 int r = 0;
1324 1524
1325 if (!apic_hw_enabled(vcpu->arch.apic)) 1525 if (!kvm_apic_hw_enabled(vcpu->arch.apic))
1326 r = 1; 1526 r = 1;
1327 if ((lvt0 & APIC_LVT_MASKED) == 0 && 1527 if ((lvt0 & APIC_LVT_MASKED) == 0 &&
1328 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT) 1528 GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
@@ -1334,7 +1534,10 @@ void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
1334{ 1534{
1335 struct kvm_lapic *apic = vcpu->arch.apic; 1535 struct kvm_lapic *apic = vcpu->arch.apic;
1336 1536
1337 if (apic && atomic_read(&apic->lapic_timer.pending) > 0) { 1537 if (!kvm_vcpu_has_lapic(vcpu))
1538 return;
1539
1540 if (atomic_read(&apic->lapic_timer.pending) > 0) {
1338 if (kvm_apic_local_deliver(apic, APIC_LVTT)) 1541 if (kvm_apic_local_deliver(apic, APIC_LVTT))
1339 atomic_dec(&apic->lapic_timer.pending); 1542 atomic_dec(&apic->lapic_timer.pending);
1340 } 1543 }
@@ -1354,12 +1557,17 @@ int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
1354 return vector; 1557 return vector;
1355} 1558}
1356 1559
1357void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu) 1560void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
1561 struct kvm_lapic_state *s)
1358{ 1562{
1359 struct kvm_lapic *apic = vcpu->arch.apic; 1563 struct kvm_lapic *apic = vcpu->arch.apic;
1360 1564
1361 apic->base_address = vcpu->arch.apic_base & 1565 kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
1362 MSR_IA32_APICBASE_BASE; 1566 /* set SPIV separately to get count of SW disabled APICs right */
1567 apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
1568 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s);
1569 /* call kvm_apic_set_id() to put apic into apic_map */
1570 kvm_apic_set_id(apic, kvm_apic_id(apic));
1363 kvm_apic_set_version(vcpu); 1571 kvm_apic_set_version(vcpu);
1364 1572
1365 apic_update_ppr(apic); 1573 apic_update_ppr(apic);
@@ -1374,13 +1582,12 @@ void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu)
1374 1582
1375void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu) 1583void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
1376{ 1584{
1377 struct kvm_lapic *apic = vcpu->arch.apic;
1378 struct hrtimer *timer; 1585 struct hrtimer *timer;
1379 1586
1380 if (!apic) 1587 if (!kvm_vcpu_has_lapic(vcpu))
1381 return; 1588 return;
1382 1589
1383 timer = &apic->lapic_timer.timer; 1590 timer = &vcpu->arch.apic->lapic_timer.timer;
1384 if (hrtimer_cancel(timer)) 1591 if (hrtimer_cancel(timer))
1385 hrtimer_start_expires(timer, HRTIMER_MODE_ABS); 1592 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
1386} 1593}
@@ -1478,7 +1685,7 @@ void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
1478 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention)) 1685 if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
1479 return; 1686 return;
1480 1687
1481 tpr = apic_get_reg(apic, APIC_TASKPRI) & 0xff; 1688 tpr = kvm_apic_get_reg(apic, APIC_TASKPRI) & 0xff;
1482 max_irr = apic_find_highest_irr(apic); 1689 max_irr = apic_find_highest_irr(apic);
1483 if (max_irr < 0) 1690 if (max_irr < 0)
1484 max_irr = 0; 1691 max_irr = 0;
@@ -1537,7 +1744,7 @@ int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
1537{ 1744{
1538 struct kvm_lapic *apic = vcpu->arch.apic; 1745 struct kvm_lapic *apic = vcpu->arch.apic;
1539 1746
1540 if (!irqchip_in_kernel(vcpu->kvm)) 1747 if (!kvm_vcpu_has_lapic(vcpu))
1541 return 1; 1748 return 1;
1542 1749
1543 /* if this is ICR write vector before command */ 1750 /* if this is ICR write vector before command */
@@ -1551,7 +1758,7 @@ int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
1551 struct kvm_lapic *apic = vcpu->arch.apic; 1758 struct kvm_lapic *apic = vcpu->arch.apic;
1552 u32 low, high = 0; 1759 u32 low, high = 0;
1553 1760
1554 if (!irqchip_in_kernel(vcpu->kvm)) 1761 if (!kvm_vcpu_has_lapic(vcpu))
1555 return 1; 1762 return 1;
1556 1763
1557 if (apic_reg_read(apic, reg, 4, &low)) 1764 if (apic_reg_read(apic, reg, 4, &low))
@@ -1576,3 +1783,10 @@ int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data)
1576 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data, 1783 return kvm_gfn_to_hva_cache_init(vcpu->kvm, &vcpu->arch.pv_eoi.data,
1577 addr); 1784 addr);
1578} 1785}
1786
1787void kvm_lapic_init(void)
1788{
1789 /* do not patch jump label more than once per second */
1790 jump_label_rate_limit(&apic_hw_disabled, HZ);
1791 jump_label_rate_limit(&apic_sw_disabled, HZ);
1792}
diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h
index 4af5405ae1e2..e5ebf9f3571f 100644
--- a/arch/x86/kvm/lapic.h
+++ b/arch/x86/kvm/lapic.h
@@ -2,10 +2,17 @@
2#define __KVM_X86_LAPIC_H 2#define __KVM_X86_LAPIC_H
3 3
4#include "iodev.h" 4#include "iodev.h"
5#include "kvm_timer.h"
6 5
7#include <linux/kvm_host.h> 6#include <linux/kvm_host.h>
8 7
8struct kvm_timer {
9 struct hrtimer timer;
10 s64 period; /* unit: ns */
11 u32 timer_mode_mask;
12 u64 tscdeadline;
13 atomic_t pending; /* accumulated triggered timers */
14};
15
9struct kvm_lapic { 16struct kvm_lapic {
10 unsigned long base_address; 17 unsigned long base_address;
11 struct kvm_io_device dev; 18 struct kvm_io_device dev;
@@ -45,11 +52,13 @@ int kvm_apic_match_logical_addr(struct kvm_lapic *apic, u8 mda);
45int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq); 52int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq);
46int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type); 53int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type);
47 54
55bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
56 struct kvm_lapic_irq *irq, int *r);
57
48u64 kvm_get_apic_base(struct kvm_vcpu *vcpu); 58u64 kvm_get_apic_base(struct kvm_vcpu *vcpu);
49void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data); 59void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data);
50void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu); 60void kvm_apic_post_state_restore(struct kvm_vcpu *vcpu,
51int kvm_lapic_enabled(struct kvm_vcpu *vcpu); 61 struct kvm_lapic_state *s);
52bool kvm_apic_present(struct kvm_vcpu *vcpu);
53int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu); 62int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu);
54 63
55u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu); 64u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu);
@@ -71,4 +80,48 @@ static inline bool kvm_hv_vapic_assist_page_enabled(struct kvm_vcpu *vcpu)
71} 80}
72 81
73int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data); 82int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data);
83void kvm_lapic_init(void);
84
85static inline u32 kvm_apic_get_reg(struct kvm_lapic *apic, int reg_off)
86{
87 return *((u32 *) (apic->regs + reg_off));
88}
89
90extern struct static_key kvm_no_apic_vcpu;
91
92static inline bool kvm_vcpu_has_lapic(struct kvm_vcpu *vcpu)
93{
94 if (static_key_false(&kvm_no_apic_vcpu))
95 return vcpu->arch.apic;
96 return true;
97}
98
99extern struct static_key_deferred apic_hw_disabled;
100
101static inline int kvm_apic_hw_enabled(struct kvm_lapic *apic)
102{
103 if (static_key_false(&apic_hw_disabled.key))
104 return apic->vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE;
105 return MSR_IA32_APICBASE_ENABLE;
106}
107
108extern struct static_key_deferred apic_sw_disabled;
109
110static inline int kvm_apic_sw_enabled(struct kvm_lapic *apic)
111{
112 if (static_key_false(&apic_sw_disabled.key))
113 return kvm_apic_get_reg(apic, APIC_SPIV) & APIC_SPIV_APIC_ENABLED;
114 return APIC_SPIV_APIC_ENABLED;
115}
116
117static inline bool kvm_apic_present(struct kvm_vcpu *vcpu)
118{
119 return kvm_vcpu_has_lapic(vcpu) && kvm_apic_hw_enabled(vcpu->arch.apic);
120}
121
122static inline int kvm_lapic_enabled(struct kvm_vcpu *vcpu)
123{
124 return kvm_apic_present(vcpu) && kvm_apic_sw_enabled(vcpu->arch.apic);
125}
126
74#endif 127#endif
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 7fbd0d273ea8..d289fee1ffb8 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -556,6 +556,14 @@ static int mmu_spte_clear_track_bits(u64 *sptep)
556 return 0; 556 return 0;
557 557
558 pfn = spte_to_pfn(old_spte); 558 pfn = spte_to_pfn(old_spte);
559
560 /*
561 * KVM does not hold the refcount of the page used by
562 * kvm mmu, before reclaiming the page, we should
563 * unmap it from mmu first.
564 */
565 WARN_ON(!kvm_is_mmio_pfn(pfn) && !page_count(pfn_to_page(pfn)));
566
559 if (!shadow_accessed_mask || old_spte & shadow_accessed_mask) 567 if (!shadow_accessed_mask || old_spte & shadow_accessed_mask)
560 kvm_set_pfn_accessed(pfn); 568 kvm_set_pfn_accessed(pfn);
561 if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask)) 569 if (!shadow_dirty_mask || (old_spte & shadow_dirty_mask))
@@ -960,13 +968,10 @@ static void pte_list_walk(unsigned long *pte_list, pte_list_walk_fn fn)
960static unsigned long *__gfn_to_rmap(gfn_t gfn, int level, 968static unsigned long *__gfn_to_rmap(gfn_t gfn, int level,
961 struct kvm_memory_slot *slot) 969 struct kvm_memory_slot *slot)
962{ 970{
963 struct kvm_lpage_info *linfo; 971 unsigned long idx;
964
965 if (likely(level == PT_PAGE_TABLE_LEVEL))
966 return &slot->rmap[gfn - slot->base_gfn];
967 972
968 linfo = lpage_info_slot(gfn, slot, level); 973 idx = gfn_to_index(gfn, slot->base_gfn, level);
969 return &linfo->rmap_pde; 974 return &slot->arch.rmap[level - PT_PAGE_TABLE_LEVEL][idx];
970} 975}
971 976
972/* 977/*
@@ -1173,7 +1178,8 @@ void kvm_mmu_write_protect_pt_masked(struct kvm *kvm,
1173 unsigned long *rmapp; 1178 unsigned long *rmapp;
1174 1179
1175 while (mask) { 1180 while (mask) {
1176 rmapp = &slot->rmap[gfn_offset + __ffs(mask)]; 1181 rmapp = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask),
1182 PT_PAGE_TABLE_LEVEL, slot);
1177 __rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false); 1183 __rmap_write_protect(kvm, rmapp, PT_PAGE_TABLE_LEVEL, false);
1178 1184
1179 /* clear the first set bit */ 1185 /* clear the first set bit */
@@ -1200,7 +1206,7 @@ static bool rmap_write_protect(struct kvm *kvm, u64 gfn)
1200} 1206}
1201 1207
1202static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp, 1208static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
1203 unsigned long data) 1209 struct kvm_memory_slot *slot, unsigned long data)
1204{ 1210{
1205 u64 *sptep; 1211 u64 *sptep;
1206 struct rmap_iterator iter; 1212 struct rmap_iterator iter;
@@ -1218,7 +1224,7 @@ static int kvm_unmap_rmapp(struct kvm *kvm, unsigned long *rmapp,
1218} 1224}
1219 1225
1220static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp, 1226static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
1221 unsigned long data) 1227 struct kvm_memory_slot *slot, unsigned long data)
1222{ 1228{
1223 u64 *sptep; 1229 u64 *sptep;
1224 struct rmap_iterator iter; 1230 struct rmap_iterator iter;
@@ -1259,43 +1265,67 @@ static int kvm_set_pte_rmapp(struct kvm *kvm, unsigned long *rmapp,
1259 return 0; 1265 return 0;
1260} 1266}
1261 1267
1262static int kvm_handle_hva(struct kvm *kvm, unsigned long hva, 1268static int kvm_handle_hva_range(struct kvm *kvm,
1263 unsigned long data, 1269 unsigned long start,
1264 int (*handler)(struct kvm *kvm, unsigned long *rmapp, 1270 unsigned long end,
1265 unsigned long data)) 1271 unsigned long data,
1272 int (*handler)(struct kvm *kvm,
1273 unsigned long *rmapp,
1274 struct kvm_memory_slot *slot,
1275 unsigned long data))
1266{ 1276{
1267 int j; 1277 int j;
1268 int ret; 1278 int ret = 0;
1269 int retval = 0;
1270 struct kvm_memslots *slots; 1279 struct kvm_memslots *slots;
1271 struct kvm_memory_slot *memslot; 1280 struct kvm_memory_slot *memslot;
1272 1281
1273 slots = kvm_memslots(kvm); 1282 slots = kvm_memslots(kvm);
1274 1283
1275 kvm_for_each_memslot(memslot, slots) { 1284 kvm_for_each_memslot(memslot, slots) {
1276 unsigned long start = memslot->userspace_addr; 1285 unsigned long hva_start, hva_end;
1277 unsigned long end; 1286 gfn_t gfn_start, gfn_end;
1278 1287
1279 end = start + (memslot->npages << PAGE_SHIFT); 1288 hva_start = max(start, memslot->userspace_addr);
1280 if (hva >= start && hva < end) { 1289 hva_end = min(end, memslot->userspace_addr +
1281 gfn_t gfn_offset = (hva - start) >> PAGE_SHIFT; 1290 (memslot->npages << PAGE_SHIFT));
1282 gfn_t gfn = memslot->base_gfn + gfn_offset; 1291 if (hva_start >= hva_end)
1292 continue;
1293 /*
1294 * {gfn(page) | page intersects with [hva_start, hva_end)} =
1295 * {gfn_start, gfn_start+1, ..., gfn_end-1}.
1296 */
1297 gfn_start = hva_to_gfn_memslot(hva_start, memslot);
1298 gfn_end = hva_to_gfn_memslot(hva_end + PAGE_SIZE - 1, memslot);
1283 1299
1284 ret = handler(kvm, &memslot->rmap[gfn_offset], data); 1300 for (j = PT_PAGE_TABLE_LEVEL;
1301 j < PT_PAGE_TABLE_LEVEL + KVM_NR_PAGE_SIZES; ++j) {
1302 unsigned long idx, idx_end;
1303 unsigned long *rmapp;
1285 1304
1286 for (j = 0; j < KVM_NR_PAGE_SIZES - 1; ++j) { 1305 /*
1287 struct kvm_lpage_info *linfo; 1306 * {idx(page_j) | page_j intersects with
1307 * [hva_start, hva_end)} = {idx, idx+1, ..., idx_end}.
1308 */
1309 idx = gfn_to_index(gfn_start, memslot->base_gfn, j);
1310 idx_end = gfn_to_index(gfn_end - 1, memslot->base_gfn, j);
1288 1311
1289 linfo = lpage_info_slot(gfn, memslot, 1312 rmapp = __gfn_to_rmap(gfn_start, j, memslot);
1290 PT_DIRECTORY_LEVEL + j); 1313
1291 ret |= handler(kvm, &linfo->rmap_pde, data); 1314 for (; idx <= idx_end; ++idx)
1292 } 1315 ret |= handler(kvm, rmapp++, memslot, data);
1293 trace_kvm_age_page(hva, memslot, ret);
1294 retval |= ret;
1295 } 1316 }
1296 } 1317 }
1297 1318
1298 return retval; 1319 return ret;
1320}
1321
1322static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
1323 unsigned long data,
1324 int (*handler)(struct kvm *kvm, unsigned long *rmapp,
1325 struct kvm_memory_slot *slot,
1326 unsigned long data))
1327{
1328 return kvm_handle_hva_range(kvm, hva, hva + 1, data, handler);
1299} 1329}
1300 1330
1301int kvm_unmap_hva(struct kvm *kvm, unsigned long hva) 1331int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
@@ -1303,13 +1333,18 @@ int kvm_unmap_hva(struct kvm *kvm, unsigned long hva)
1303 return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp); 1333 return kvm_handle_hva(kvm, hva, 0, kvm_unmap_rmapp);
1304} 1334}
1305 1335
1336int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
1337{
1338 return kvm_handle_hva_range(kvm, start, end, 0, kvm_unmap_rmapp);
1339}
1340
1306void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte) 1341void kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte)
1307{ 1342{
1308 kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp); 1343 kvm_handle_hva(kvm, hva, (unsigned long)&pte, kvm_set_pte_rmapp);
1309} 1344}
1310 1345
1311static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp, 1346static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1312 unsigned long data) 1347 struct kvm_memory_slot *slot, unsigned long data)
1313{ 1348{
1314 u64 *sptep; 1349 u64 *sptep;
1315 struct rmap_iterator uninitialized_var(iter); 1350 struct rmap_iterator uninitialized_var(iter);
@@ -1323,8 +1358,10 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1323 * This has some overhead, but not as much as the cost of swapping 1358 * This has some overhead, but not as much as the cost of swapping
1324 * out actively used pages or breaking up actively used hugepages. 1359 * out actively used pages or breaking up actively used hugepages.
1325 */ 1360 */
1326 if (!shadow_accessed_mask) 1361 if (!shadow_accessed_mask) {
1327 return kvm_unmap_rmapp(kvm, rmapp, data); 1362 young = kvm_unmap_rmapp(kvm, rmapp, slot, data);
1363 goto out;
1364 }
1328 1365
1329 for (sptep = rmap_get_first(*rmapp, &iter); sptep; 1366 for (sptep = rmap_get_first(*rmapp, &iter); sptep;
1330 sptep = rmap_get_next(&iter)) { 1367 sptep = rmap_get_next(&iter)) {
@@ -1336,12 +1373,14 @@ static int kvm_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1336 (unsigned long *)sptep); 1373 (unsigned long *)sptep);
1337 } 1374 }
1338 } 1375 }
1339 1376out:
1377 /* @data has hva passed to kvm_age_hva(). */
1378 trace_kvm_age_page(data, slot, young);
1340 return young; 1379 return young;
1341} 1380}
1342 1381
1343static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp, 1382static int kvm_test_age_rmapp(struct kvm *kvm, unsigned long *rmapp,
1344 unsigned long data) 1383 struct kvm_memory_slot *slot, unsigned long data)
1345{ 1384{
1346 u64 *sptep; 1385 u64 *sptep;
1347 struct rmap_iterator iter; 1386 struct rmap_iterator iter;
@@ -1379,13 +1418,13 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
1379 1418
1380 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level); 1419 rmapp = gfn_to_rmap(vcpu->kvm, gfn, sp->role.level);
1381 1420
1382 kvm_unmap_rmapp(vcpu->kvm, rmapp, 0); 1421 kvm_unmap_rmapp(vcpu->kvm, rmapp, NULL, 0);
1383 kvm_flush_remote_tlbs(vcpu->kvm); 1422 kvm_flush_remote_tlbs(vcpu->kvm);
1384} 1423}
1385 1424
1386int kvm_age_hva(struct kvm *kvm, unsigned long hva) 1425int kvm_age_hva(struct kvm *kvm, unsigned long hva)
1387{ 1426{
1388 return kvm_handle_hva(kvm, hva, 0, kvm_age_rmapp); 1427 return kvm_handle_hva(kvm, hva, hva, kvm_age_rmapp);
1389} 1428}
1390 1429
1391int kvm_test_age_hva(struct kvm *kvm, unsigned long hva) 1430int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
@@ -2457,7 +2496,9 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
2457 rmap_recycle(vcpu, sptep, gfn); 2496 rmap_recycle(vcpu, sptep, gfn);
2458 } 2497 }
2459 } 2498 }
2460 kvm_release_pfn_clean(pfn); 2499
2500 if (!is_error_pfn(pfn))
2501 kvm_release_pfn_clean(pfn);
2461} 2502}
2462 2503
2463static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) 2504static void nonpaging_new_cr3(struct kvm_vcpu *vcpu)
@@ -2469,17 +2510,12 @@ static pfn_t pte_prefetch_gfn_to_pfn(struct kvm_vcpu *vcpu, gfn_t gfn,
2469 bool no_dirty_log) 2510 bool no_dirty_log)
2470{ 2511{
2471 struct kvm_memory_slot *slot; 2512 struct kvm_memory_slot *slot;
2472 unsigned long hva;
2473 2513
2474 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log); 2514 slot = gfn_to_memslot_dirty_bitmap(vcpu, gfn, no_dirty_log);
2475 if (!slot) { 2515 if (!slot)
2476 get_page(fault_page); 2516 return KVM_PFN_ERR_FAULT;
2477 return page_to_pfn(fault_page);
2478 }
2479 2517
2480 hva = gfn_to_hva_memslot(slot, gfn); 2518 return gfn_to_pfn_memslot_atomic(slot, gfn);
2481
2482 return hva_to_pfn_atomic(vcpu->kvm, hva);
2483} 2519}
2484 2520
2485static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu, 2521static int direct_pte_prefetch_many(struct kvm_vcpu *vcpu,
@@ -2580,11 +2616,6 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write,
2580 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr, 2616 sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr,
2581 iterator.level - 1, 2617 iterator.level - 1,
2582 1, ACC_ALL, iterator.sptep); 2618 1, ACC_ALL, iterator.sptep);
2583 if (!sp) {
2584 pgprintk("nonpaging_map: ENOMEM\n");
2585 kvm_release_pfn_clean(pfn);
2586 return -ENOMEM;
2587 }
2588 2619
2589 mmu_spte_set(iterator.sptep, 2620 mmu_spte_set(iterator.sptep,
2590 __pa(sp->spt) 2621 __pa(sp->spt)
@@ -2611,8 +2642,16 @@ static void kvm_send_hwpoison_signal(unsigned long address, struct task_struct *
2611 2642
2612static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn) 2643static int kvm_handle_bad_page(struct kvm_vcpu *vcpu, gfn_t gfn, pfn_t pfn)
2613{ 2644{
2614 kvm_release_pfn_clean(pfn); 2645 /*
2615 if (is_hwpoison_pfn(pfn)) { 2646 * Do not cache the mmio info caused by writing the readonly gfn
2647 * into the spte otherwise read access on readonly gfn also can
2648 * caused mmio page fault and treat it as mmio access.
2649 * Return 1 to tell kvm to emulate it.
2650 */
2651 if (pfn == KVM_PFN_ERR_RO_FAULT)
2652 return 1;
2653
2654 if (pfn == KVM_PFN_ERR_HWPOISON) {
2616 kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current); 2655 kvm_send_hwpoison_signal(gfn_to_hva(vcpu->kvm, gfn), current);
2617 return 0; 2656 return 0;
2618 } 2657 }
@@ -3236,8 +3275,6 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn,
3236 if (!async) 3275 if (!async)
3237 return false; /* *pfn has correct page already */ 3276 return false; /* *pfn has correct page already */
3238 3277
3239 put_page(pfn_to_page(*pfn));
3240
3241 if (!prefault && can_do_async_pf(vcpu)) { 3278 if (!prefault && can_do_async_pf(vcpu)) {
3242 trace_kvm_try_async_get_page(gva, gfn); 3279 trace_kvm_try_async_get_page(gva, gfn);
3243 if (kvm_find_async_pf_gfn(vcpu, gfn)) { 3280 if (kvm_find_async_pf_gfn(vcpu, gfn)) {
@@ -3371,6 +3408,18 @@ static bool is_rsvd_bits_set(struct kvm_mmu *mmu, u64 gpte, int level)
3371 return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0; 3408 return (gpte & mmu->rsvd_bits_mask[bit7][level-1]) != 0;
3372} 3409}
3373 3410
3411static inline void protect_clean_gpte(unsigned *access, unsigned gpte)
3412{
3413 unsigned mask;
3414
3415 BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
3416
3417 mask = (unsigned)~ACC_WRITE_MASK;
3418 /* Allow write access to dirty gptes */
3419 mask |= (gpte >> (PT_DIRTY_SHIFT - PT_WRITABLE_SHIFT)) & PT_WRITABLE_MASK;
3420 *access &= mask;
3421}
3422
3374static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access, 3423static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
3375 int *nr_present) 3424 int *nr_present)
3376{ 3425{
@@ -3388,6 +3437,25 @@ static bool sync_mmio_spte(u64 *sptep, gfn_t gfn, unsigned access,
3388 return false; 3437 return false;
3389} 3438}
3390 3439
3440static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
3441{
3442 unsigned access;
3443
3444 access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK;
3445 access &= ~(gpte >> PT64_NX_SHIFT);
3446
3447 return access;
3448}
3449
3450static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
3451{
3452 unsigned index;
3453
3454 index = level - 1;
3455 index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
3456 return mmu->last_pte_bitmap & (1 << index);
3457}
3458
3391#define PTTYPE 64 3459#define PTTYPE 64
3392#include "paging_tmpl.h" 3460#include "paging_tmpl.h"
3393#undef PTTYPE 3461#undef PTTYPE
@@ -3457,6 +3525,56 @@ static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu,
3457 } 3525 }
3458} 3526}
3459 3527
3528static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
3529{
3530 unsigned bit, byte, pfec;
3531 u8 map;
3532 bool fault, x, w, u, wf, uf, ff, smep;
3533
3534 smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP);
3535 for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) {
3536 pfec = byte << 1;
3537 map = 0;
3538 wf = pfec & PFERR_WRITE_MASK;
3539 uf = pfec & PFERR_USER_MASK;
3540 ff = pfec & PFERR_FETCH_MASK;
3541 for (bit = 0; bit < 8; ++bit) {
3542 x = bit & ACC_EXEC_MASK;
3543 w = bit & ACC_WRITE_MASK;
3544 u = bit & ACC_USER_MASK;
3545
3546 /* Not really needed: !nx will cause pte.nx to fault */
3547 x |= !mmu->nx;
3548 /* Allow supervisor writes if !cr0.wp */
3549 w |= !is_write_protection(vcpu) && !uf;
3550 /* Disallow supervisor fetches of user code if cr4.smep */
3551 x &= !(smep && u && !uf);
3552
3553 fault = (ff && !x) || (uf && !u) || (wf && !w);
3554 map |= fault << bit;
3555 }
3556 mmu->permissions[byte] = map;
3557 }
3558}
3559
3560static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
3561{
3562 u8 map;
3563 unsigned level, root_level = mmu->root_level;
3564 const unsigned ps_set_index = 1 << 2; /* bit 2 of index: ps */
3565
3566 if (root_level == PT32E_ROOT_LEVEL)
3567 --root_level;
3568 /* PT_PAGE_TABLE_LEVEL always terminates */
3569 map = 1 | (1 << ps_set_index);
3570 for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
3571 if (level <= PT_PDPE_LEVEL
3572 && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
3573 map |= 1 << (ps_set_index | (level - 1));
3574 }
3575 mmu->last_pte_bitmap = map;
3576}
3577
3460static int paging64_init_context_common(struct kvm_vcpu *vcpu, 3578static int paging64_init_context_common(struct kvm_vcpu *vcpu,
3461 struct kvm_mmu *context, 3579 struct kvm_mmu *context,
3462 int level) 3580 int level)
@@ -3465,6 +3583,8 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu,
3465 context->root_level = level; 3583 context->root_level = level;
3466 3584
3467 reset_rsvds_bits_mask(vcpu, context); 3585 reset_rsvds_bits_mask(vcpu, context);
3586 update_permission_bitmask(vcpu, context);
3587 update_last_pte_bitmap(vcpu, context);
3468 3588
3469 ASSERT(is_pae(vcpu)); 3589 ASSERT(is_pae(vcpu));
3470 context->new_cr3 = paging_new_cr3; 3590 context->new_cr3 = paging_new_cr3;
@@ -3493,6 +3613,8 @@ static int paging32_init_context(struct kvm_vcpu *vcpu,
3493 context->root_level = PT32_ROOT_LEVEL; 3613 context->root_level = PT32_ROOT_LEVEL;
3494 3614
3495 reset_rsvds_bits_mask(vcpu, context); 3615 reset_rsvds_bits_mask(vcpu, context);
3616 update_permission_bitmask(vcpu, context);
3617 update_last_pte_bitmap(vcpu, context);
3496 3618
3497 context->new_cr3 = paging_new_cr3; 3619 context->new_cr3 = paging_new_cr3;
3498 context->page_fault = paging32_page_fault; 3620 context->page_fault = paging32_page_fault;
@@ -3553,6 +3675,9 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
3553 context->gva_to_gpa = paging32_gva_to_gpa; 3675 context->gva_to_gpa = paging32_gva_to_gpa;
3554 } 3676 }
3555 3677
3678 update_permission_bitmask(vcpu, context);
3679 update_last_pte_bitmap(vcpu, context);
3680
3556 return 0; 3681 return 0;
3557} 3682}
3558 3683
@@ -3628,6 +3753,9 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
3628 g_context->gva_to_gpa = paging32_gva_to_gpa_nested; 3753 g_context->gva_to_gpa = paging32_gva_to_gpa_nested;
3629 } 3754 }
3630 3755
3756 update_permission_bitmask(vcpu, g_context);
3757 update_last_pte_bitmap(vcpu, g_context);
3758
3631 return 0; 3759 return 0;
3632} 3760}
3633 3761
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index e374db9af021..69871080e866 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -18,8 +18,10 @@
18#define PT_PCD_MASK (1ULL << 4) 18#define PT_PCD_MASK (1ULL << 4)
19#define PT_ACCESSED_SHIFT 5 19#define PT_ACCESSED_SHIFT 5
20#define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT) 20#define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
21#define PT_DIRTY_MASK (1ULL << 6) 21#define PT_DIRTY_SHIFT 6
22#define PT_PAGE_SIZE_MASK (1ULL << 7) 22#define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
23#define PT_PAGE_SIZE_SHIFT 7
24#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
23#define PT_PAT_MASK (1ULL << 7) 25#define PT_PAT_MASK (1ULL << 7)
24#define PT_GLOBAL_MASK (1ULL << 8) 26#define PT_GLOBAL_MASK (1ULL << 8)
25#define PT64_NX_SHIFT 63 27#define PT64_NX_SHIFT 63
@@ -88,17 +90,14 @@ static inline bool is_write_protection(struct kvm_vcpu *vcpu)
88 return kvm_read_cr0_bits(vcpu, X86_CR0_WP); 90 return kvm_read_cr0_bits(vcpu, X86_CR0_WP);
89} 91}
90 92
91static inline bool check_write_user_access(struct kvm_vcpu *vcpu, 93/*
92 bool write_fault, bool user_fault, 94 * Will a fault with a given page-fault error code (pfec) cause a permission
93 unsigned long pte) 95 * fault with the given access (in ACC_* format)?
96 */
97static inline bool permission_fault(struct kvm_mmu *mmu, unsigned pte_access,
98 unsigned pfec)
94{ 99{
95 if (unlikely(write_fault && !is_writable_pte(pte) 100 return (mmu->permissions[pfec >> 1] >> pte_access) & 1;
96 && (user_fault || is_write_protection(vcpu))))
97 return false;
98
99 if (unlikely(user_fault && !(pte & PT_USER_MASK)))
100 return false;
101
102 return true;
103} 101}
102
104#endif 103#endif
diff --git a/arch/x86/kvm/mmu_audit.c b/arch/x86/kvm/mmu_audit.c
index 7d7d0b9e23eb..daff69e21150 100644
--- a/arch/x86/kvm/mmu_audit.c
+++ b/arch/x86/kvm/mmu_audit.c
@@ -116,10 +116,8 @@ static void audit_mappings(struct kvm_vcpu *vcpu, u64 *sptep, int level)
116 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt); 116 gfn = kvm_mmu_page_get_gfn(sp, sptep - sp->spt);
117 pfn = gfn_to_pfn_atomic(vcpu->kvm, gfn); 117 pfn = gfn_to_pfn_atomic(vcpu->kvm, gfn);
118 118
119 if (is_error_pfn(pfn)) { 119 if (is_error_pfn(pfn))
120 kvm_release_pfn_clean(pfn);
121 return; 120 return;
122 }
123 121
124 hpa = pfn << PAGE_SHIFT; 122 hpa = pfn << PAGE_SHIFT;
125 if ((*sptep & PT64_BASE_ADDR_MASK) != hpa) 123 if ((*sptep & PT64_BASE_ADDR_MASK) != hpa)
@@ -190,7 +188,6 @@ static void check_mappings_rmap(struct kvm *kvm, struct kvm_mmu_page *sp)
190 188
191static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp) 189static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
192{ 190{
193 struct kvm_memory_slot *slot;
194 unsigned long *rmapp; 191 unsigned long *rmapp;
195 u64 *sptep; 192 u64 *sptep;
196 struct rmap_iterator iter; 193 struct rmap_iterator iter;
@@ -198,8 +195,7 @@ static void audit_write_protection(struct kvm *kvm, struct kvm_mmu_page *sp)
198 if (sp->role.direct || sp->unsync || sp->role.invalid) 195 if (sp->role.direct || sp->unsync || sp->role.invalid)
199 return; 196 return;
200 197
201 slot = gfn_to_memslot(kvm, sp->gfn); 198 rmapp = gfn_to_rmap(kvm, sp->gfn, PT_PAGE_TABLE_LEVEL);
202 rmapp = &slot->rmap[sp->gfn - slot->base_gfn];
203 199
204 for (sptep = rmap_get_first(*rmapp, &iter); sptep; 200 for (sptep = rmap_get_first(*rmapp, &iter); sptep;
205 sptep = rmap_get_next(&iter)) { 201 sptep = rmap_get_next(&iter)) {
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index bb7cf01cae76..714e2c01a6fe 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -63,10 +63,12 @@
63 */ 63 */
64struct guest_walker { 64struct guest_walker {
65 int level; 65 int level;
66 unsigned max_level;
66 gfn_t table_gfn[PT_MAX_FULL_LEVELS]; 67 gfn_t table_gfn[PT_MAX_FULL_LEVELS];
67 pt_element_t ptes[PT_MAX_FULL_LEVELS]; 68 pt_element_t ptes[PT_MAX_FULL_LEVELS];
68 pt_element_t prefetch_ptes[PTE_PREFETCH_NUM]; 69 pt_element_t prefetch_ptes[PTE_PREFETCH_NUM];
69 gpa_t pte_gpa[PT_MAX_FULL_LEVELS]; 70 gpa_t pte_gpa[PT_MAX_FULL_LEVELS];
71 pt_element_t __user *ptep_user[PT_MAX_FULL_LEVELS];
70 unsigned pt_access; 72 unsigned pt_access;
71 unsigned pte_access; 73 unsigned pte_access;
72 gfn_t gfn; 74 gfn_t gfn;
@@ -101,38 +103,41 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
101 return (ret != orig_pte); 103 return (ret != orig_pte);
102} 104}
103 105
104static unsigned FNAME(gpte_access)(struct kvm_vcpu *vcpu, pt_element_t gpte, 106static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
105 bool last) 107 struct kvm_mmu *mmu,
108 struct guest_walker *walker,
109 int write_fault)
106{ 110{
107 unsigned access; 111 unsigned level, index;
108 112 pt_element_t pte, orig_pte;
109 access = (gpte & (PT_WRITABLE_MASK | PT_USER_MASK)) | ACC_EXEC_MASK; 113 pt_element_t __user *ptep_user;
110 if (last && !is_dirty_gpte(gpte)) 114 gfn_t table_gfn;
111 access &= ~ACC_WRITE_MASK; 115 int ret;
112 116
113#if PTTYPE == 64 117 for (level = walker->max_level; level >= walker->level; --level) {
114 if (vcpu->arch.mmu.nx) 118 pte = orig_pte = walker->ptes[level - 1];
115 access &= ~(gpte >> PT64_NX_SHIFT); 119 table_gfn = walker->table_gfn[level - 1];
116#endif 120 ptep_user = walker->ptep_user[level - 1];
117 return access; 121 index = offset_in_page(ptep_user) / sizeof(pt_element_t);
118} 122 if (!(pte & PT_ACCESSED_MASK)) {
119 123 trace_kvm_mmu_set_accessed_bit(table_gfn, index, sizeof(pte));
120static bool FNAME(is_last_gpte)(struct guest_walker *walker, 124 pte |= PT_ACCESSED_MASK;
121 struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 125 }
122 pt_element_t gpte) 126 if (level == walker->level && write_fault && !is_dirty_gpte(pte)) {
123{ 127 trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte));
124 if (walker->level == PT_PAGE_TABLE_LEVEL) 128 pte |= PT_DIRTY_MASK;
125 return true; 129 }
126 130 if (pte == orig_pte)
127 if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) && 131 continue;
128 (PTTYPE == 64 || is_pse(vcpu)))
129 return true;
130 132
131 if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) && 133 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, orig_pte, pte);
132 (mmu->root_level == PT64_ROOT_LEVEL)) 134 if (ret)
133 return true; 135 return ret;
134 136
135 return false; 137 mark_page_dirty(vcpu->kvm, table_gfn);
138 walker->ptes[level] = pte;
139 }
140 return 0;
136} 141}
137 142
138/* 143/*
@@ -142,21 +147,22 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
142 struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, 147 struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
143 gva_t addr, u32 access) 148 gva_t addr, u32 access)
144{ 149{
150 int ret;
145 pt_element_t pte; 151 pt_element_t pte;
146 pt_element_t __user *uninitialized_var(ptep_user); 152 pt_element_t __user *uninitialized_var(ptep_user);
147 gfn_t table_gfn; 153 gfn_t table_gfn;
148 unsigned index, pt_access, uninitialized_var(pte_access); 154 unsigned index, pt_access, pte_access, accessed_dirty, shift;
149 gpa_t pte_gpa; 155 gpa_t pte_gpa;
150 bool eperm, last_gpte;
151 int offset; 156 int offset;
152 const int write_fault = access & PFERR_WRITE_MASK; 157 const int write_fault = access & PFERR_WRITE_MASK;
153 const int user_fault = access & PFERR_USER_MASK; 158 const int user_fault = access & PFERR_USER_MASK;
154 const int fetch_fault = access & PFERR_FETCH_MASK; 159 const int fetch_fault = access & PFERR_FETCH_MASK;
155 u16 errcode = 0; 160 u16 errcode = 0;
161 gpa_t real_gpa;
162 gfn_t gfn;
156 163
157 trace_kvm_mmu_pagetable_walk(addr, access); 164 trace_kvm_mmu_pagetable_walk(addr, access);
158retry_walk: 165retry_walk:
159 eperm = false;
160 walker->level = mmu->root_level; 166 walker->level = mmu->root_level;
161 pte = mmu->get_cr3(vcpu); 167 pte = mmu->get_cr3(vcpu);
162 168
@@ -169,15 +175,21 @@ retry_walk:
169 --walker->level; 175 --walker->level;
170 } 176 }
171#endif 177#endif
178 walker->max_level = walker->level;
172 ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) || 179 ASSERT((!is_long_mode(vcpu) && is_pae(vcpu)) ||
173 (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0); 180 (mmu->get_cr3(vcpu) & CR3_NONPAE_RESERVED_BITS) == 0);
174 181
175 pt_access = ACC_ALL; 182 accessed_dirty = PT_ACCESSED_MASK;
183 pt_access = pte_access = ACC_ALL;
184 ++walker->level;
176 185
177 for (;;) { 186 do {
178 gfn_t real_gfn; 187 gfn_t real_gfn;
179 unsigned long host_addr; 188 unsigned long host_addr;
180 189
190 pt_access &= pte_access;
191 --walker->level;
192
181 index = PT_INDEX(addr, walker->level); 193 index = PT_INDEX(addr, walker->level);
182 194
183 table_gfn = gpte_to_gfn(pte); 195 table_gfn = gpte_to_gfn(pte);
@@ -199,6 +211,7 @@ retry_walk:
199 ptep_user = (pt_element_t __user *)((void *)host_addr + offset); 211 ptep_user = (pt_element_t __user *)((void *)host_addr + offset);
200 if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte)))) 212 if (unlikely(__copy_from_user(&pte, ptep_user, sizeof(pte))))
201 goto error; 213 goto error;
214 walker->ptep_user[walker->level - 1] = ptep_user;
202 215
203 trace_kvm_mmu_paging_element(pte, walker->level); 216 trace_kvm_mmu_paging_element(pte, walker->level);
204 217
@@ -211,92 +224,48 @@ retry_walk:
211 goto error; 224 goto error;
212 } 225 }
213 226
214 if (!check_write_user_access(vcpu, write_fault, user_fault, 227 accessed_dirty &= pte;
215 pte)) 228 pte_access = pt_access & gpte_access(vcpu, pte);
216 eperm = true;
217
218#if PTTYPE == 64
219 if (unlikely(fetch_fault && (pte & PT64_NX_MASK)))
220 eperm = true;
221#endif
222
223 last_gpte = FNAME(is_last_gpte)(walker, vcpu, mmu, pte);
224 if (last_gpte) {
225 pte_access = pt_access &
226 FNAME(gpte_access)(vcpu, pte, true);
227 /* check if the kernel is fetching from user page */
228 if (unlikely(pte_access & PT_USER_MASK) &&
229 kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))
230 if (fetch_fault && !user_fault)
231 eperm = true;
232 }
233
234 if (!eperm && unlikely(!(pte & PT_ACCESSED_MASK))) {
235 int ret;
236 trace_kvm_mmu_set_accessed_bit(table_gfn, index,
237 sizeof(pte));
238 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index,
239 pte, pte|PT_ACCESSED_MASK);
240 if (unlikely(ret < 0))
241 goto error;
242 else if (ret)
243 goto retry_walk;
244
245 mark_page_dirty(vcpu->kvm, table_gfn);
246 pte |= PT_ACCESSED_MASK;
247 }
248 229
249 walker->ptes[walker->level - 1] = pte; 230 walker->ptes[walker->level - 1] = pte;
231 } while (!is_last_gpte(mmu, walker->level, pte));
250 232
251 if (last_gpte) { 233 if (unlikely(permission_fault(mmu, pte_access, access))) {
252 int lvl = walker->level; 234 errcode |= PFERR_PRESENT_MASK;
253 gpa_t real_gpa; 235 goto error;
254 gfn_t gfn; 236 }
255 u32 ac;
256
257 gfn = gpte_to_gfn_lvl(pte, lvl);
258 gfn += (addr & PT_LVL_OFFSET_MASK(lvl)) >> PAGE_SHIFT;
259
260 if (PTTYPE == 32 &&
261 walker->level == PT_DIRECTORY_LEVEL &&
262 is_cpuid_PSE36())
263 gfn += pse36_gfn_delta(pte);
264
265 ac = write_fault | fetch_fault | user_fault;
266 237
267 real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), 238 gfn = gpte_to_gfn_lvl(pte, walker->level);
268 ac); 239 gfn += (addr & PT_LVL_OFFSET_MASK(walker->level)) >> PAGE_SHIFT;
269 if (real_gpa == UNMAPPED_GVA)
270 return 0;
271 240
272 walker->gfn = real_gpa >> PAGE_SHIFT; 241 if (PTTYPE == 32 && walker->level == PT_DIRECTORY_LEVEL && is_cpuid_PSE36())
242 gfn += pse36_gfn_delta(pte);
273 243
274 break; 244 real_gpa = mmu->translate_gpa(vcpu, gfn_to_gpa(gfn), access);
275 } 245 if (real_gpa == UNMAPPED_GVA)
246 return 0;
276 247
277 pt_access &= FNAME(gpte_access)(vcpu, pte, false); 248 walker->gfn = real_gpa >> PAGE_SHIFT;
278 --walker->level;
279 }
280 249
281 if (unlikely(eperm)) { 250 if (!write_fault)
282 errcode |= PFERR_PRESENT_MASK; 251 protect_clean_gpte(&pte_access, pte);
283 goto error;
284 }
285 252
286 if (write_fault && unlikely(!is_dirty_gpte(pte))) { 253 /*
287 int ret; 254 * On a write fault, fold the dirty bit into accessed_dirty by shifting it one
255 * place right.
256 *
257 * On a read fault, do nothing.
258 */
259 shift = write_fault >> ilog2(PFERR_WRITE_MASK);
260 shift *= PT_DIRTY_SHIFT - PT_ACCESSED_SHIFT;
261 accessed_dirty &= pte >> shift;
288 262
289 trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte)); 263 if (unlikely(!accessed_dirty)) {
290 ret = FNAME(cmpxchg_gpte)(vcpu, mmu, ptep_user, index, 264 ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker, write_fault);
291 pte, pte|PT_DIRTY_MASK);
292 if (unlikely(ret < 0)) 265 if (unlikely(ret < 0))
293 goto error; 266 goto error;
294 else if (ret) 267 else if (ret)
295 goto retry_walk; 268 goto retry_walk;
296
297 mark_page_dirty(vcpu->kvm, table_gfn);
298 pte |= PT_DIRTY_MASK;
299 walker->ptes[walker->level - 1] = pte;
300 } 269 }
301 270
302 walker->pt_access = pt_access; 271 walker->pt_access = pt_access;
@@ -368,12 +337,11 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
368 return; 337 return;
369 338
370 pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte); 339 pgprintk("%s: gpte %llx spte %p\n", __func__, (u64)gpte, spte);
371 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, true); 340 pte_access = sp->role.access & gpte_access(vcpu, gpte);
341 protect_clean_gpte(&pte_access, gpte);
372 pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte)); 342 pfn = gfn_to_pfn_atomic(vcpu->kvm, gpte_to_gfn(gpte));
373 if (mmu_invalid_pfn(pfn)) { 343 if (mmu_invalid_pfn(pfn))
374 kvm_release_pfn_clean(pfn);
375 return; 344 return;
376 }
377 345
378 /* 346 /*
379 * we call mmu_set_spte() with host_writable = true because that 347 * we call mmu_set_spte() with host_writable = true because that
@@ -443,15 +411,13 @@ static void FNAME(pte_prefetch)(struct kvm_vcpu *vcpu, struct guest_walker *gw,
443 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte)) 411 if (FNAME(prefetch_invalid_gpte)(vcpu, sp, spte, gpte))
444 continue; 412 continue;
445 413
446 pte_access = sp->role.access & FNAME(gpte_access)(vcpu, gpte, 414 pte_access = sp->role.access & gpte_access(vcpu, gpte);
447 true); 415 protect_clean_gpte(&pte_access, gpte);
448 gfn = gpte_to_gfn(gpte); 416 gfn = gpte_to_gfn(gpte);
449 pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn, 417 pfn = pte_prefetch_gfn_to_pfn(vcpu, gfn,
450 pte_access & ACC_WRITE_MASK); 418 pte_access & ACC_WRITE_MASK);
451 if (mmu_invalid_pfn(pfn)) { 419 if (mmu_invalid_pfn(pfn))
452 kvm_release_pfn_clean(pfn);
453 break; 420 break;
454 }
455 421
456 mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0, 422 mmu_set_spte(vcpu, spte, sp->role.access, pte_access, 0, 0,
457 NULL, PT_PAGE_TABLE_LEVEL, gfn, 423 NULL, PT_PAGE_TABLE_LEVEL, gfn,
@@ -798,7 +764,8 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
798 764
799 gfn = gpte_to_gfn(gpte); 765 gfn = gpte_to_gfn(gpte);
800 pte_access = sp->role.access; 766 pte_access = sp->role.access;
801 pte_access &= FNAME(gpte_access)(vcpu, gpte, true); 767 pte_access &= gpte_access(vcpu, gpte);
768 protect_clean_gpte(&pte_access, gpte);
802 769
803 if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present)) 770 if (sync_mmio_spte(&sp->spt[i], gfn, pte_access, &nr_present))
804 continue; 771 continue;
diff --git a/arch/x86/kvm/pmu.c b/arch/x86/kvm/pmu.c
index 9b7ec1150ab0..cfc258a6bf97 100644
--- a/arch/x86/kvm/pmu.c
+++ b/arch/x86/kvm/pmu.c
@@ -1,5 +1,5 @@
1/* 1/*
2 * Kernel-based Virtual Machine -- Performane Monitoring Unit support 2 * Kernel-based Virtual Machine -- Performance Monitoring Unit support
3 * 3 *
4 * Copyright 2011 Red Hat, Inc. and/or its affiliates. 4 * Copyright 2011 Red Hat, Inc. and/or its affiliates.
5 * 5 *
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c
index baead950d6c8..d017df3899ef 100644
--- a/arch/x86/kvm/svm.c
+++ b/arch/x86/kvm/svm.c
@@ -163,7 +163,7 @@ static DEFINE_PER_CPU(u64, current_tsc_ratio);
163 163
164#define MSR_INVALID 0xffffffffU 164#define MSR_INVALID 0xffffffffU
165 165
166static struct svm_direct_access_msrs { 166static const struct svm_direct_access_msrs {
167 u32 index; /* Index of the MSR */ 167 u32 index; /* Index of the MSR */
168 bool always; /* True if intercept is always on */ 168 bool always; /* True if intercept is always on */
169} direct_access_msrs[] = { 169} direct_access_msrs[] = {
@@ -400,7 +400,7 @@ struct svm_init_data {
400 int r; 400 int r;
401}; 401};
402 402
403static u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000}; 403static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
404 404
405#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges) 405#define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
406#define MSRS_RANGE_SIZE 2048 406#define MSRS_RANGE_SIZE 2048
@@ -1146,7 +1146,6 @@ static void init_vmcb(struct vcpu_svm *svm)
1146 1146
1147 svm_set_efer(&svm->vcpu, 0); 1147 svm_set_efer(&svm->vcpu, 0);
1148 save->dr6 = 0xffff0ff0; 1148 save->dr6 = 0xffff0ff0;
1149 save->dr7 = 0x400;
1150 kvm_set_rflags(&svm->vcpu, 2); 1149 kvm_set_rflags(&svm->vcpu, 2);
1151 save->rip = 0x0000fff0; 1150 save->rip = 0x0000fff0;
1152 svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip; 1151 svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
@@ -1643,7 +1642,7 @@ static void svm_set_segment(struct kvm_vcpu *vcpu,
1643 mark_dirty(svm->vmcb, VMCB_SEG); 1642 mark_dirty(svm->vmcb, VMCB_SEG);
1644} 1643}
1645 1644
1646static void update_db_intercept(struct kvm_vcpu *vcpu) 1645static void update_db_bp_intercept(struct kvm_vcpu *vcpu)
1647{ 1646{
1648 struct vcpu_svm *svm = to_svm(vcpu); 1647 struct vcpu_svm *svm = to_svm(vcpu);
1649 1648
@@ -1663,20 +1662,6 @@ static void update_db_intercept(struct kvm_vcpu *vcpu)
1663 vcpu->guest_debug = 0; 1662 vcpu->guest_debug = 0;
1664} 1663}
1665 1664
1666static void svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
1667{
1668 struct vcpu_svm *svm = to_svm(vcpu);
1669
1670 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
1671 svm->vmcb->save.dr7 = dbg->arch.debugreg[7];
1672 else
1673 svm->vmcb->save.dr7 = vcpu->arch.dr7;
1674
1675 mark_dirty(svm->vmcb, VMCB_DR);
1676
1677 update_db_intercept(vcpu);
1678}
1679
1680static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd) 1665static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1681{ 1666{
1682 if (sd->next_asid > sd->max_asid) { 1667 if (sd->next_asid > sd->max_asid) {
@@ -1748,7 +1733,7 @@ static int db_interception(struct vcpu_svm *svm)
1748 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) 1733 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1749 svm->vmcb->save.rflags &= 1734 svm->vmcb->save.rflags &=
1750 ~(X86_EFLAGS_TF | X86_EFLAGS_RF); 1735 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1751 update_db_intercept(&svm->vcpu); 1736 update_db_bp_intercept(&svm->vcpu);
1752 } 1737 }
1753 1738
1754 if (svm->vcpu.guest_debug & 1739 if (svm->vcpu.guest_debug &
@@ -2063,7 +2048,7 @@ static inline bool nested_svm_intr(struct vcpu_svm *svm)
2063 if (svm->nested.intercept & 1ULL) { 2048 if (svm->nested.intercept & 1ULL) {
2064 /* 2049 /*
2065 * The #vmexit can't be emulated here directly because this 2050 * The #vmexit can't be emulated here directly because this
2066 * code path runs with irqs and preemtion disabled. A 2051 * code path runs with irqs and preemption disabled. A
2067 * #vmexit emulation might sleep. Only signal request for 2052 * #vmexit emulation might sleep. Only signal request for
2068 * the #vmexit here. 2053 * the #vmexit here.
2069 */ 2054 */
@@ -2105,7 +2090,6 @@ static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2105 return kmap(page); 2090 return kmap(page);
2106 2091
2107error: 2092error:
2108 kvm_release_page_clean(page);
2109 kvm_inject_gp(&svm->vcpu, 0); 2093 kvm_inject_gp(&svm->vcpu, 0);
2110 2094
2111 return NULL; 2095 return NULL;
@@ -2409,7 +2393,7 @@ static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2409{ 2393{
2410 /* 2394 /*
2411 * This function merges the msr permission bitmaps of kvm and the 2395 * This function merges the msr permission bitmaps of kvm and the
2412 * nested vmcb. It is omptimized in that it only merges the parts where 2396 * nested vmcb. It is optimized in that it only merges the parts where
2413 * the kvm msr permission bitmap may contain zero bits 2397 * the kvm msr permission bitmap may contain zero bits
2414 */ 2398 */
2415 int i; 2399 int i;
@@ -3268,7 +3252,7 @@ static int pause_interception(struct vcpu_svm *svm)
3268 return 1; 3252 return 1;
3269} 3253}
3270 3254
3271static int (*svm_exit_handlers[])(struct vcpu_svm *svm) = { 3255static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3272 [SVM_EXIT_READ_CR0] = cr_interception, 3256 [SVM_EXIT_READ_CR0] = cr_interception,
3273 [SVM_EXIT_READ_CR3] = cr_interception, 3257 [SVM_EXIT_READ_CR3] = cr_interception,
3274 [SVM_EXIT_READ_CR4] = cr_interception, 3258 [SVM_EXIT_READ_CR4] = cr_interception,
@@ -3660,7 +3644,7 @@ static void enable_nmi_window(struct kvm_vcpu *vcpu)
3660 */ 3644 */
3661 svm->nmi_singlestep = true; 3645 svm->nmi_singlestep = true;
3662 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); 3646 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3663 update_db_intercept(vcpu); 3647 update_db_bp_intercept(vcpu);
3664} 3648}
3665 3649
3666static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) 3650static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
@@ -3783,12 +3767,6 @@ static void svm_cancel_injection(struct kvm_vcpu *vcpu)
3783 svm_complete_interrupts(svm); 3767 svm_complete_interrupts(svm);
3784} 3768}
3785 3769
3786#ifdef CONFIG_X86_64
3787#define R "r"
3788#else
3789#define R "e"
3790#endif
3791
3792static void svm_vcpu_run(struct kvm_vcpu *vcpu) 3770static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3793{ 3771{
3794 struct vcpu_svm *svm = to_svm(vcpu); 3772 struct vcpu_svm *svm = to_svm(vcpu);
@@ -3815,13 +3793,13 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3815 local_irq_enable(); 3793 local_irq_enable();
3816 3794
3817 asm volatile ( 3795 asm volatile (
3818 "push %%"R"bp; \n\t" 3796 "push %%" _ASM_BP "; \n\t"
3819 "mov %c[rbx](%[svm]), %%"R"bx \n\t" 3797 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
3820 "mov %c[rcx](%[svm]), %%"R"cx \n\t" 3798 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
3821 "mov %c[rdx](%[svm]), %%"R"dx \n\t" 3799 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
3822 "mov %c[rsi](%[svm]), %%"R"si \n\t" 3800 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
3823 "mov %c[rdi](%[svm]), %%"R"di \n\t" 3801 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
3824 "mov %c[rbp](%[svm]), %%"R"bp \n\t" 3802 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
3825#ifdef CONFIG_X86_64 3803#ifdef CONFIG_X86_64
3826 "mov %c[r8](%[svm]), %%r8 \n\t" 3804 "mov %c[r8](%[svm]), %%r8 \n\t"
3827 "mov %c[r9](%[svm]), %%r9 \n\t" 3805 "mov %c[r9](%[svm]), %%r9 \n\t"
@@ -3834,20 +3812,20 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3834#endif 3812#endif
3835 3813
3836 /* Enter guest mode */ 3814 /* Enter guest mode */
3837 "push %%"R"ax \n\t" 3815 "push %%" _ASM_AX " \n\t"
3838 "mov %c[vmcb](%[svm]), %%"R"ax \n\t" 3816 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
3839 __ex(SVM_VMLOAD) "\n\t" 3817 __ex(SVM_VMLOAD) "\n\t"
3840 __ex(SVM_VMRUN) "\n\t" 3818 __ex(SVM_VMRUN) "\n\t"
3841 __ex(SVM_VMSAVE) "\n\t" 3819 __ex(SVM_VMSAVE) "\n\t"
3842 "pop %%"R"ax \n\t" 3820 "pop %%" _ASM_AX " \n\t"
3843 3821
3844 /* Save guest registers, load host registers */ 3822 /* Save guest registers, load host registers */
3845 "mov %%"R"bx, %c[rbx](%[svm]) \n\t" 3823 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
3846 "mov %%"R"cx, %c[rcx](%[svm]) \n\t" 3824 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
3847 "mov %%"R"dx, %c[rdx](%[svm]) \n\t" 3825 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
3848 "mov %%"R"si, %c[rsi](%[svm]) \n\t" 3826 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
3849 "mov %%"R"di, %c[rdi](%[svm]) \n\t" 3827 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
3850 "mov %%"R"bp, %c[rbp](%[svm]) \n\t" 3828 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
3851#ifdef CONFIG_X86_64 3829#ifdef CONFIG_X86_64
3852 "mov %%r8, %c[r8](%[svm]) \n\t" 3830 "mov %%r8, %c[r8](%[svm]) \n\t"
3853 "mov %%r9, %c[r9](%[svm]) \n\t" 3831 "mov %%r9, %c[r9](%[svm]) \n\t"
@@ -3858,7 +3836,7 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3858 "mov %%r14, %c[r14](%[svm]) \n\t" 3836 "mov %%r14, %c[r14](%[svm]) \n\t"
3859 "mov %%r15, %c[r15](%[svm]) \n\t" 3837 "mov %%r15, %c[r15](%[svm]) \n\t"
3860#endif 3838#endif
3861 "pop %%"R"bp" 3839 "pop %%" _ASM_BP
3862 : 3840 :
3863 : [svm]"a"(svm), 3841 : [svm]"a"(svm),
3864 [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)), 3842 [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
@@ -3879,9 +3857,11 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3879 [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15])) 3857 [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
3880#endif 3858#endif
3881 : "cc", "memory" 3859 : "cc", "memory"
3882 , R"bx", R"cx", R"dx", R"si", R"di"
3883#ifdef CONFIG_X86_64 3860#ifdef CONFIG_X86_64
3861 , "rbx", "rcx", "rdx", "rsi", "rdi"
3884 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15" 3862 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
3863#else
3864 , "ebx", "ecx", "edx", "esi", "edi"
3885#endif 3865#endif
3886 ); 3866 );
3887 3867
@@ -3941,8 +3921,6 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu)
3941 mark_all_clean(svm->vmcb); 3921 mark_all_clean(svm->vmcb);
3942} 3922}
3943 3923
3944#undef R
3945
3946static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root) 3924static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
3947{ 3925{
3948 struct vcpu_svm *svm = to_svm(vcpu); 3926 struct vcpu_svm *svm = to_svm(vcpu);
@@ -4069,7 +4047,7 @@ static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
4069#define POST_MEM(exit) { .exit_code = (exit), \ 4047#define POST_MEM(exit) { .exit_code = (exit), \
4070 .stage = X86_ICPT_POST_MEMACCESS, } 4048 .stage = X86_ICPT_POST_MEMACCESS, }
4071 4049
4072static struct __x86_intercept { 4050static const struct __x86_intercept {
4073 u32 exit_code; 4051 u32 exit_code;
4074 enum x86_intercept_stage stage; 4052 enum x86_intercept_stage stage;
4075} x86_intercept_map[] = { 4053} x86_intercept_map[] = {
@@ -4260,7 +4238,7 @@ static struct kvm_x86_ops svm_x86_ops = {
4260 .vcpu_load = svm_vcpu_load, 4238 .vcpu_load = svm_vcpu_load,
4261 .vcpu_put = svm_vcpu_put, 4239 .vcpu_put = svm_vcpu_put,
4262 4240
4263 .set_guest_debug = svm_guest_debug, 4241 .update_db_bp_intercept = update_db_bp_intercept,
4264 .get_msr = svm_get_msr, 4242 .get_msr = svm_get_msr,
4265 .set_msr = svm_set_msr, 4243 .set_msr = svm_set_msr,
4266 .get_segment_base = svm_get_segment_base, 4244 .get_segment_base = svm_get_segment_base,
diff --git a/arch/x86/kvm/timer.c b/arch/x86/kvm/timer.c
deleted file mode 100644
index 6b85cc647f34..000000000000
--- a/arch/x86/kvm/timer.c
+++ /dev/null
@@ -1,47 +0,0 @@
1/*
2 * Kernel-based Virtual Machine driver for Linux
3 *
4 * This module enables machines with Intel VT-x extensions to run virtual
5 * machines without emulation or binary translation.
6 *
7 * timer support
8 *
9 * Copyright 2010 Red Hat, Inc. and/or its affiliates.
10 *
11 * This work is licensed under the terms of the GNU GPL, version 2. See
12 * the COPYING file in the top-level directory.
13 */
14
15#include <linux/kvm_host.h>
16#include <linux/kvm.h>
17#include <linux/hrtimer.h>
18#include <linux/atomic.h>
19#include "kvm_timer.h"
20
21enum hrtimer_restart kvm_timer_fn(struct hrtimer *data)
22{
23 struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
24 struct kvm_vcpu *vcpu = ktimer->vcpu;
25 wait_queue_head_t *q = &vcpu->wq;
26
27 /*
28 * There is a race window between reading and incrementing, but we do
29 * not care about potentially losing timer events in the !reinject
30 * case anyway. Note: KVM_REQ_PENDING_TIMER is implicitly checked
31 * in vcpu_enter_guest.
32 */
33 if (ktimer->reinject || !atomic_read(&ktimer->pending)) {
34 atomic_inc(&ktimer->pending);
35 /* FIXME: this code should not know anything about vcpus */
36 kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
37 }
38
39 if (waitqueue_active(q))
40 wake_up_interruptible(q);
41
42 if (ktimer->t_ops->is_periodic(ktimer)) {
43 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
44 return HRTIMER_RESTART;
45 } else
46 return HRTIMER_NORESTART;
47}
diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h
index a71faf727ff3..bca63f04dccb 100644
--- a/arch/x86/kvm/trace.h
+++ b/arch/x86/kvm/trace.h
@@ -183,95 +183,6 @@ TRACE_EVENT(kvm_apic,
183#define KVM_ISA_VMX 1 183#define KVM_ISA_VMX 1
184#define KVM_ISA_SVM 2 184#define KVM_ISA_SVM 2
185 185
186#define VMX_EXIT_REASONS \
187 { EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \
188 { EXIT_REASON_EXTERNAL_INTERRUPT, "EXTERNAL_INTERRUPT" }, \
189 { EXIT_REASON_TRIPLE_FAULT, "TRIPLE_FAULT" }, \
190 { EXIT_REASON_PENDING_INTERRUPT, "PENDING_INTERRUPT" }, \
191 { EXIT_REASON_NMI_WINDOW, "NMI_WINDOW" }, \
192 { EXIT_REASON_TASK_SWITCH, "TASK_SWITCH" }, \
193 { EXIT_REASON_CPUID, "CPUID" }, \
194 { EXIT_REASON_HLT, "HLT" }, \
195 { EXIT_REASON_INVLPG, "INVLPG" }, \
196 { EXIT_REASON_RDPMC, "RDPMC" }, \
197 { EXIT_REASON_RDTSC, "RDTSC" }, \
198 { EXIT_REASON_VMCALL, "VMCALL" }, \
199 { EXIT_REASON_VMCLEAR, "VMCLEAR" }, \
200 { EXIT_REASON_VMLAUNCH, "VMLAUNCH" }, \
201 { EXIT_REASON_VMPTRLD, "VMPTRLD" }, \
202 { EXIT_REASON_VMPTRST, "VMPTRST" }, \
203 { EXIT_REASON_VMREAD, "VMREAD" }, \
204 { EXIT_REASON_VMRESUME, "VMRESUME" }, \
205 { EXIT_REASON_VMWRITE, "VMWRITE" }, \
206 { EXIT_REASON_VMOFF, "VMOFF" }, \
207 { EXIT_REASON_VMON, "VMON" }, \
208 { EXIT_REASON_CR_ACCESS, "CR_ACCESS" }, \
209 { EXIT_REASON_DR_ACCESS, "DR_ACCESS" }, \
210 { EXIT_REASON_IO_INSTRUCTION, "IO_INSTRUCTION" }, \
211 { EXIT_REASON_MSR_READ, "MSR_READ" }, \
212 { EXIT_REASON_MSR_WRITE, "MSR_WRITE" }, \
213 { EXIT_REASON_MWAIT_INSTRUCTION, "MWAIT_INSTRUCTION" }, \
214 { EXIT_REASON_MONITOR_INSTRUCTION, "MONITOR_INSTRUCTION" }, \
215 { EXIT_REASON_PAUSE_INSTRUCTION, "PAUSE_INSTRUCTION" }, \
216 { EXIT_REASON_MCE_DURING_VMENTRY, "MCE_DURING_VMENTRY" }, \
217 { EXIT_REASON_TPR_BELOW_THRESHOLD, "TPR_BELOW_THRESHOLD" }, \
218 { EXIT_REASON_APIC_ACCESS, "APIC_ACCESS" }, \
219 { EXIT_REASON_EPT_VIOLATION, "EPT_VIOLATION" }, \
220 { EXIT_REASON_EPT_MISCONFIG, "EPT_MISCONFIG" }, \
221 { EXIT_REASON_WBINVD, "WBINVD" }
222
223#define SVM_EXIT_REASONS \
224 { SVM_EXIT_READ_CR0, "read_cr0" }, \
225 { SVM_EXIT_READ_CR3, "read_cr3" }, \
226 { SVM_EXIT_READ_CR4, "read_cr4" }, \
227 { SVM_EXIT_READ_CR8, "read_cr8" }, \
228 { SVM_EXIT_WRITE_CR0, "write_cr0" }, \
229 { SVM_EXIT_WRITE_CR3, "write_cr3" }, \
230 { SVM_EXIT_WRITE_CR4, "write_cr4" }, \
231 { SVM_EXIT_WRITE_CR8, "write_cr8" }, \
232 { SVM_EXIT_READ_DR0, "read_dr0" }, \
233 { SVM_EXIT_READ_DR1, "read_dr1" }, \
234 { SVM_EXIT_READ_DR2, "read_dr2" }, \
235 { SVM_EXIT_READ_DR3, "read_dr3" }, \
236 { SVM_EXIT_WRITE_DR0, "write_dr0" }, \
237 { SVM_EXIT_WRITE_DR1, "write_dr1" }, \
238 { SVM_EXIT_WRITE_DR2, "write_dr2" }, \
239 { SVM_EXIT_WRITE_DR3, "write_dr3" }, \
240 { SVM_EXIT_WRITE_DR5, "write_dr5" }, \
241 { SVM_EXIT_WRITE_DR7, "write_dr7" }, \
242 { SVM_EXIT_EXCP_BASE + DB_VECTOR, "DB excp" }, \
243 { SVM_EXIT_EXCP_BASE + BP_VECTOR, "BP excp" }, \
244 { SVM_EXIT_EXCP_BASE + UD_VECTOR, "UD excp" }, \
245 { SVM_EXIT_EXCP_BASE + PF_VECTOR, "PF excp" }, \
246 { SVM_EXIT_EXCP_BASE + NM_VECTOR, "NM excp" }, \
247 { SVM_EXIT_EXCP_BASE + MC_VECTOR, "MC excp" }, \
248 { SVM_EXIT_INTR, "interrupt" }, \
249 { SVM_EXIT_NMI, "nmi" }, \
250 { SVM_EXIT_SMI, "smi" }, \
251 { SVM_EXIT_INIT, "init" }, \
252 { SVM_EXIT_VINTR, "vintr" }, \
253 { SVM_EXIT_CPUID, "cpuid" }, \
254 { SVM_EXIT_INVD, "invd" }, \
255 { SVM_EXIT_HLT, "hlt" }, \
256 { SVM_EXIT_INVLPG, "invlpg" }, \
257 { SVM_EXIT_INVLPGA, "invlpga" }, \
258 { SVM_EXIT_IOIO, "io" }, \
259 { SVM_EXIT_MSR, "msr" }, \
260 { SVM_EXIT_TASK_SWITCH, "task_switch" }, \
261 { SVM_EXIT_SHUTDOWN, "shutdown" }, \
262 { SVM_EXIT_VMRUN, "vmrun" }, \
263 { SVM_EXIT_VMMCALL, "hypercall" }, \
264 { SVM_EXIT_VMLOAD, "vmload" }, \
265 { SVM_EXIT_VMSAVE, "vmsave" }, \
266 { SVM_EXIT_STGI, "stgi" }, \
267 { SVM_EXIT_CLGI, "clgi" }, \
268 { SVM_EXIT_SKINIT, "skinit" }, \
269 { SVM_EXIT_WBINVD, "wbinvd" }, \
270 { SVM_EXIT_MONITOR, "monitor" }, \
271 { SVM_EXIT_MWAIT, "mwait" }, \
272 { SVM_EXIT_XSETBV, "xsetbv" }, \
273 { SVM_EXIT_NPF, "npf" }
274
275/* 186/*
276 * Tracepoint for kvm guest exit: 187 * Tracepoint for kvm guest exit:
277 */ 188 */
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index c00f03de1b79..ad6b1dd06f8b 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -127,6 +127,8 @@ module_param(ple_gap, int, S_IRUGO);
127static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW; 127static int ple_window = KVM_VMX_DEFAULT_PLE_WINDOW;
128module_param(ple_window, int, S_IRUGO); 128module_param(ple_window, int, S_IRUGO);
129 129
130extern const ulong vmx_return;
131
130#define NR_AUTOLOAD_MSRS 8 132#define NR_AUTOLOAD_MSRS 8
131#define VMCS02_POOL_SIZE 1 133#define VMCS02_POOL_SIZE 1
132 134
@@ -405,16 +407,16 @@ struct vcpu_vmx {
405 struct { 407 struct {
406 int vm86_active; 408 int vm86_active;
407 ulong save_rflags; 409 ulong save_rflags;
410 struct kvm_segment segs[8];
411 } rmode;
412 struct {
413 u32 bitmask; /* 4 bits per segment (1 bit per field) */
408 struct kvm_save_segment { 414 struct kvm_save_segment {
409 u16 selector; 415 u16 selector;
410 unsigned long base; 416 unsigned long base;
411 u32 limit; 417 u32 limit;
412 u32 ar; 418 u32 ar;
413 } tr, es, ds, fs, gs; 419 } seg[8];
414 } rmode;
415 struct {
416 u32 bitmask; /* 4 bits per segment (1 bit per field) */
417 struct kvm_save_segment seg[8];
418 } segment_cache; 420 } segment_cache;
419 int vpid; 421 int vpid;
420 bool emulation_required; 422 bool emulation_required;
@@ -450,7 +452,7 @@ static inline struct vcpu_vmx *to_vmx(struct kvm_vcpu *vcpu)
450#define FIELD64(number, name) [number] = VMCS12_OFFSET(name), \ 452#define FIELD64(number, name) [number] = VMCS12_OFFSET(name), \
451 [number##_HIGH] = VMCS12_OFFSET(name)+4 453 [number##_HIGH] = VMCS12_OFFSET(name)+4
452 454
453static unsigned short vmcs_field_to_offset_table[] = { 455static const unsigned short vmcs_field_to_offset_table[] = {
454 FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id), 456 FIELD(VIRTUAL_PROCESSOR_ID, virtual_processor_id),
455 FIELD(GUEST_ES_SELECTOR, guest_es_selector), 457 FIELD(GUEST_ES_SELECTOR, guest_es_selector),
456 FIELD(GUEST_CS_SELECTOR, guest_cs_selector), 458 FIELD(GUEST_CS_SELECTOR, guest_cs_selector),
@@ -596,10 +598,9 @@ static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu)
596static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr) 598static struct page *nested_get_page(struct kvm_vcpu *vcpu, gpa_t addr)
597{ 599{
598 struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT); 600 struct page *page = gfn_to_page(vcpu->kvm, addr >> PAGE_SHIFT);
599 if (is_error_page(page)) { 601 if (is_error_page(page))
600 kvm_release_page_clean(page);
601 return NULL; 602 return NULL;
602 } 603
603 return page; 604 return page;
604} 605}
605 606
@@ -667,7 +668,7 @@ static struct vmx_capability {
667 .ar_bytes = GUEST_##seg##_AR_BYTES, \ 668 .ar_bytes = GUEST_##seg##_AR_BYTES, \
668 } 669 }
669 670
670static struct kvm_vmx_segment_field { 671static const struct kvm_vmx_segment_field {
671 unsigned selector; 672 unsigned selector;
672 unsigned base; 673 unsigned base;
673 unsigned limit; 674 unsigned limit;
@@ -1343,7 +1344,7 @@ static bool update_transition_efer(struct vcpu_vmx *vmx, int efer_offset)
1343 guest_efer = vmx->vcpu.arch.efer; 1344 guest_efer = vmx->vcpu.arch.efer;
1344 1345
1345 /* 1346 /*
1346 * NX is emulated; LMA and LME handled by hardware; SCE meaninless 1347 * NX is emulated; LMA and LME handled by hardware; SCE meaningless
1347 * outside long mode 1348 * outside long mode
1348 */ 1349 */
1349 ignore_bits = EFER_NX | EFER_SCE; 1350 ignore_bits = EFER_NX | EFER_SCE;
@@ -1493,8 +1494,12 @@ static void __vmx_load_host_state(struct vcpu_vmx *vmx)
1493#ifdef CONFIG_X86_64 1494#ifdef CONFIG_X86_64
1494 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base); 1495 wrmsrl(MSR_KERNEL_GS_BASE, vmx->msr_host_kernel_gs_base);
1495#endif 1496#endif
1496 if (user_has_fpu()) 1497 /*
1497 clts(); 1498 * If the FPU is not active (through the host task or
1499 * the guest vcpu), then restore the cr0.TS bit.
1500 */
1501 if (!user_has_fpu() && !vmx->vcpu.guest_fpu_loaded)
1502 stts();
1498 load_gdt(&__get_cpu_var(host_gdt)); 1503 load_gdt(&__get_cpu_var(host_gdt));
1499} 1504}
1500 1505
@@ -1991,7 +1996,7 @@ static __init void nested_vmx_setup_ctls_msrs(void)
1991#endif 1996#endif
1992 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING | 1997 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
1993 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING | 1998 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_EXITING |
1994 CPU_BASED_RDPMC_EXITING | 1999 CPU_BASED_RDPMC_EXITING | CPU_BASED_RDTSC_EXITING |
1995 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS; 2000 CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
1996 /* 2001 /*
1997 * We can allow some features even when not supported by the 2002 * We can allow some features even when not supported by the
@@ -2287,16 +2292,6 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2287 } 2292 }
2288} 2293}
2289 2294
2290static void set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg)
2291{
2292 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
2293 vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]);
2294 else
2295 vmcs_writel(GUEST_DR7, vcpu->arch.dr7);
2296
2297 update_exception_bitmap(vcpu);
2298}
2299
2300static __init int cpu_has_kvm_support(void) 2295static __init int cpu_has_kvm_support(void)
2301{ 2296{
2302 return cpu_has_vmx(); 2297 return cpu_has_vmx();
@@ -2694,20 +2689,17 @@ static __exit void hardware_unsetup(void)
2694 free_kvm_area(); 2689 free_kvm_area();
2695} 2690}
2696 2691
2697static void fix_pmode_dataseg(int seg, struct kvm_save_segment *save) 2692static void fix_pmode_dataseg(struct kvm_vcpu *vcpu, int seg, struct kvm_segment *save)
2698{ 2693{
2699 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 2694 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2695 struct kvm_segment tmp = *save;
2700 2696
2701 if (vmcs_readl(sf->base) == save->base && (save->base & AR_S_MASK)) { 2697 if (!(vmcs_readl(sf->base) == tmp.base && tmp.s)) {
2702 vmcs_write16(sf->selector, save->selector); 2698 tmp.base = vmcs_readl(sf->base);
2703 vmcs_writel(sf->base, save->base); 2699 tmp.selector = vmcs_read16(sf->selector);
2704 vmcs_write32(sf->limit, save->limit); 2700 tmp.s = 1;
2705 vmcs_write32(sf->ar_bytes, save->ar);
2706 } else {
2707 u32 dpl = (vmcs_read16(sf->selector) & SELECTOR_RPL_MASK)
2708 << AR_DPL_SHIFT;
2709 vmcs_write32(sf->ar_bytes, 0x93 | dpl);
2710 } 2701 }
2702 vmx_set_segment(vcpu, &tmp, seg);
2711} 2703}
2712 2704
2713static void enter_pmode(struct kvm_vcpu *vcpu) 2705static void enter_pmode(struct kvm_vcpu *vcpu)
@@ -2720,10 +2712,7 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
2720 2712
2721 vmx_segment_cache_clear(vmx); 2713 vmx_segment_cache_clear(vmx);
2722 2714
2723 vmcs_write16(GUEST_TR_SELECTOR, vmx->rmode.tr.selector); 2715 vmx_set_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2724 vmcs_writel(GUEST_TR_BASE, vmx->rmode.tr.base);
2725 vmcs_write32(GUEST_TR_LIMIT, vmx->rmode.tr.limit);
2726 vmcs_write32(GUEST_TR_AR_BYTES, vmx->rmode.tr.ar);
2727 2716
2728 flags = vmcs_readl(GUEST_RFLAGS); 2717 flags = vmcs_readl(GUEST_RFLAGS);
2729 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS; 2718 flags &= RMODE_GUEST_OWNED_EFLAGS_BITS;
@@ -2738,10 +2727,10 @@ static void enter_pmode(struct kvm_vcpu *vcpu)
2738 if (emulate_invalid_guest_state) 2727 if (emulate_invalid_guest_state)
2739 return; 2728 return;
2740 2729
2741 fix_pmode_dataseg(VCPU_SREG_ES, &vmx->rmode.es); 2730 fix_pmode_dataseg(vcpu, VCPU_SREG_ES, &vmx->rmode.segs[VCPU_SREG_ES]);
2742 fix_pmode_dataseg(VCPU_SREG_DS, &vmx->rmode.ds); 2731 fix_pmode_dataseg(vcpu, VCPU_SREG_DS, &vmx->rmode.segs[VCPU_SREG_DS]);
2743 fix_pmode_dataseg(VCPU_SREG_GS, &vmx->rmode.gs); 2732 fix_pmode_dataseg(vcpu, VCPU_SREG_FS, &vmx->rmode.segs[VCPU_SREG_FS]);
2744 fix_pmode_dataseg(VCPU_SREG_FS, &vmx->rmode.fs); 2733 fix_pmode_dataseg(vcpu, VCPU_SREG_GS, &vmx->rmode.segs[VCPU_SREG_GS]);
2745 2734
2746 vmx_segment_cache_clear(vmx); 2735 vmx_segment_cache_clear(vmx);
2747 2736
@@ -2769,14 +2758,10 @@ static gva_t rmode_tss_base(struct kvm *kvm)
2769 return kvm->arch.tss_addr; 2758 return kvm->arch.tss_addr;
2770} 2759}
2771 2760
2772static void fix_rmode_seg(int seg, struct kvm_save_segment *save) 2761static void fix_rmode_seg(int seg, struct kvm_segment *save)
2773{ 2762{
2774 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 2763 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
2775 2764
2776 save->selector = vmcs_read16(sf->selector);
2777 save->base = vmcs_readl(sf->base);
2778 save->limit = vmcs_read32(sf->limit);
2779 save->ar = vmcs_read32(sf->ar_bytes);
2780 vmcs_write16(sf->selector, save->base >> 4); 2765 vmcs_write16(sf->selector, save->base >> 4);
2781 vmcs_write32(sf->base, save->base & 0xffff0); 2766 vmcs_write32(sf->base, save->base & 0xffff0);
2782 vmcs_write32(sf->limit, 0xffff); 2767 vmcs_write32(sf->limit, 0xffff);
@@ -2796,9 +2781,16 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
2796 if (enable_unrestricted_guest) 2781 if (enable_unrestricted_guest)
2797 return; 2782 return;
2798 2783
2784 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_TR], VCPU_SREG_TR);
2785 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_ES], VCPU_SREG_ES);
2786 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_DS], VCPU_SREG_DS);
2787 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_FS], VCPU_SREG_FS);
2788 vmx_get_segment(vcpu, &vmx->rmode.segs[VCPU_SREG_GS], VCPU_SREG_GS);
2789
2799 vmx->emulation_required = 1; 2790 vmx->emulation_required = 1;
2800 vmx->rmode.vm86_active = 1; 2791 vmx->rmode.vm86_active = 1;
2801 2792
2793
2802 /* 2794 /*
2803 * Very old userspace does not call KVM_SET_TSS_ADDR before entering 2795 * Very old userspace does not call KVM_SET_TSS_ADDR before entering
2804 * vcpu. Call it here with phys address pointing 16M below 4G. 2796 * vcpu. Call it here with phys address pointing 16M below 4G.
@@ -2813,14 +2805,8 @@ static void enter_rmode(struct kvm_vcpu *vcpu)
2813 2805
2814 vmx_segment_cache_clear(vmx); 2806 vmx_segment_cache_clear(vmx);
2815 2807
2816 vmx->rmode.tr.selector = vmcs_read16(GUEST_TR_SELECTOR);
2817 vmx->rmode.tr.base = vmcs_readl(GUEST_TR_BASE);
2818 vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm)); 2808 vmcs_writel(GUEST_TR_BASE, rmode_tss_base(vcpu->kvm));
2819
2820 vmx->rmode.tr.limit = vmcs_read32(GUEST_TR_LIMIT);
2821 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1); 2809 vmcs_write32(GUEST_TR_LIMIT, RMODE_TSS_SIZE - 1);
2822
2823 vmx->rmode.tr.ar = vmcs_read32(GUEST_TR_AR_BYTES);
2824 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b); 2810 vmcs_write32(GUEST_TR_AR_BYTES, 0x008b);
2825 2811
2826 flags = vmcs_readl(GUEST_RFLAGS); 2812 flags = vmcs_readl(GUEST_RFLAGS);
@@ -3113,35 +3099,24 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu,
3113 struct kvm_segment *var, int seg) 3099 struct kvm_segment *var, int seg)
3114{ 3100{
3115 struct vcpu_vmx *vmx = to_vmx(vcpu); 3101 struct vcpu_vmx *vmx = to_vmx(vcpu);
3116 struct kvm_save_segment *save;
3117 u32 ar; 3102 u32 ar;
3118 3103
3119 if (vmx->rmode.vm86_active 3104 if (vmx->rmode.vm86_active
3120 && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES 3105 && (seg == VCPU_SREG_TR || seg == VCPU_SREG_ES
3121 || seg == VCPU_SREG_DS || seg == VCPU_SREG_FS 3106 || seg == VCPU_SREG_DS || seg == VCPU_SREG_FS
3122 || seg == VCPU_SREG_GS) 3107 || seg == VCPU_SREG_GS)) {
3123 && !emulate_invalid_guest_state) { 3108 *var = vmx->rmode.segs[seg];
3124 switch (seg) {
3125 case VCPU_SREG_TR: save = &vmx->rmode.tr; break;
3126 case VCPU_SREG_ES: save = &vmx->rmode.es; break;
3127 case VCPU_SREG_DS: save = &vmx->rmode.ds; break;
3128 case VCPU_SREG_FS: save = &vmx->rmode.fs; break;
3129 case VCPU_SREG_GS: save = &vmx->rmode.gs; break;
3130 default: BUG();
3131 }
3132 var->selector = save->selector;
3133 var->base = save->base;
3134 var->limit = save->limit;
3135 ar = save->ar;
3136 if (seg == VCPU_SREG_TR 3109 if (seg == VCPU_SREG_TR
3137 || var->selector == vmx_read_guest_seg_selector(vmx, seg)) 3110 || var->selector == vmx_read_guest_seg_selector(vmx, seg))
3138 goto use_saved_rmode_seg; 3111 return;
3112 var->base = vmx_read_guest_seg_base(vmx, seg);
3113 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3114 return;
3139 } 3115 }
3140 var->base = vmx_read_guest_seg_base(vmx, seg); 3116 var->base = vmx_read_guest_seg_base(vmx, seg);
3141 var->limit = vmx_read_guest_seg_limit(vmx, seg); 3117 var->limit = vmx_read_guest_seg_limit(vmx, seg);
3142 var->selector = vmx_read_guest_seg_selector(vmx, seg); 3118 var->selector = vmx_read_guest_seg_selector(vmx, seg);
3143 ar = vmx_read_guest_seg_ar(vmx, seg); 3119 ar = vmx_read_guest_seg_ar(vmx, seg);
3144use_saved_rmode_seg:
3145 if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state) 3120 if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state)
3146 ar = 0; 3121 ar = 0;
3147 var->type = ar & 15; 3122 var->type = ar & 15;
@@ -3223,23 +3198,21 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
3223 struct kvm_segment *var, int seg) 3198 struct kvm_segment *var, int seg)
3224{ 3199{
3225 struct vcpu_vmx *vmx = to_vmx(vcpu); 3200 struct vcpu_vmx *vmx = to_vmx(vcpu);
3226 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3201 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3227 u32 ar; 3202 u32 ar;
3228 3203
3229 vmx_segment_cache_clear(vmx); 3204 vmx_segment_cache_clear(vmx);
3230 3205
3231 if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) { 3206 if (vmx->rmode.vm86_active && seg == VCPU_SREG_TR) {
3232 vmcs_write16(sf->selector, var->selector); 3207 vmcs_write16(sf->selector, var->selector);
3233 vmx->rmode.tr.selector = var->selector; 3208 vmx->rmode.segs[VCPU_SREG_TR] = *var;
3234 vmx->rmode.tr.base = var->base;
3235 vmx->rmode.tr.limit = var->limit;
3236 vmx->rmode.tr.ar = vmx_segment_access_rights(var);
3237 return; 3209 return;
3238 } 3210 }
3239 vmcs_writel(sf->base, var->base); 3211 vmcs_writel(sf->base, var->base);
3240 vmcs_write32(sf->limit, var->limit); 3212 vmcs_write32(sf->limit, var->limit);
3241 vmcs_write16(sf->selector, var->selector); 3213 vmcs_write16(sf->selector, var->selector);
3242 if (vmx->rmode.vm86_active && var->s) { 3214 if (vmx->rmode.vm86_active && var->s) {
3215 vmx->rmode.segs[seg] = *var;
3243 /* 3216 /*
3244 * Hack real-mode segments into vm86 compatibility. 3217 * Hack real-mode segments into vm86 compatibility.
3245 */ 3218 */
@@ -3254,7 +3227,7 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
3254 * qemu binaries. 3227 * qemu binaries.
3255 * IA32 arch specifies that at the time of processor reset the 3228 * IA32 arch specifies that at the time of processor reset the
3256 * "Accessed" bit in the AR field of segment registers is 1. And qemu 3229 * "Accessed" bit in the AR field of segment registers is 1. And qemu
3257 * is setting it to 0 in the usedland code. This causes invalid guest 3230 * is setting it to 0 in the userland code. This causes invalid guest
3258 * state vmexit when "unrestricted guest" mode is turned on. 3231 * state vmexit when "unrestricted guest" mode is turned on.
3259 * Fix for this setup issue in cpu_reset is being pushed in the qemu 3232 * Fix for this setup issue in cpu_reset is being pushed in the qemu
3260 * tree. Newer qemu binaries with that qemu fix would not need this 3233 * tree. Newer qemu binaries with that qemu fix would not need this
@@ -3284,16 +3257,10 @@ static void vmx_set_segment(struct kvm_vcpu *vcpu,
3284 vmcs_readl(GUEST_CS_BASE) >> 4); 3257 vmcs_readl(GUEST_CS_BASE) >> 4);
3285 break; 3258 break;
3286 case VCPU_SREG_ES: 3259 case VCPU_SREG_ES:
3287 fix_rmode_seg(VCPU_SREG_ES, &vmx->rmode.es);
3288 break;
3289 case VCPU_SREG_DS: 3260 case VCPU_SREG_DS:
3290 fix_rmode_seg(VCPU_SREG_DS, &vmx->rmode.ds);
3291 break;
3292 case VCPU_SREG_GS: 3261 case VCPU_SREG_GS:
3293 fix_rmode_seg(VCPU_SREG_GS, &vmx->rmode.gs);
3294 break;
3295 case VCPU_SREG_FS: 3262 case VCPU_SREG_FS:
3296 fix_rmode_seg(VCPU_SREG_FS, &vmx->rmode.fs); 3263 fix_rmode_seg(seg, &vmx->rmode.segs[seg]);
3297 break; 3264 break;
3298 case VCPU_SREG_SS: 3265 case VCPU_SREG_SS:
3299 vmcs_write16(GUEST_SS_SELECTOR, 3266 vmcs_write16(GUEST_SS_SELECTOR,
@@ -3347,9 +3314,9 @@ static bool rmode_segment_valid(struct kvm_vcpu *vcpu, int seg)
3347 3314
3348 if (var.base != (var.selector << 4)) 3315 if (var.base != (var.selector << 4))
3349 return false; 3316 return false;
3350 if (var.limit != 0xffff) 3317 if (var.limit < 0xffff)
3351 return false; 3318 return false;
3352 if (ar != 0xf3) 3319 if (((ar | (3 << AR_DPL_SHIFT)) & ~(AR_G_MASK | AR_DB_MASK)) != 0xf3)
3353 return false; 3320 return false;
3354 3321
3355 return true; 3322 return true;
@@ -3601,7 +3568,7 @@ out:
3601 3568
3602static void seg_setup(int seg) 3569static void seg_setup(int seg)
3603{ 3570{
3604 struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; 3571 const struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg];
3605 unsigned int ar; 3572 unsigned int ar;
3606 3573
3607 vmcs_write16(sf->selector, 0); 3574 vmcs_write16(sf->selector, 0);
@@ -3619,6 +3586,7 @@ static void seg_setup(int seg)
3619 3586
3620static int alloc_apic_access_page(struct kvm *kvm) 3587static int alloc_apic_access_page(struct kvm *kvm)
3621{ 3588{
3589 struct page *page;
3622 struct kvm_userspace_memory_region kvm_userspace_mem; 3590 struct kvm_userspace_memory_region kvm_userspace_mem;
3623 int r = 0; 3591 int r = 0;
3624 3592
@@ -3633,7 +3601,13 @@ static int alloc_apic_access_page(struct kvm *kvm)
3633 if (r) 3601 if (r)
3634 goto out; 3602 goto out;
3635 3603
3636 kvm->arch.apic_access_page = gfn_to_page(kvm, 0xfee00); 3604 page = gfn_to_page(kvm, 0xfee00);
3605 if (is_error_page(page)) {
3606 r = -EFAULT;
3607 goto out;
3608 }
3609
3610 kvm->arch.apic_access_page = page;
3637out: 3611out:
3638 mutex_unlock(&kvm->slots_lock); 3612 mutex_unlock(&kvm->slots_lock);
3639 return r; 3613 return r;
@@ -3641,6 +3615,7 @@ out:
3641 3615
3642static int alloc_identity_pagetable(struct kvm *kvm) 3616static int alloc_identity_pagetable(struct kvm *kvm)
3643{ 3617{
3618 struct page *page;
3644 struct kvm_userspace_memory_region kvm_userspace_mem; 3619 struct kvm_userspace_memory_region kvm_userspace_mem;
3645 int r = 0; 3620 int r = 0;
3646 3621
@@ -3656,8 +3631,13 @@ static int alloc_identity_pagetable(struct kvm *kvm)
3656 if (r) 3631 if (r)
3657 goto out; 3632 goto out;
3658 3633
3659 kvm->arch.ept_identity_pagetable = gfn_to_page(kvm, 3634 page = gfn_to_page(kvm, kvm->arch.ept_identity_map_addr >> PAGE_SHIFT);
3660 kvm->arch.ept_identity_map_addr >> PAGE_SHIFT); 3635 if (is_error_page(page)) {
3636 r = -EFAULT;
3637 goto out;
3638 }
3639
3640 kvm->arch.ept_identity_pagetable = page;
3661out: 3641out:
3662 mutex_unlock(&kvm->slots_lock); 3642 mutex_unlock(&kvm->slots_lock);
3663 return r; 3643 return r;
@@ -3730,7 +3710,7 @@ static void vmx_set_constant_host_state(void)
3730 unsigned long tmpl; 3710 unsigned long tmpl;
3731 struct desc_ptr dt; 3711 struct desc_ptr dt;
3732 3712
3733 vmcs_writel(HOST_CR0, read_cr0() | X86_CR0_TS); /* 22.2.3 */ 3713 vmcs_writel(HOST_CR0, read_cr0() & ~X86_CR0_TS); /* 22.2.3 */
3734 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */ 3714 vmcs_writel(HOST_CR4, read_cr4()); /* 22.2.3, 22.2.5 */
3735 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */ 3715 vmcs_writel(HOST_CR3, read_cr3()); /* 22.2.3 FIXME: shadow tables */
3736 3716
@@ -3753,8 +3733,7 @@ static void vmx_set_constant_host_state(void)
3753 native_store_idt(&dt); 3733 native_store_idt(&dt);
3754 vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */ 3734 vmcs_writel(HOST_IDTR_BASE, dt.address); /* 22.2.4 */
3755 3735
3756 asm("mov $.Lkvm_vmx_return, %0" : "=r"(tmpl)); 3736 vmcs_writel(HOST_RIP, vmx_return); /* 22.2.5 */
3757 vmcs_writel(HOST_RIP, tmpl); /* 22.2.5 */
3758 3737
3759 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32); 3738 rdmsr(MSR_IA32_SYSENTER_CS, low32, high32);
3760 vmcs_write32(HOST_IA32_SYSENTER_CS, low32); 3739 vmcs_write32(HOST_IA32_SYSENTER_CS, low32);
@@ -3988,8 +3967,6 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu)
3988 kvm_rip_write(vcpu, 0); 3967 kvm_rip_write(vcpu, 0);
3989 kvm_register_write(vcpu, VCPU_REGS_RSP, 0); 3968 kvm_register_write(vcpu, VCPU_REGS_RSP, 0);
3990 3969
3991 vmcs_writel(GUEST_DR7, 0x400);
3992
3993 vmcs_writel(GUEST_GDTR_BASE, 0); 3970 vmcs_writel(GUEST_GDTR_BASE, 0);
3994 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff); 3971 vmcs_write32(GUEST_GDTR_LIMIT, 0xffff);
3995 3972
@@ -4439,7 +4416,7 @@ vmx_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4439 hypercall[2] = 0xc1; 4416 hypercall[2] = 0xc1;
4440} 4417}
4441 4418
4442/* called to set cr0 as approriate for a mov-to-cr0 exit. */ 4419/* called to set cr0 as appropriate for a mov-to-cr0 exit. */
4443static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val) 4420static int handle_set_cr0(struct kvm_vcpu *vcpu, unsigned long val)
4444{ 4421{
4445 if (to_vmx(vcpu)->nested.vmxon && 4422 if (to_vmx(vcpu)->nested.vmxon &&
@@ -4530,7 +4507,7 @@ static int handle_cr(struct kvm_vcpu *vcpu)
4530 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; 4507 vcpu->run->exit_reason = KVM_EXIT_SET_TPR;
4531 return 0; 4508 return 0;
4532 } 4509 }
4533 }; 4510 }
4534 break; 4511 break;
4535 case 2: /* clts */ 4512 case 2: /* clts */
4536 handle_clts(vcpu); 4513 handle_clts(vcpu);
@@ -5684,7 +5661,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu)
5684 * may resume. Otherwise they set the kvm_run parameter to indicate what needs 5661 * may resume. Otherwise they set the kvm_run parameter to indicate what needs
5685 * to be done to userspace and return 0. 5662 * to be done to userspace and return 0.
5686 */ 5663 */
5687static int (*kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = { 5664static int (*const kvm_vmx_exit_handlers[])(struct kvm_vcpu *vcpu) = {
5688 [EXIT_REASON_EXCEPTION_NMI] = handle_exception, 5665 [EXIT_REASON_EXCEPTION_NMI] = handle_exception,
5689 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt, 5666 [EXIT_REASON_EXTERNAL_INTERRUPT] = handle_external_interrupt,
5690 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault, 5667 [EXIT_REASON_TRIPLE_FAULT] = handle_triple_fault,
@@ -6212,17 +6189,10 @@ static void atomic_switch_perf_msrs(struct vcpu_vmx *vmx)
6212 msrs[i].host); 6189 msrs[i].host);
6213} 6190}
6214 6191
6215#ifdef CONFIG_X86_64
6216#define R "r"
6217#define Q "q"
6218#else
6219#define R "e"
6220#define Q "l"
6221#endif
6222
6223static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu) 6192static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
6224{ 6193{
6225 struct vcpu_vmx *vmx = to_vmx(vcpu); 6194 struct vcpu_vmx *vmx = to_vmx(vcpu);
6195 unsigned long debugctlmsr;
6226 6196
6227 if (is_guest_mode(vcpu) && !vmx->nested.nested_run_pending) { 6197 if (is_guest_mode(vcpu) && !vmx->nested.nested_run_pending) {
6228 struct vmcs12 *vmcs12 = get_vmcs12(vcpu); 6198 struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
@@ -6262,34 +6232,35 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
6262 vmx_set_interrupt_shadow(vcpu, 0); 6232 vmx_set_interrupt_shadow(vcpu, 0);
6263 6233
6264 atomic_switch_perf_msrs(vmx); 6234 atomic_switch_perf_msrs(vmx);
6235 debugctlmsr = get_debugctlmsr();
6265 6236
6266 vmx->__launched = vmx->loaded_vmcs->launched; 6237 vmx->__launched = vmx->loaded_vmcs->launched;
6267 asm( 6238 asm(
6268 /* Store host registers */ 6239 /* Store host registers */
6269 "push %%"R"dx; push %%"R"bp;" 6240 "push %%" _ASM_DX "; push %%" _ASM_BP ";"
6270 "push %%"R"cx \n\t" /* placeholder for guest rcx */ 6241 "push %%" _ASM_CX " \n\t" /* placeholder for guest rcx */
6271 "push %%"R"cx \n\t" 6242 "push %%" _ASM_CX " \n\t"
6272 "cmp %%"R"sp, %c[host_rsp](%0) \n\t" 6243 "cmp %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
6273 "je 1f \n\t" 6244 "je 1f \n\t"
6274 "mov %%"R"sp, %c[host_rsp](%0) \n\t" 6245 "mov %%" _ASM_SP ", %c[host_rsp](%0) \n\t"
6275 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t" 6246 __ex(ASM_VMX_VMWRITE_RSP_RDX) "\n\t"
6276 "1: \n\t" 6247 "1: \n\t"
6277 /* Reload cr2 if changed */ 6248 /* Reload cr2 if changed */
6278 "mov %c[cr2](%0), %%"R"ax \n\t" 6249 "mov %c[cr2](%0), %%" _ASM_AX " \n\t"
6279 "mov %%cr2, %%"R"dx \n\t" 6250 "mov %%cr2, %%" _ASM_DX " \n\t"
6280 "cmp %%"R"ax, %%"R"dx \n\t" 6251 "cmp %%" _ASM_AX ", %%" _ASM_DX " \n\t"
6281 "je 2f \n\t" 6252 "je 2f \n\t"
6282 "mov %%"R"ax, %%cr2 \n\t" 6253 "mov %%" _ASM_AX", %%cr2 \n\t"
6283 "2: \n\t" 6254 "2: \n\t"
6284 /* Check if vmlaunch of vmresume is needed */ 6255 /* Check if vmlaunch of vmresume is needed */
6285 "cmpl $0, %c[launched](%0) \n\t" 6256 "cmpl $0, %c[launched](%0) \n\t"
6286 /* Load guest registers. Don't clobber flags. */ 6257 /* Load guest registers. Don't clobber flags. */
6287 "mov %c[rax](%0), %%"R"ax \n\t" 6258 "mov %c[rax](%0), %%" _ASM_AX " \n\t"
6288 "mov %c[rbx](%0), %%"R"bx \n\t" 6259 "mov %c[rbx](%0), %%" _ASM_BX " \n\t"
6289 "mov %c[rdx](%0), %%"R"dx \n\t" 6260 "mov %c[rdx](%0), %%" _ASM_DX " \n\t"
6290 "mov %c[rsi](%0), %%"R"si \n\t" 6261 "mov %c[rsi](%0), %%" _ASM_SI " \n\t"
6291 "mov %c[rdi](%0), %%"R"di \n\t" 6262 "mov %c[rdi](%0), %%" _ASM_DI " \n\t"
6292 "mov %c[rbp](%0), %%"R"bp \n\t" 6263 "mov %c[rbp](%0), %%" _ASM_BP " \n\t"
6293#ifdef CONFIG_X86_64 6264#ifdef CONFIG_X86_64
6294 "mov %c[r8](%0), %%r8 \n\t" 6265 "mov %c[r8](%0), %%r8 \n\t"
6295 "mov %c[r9](%0), %%r9 \n\t" 6266 "mov %c[r9](%0), %%r9 \n\t"
@@ -6300,24 +6271,24 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
6300 "mov %c[r14](%0), %%r14 \n\t" 6271 "mov %c[r14](%0), %%r14 \n\t"
6301 "mov %c[r15](%0), %%r15 \n\t" 6272 "mov %c[r15](%0), %%r15 \n\t"
6302#endif 6273#endif
6303 "mov %c[rcx](%0), %%"R"cx \n\t" /* kills %0 (ecx) */ 6274 "mov %c[rcx](%0), %%" _ASM_CX " \n\t" /* kills %0 (ecx) */
6304 6275
6305 /* Enter guest mode */ 6276 /* Enter guest mode */
6306 "jne .Llaunched \n\t" 6277 "jne 1f \n\t"
6307 __ex(ASM_VMX_VMLAUNCH) "\n\t" 6278 __ex(ASM_VMX_VMLAUNCH) "\n\t"
6308 "jmp .Lkvm_vmx_return \n\t" 6279 "jmp 2f \n\t"
6309 ".Llaunched: " __ex(ASM_VMX_VMRESUME) "\n\t" 6280 "1: " __ex(ASM_VMX_VMRESUME) "\n\t"
6310 ".Lkvm_vmx_return: " 6281 "2: "
6311 /* Save guest registers, load host registers, keep flags */ 6282 /* Save guest registers, load host registers, keep flags */
6312 "mov %0, %c[wordsize](%%"R"sp) \n\t" 6283 "mov %0, %c[wordsize](%%" _ASM_SP ") \n\t"
6313 "pop %0 \n\t" 6284 "pop %0 \n\t"
6314 "mov %%"R"ax, %c[rax](%0) \n\t" 6285 "mov %%" _ASM_AX ", %c[rax](%0) \n\t"
6315 "mov %%"R"bx, %c[rbx](%0) \n\t" 6286 "mov %%" _ASM_BX ", %c[rbx](%0) \n\t"
6316 "pop"Q" %c[rcx](%0) \n\t" 6287 __ASM_SIZE(pop) " %c[rcx](%0) \n\t"
6317 "mov %%"R"dx, %c[rdx](%0) \n\t" 6288 "mov %%" _ASM_DX ", %c[rdx](%0) \n\t"
6318 "mov %%"R"si, %c[rsi](%0) \n\t" 6289 "mov %%" _ASM_SI ", %c[rsi](%0) \n\t"
6319 "mov %%"R"di, %c[rdi](%0) \n\t" 6290 "mov %%" _ASM_DI ", %c[rdi](%0) \n\t"
6320 "mov %%"R"bp, %c[rbp](%0) \n\t" 6291 "mov %%" _ASM_BP ", %c[rbp](%0) \n\t"
6321#ifdef CONFIG_X86_64 6292#ifdef CONFIG_X86_64
6322 "mov %%r8, %c[r8](%0) \n\t" 6293 "mov %%r8, %c[r8](%0) \n\t"
6323 "mov %%r9, %c[r9](%0) \n\t" 6294 "mov %%r9, %c[r9](%0) \n\t"
@@ -6328,11 +6299,15 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
6328 "mov %%r14, %c[r14](%0) \n\t" 6299 "mov %%r14, %c[r14](%0) \n\t"
6329 "mov %%r15, %c[r15](%0) \n\t" 6300 "mov %%r15, %c[r15](%0) \n\t"
6330#endif 6301#endif
6331 "mov %%cr2, %%"R"ax \n\t" 6302 "mov %%cr2, %%" _ASM_AX " \n\t"
6332 "mov %%"R"ax, %c[cr2](%0) \n\t" 6303 "mov %%" _ASM_AX ", %c[cr2](%0) \n\t"
6333 6304
6334 "pop %%"R"bp; pop %%"R"dx \n\t" 6305 "pop %%" _ASM_BP "; pop %%" _ASM_DX " \n\t"
6335 "setbe %c[fail](%0) \n\t" 6306 "setbe %c[fail](%0) \n\t"
6307 ".pushsection .rodata \n\t"
6308 ".global vmx_return \n\t"
6309 "vmx_return: " _ASM_PTR " 2b \n\t"
6310 ".popsection"
6336 : : "c"(vmx), "d"((unsigned long)HOST_RSP), 6311 : : "c"(vmx), "d"((unsigned long)HOST_RSP),
6337 [launched]"i"(offsetof(struct vcpu_vmx, __launched)), 6312 [launched]"i"(offsetof(struct vcpu_vmx, __launched)),
6338 [fail]"i"(offsetof(struct vcpu_vmx, fail)), 6313 [fail]"i"(offsetof(struct vcpu_vmx, fail)),
@@ -6357,12 +6332,18 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
6357 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)), 6332 [cr2]"i"(offsetof(struct vcpu_vmx, vcpu.arch.cr2)),
6358 [wordsize]"i"(sizeof(ulong)) 6333 [wordsize]"i"(sizeof(ulong))
6359 : "cc", "memory" 6334 : "cc", "memory"
6360 , R"ax", R"bx", R"di", R"si"
6361#ifdef CONFIG_X86_64 6335#ifdef CONFIG_X86_64
6336 , "rax", "rbx", "rdi", "rsi"
6362 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15" 6337 , "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"
6338#else
6339 , "eax", "ebx", "edi", "esi"
6363#endif 6340#endif
6364 ); 6341 );
6365 6342
6343 /* MSR_IA32_DEBUGCTLMSR is zeroed on vmexit. Restore it if needed */
6344 if (debugctlmsr)
6345 update_debugctlmsr(debugctlmsr);
6346
6366#ifndef CONFIG_X86_64 6347#ifndef CONFIG_X86_64
6367 /* 6348 /*
6368 * The sysexit path does not restore ds/es, so we must set them to 6349 * The sysexit path does not restore ds/es, so we must set them to
@@ -6407,9 +6388,6 @@ static void __noclone vmx_vcpu_run(struct kvm_vcpu *vcpu)
6407 vmx_complete_interrupts(vmx); 6388 vmx_complete_interrupts(vmx);
6408} 6389}
6409 6390
6410#undef R
6411#undef Q
6412
6413static void vmx_free_vcpu(struct kvm_vcpu *vcpu) 6391static void vmx_free_vcpu(struct kvm_vcpu *vcpu)
6414{ 6392{
6415 struct vcpu_vmx *vmx = to_vmx(vcpu); 6393 struct vcpu_vmx *vmx = to_vmx(vcpu);
@@ -6575,7 +6553,7 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
6575 /* Exposing INVPCID only when PCID is exposed */ 6553 /* Exposing INVPCID only when PCID is exposed */
6576 best = kvm_find_cpuid_entry(vcpu, 0x7, 0); 6554 best = kvm_find_cpuid_entry(vcpu, 0x7, 0);
6577 if (vmx_invpcid_supported() && 6555 if (vmx_invpcid_supported() &&
6578 best && (best->ecx & bit(X86_FEATURE_INVPCID)) && 6556 best && (best->ebx & bit(X86_FEATURE_INVPCID)) &&
6579 guest_cpuid_has_pcid(vcpu)) { 6557 guest_cpuid_has_pcid(vcpu)) {
6580 exec_control |= SECONDARY_EXEC_ENABLE_INVPCID; 6558 exec_control |= SECONDARY_EXEC_ENABLE_INVPCID;
6581 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, 6559 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
@@ -6585,7 +6563,7 @@ static void vmx_cpuid_update(struct kvm_vcpu *vcpu)
6585 vmcs_write32(SECONDARY_VM_EXEC_CONTROL, 6563 vmcs_write32(SECONDARY_VM_EXEC_CONTROL,
6586 exec_control); 6564 exec_control);
6587 if (best) 6565 if (best)
6588 best->ecx &= ~bit(X86_FEATURE_INVPCID); 6566 best->ebx &= ~bit(X86_FEATURE_INVPCID);
6589 } 6567 }
6590} 6568}
6591 6569
@@ -7264,7 +7242,7 @@ static struct kvm_x86_ops vmx_x86_ops = {
7264 .vcpu_load = vmx_vcpu_load, 7242 .vcpu_load = vmx_vcpu_load,
7265 .vcpu_put = vmx_vcpu_put, 7243 .vcpu_put = vmx_vcpu_put,
7266 7244
7267 .set_guest_debug = set_guest_debug, 7245 .update_db_bp_intercept = update_exception_bitmap,
7268 .get_msr = vmx_get_msr, 7246 .get_msr = vmx_get_msr,
7269 .set_msr = vmx_set_msr, 7247 .set_msr = vmx_set_msr,
7270 .get_segment_base = vmx_get_segment_base, 7248 .get_segment_base = vmx_get_segment_base,
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 148ed666e311..1eefebe5d727 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -246,20 +246,14 @@ static void drop_user_return_notifiers(void *ignore)
246 246
247u64 kvm_get_apic_base(struct kvm_vcpu *vcpu) 247u64 kvm_get_apic_base(struct kvm_vcpu *vcpu)
248{ 248{
249 if (irqchip_in_kernel(vcpu->kvm)) 249 return vcpu->arch.apic_base;
250 return vcpu->arch.apic_base;
251 else
252 return vcpu->arch.apic_base;
253} 250}
254EXPORT_SYMBOL_GPL(kvm_get_apic_base); 251EXPORT_SYMBOL_GPL(kvm_get_apic_base);
255 252
256void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data) 253void kvm_set_apic_base(struct kvm_vcpu *vcpu, u64 data)
257{ 254{
258 /* TODO: reserve bits check */ 255 /* TODO: reserve bits check */
259 if (irqchip_in_kernel(vcpu->kvm)) 256 kvm_lapic_set_base(vcpu, data);
260 kvm_lapic_set_base(vcpu, data);
261 else
262 vcpu->arch.apic_base = data;
263} 257}
264EXPORT_SYMBOL_GPL(kvm_set_apic_base); 258EXPORT_SYMBOL_GPL(kvm_set_apic_base);
265 259
@@ -698,6 +692,18 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu)
698} 692}
699EXPORT_SYMBOL_GPL(kvm_get_cr8); 693EXPORT_SYMBOL_GPL(kvm_get_cr8);
700 694
695static void kvm_update_dr7(struct kvm_vcpu *vcpu)
696{
697 unsigned long dr7;
698
699 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
700 dr7 = vcpu->arch.guest_debug_dr7;
701 else
702 dr7 = vcpu->arch.dr7;
703 kvm_x86_ops->set_dr7(vcpu, dr7);
704 vcpu->arch.switch_db_regs = (dr7 & DR7_BP_EN_MASK);
705}
706
701static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val) 707static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
702{ 708{
703 switch (dr) { 709 switch (dr) {
@@ -723,10 +729,7 @@ static int __kvm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long val)
723 if (val & 0xffffffff00000000ULL) 729 if (val & 0xffffffff00000000ULL)
724 return -1; /* #GP */ 730 return -1; /* #GP */
725 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; 731 vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1;
726 if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { 732 kvm_update_dr7(vcpu);
727 kvm_x86_ops->set_dr7(vcpu, vcpu->arch.dr7);
728 vcpu->arch.switch_db_regs = (val & DR7_BP_EN_MASK);
729 }
730 break; 733 break;
731 } 734 }
732 735
@@ -823,7 +826,7 @@ static u32 msrs_to_save[] = {
823 826
824static unsigned num_msrs_to_save; 827static unsigned num_msrs_to_save;
825 828
826static u32 emulated_msrs[] = { 829static const u32 emulated_msrs[] = {
827 MSR_IA32_TSCDEADLINE, 830 MSR_IA32_TSCDEADLINE,
828 MSR_IA32_MISC_ENABLE, 831 MSR_IA32_MISC_ENABLE,
829 MSR_IA32_MCG_STATUS, 832 MSR_IA32_MCG_STATUS,
@@ -1097,7 +1100,7 @@ void kvm_write_tsc(struct kvm_vcpu *vcpu, u64 data)
1097 * For each generation, we track the original measured 1100 * For each generation, we track the original measured
1098 * nanosecond time, offset, and write, so if TSCs are in 1101 * nanosecond time, offset, and write, so if TSCs are in
1099 * sync, we can match exact offset, and if not, we can match 1102 * sync, we can match exact offset, and if not, we can match
1100 * exact software computaion in compute_guest_tsc() 1103 * exact software computation in compute_guest_tsc()
1101 * 1104 *
1102 * These values are tracked in kvm->arch.cur_xxx variables. 1105 * These values are tracked in kvm->arch.cur_xxx variables.
1103 */ 1106 */
@@ -1140,6 +1143,7 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
1140 unsigned long this_tsc_khz; 1143 unsigned long this_tsc_khz;
1141 s64 kernel_ns, max_kernel_ns; 1144 s64 kernel_ns, max_kernel_ns;
1142 u64 tsc_timestamp; 1145 u64 tsc_timestamp;
1146 u8 pvclock_flags;
1143 1147
1144 /* Keep irq disabled to prevent changes to the clock */ 1148 /* Keep irq disabled to prevent changes to the clock */
1145 local_irq_save(flags); 1149 local_irq_save(flags);
@@ -1221,7 +1225,14 @@ static int kvm_guest_time_update(struct kvm_vcpu *v)
1221 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset; 1225 vcpu->hv_clock.system_time = kernel_ns + v->kvm->arch.kvmclock_offset;
1222 vcpu->last_kernel_ns = kernel_ns; 1226 vcpu->last_kernel_ns = kernel_ns;
1223 vcpu->last_guest_tsc = tsc_timestamp; 1227 vcpu->last_guest_tsc = tsc_timestamp;
1224 vcpu->hv_clock.flags = 0; 1228
1229 pvclock_flags = 0;
1230 if (vcpu->pvclock_set_guest_stopped_request) {
1231 pvclock_flags |= PVCLOCK_GUEST_STOPPED;
1232 vcpu->pvclock_set_guest_stopped_request = false;
1233 }
1234
1235 vcpu->hv_clock.flags = pvclock_flags;
1225 1236
1226 /* 1237 /*
1227 * The interface expects us to write an even number signaling that the 1238 * The interface expects us to write an even number signaling that the
@@ -1504,7 +1515,7 @@ static int kvm_pv_enable_async_pf(struct kvm_vcpu *vcpu, u64 data)
1504{ 1515{
1505 gpa_t gpa = data & ~0x3f; 1516 gpa_t gpa = data & ~0x3f;
1506 1517
1507 /* Bits 2:5 are resrved, Should be zero */ 1518 /* Bits 2:5 are reserved, Should be zero */
1508 if (data & 0x3c) 1519 if (data & 0x3c)
1509 return 1; 1520 return 1;
1510 1521
@@ -1639,10 +1650,9 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1639 vcpu->arch.time_page = 1650 vcpu->arch.time_page =
1640 gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT); 1651 gfn_to_page(vcpu->kvm, data >> PAGE_SHIFT);
1641 1652
1642 if (is_error_page(vcpu->arch.time_page)) { 1653 if (is_error_page(vcpu->arch.time_page))
1643 kvm_release_page_clean(vcpu->arch.time_page);
1644 vcpu->arch.time_page = NULL; 1654 vcpu->arch.time_page = NULL;
1645 } 1655
1646 break; 1656 break;
1647 } 1657 }
1648 case MSR_KVM_ASYNC_PF_EN: 1658 case MSR_KVM_ASYNC_PF_EN:
@@ -1727,7 +1737,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data)
1727 * Ignore all writes to this no longer documented MSR. 1737 * Ignore all writes to this no longer documented MSR.
1728 * Writes are only relevant for old K7 processors, 1738 * Writes are only relevant for old K7 processors,
1729 * all pre-dating SVM, but a recommended workaround from 1739 * all pre-dating SVM, but a recommended workaround from
1730 * AMD for these chips. It is possible to speicify the 1740 * AMD for these chips. It is possible to specify the
1731 * affected processor models on the command line, hence 1741 * affected processor models on the command line, hence
1732 * the need to ignore the workaround. 1742 * the need to ignore the workaround.
1733 */ 1743 */
@@ -2177,6 +2187,8 @@ int kvm_dev_ioctl_check_extension(long ext)
2177 case KVM_CAP_GET_TSC_KHZ: 2187 case KVM_CAP_GET_TSC_KHZ:
2178 case KVM_CAP_PCI_2_3: 2188 case KVM_CAP_PCI_2_3:
2179 case KVM_CAP_KVMCLOCK_CTRL: 2189 case KVM_CAP_KVMCLOCK_CTRL:
2190 case KVM_CAP_READONLY_MEM:
2191 case KVM_CAP_IRQFD_RESAMPLE:
2180 r = 1; 2192 r = 1;
2181 break; 2193 break;
2182 case KVM_CAP_COALESCED_MMIO: 2194 case KVM_CAP_COALESCED_MMIO:
@@ -2358,8 +2370,7 @@ static int kvm_vcpu_ioctl_get_lapic(struct kvm_vcpu *vcpu,
2358static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu, 2370static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
2359 struct kvm_lapic_state *s) 2371 struct kvm_lapic_state *s)
2360{ 2372{
2361 memcpy(vcpu->arch.apic->regs, s->regs, sizeof *s); 2373 kvm_apic_post_state_restore(vcpu, s);
2362 kvm_apic_post_state_restore(vcpu);
2363 update_cr8_intercept(vcpu); 2374 update_cr8_intercept(vcpu);
2364 2375
2365 return 0; 2376 return 0;
@@ -2368,7 +2379,7 @@ static int kvm_vcpu_ioctl_set_lapic(struct kvm_vcpu *vcpu,
2368static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, 2379static int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu,
2369 struct kvm_interrupt *irq) 2380 struct kvm_interrupt *irq)
2370{ 2381{
2371 if (irq->irq < 0 || irq->irq >= 256) 2382 if (irq->irq < 0 || irq->irq >= KVM_NR_INTERRUPTS)
2372 return -EINVAL; 2383 return -EINVAL;
2373 if (irqchip_in_kernel(vcpu->kvm)) 2384 if (irqchip_in_kernel(vcpu->kvm))
2374 return -ENXIO; 2385 return -ENXIO;
@@ -2635,11 +2646,9 @@ static int kvm_vcpu_ioctl_x86_set_xcrs(struct kvm_vcpu *vcpu,
2635 */ 2646 */
2636static int kvm_set_guest_paused(struct kvm_vcpu *vcpu) 2647static int kvm_set_guest_paused(struct kvm_vcpu *vcpu)
2637{ 2648{
2638 struct pvclock_vcpu_time_info *src = &vcpu->arch.hv_clock;
2639 if (!vcpu->arch.time_page) 2649 if (!vcpu->arch.time_page)
2640 return -EINVAL; 2650 return -EINVAL;
2641 src->flags |= PVCLOCK_GUEST_STOPPED; 2651 vcpu->arch.pvclock_set_guest_stopped_request = true;
2642 mark_page_dirty(vcpu->kvm, vcpu->arch.time >> PAGE_SHIFT);
2643 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu); 2652 kvm_make_request(KVM_REQ_CLOCK_UPDATE, vcpu);
2644 return 0; 2653 return 0;
2645} 2654}
@@ -3090,7 +3099,7 @@ static int kvm_vm_ioctl_reinject(struct kvm *kvm,
3090 if (!kvm->arch.vpit) 3099 if (!kvm->arch.vpit)
3091 return -ENXIO; 3100 return -ENXIO;
3092 mutex_lock(&kvm->arch.vpit->pit_state.lock); 3101 mutex_lock(&kvm->arch.vpit->pit_state.lock);
3093 kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject; 3102 kvm->arch.vpit->pit_state.reinject = control->pit_reinject;
3094 mutex_unlock(&kvm->arch.vpit->pit_state.lock); 3103 mutex_unlock(&kvm->arch.vpit->pit_state.lock);
3095 return 0; 3104 return 0;
3096} 3105}
@@ -3173,6 +3182,16 @@ out:
3173 return r; 3182 return r;
3174} 3183}
3175 3184
3185int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event)
3186{
3187 if (!irqchip_in_kernel(kvm))
3188 return -ENXIO;
3189
3190 irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
3191 irq_event->irq, irq_event->level);
3192 return 0;
3193}
3194
3176long kvm_arch_vm_ioctl(struct file *filp, 3195long kvm_arch_vm_ioctl(struct file *filp,
3177 unsigned int ioctl, unsigned long arg) 3196 unsigned int ioctl, unsigned long arg)
3178{ 3197{
@@ -3279,29 +3298,6 @@ long kvm_arch_vm_ioctl(struct file *filp,
3279 create_pit_unlock: 3298 create_pit_unlock:
3280 mutex_unlock(&kvm->slots_lock); 3299 mutex_unlock(&kvm->slots_lock);
3281 break; 3300 break;
3282 case KVM_IRQ_LINE_STATUS:
3283 case KVM_IRQ_LINE: {
3284 struct kvm_irq_level irq_event;
3285
3286 r = -EFAULT;
3287 if (copy_from_user(&irq_event, argp, sizeof irq_event))
3288 goto out;
3289 r = -ENXIO;
3290 if (irqchip_in_kernel(kvm)) {
3291 __s32 status;
3292 status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
3293 irq_event.irq, irq_event.level);
3294 if (ioctl == KVM_IRQ_LINE_STATUS) {
3295 r = -EFAULT;
3296 irq_event.status = status;
3297 if (copy_to_user(argp, &irq_event,
3298 sizeof irq_event))
3299 goto out;
3300 }
3301 r = 0;
3302 }
3303 break;
3304 }
3305 case KVM_GET_IRQCHIP: { 3301 case KVM_GET_IRQCHIP: {
3306 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */ 3302 /* 0: PIC master, 1: PIC slave, 2: IOAPIC */
3307 struct kvm_irqchip *chip; 3303 struct kvm_irqchip *chip;
@@ -3689,20 +3685,17 @@ static int vcpu_mmio_gva_to_gpa(struct kvm_vcpu *vcpu, unsigned long gva,
3689 gpa_t *gpa, struct x86_exception *exception, 3685 gpa_t *gpa, struct x86_exception *exception,
3690 bool write) 3686 bool write)
3691{ 3687{
3692 u32 access = (kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0; 3688 u32 access = ((kvm_x86_ops->get_cpl(vcpu) == 3) ? PFERR_USER_MASK : 0)
3689 | (write ? PFERR_WRITE_MASK : 0);
3693 3690
3694 if (vcpu_match_mmio_gva(vcpu, gva) && 3691 if (vcpu_match_mmio_gva(vcpu, gva)
3695 check_write_user_access(vcpu, write, access, 3692 && !permission_fault(vcpu->arch.walk_mmu, vcpu->arch.access, access)) {
3696 vcpu->arch.access)) {
3697 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT | 3693 *gpa = vcpu->arch.mmio_gfn << PAGE_SHIFT |
3698 (gva & (PAGE_SIZE - 1)); 3694 (gva & (PAGE_SIZE - 1));
3699 trace_vcpu_match_mmio(gva, *gpa, write, false); 3695 trace_vcpu_match_mmio(gva, *gpa, write, false);
3700 return 1; 3696 return 1;
3701 } 3697 }
3702 3698
3703 if (write)
3704 access |= PFERR_WRITE_MASK;
3705
3706 *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception); 3699 *gpa = vcpu->arch.walk_mmu->gva_to_gpa(vcpu, gva, access, exception);
3707 3700
3708 if (*gpa == UNMAPPED_GVA) 3701 if (*gpa == UNMAPPED_GVA)
@@ -3790,14 +3783,14 @@ static int write_exit_mmio(struct kvm_vcpu *vcpu, gpa_t gpa,
3790 return X86EMUL_CONTINUE; 3783 return X86EMUL_CONTINUE;
3791} 3784}
3792 3785
3793static struct read_write_emulator_ops read_emultor = { 3786static const struct read_write_emulator_ops read_emultor = {
3794 .read_write_prepare = read_prepare, 3787 .read_write_prepare = read_prepare,
3795 .read_write_emulate = read_emulate, 3788 .read_write_emulate = read_emulate,
3796 .read_write_mmio = vcpu_mmio_read, 3789 .read_write_mmio = vcpu_mmio_read,
3797 .read_write_exit_mmio = read_exit_mmio, 3790 .read_write_exit_mmio = read_exit_mmio,
3798}; 3791};
3799 3792
3800static struct read_write_emulator_ops write_emultor = { 3793static const struct read_write_emulator_ops write_emultor = {
3801 .read_write_emulate = write_emulate, 3794 .read_write_emulate = write_emulate,
3802 .read_write_mmio = write_mmio, 3795 .read_write_mmio = write_mmio,
3803 .read_write_exit_mmio = write_exit_mmio, 3796 .read_write_exit_mmio = write_exit_mmio,
@@ -3808,7 +3801,7 @@ static int emulator_read_write_onepage(unsigned long addr, void *val,
3808 unsigned int bytes, 3801 unsigned int bytes,
3809 struct x86_exception *exception, 3802 struct x86_exception *exception,
3810 struct kvm_vcpu *vcpu, 3803 struct kvm_vcpu *vcpu,
3811 struct read_write_emulator_ops *ops) 3804 const struct read_write_emulator_ops *ops)
3812{ 3805{
3813 gpa_t gpa; 3806 gpa_t gpa;
3814 int handled, ret; 3807 int handled, ret;
@@ -3857,7 +3850,7 @@ mmio:
3857int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr, 3850int emulator_read_write(struct x86_emulate_ctxt *ctxt, unsigned long addr,
3858 void *val, unsigned int bytes, 3851 void *val, unsigned int bytes,
3859 struct x86_exception *exception, 3852 struct x86_exception *exception,
3860 struct read_write_emulator_ops *ops) 3853 const struct read_write_emulator_ops *ops)
3861{ 3854{
3862 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); 3855 struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt);
3863 gpa_t gpa; 3856 gpa_t gpa;
@@ -3962,10 +3955,8 @@ static int emulator_cmpxchg_emulated(struct x86_emulate_ctxt *ctxt,
3962 goto emul_write; 3955 goto emul_write;
3963 3956
3964 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT); 3957 page = gfn_to_page(vcpu->kvm, gpa >> PAGE_SHIFT);
3965 if (is_error_page(page)) { 3958 if (is_error_page(page))
3966 kvm_release_page_clean(page);
3967 goto emul_write; 3959 goto emul_write;
3968 }
3969 3960
3970 kaddr = kmap_atomic(page); 3961 kaddr = kmap_atomic(page);
3971 kaddr += offset_in_page(gpa); 3962 kaddr += offset_in_page(gpa);
@@ -4332,7 +4323,19 @@ static void emulator_get_cpuid(struct x86_emulate_ctxt *ctxt,
4332 kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx); 4323 kvm_cpuid(emul_to_vcpu(ctxt), eax, ebx, ecx, edx);
4333} 4324}
4334 4325
4335static struct x86_emulate_ops emulate_ops = { 4326static ulong emulator_read_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg)
4327{
4328 return kvm_register_read(emul_to_vcpu(ctxt), reg);
4329}
4330
4331static void emulator_write_gpr(struct x86_emulate_ctxt *ctxt, unsigned reg, ulong val)
4332{
4333 kvm_register_write(emul_to_vcpu(ctxt), reg, val);
4334}
4335
4336static const struct x86_emulate_ops emulate_ops = {
4337 .read_gpr = emulator_read_gpr,
4338 .write_gpr = emulator_write_gpr,
4336 .read_std = kvm_read_guest_virt_system, 4339 .read_std = kvm_read_guest_virt_system,
4337 .write_std = kvm_write_guest_virt_system, 4340 .write_std = kvm_write_guest_virt_system,
4338 .fetch = kvm_fetch_guest_virt, 4341 .fetch = kvm_fetch_guest_virt,
@@ -4367,14 +4370,6 @@ static struct x86_emulate_ops emulate_ops = {
4367 .get_cpuid = emulator_get_cpuid, 4370 .get_cpuid = emulator_get_cpuid,
4368}; 4371};
4369 4372
4370static void cache_all_regs(struct kvm_vcpu *vcpu)
4371{
4372 kvm_register_read(vcpu, VCPU_REGS_RAX);
4373 kvm_register_read(vcpu, VCPU_REGS_RSP);
4374 kvm_register_read(vcpu, VCPU_REGS_RIP);
4375 vcpu->arch.regs_dirty = ~0;
4376}
4377
4378static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask) 4373static void toggle_interruptibility(struct kvm_vcpu *vcpu, u32 mask)
4379{ 4374{
4380 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask); 4375 u32 int_shadow = kvm_x86_ops->get_interrupt_shadow(vcpu, mask);
@@ -4401,12 +4396,10 @@ static void inject_emulated_exception(struct kvm_vcpu *vcpu)
4401 kvm_queue_exception(vcpu, ctxt->exception.vector); 4396 kvm_queue_exception(vcpu, ctxt->exception.vector);
4402} 4397}
4403 4398
4404static void init_decode_cache(struct x86_emulate_ctxt *ctxt, 4399static void init_decode_cache(struct x86_emulate_ctxt *ctxt)
4405 const unsigned long *regs)
4406{ 4400{
4407 memset(&ctxt->twobyte, 0, 4401 memset(&ctxt->twobyte, 0,
4408 (void *)&ctxt->regs - (void *)&ctxt->twobyte); 4402 (void *)&ctxt->_regs - (void *)&ctxt->twobyte);
4409 memcpy(ctxt->regs, regs, sizeof(ctxt->regs));
4410 4403
4411 ctxt->fetch.start = 0; 4404 ctxt->fetch.start = 0;
4412 ctxt->fetch.end = 0; 4405 ctxt->fetch.end = 0;
@@ -4421,14 +4414,6 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
4421 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 4414 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt;
4422 int cs_db, cs_l; 4415 int cs_db, cs_l;
4423 4416
4424 /*
4425 * TODO: fix emulate.c to use guest_read/write_register
4426 * instead of direct ->regs accesses, can save hundred cycles
4427 * on Intel for instructions that don't read/change RSP, for
4428 * for example.
4429 */
4430 cache_all_regs(vcpu);
4431
4432 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l); 4417 kvm_x86_ops->get_cs_db_l_bits(vcpu, &cs_db, &cs_l);
4433 4418
4434 ctxt->eflags = kvm_get_rflags(vcpu); 4419 ctxt->eflags = kvm_get_rflags(vcpu);
@@ -4440,7 +4425,7 @@ static void init_emulate_ctxt(struct kvm_vcpu *vcpu)
4440 X86EMUL_MODE_PROT16; 4425 X86EMUL_MODE_PROT16;
4441 ctxt->guest_mode = is_guest_mode(vcpu); 4426 ctxt->guest_mode = is_guest_mode(vcpu);
4442 4427
4443 init_decode_cache(ctxt, vcpu->arch.regs); 4428 init_decode_cache(ctxt);
4444 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 4429 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4445} 4430}
4446 4431
@@ -4460,7 +4445,6 @@ int kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip)
4460 return EMULATE_FAIL; 4445 return EMULATE_FAIL;
4461 4446
4462 ctxt->eip = ctxt->_eip; 4447 ctxt->eip = ctxt->_eip;
4463 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
4464 kvm_rip_write(vcpu, ctxt->eip); 4448 kvm_rip_write(vcpu, ctxt->eip);
4465 kvm_set_rflags(vcpu, ctxt->eflags); 4449 kvm_set_rflags(vcpu, ctxt->eflags);
4466 4450
@@ -4493,13 +4477,14 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu)
4493static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva) 4477static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
4494{ 4478{
4495 gpa_t gpa; 4479 gpa_t gpa;
4480 pfn_t pfn;
4496 4481
4497 if (tdp_enabled) 4482 if (tdp_enabled)
4498 return false; 4483 return false;
4499 4484
4500 /* 4485 /*
4501 * if emulation was due to access to shadowed page table 4486 * if emulation was due to access to shadowed page table
4502 * and it failed try to unshadow page and re-entetr the 4487 * and it failed try to unshadow page and re-enter the
4503 * guest to let CPU execute the instruction. 4488 * guest to let CPU execute the instruction.
4504 */ 4489 */
4505 if (kvm_mmu_unprotect_page_virt(vcpu, gva)) 4490 if (kvm_mmu_unprotect_page_virt(vcpu, gva))
@@ -4510,8 +4495,17 @@ static bool reexecute_instruction(struct kvm_vcpu *vcpu, gva_t gva)
4510 if (gpa == UNMAPPED_GVA) 4495 if (gpa == UNMAPPED_GVA)
4511 return true; /* let cpu generate fault */ 4496 return true; /* let cpu generate fault */
4512 4497
4513 if (!kvm_is_error_hva(gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT))) 4498 /*
4499 * Do not retry the unhandleable instruction if it faults on the
4500 * readonly host memory, otherwise it will goto a infinite loop:
4501 * retry instruction -> write #PF -> emulation fail -> retry
4502 * instruction -> ...
4503 */
4504 pfn = gfn_to_pfn(vcpu->kvm, gpa_to_gfn(gpa));
4505 if (!is_error_pfn(pfn)) {
4506 kvm_release_pfn_clean(pfn);
4514 return true; 4507 return true;
4508 }
4515 4509
4516 return false; 4510 return false;
4517} 4511}
@@ -4560,6 +4554,9 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt,
4560 return true; 4554 return true;
4561} 4555}
4562 4556
4557static int complete_emulated_mmio(struct kvm_vcpu *vcpu);
4558static int complete_emulated_pio(struct kvm_vcpu *vcpu);
4559
4563int x86_emulate_instruction(struct kvm_vcpu *vcpu, 4560int x86_emulate_instruction(struct kvm_vcpu *vcpu,
4564 unsigned long cr2, 4561 unsigned long cr2,
4565 int emulation_type, 4562 int emulation_type,
@@ -4608,7 +4605,7 @@ int x86_emulate_instruction(struct kvm_vcpu *vcpu,
4608 changes registers values during IO operation */ 4605 changes registers values during IO operation */
4609 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) { 4606 if (vcpu->arch.emulate_regs_need_sync_from_vcpu) {
4610 vcpu->arch.emulate_regs_need_sync_from_vcpu = false; 4607 vcpu->arch.emulate_regs_need_sync_from_vcpu = false;
4611 memcpy(ctxt->regs, vcpu->arch.regs, sizeof ctxt->regs); 4608 emulator_invalidate_register_cache(ctxt);
4612 } 4609 }
4613 4610
4614restart: 4611restart:
@@ -4630,13 +4627,16 @@ restart:
4630 } else if (vcpu->arch.pio.count) { 4627 } else if (vcpu->arch.pio.count) {
4631 if (!vcpu->arch.pio.in) 4628 if (!vcpu->arch.pio.in)
4632 vcpu->arch.pio.count = 0; 4629 vcpu->arch.pio.count = 0;
4633 else 4630 else {
4634 writeback = false; 4631 writeback = false;
4632 vcpu->arch.complete_userspace_io = complete_emulated_pio;
4633 }
4635 r = EMULATE_DO_MMIO; 4634 r = EMULATE_DO_MMIO;
4636 } else if (vcpu->mmio_needed) { 4635 } else if (vcpu->mmio_needed) {
4637 if (!vcpu->mmio_is_write) 4636 if (!vcpu->mmio_is_write)
4638 writeback = false; 4637 writeback = false;
4639 r = EMULATE_DO_MMIO; 4638 r = EMULATE_DO_MMIO;
4639 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
4640 } else if (r == EMULATION_RESTART) 4640 } else if (r == EMULATION_RESTART)
4641 goto restart; 4641 goto restart;
4642 else 4642 else
@@ -4646,7 +4646,6 @@ restart:
4646 toggle_interruptibility(vcpu, ctxt->interruptibility); 4646 toggle_interruptibility(vcpu, ctxt->interruptibility);
4647 kvm_set_rflags(vcpu, ctxt->eflags); 4647 kvm_set_rflags(vcpu, ctxt->eflags);
4648 kvm_make_request(KVM_REQ_EVENT, vcpu); 4648 kvm_make_request(KVM_REQ_EVENT, vcpu);
4649 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
4650 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 4649 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
4651 kvm_rip_write(vcpu, ctxt->eip); 4650 kvm_rip_write(vcpu, ctxt->eip);
4652 } else 4651 } else
@@ -4929,6 +4928,7 @@ int kvm_arch_init(void *opaque)
4929 if (cpu_has_xsave) 4928 if (cpu_has_xsave)
4930 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); 4929 host_xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
4931 4930
4931 kvm_lapic_init();
4932 return 0; 4932 return 0;
4933 4933
4934out: 4934out:
@@ -5113,17 +5113,20 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu)
5113 !kvm_event_needs_reinjection(vcpu); 5113 !kvm_event_needs_reinjection(vcpu);
5114} 5114}
5115 5115
5116static void vapic_enter(struct kvm_vcpu *vcpu) 5116static int vapic_enter(struct kvm_vcpu *vcpu)
5117{ 5117{
5118 struct kvm_lapic *apic = vcpu->arch.apic; 5118 struct kvm_lapic *apic = vcpu->arch.apic;
5119 struct page *page; 5119 struct page *page;
5120 5120
5121 if (!apic || !apic->vapic_addr) 5121 if (!apic || !apic->vapic_addr)
5122 return; 5122 return 0;
5123 5123
5124 page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT); 5124 page = gfn_to_page(vcpu->kvm, apic->vapic_addr >> PAGE_SHIFT);
5125 if (is_error_page(page))
5126 return -EFAULT;
5125 5127
5126 vcpu->arch.apic->vapic_page = page; 5128 vcpu->arch.apic->vapic_page = page;
5129 return 0;
5127} 5130}
5128 5131
5129static void vapic_exit(struct kvm_vcpu *vcpu) 5132static void vapic_exit(struct kvm_vcpu *vcpu)
@@ -5430,7 +5433,11 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
5430 } 5433 }
5431 5434
5432 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu); 5435 vcpu->srcu_idx = srcu_read_lock(&kvm->srcu);
5433 vapic_enter(vcpu); 5436 r = vapic_enter(vcpu);
5437 if (r) {
5438 srcu_read_unlock(&kvm->srcu, vcpu->srcu_idx);
5439 return r;
5440 }
5434 5441
5435 r = 1; 5442 r = 1;
5436 while (r > 0) { 5443 while (r > 0) {
@@ -5492,6 +5499,24 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
5492 return r; 5499 return r;
5493} 5500}
5494 5501
5502static inline int complete_emulated_io(struct kvm_vcpu *vcpu)
5503{
5504 int r;
5505 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5506 r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE);
5507 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
5508 if (r != EMULATE_DONE)
5509 return 0;
5510 return 1;
5511}
5512
5513static int complete_emulated_pio(struct kvm_vcpu *vcpu)
5514{
5515 BUG_ON(!vcpu->arch.pio.count);
5516
5517 return complete_emulated_io(vcpu);
5518}
5519
5495/* 5520/*
5496 * Implements the following, as a state machine: 5521 * Implements the following, as a state machine:
5497 * 5522 *
@@ -5508,47 +5533,37 @@ static int __vcpu_run(struct kvm_vcpu *vcpu)
5508 * copy data 5533 * copy data
5509 * exit 5534 * exit
5510 */ 5535 */
5511static int complete_mmio(struct kvm_vcpu *vcpu) 5536static int complete_emulated_mmio(struct kvm_vcpu *vcpu)
5512{ 5537{
5513 struct kvm_run *run = vcpu->run; 5538 struct kvm_run *run = vcpu->run;
5514 struct kvm_mmio_fragment *frag; 5539 struct kvm_mmio_fragment *frag;
5515 int r;
5516 5540
5517 if (!(vcpu->arch.pio.count || vcpu->mmio_needed)) 5541 BUG_ON(!vcpu->mmio_needed);
5518 return 1;
5519 5542
5520 if (vcpu->mmio_needed) { 5543 /* Complete previous fragment */
5521 /* Complete previous fragment */ 5544 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment++];
5522 frag = &vcpu->mmio_fragments[vcpu->mmio_cur_fragment++]; 5545 if (!vcpu->mmio_is_write)
5523 if (!vcpu->mmio_is_write) 5546 memcpy(frag->data, run->mmio.data, frag->len);
5524 memcpy(frag->data, run->mmio.data, frag->len); 5547 if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) {
5525 if (vcpu->mmio_cur_fragment == vcpu->mmio_nr_fragments) { 5548 vcpu->mmio_needed = 0;
5526 vcpu->mmio_needed = 0;
5527 if (vcpu->mmio_is_write)
5528 return 1;
5529 vcpu->mmio_read_completed = 1;
5530 goto done;
5531 }
5532 /* Initiate next fragment */
5533 ++frag;
5534 run->exit_reason = KVM_EXIT_MMIO;
5535 run->mmio.phys_addr = frag->gpa;
5536 if (vcpu->mmio_is_write) 5549 if (vcpu->mmio_is_write)
5537 memcpy(run->mmio.data, frag->data, frag->len); 5550 return 1;
5538 run->mmio.len = frag->len; 5551 vcpu->mmio_read_completed = 1;
5539 run->mmio.is_write = vcpu->mmio_is_write; 5552 return complete_emulated_io(vcpu);
5540 return 0; 5553 }
5541 5554 /* Initiate next fragment */
5542 } 5555 ++frag;
5543done: 5556 run->exit_reason = KVM_EXIT_MMIO;
5544 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 5557 run->mmio.phys_addr = frag->gpa;
5545 r = emulate_instruction(vcpu, EMULTYPE_NO_DECODE); 5558 if (vcpu->mmio_is_write)
5546 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 5559 memcpy(run->mmio.data, frag->data, frag->len);
5547 if (r != EMULATE_DONE) 5560 run->mmio.len = frag->len;
5548 return 0; 5561 run->mmio.is_write = vcpu->mmio_is_write;
5549 return 1; 5562 vcpu->arch.complete_userspace_io = complete_emulated_mmio;
5563 return 0;
5550} 5564}
5551 5565
5566
5552int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 5567int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
5553{ 5568{
5554 int r; 5569 int r;
@@ -5575,9 +5590,14 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run)
5575 } 5590 }
5576 } 5591 }
5577 5592
5578 r = complete_mmio(vcpu); 5593 if (unlikely(vcpu->arch.complete_userspace_io)) {
5579 if (r <= 0) 5594 int (*cui)(struct kvm_vcpu *) = vcpu->arch.complete_userspace_io;
5580 goto out; 5595 vcpu->arch.complete_userspace_io = NULL;
5596 r = cui(vcpu);
5597 if (r <= 0)
5598 goto out;
5599 } else
5600 WARN_ON(vcpu->arch.pio.count || vcpu->mmio_needed);
5581 5601
5582 r = __vcpu_run(vcpu); 5602 r = __vcpu_run(vcpu);
5583 5603
@@ -5595,12 +5615,11 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
5595 /* 5615 /*
5596 * We are here if userspace calls get_regs() in the middle of 5616 * We are here if userspace calls get_regs() in the middle of
5597 * instruction emulation. Registers state needs to be copied 5617 * instruction emulation. Registers state needs to be copied
5598 * back from emulation context to vcpu. Usrapace shouldn't do 5618 * back from emulation context to vcpu. Userspace shouldn't do
5599 * that usually, but some bad designed PV devices (vmware 5619 * that usually, but some bad designed PV devices (vmware
5600 * backdoor interface) need this to work 5620 * backdoor interface) need this to work
5601 */ 5621 */
5602 struct x86_emulate_ctxt *ctxt = &vcpu->arch.emulate_ctxt; 5622 emulator_writeback_register_cache(&vcpu->arch.emulate_ctxt);
5603 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
5604 vcpu->arch.emulate_regs_need_sync_to_vcpu = false; 5623 vcpu->arch.emulate_regs_need_sync_to_vcpu = false;
5605 } 5624 }
5606 regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX); 5625 regs->rax = kvm_register_read(vcpu, VCPU_REGS_RAX);
@@ -5740,7 +5759,6 @@ int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index,
5740 if (ret) 5759 if (ret)
5741 return EMULATE_FAIL; 5760 return EMULATE_FAIL;
5742 5761
5743 memcpy(vcpu->arch.regs, ctxt->regs, sizeof ctxt->regs);
5744 kvm_rip_write(vcpu, ctxt->eip); 5762 kvm_rip_write(vcpu, ctxt->eip);
5745 kvm_set_rflags(vcpu, ctxt->eflags); 5763 kvm_set_rflags(vcpu, ctxt->eflags);
5746 kvm_make_request(KVM_REQ_EVENT, vcpu); 5764 kvm_make_request(KVM_REQ_EVENT, vcpu);
@@ -5792,7 +5810,7 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
5792 if (mmu_reset_needed) 5810 if (mmu_reset_needed)
5793 kvm_mmu_reset_context(vcpu); 5811 kvm_mmu_reset_context(vcpu);
5794 5812
5795 max_bits = (sizeof sregs->interrupt_bitmap) << 3; 5813 max_bits = KVM_NR_INTERRUPTS;
5796 pending_vec = find_first_bit( 5814 pending_vec = find_first_bit(
5797 (const unsigned long *)sregs->interrupt_bitmap, max_bits); 5815 (const unsigned long *)sregs->interrupt_bitmap, max_bits);
5798 if (pending_vec < max_bits) { 5816 if (pending_vec < max_bits) {
@@ -5852,13 +5870,12 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
5852 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { 5870 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) {
5853 for (i = 0; i < KVM_NR_DB_REGS; ++i) 5871 for (i = 0; i < KVM_NR_DB_REGS; ++i)
5854 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; 5872 vcpu->arch.eff_db[i] = dbg->arch.debugreg[i];
5855 vcpu->arch.switch_db_regs = 5873 vcpu->arch.guest_debug_dr7 = dbg->arch.debugreg[7];
5856 (dbg->arch.debugreg[7] & DR7_BP_EN_MASK);
5857 } else { 5874 } else {
5858 for (i = 0; i < KVM_NR_DB_REGS; i++) 5875 for (i = 0; i < KVM_NR_DB_REGS; i++)
5859 vcpu->arch.eff_db[i] = vcpu->arch.db[i]; 5876 vcpu->arch.eff_db[i] = vcpu->arch.db[i];
5860 vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK);
5861 } 5877 }
5878 kvm_update_dr7(vcpu);
5862 5879
5863 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) 5880 if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
5864 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) + 5881 vcpu->arch.singlestep_rip = kvm_rip_read(vcpu) +
@@ -5870,7 +5887,7 @@ int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
5870 */ 5887 */
5871 kvm_set_rflags(vcpu, rflags); 5888 kvm_set_rflags(vcpu, rflags);
5872 5889
5873 kvm_x86_ops->set_guest_debug(vcpu, dbg); 5890 kvm_x86_ops->update_db_bp_intercept(vcpu);
5874 5891
5875 r = 0; 5892 r = 0;
5876 5893
@@ -5972,7 +5989,7 @@ void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
5972 */ 5989 */
5973 kvm_put_guest_xcr0(vcpu); 5990 kvm_put_guest_xcr0(vcpu);
5974 vcpu->guest_fpu_loaded = 1; 5991 vcpu->guest_fpu_loaded = 1;
5975 unlazy_fpu(current); 5992 __kernel_fpu_begin();
5976 fpu_restore_checking(&vcpu->arch.guest_fpu); 5993 fpu_restore_checking(&vcpu->arch.guest_fpu);
5977 trace_kvm_fpu(1); 5994 trace_kvm_fpu(1);
5978} 5995}
@@ -5986,6 +6003,7 @@ void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
5986 6003
5987 vcpu->guest_fpu_loaded = 0; 6004 vcpu->guest_fpu_loaded = 0;
5988 fpu_save_init(&vcpu->arch.guest_fpu); 6005 fpu_save_init(&vcpu->arch.guest_fpu);
6006 __kernel_fpu_end();
5989 ++vcpu->stat.fpu_reload; 6007 ++vcpu->stat.fpu_reload;
5990 kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu); 6008 kvm_make_request(KVM_REQ_DEACTIVATE_FPU, vcpu);
5991 trace_kvm_fpu(0); 6009 trace_kvm_fpu(0);
@@ -6015,7 +6033,9 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
6015 int r; 6033 int r;
6016 6034
6017 vcpu->arch.mtrr_state.have_fixed = 1; 6035 vcpu->arch.mtrr_state.have_fixed = 1;
6018 vcpu_load(vcpu); 6036 r = vcpu_load(vcpu);
6037 if (r)
6038 return r;
6019 r = kvm_arch_vcpu_reset(vcpu); 6039 r = kvm_arch_vcpu_reset(vcpu);
6020 if (r == 0) 6040 if (r == 0)
6021 r = kvm_mmu_setup(vcpu); 6041 r = kvm_mmu_setup(vcpu);
@@ -6026,9 +6046,11 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
6026 6046
6027void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 6047void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
6028{ 6048{
6049 int r;
6029 vcpu->arch.apf.msr_val = 0; 6050 vcpu->arch.apf.msr_val = 0;
6030 6051
6031 vcpu_load(vcpu); 6052 r = vcpu_load(vcpu);
6053 BUG_ON(r);
6032 kvm_mmu_unload(vcpu); 6054 kvm_mmu_unload(vcpu);
6033 vcpu_put(vcpu); 6055 vcpu_put(vcpu);
6034 6056
@@ -6042,10 +6064,10 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu)
6042 vcpu->arch.nmi_pending = 0; 6064 vcpu->arch.nmi_pending = 0;
6043 vcpu->arch.nmi_injected = false; 6065 vcpu->arch.nmi_injected = false;
6044 6066
6045 vcpu->arch.switch_db_regs = 0;
6046 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); 6067 memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db));
6047 vcpu->arch.dr6 = DR6_FIXED_1; 6068 vcpu->arch.dr6 = DR6_FIXED_1;
6048 vcpu->arch.dr7 = DR7_FIXED_1; 6069 vcpu->arch.dr7 = DR7_FIXED_1;
6070 kvm_update_dr7(vcpu);
6049 6071
6050 kvm_make_request(KVM_REQ_EVENT, vcpu); 6072 kvm_make_request(KVM_REQ_EVENT, vcpu);
6051 vcpu->arch.apf.msr_val = 0; 6073 vcpu->arch.apf.msr_val = 0;
@@ -6124,7 +6146,7 @@ int kvm_arch_hardware_enable(void *garbage)
6124 * as we reset last_host_tsc on all VCPUs to stop this from being 6146 * as we reset last_host_tsc on all VCPUs to stop this from being
6125 * called multiple times (one for each physical CPU bringup). 6147 * called multiple times (one for each physical CPU bringup).
6126 * 6148 *
6127 * Platforms with unnreliable TSCs don't have to deal with this, they 6149 * Platforms with unreliable TSCs don't have to deal with this, they
6128 * will be compensated by the logic in vcpu_load, which sets the TSC to 6150 * will be compensated by the logic in vcpu_load, which sets the TSC to
6129 * catchup mode. This will catchup all VCPUs to real time, but cannot 6151 * catchup mode. This will catchup all VCPUs to real time, but cannot
6130 * guarantee that they stay in perfect synchronization. 6152 * guarantee that they stay in perfect synchronization.
@@ -6177,6 +6199,8 @@ bool kvm_vcpu_compatible(struct kvm_vcpu *vcpu)
6177 return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL); 6199 return irqchip_in_kernel(vcpu->kvm) == (vcpu->arch.apic != NULL);
6178} 6200}
6179 6201
6202struct static_key kvm_no_apic_vcpu __read_mostly;
6203
6180int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 6204int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
6181{ 6205{
6182 struct page *page; 6206 struct page *page;
@@ -6209,7 +6233,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu)
6209 r = kvm_create_lapic(vcpu); 6233 r = kvm_create_lapic(vcpu);
6210 if (r < 0) 6234 if (r < 0)
6211 goto fail_mmu_destroy; 6235 goto fail_mmu_destroy;
6212 } 6236 } else
6237 static_key_slow_inc(&kvm_no_apic_vcpu);
6213 6238
6214 vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4, 6239 vcpu->arch.mce_banks = kzalloc(KVM_MAX_MCE_BANKS * sizeof(u64) * 4,
6215 GFP_KERNEL); 6240 GFP_KERNEL);
@@ -6249,6 +6274,8 @@ void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu)
6249 kvm_mmu_destroy(vcpu); 6274 kvm_mmu_destroy(vcpu);
6250 srcu_read_unlock(&vcpu->kvm->srcu, idx); 6275 srcu_read_unlock(&vcpu->kvm->srcu, idx);
6251 free_page((unsigned long)vcpu->arch.pio_data); 6276 free_page((unsigned long)vcpu->arch.pio_data);
6277 if (!irqchip_in_kernel(vcpu->kvm))
6278 static_key_slow_dec(&kvm_no_apic_vcpu);
6252} 6279}
6253 6280
6254int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 6281int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
@@ -6261,15 +6288,21 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
6261 6288
6262 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ 6289 /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */
6263 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); 6290 set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap);
6291 /* Reserve bit 1 of irq_sources_bitmap for irqfd-resampler */
6292 set_bit(KVM_IRQFD_RESAMPLE_IRQ_SOURCE_ID,
6293 &kvm->arch.irq_sources_bitmap);
6264 6294
6265 raw_spin_lock_init(&kvm->arch.tsc_write_lock); 6295 raw_spin_lock_init(&kvm->arch.tsc_write_lock);
6296 mutex_init(&kvm->arch.apic_map_lock);
6266 6297
6267 return 0; 6298 return 0;
6268} 6299}
6269 6300
6270static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu) 6301static void kvm_unload_vcpu_mmu(struct kvm_vcpu *vcpu)
6271{ 6302{
6272 vcpu_load(vcpu); 6303 int r;
6304 r = vcpu_load(vcpu);
6305 BUG_ON(r);
6273 kvm_mmu_unload(vcpu); 6306 kvm_mmu_unload(vcpu);
6274 vcpu_put(vcpu); 6307 vcpu_put(vcpu);
6275} 6308}
@@ -6313,6 +6346,7 @@ void kvm_arch_destroy_vm(struct kvm *kvm)
6313 put_page(kvm->arch.apic_access_page); 6346 put_page(kvm->arch.apic_access_page);
6314 if (kvm->arch.ept_identity_pagetable) 6347 if (kvm->arch.ept_identity_pagetable)
6315 put_page(kvm->arch.ept_identity_pagetable); 6348 put_page(kvm->arch.ept_identity_pagetable);
6349 kfree(rcu_dereference_check(kvm->arch.apic_map, 1));
6316} 6350}
6317 6351
6318void kvm_arch_free_memslot(struct kvm_memory_slot *free, 6352void kvm_arch_free_memslot(struct kvm_memory_slot *free,
@@ -6320,10 +6354,18 @@ void kvm_arch_free_memslot(struct kvm_memory_slot *free,
6320{ 6354{
6321 int i; 6355 int i;
6322 6356
6323 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { 6357 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
6324 if (!dont || free->arch.lpage_info[i] != dont->arch.lpage_info[i]) { 6358 if (!dont || free->arch.rmap[i] != dont->arch.rmap[i]) {
6325 kvm_kvfree(free->arch.lpage_info[i]); 6359 kvm_kvfree(free->arch.rmap[i]);
6326 free->arch.lpage_info[i] = NULL; 6360 free->arch.rmap[i] = NULL;
6361 }
6362 if (i == 0)
6363 continue;
6364
6365 if (!dont || free->arch.lpage_info[i - 1] !=
6366 dont->arch.lpage_info[i - 1]) {
6367 kvm_kvfree(free->arch.lpage_info[i - 1]);
6368 free->arch.lpage_info[i - 1] = NULL;
6327 } 6369 }
6328 } 6370 }
6329} 6371}
@@ -6332,23 +6374,30 @@ int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
6332{ 6374{
6333 int i; 6375 int i;
6334 6376
6335 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { 6377 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
6336 unsigned long ugfn; 6378 unsigned long ugfn;
6337 int lpages; 6379 int lpages;
6338 int level = i + 2; 6380 int level = i + 1;
6339 6381
6340 lpages = gfn_to_index(slot->base_gfn + npages - 1, 6382 lpages = gfn_to_index(slot->base_gfn + npages - 1,
6341 slot->base_gfn, level) + 1; 6383 slot->base_gfn, level) + 1;
6342 6384
6343 slot->arch.lpage_info[i] = 6385 slot->arch.rmap[i] =
6344 kvm_kvzalloc(lpages * sizeof(*slot->arch.lpage_info[i])); 6386 kvm_kvzalloc(lpages * sizeof(*slot->arch.rmap[i]));
6345 if (!slot->arch.lpage_info[i]) 6387 if (!slot->arch.rmap[i])
6388 goto out_free;
6389 if (i == 0)
6390 continue;
6391
6392 slot->arch.lpage_info[i - 1] = kvm_kvzalloc(lpages *
6393 sizeof(*slot->arch.lpage_info[i - 1]));
6394 if (!slot->arch.lpage_info[i - 1])
6346 goto out_free; 6395 goto out_free;
6347 6396
6348 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1)) 6397 if (slot->base_gfn & (KVM_PAGES_PER_HPAGE(level) - 1))
6349 slot->arch.lpage_info[i][0].write_count = 1; 6398 slot->arch.lpage_info[i - 1][0].write_count = 1;
6350 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1)) 6399 if ((slot->base_gfn + npages) & (KVM_PAGES_PER_HPAGE(level) - 1))
6351 slot->arch.lpage_info[i][lpages - 1].write_count = 1; 6400 slot->arch.lpage_info[i - 1][lpages - 1].write_count = 1;
6352 ugfn = slot->userspace_addr >> PAGE_SHIFT; 6401 ugfn = slot->userspace_addr >> PAGE_SHIFT;
6353 /* 6402 /*
6354 * If the gfn and userspace address are not aligned wrt each 6403 * If the gfn and userspace address are not aligned wrt each
@@ -6360,16 +6409,21 @@ int kvm_arch_create_memslot(struct kvm_memory_slot *slot, unsigned long npages)
6360 unsigned long j; 6409 unsigned long j;
6361 6410
6362 for (j = 0; j < lpages; ++j) 6411 for (j = 0; j < lpages; ++j)
6363 slot->arch.lpage_info[i][j].write_count = 1; 6412 slot->arch.lpage_info[i - 1][j].write_count = 1;
6364 } 6413 }
6365 } 6414 }
6366 6415
6367 return 0; 6416 return 0;
6368 6417
6369out_free: 6418out_free:
6370 for (i = 0; i < KVM_NR_PAGE_SIZES - 1; ++i) { 6419 for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) {
6371 kvm_kvfree(slot->arch.lpage_info[i]); 6420 kvm_kvfree(slot->arch.rmap[i]);
6372 slot->arch.lpage_info[i] = NULL; 6421 slot->arch.rmap[i] = NULL;
6422 if (i == 0)
6423 continue;
6424
6425 kvm_kvfree(slot->arch.lpage_info[i - 1]);
6426 slot->arch.lpage_info[i - 1] = NULL;
6373 } 6427 }
6374 return -ENOMEM; 6428 return -ENOMEM;
6375} 6429}
@@ -6388,10 +6442,10 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
6388 map_flags = MAP_SHARED | MAP_ANONYMOUS; 6442 map_flags = MAP_SHARED | MAP_ANONYMOUS;
6389 6443
6390 /*To keep backward compatibility with older userspace, 6444 /*To keep backward compatibility with older userspace,
6391 *x86 needs to hanlde !user_alloc case. 6445 *x86 needs to handle !user_alloc case.
6392 */ 6446 */
6393 if (!user_alloc) { 6447 if (!user_alloc) {
6394 if (npages && !old.rmap) { 6448 if (npages && !old.npages) {
6395 unsigned long userspace_addr; 6449 unsigned long userspace_addr;
6396 6450
6397 userspace_addr = vm_mmap(NULL, 0, 6451 userspace_addr = vm_mmap(NULL, 0,
@@ -6419,7 +6473,7 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
6419 6473
6420 int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT; 6474 int nr_mmu_pages = 0, npages = mem->memory_size >> PAGE_SHIFT;
6421 6475
6422 if (!user_alloc && !old.user_alloc && old.rmap && !npages) { 6476 if (!user_alloc && !old.user_alloc && old.npages && !npages) {
6423 int ret; 6477 int ret;
6424 6478
6425 ret = vm_munmap(old.userspace_addr, 6479 ret = vm_munmap(old.userspace_addr,
@@ -6438,14 +6492,28 @@ void kvm_arch_commit_memory_region(struct kvm *kvm,
6438 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages); 6492 kvm_mmu_change_mmu_pages(kvm, nr_mmu_pages);
6439 kvm_mmu_slot_remove_write_access(kvm, mem->slot); 6493 kvm_mmu_slot_remove_write_access(kvm, mem->slot);
6440 spin_unlock(&kvm->mmu_lock); 6494 spin_unlock(&kvm->mmu_lock);
6495 /*
6496 * If memory slot is created, or moved, we need to clear all
6497 * mmio sptes.
6498 */
6499 if (npages && old.base_gfn != mem->guest_phys_addr >> PAGE_SHIFT) {
6500 kvm_mmu_zap_all(kvm);
6501 kvm_reload_remote_mmus(kvm);
6502 }
6441} 6503}
6442 6504
6443void kvm_arch_flush_shadow(struct kvm *kvm) 6505void kvm_arch_flush_shadow_all(struct kvm *kvm)
6444{ 6506{
6445 kvm_mmu_zap_all(kvm); 6507 kvm_mmu_zap_all(kvm);
6446 kvm_reload_remote_mmus(kvm); 6508 kvm_reload_remote_mmus(kvm);
6447} 6509}
6448 6510
6511void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
6512 struct kvm_memory_slot *slot)
6513{
6514 kvm_arch_flush_shadow_all(kvm);
6515}
6516
6449int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 6517int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
6450{ 6518{
6451 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE && 6519 return (vcpu->arch.mp_state == KVM_MP_STATE_RUNNABLE &&
diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h
index 3d1134ddb885..2b5219c12ac8 100644
--- a/arch/x86/kvm/x86.h
+++ b/arch/x86/kvm/x86.h
@@ -124,4 +124,5 @@ int kvm_write_guest_virt_system(struct x86_emulate_ctxt *ctxt,
124 124
125extern u64 host_xcr0; 125extern u64 host_xcr0;
126 126
127extern struct static_key kvm_no_apic_vcpu;
127#endif 128#endif
diff --git a/arch/x86/lguest/Kconfig b/arch/x86/lguest/Kconfig
index 6e121a2a49e1..7872a3330fb5 100644
--- a/arch/x86/lguest/Kconfig
+++ b/arch/x86/lguest/Kconfig
@@ -4,7 +4,6 @@ config LGUEST_GUEST
4 depends on X86_32 4 depends on X86_32
5 select VIRTUALIZATION 5 select VIRTUALIZATION
6 select VIRTIO 6 select VIRTIO
7 select VIRTIO_RING
8 select VIRTIO_CONSOLE 7 select VIRTIO_CONSOLE
9 help 8 help
10 Lguest is a tiny in-kernel hypervisor. Selecting this will 9 Lguest is a tiny in-kernel hypervisor. Selecting this will
diff --git a/arch/x86/lib/copy_user_64.S b/arch/x86/lib/copy_user_64.S
index 5b2995f4557a..a30ca15be21c 100644
--- a/arch/x86/lib/copy_user_64.S
+++ b/arch/x86/lib/copy_user_64.S
@@ -17,6 +17,7 @@
17#include <asm/cpufeature.h> 17#include <asm/cpufeature.h>
18#include <asm/alternative-asm.h> 18#include <asm/alternative-asm.h>
19#include <asm/asm.h> 19#include <asm/asm.h>
20#include <asm/smap.h>
20 21
21/* 22/*
22 * By placing feature2 after feature1 in altinstructions section, we logically 23 * By placing feature2 after feature1 in altinstructions section, we logically
@@ -130,6 +131,7 @@ ENDPROC(bad_from_user)
130 */ 131 */
131ENTRY(copy_user_generic_unrolled) 132ENTRY(copy_user_generic_unrolled)
132 CFI_STARTPROC 133 CFI_STARTPROC
134 ASM_STAC
133 cmpl $8,%edx 135 cmpl $8,%edx
134 jb 20f /* less then 8 bytes, go to byte copy loop */ 136 jb 20f /* less then 8 bytes, go to byte copy loop */
135 ALIGN_DESTINATION 137 ALIGN_DESTINATION
@@ -177,6 +179,7 @@ ENTRY(copy_user_generic_unrolled)
177 decl %ecx 179 decl %ecx
178 jnz 21b 180 jnz 21b
17923: xor %eax,%eax 18123: xor %eax,%eax
182 ASM_CLAC
180 ret 183 ret
181 184
182 .section .fixup,"ax" 185 .section .fixup,"ax"
@@ -232,6 +235,7 @@ ENDPROC(copy_user_generic_unrolled)
232 */ 235 */
233ENTRY(copy_user_generic_string) 236ENTRY(copy_user_generic_string)
234 CFI_STARTPROC 237 CFI_STARTPROC
238 ASM_STAC
235 andl %edx,%edx 239 andl %edx,%edx
236 jz 4f 240 jz 4f
237 cmpl $8,%edx 241 cmpl $8,%edx
@@ -246,6 +250,7 @@ ENTRY(copy_user_generic_string)
2463: rep 2503: rep
247 movsb 251 movsb
2484: xorl %eax,%eax 2524: xorl %eax,%eax
253 ASM_CLAC
249 ret 254 ret
250 255
251 .section .fixup,"ax" 256 .section .fixup,"ax"
@@ -273,12 +278,14 @@ ENDPROC(copy_user_generic_string)
273 */ 278 */
274ENTRY(copy_user_enhanced_fast_string) 279ENTRY(copy_user_enhanced_fast_string)
275 CFI_STARTPROC 280 CFI_STARTPROC
281 ASM_STAC
276 andl %edx,%edx 282 andl %edx,%edx
277 jz 2f 283 jz 2f
278 movl %edx,%ecx 284 movl %edx,%ecx
2791: rep 2851: rep
280 movsb 286 movsb
2812: xorl %eax,%eax 2872: xorl %eax,%eax
288 ASM_CLAC
282 ret 289 ret
283 290
284 .section .fixup,"ax" 291 .section .fixup,"ax"
diff --git a/arch/x86/lib/copy_user_nocache_64.S b/arch/x86/lib/copy_user_nocache_64.S
index cacddc7163eb..6a4f43c2d9e6 100644
--- a/arch/x86/lib/copy_user_nocache_64.S
+++ b/arch/x86/lib/copy_user_nocache_64.S
@@ -15,6 +15,7 @@
15#include <asm/asm-offsets.h> 15#include <asm/asm-offsets.h>
16#include <asm/thread_info.h> 16#include <asm/thread_info.h>
17#include <asm/asm.h> 17#include <asm/asm.h>
18#include <asm/smap.h>
18 19
19 .macro ALIGN_DESTINATION 20 .macro ALIGN_DESTINATION
20#ifdef FIX_ALIGNMENT 21#ifdef FIX_ALIGNMENT
@@ -48,6 +49,7 @@
48 */ 49 */
49ENTRY(__copy_user_nocache) 50ENTRY(__copy_user_nocache)
50 CFI_STARTPROC 51 CFI_STARTPROC
52 ASM_STAC
51 cmpl $8,%edx 53 cmpl $8,%edx
52 jb 20f /* less then 8 bytes, go to byte copy loop */ 54 jb 20f /* less then 8 bytes, go to byte copy loop */
53 ALIGN_DESTINATION 55 ALIGN_DESTINATION
@@ -95,6 +97,7 @@ ENTRY(__copy_user_nocache)
95 decl %ecx 97 decl %ecx
96 jnz 21b 98 jnz 21b
9723: xorl %eax,%eax 9923: xorl %eax,%eax
100 ASM_CLAC
98 sfence 101 sfence
99 ret 102 ret
100 103
diff --git a/arch/x86/lib/getuser.S b/arch/x86/lib/getuser.S
index b33b1fb1e6d4..156b9c804670 100644
--- a/arch/x86/lib/getuser.S
+++ b/arch/x86/lib/getuser.S
@@ -33,6 +33,7 @@
33#include <asm/asm-offsets.h> 33#include <asm/asm-offsets.h>
34#include <asm/thread_info.h> 34#include <asm/thread_info.h>
35#include <asm/asm.h> 35#include <asm/asm.h>
36#include <asm/smap.h>
36 37
37 .text 38 .text
38ENTRY(__get_user_1) 39ENTRY(__get_user_1)
@@ -40,8 +41,10 @@ ENTRY(__get_user_1)
40 GET_THREAD_INFO(%_ASM_DX) 41 GET_THREAD_INFO(%_ASM_DX)
41 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 42 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
42 jae bad_get_user 43 jae bad_get_user
44 ASM_STAC
431: movzb (%_ASM_AX),%edx 451: movzb (%_ASM_AX),%edx
44 xor %eax,%eax 46 xor %eax,%eax
47 ASM_CLAC
45 ret 48 ret
46 CFI_ENDPROC 49 CFI_ENDPROC
47ENDPROC(__get_user_1) 50ENDPROC(__get_user_1)
@@ -53,8 +56,10 @@ ENTRY(__get_user_2)
53 GET_THREAD_INFO(%_ASM_DX) 56 GET_THREAD_INFO(%_ASM_DX)
54 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 57 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
55 jae bad_get_user 58 jae bad_get_user
59 ASM_STAC
562: movzwl -1(%_ASM_AX),%edx 602: movzwl -1(%_ASM_AX),%edx
57 xor %eax,%eax 61 xor %eax,%eax
62 ASM_CLAC
58 ret 63 ret
59 CFI_ENDPROC 64 CFI_ENDPROC
60ENDPROC(__get_user_2) 65ENDPROC(__get_user_2)
@@ -66,8 +71,10 @@ ENTRY(__get_user_4)
66 GET_THREAD_INFO(%_ASM_DX) 71 GET_THREAD_INFO(%_ASM_DX)
67 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 72 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
68 jae bad_get_user 73 jae bad_get_user
74 ASM_STAC
693: mov -3(%_ASM_AX),%edx 753: mov -3(%_ASM_AX),%edx
70 xor %eax,%eax 76 xor %eax,%eax
77 ASM_CLAC
71 ret 78 ret
72 CFI_ENDPROC 79 CFI_ENDPROC
73ENDPROC(__get_user_4) 80ENDPROC(__get_user_4)
@@ -80,8 +87,10 @@ ENTRY(__get_user_8)
80 GET_THREAD_INFO(%_ASM_DX) 87 GET_THREAD_INFO(%_ASM_DX)
81 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX 88 cmp TI_addr_limit(%_ASM_DX),%_ASM_AX
82 jae bad_get_user 89 jae bad_get_user
90 ASM_STAC
834: movq -7(%_ASM_AX),%_ASM_DX 914: movq -7(%_ASM_AX),%_ASM_DX
84 xor %eax,%eax 92 xor %eax,%eax
93 ASM_CLAC
85 ret 94 ret
86 CFI_ENDPROC 95 CFI_ENDPROC
87ENDPROC(__get_user_8) 96ENDPROC(__get_user_8)
@@ -91,6 +100,7 @@ bad_get_user:
91 CFI_STARTPROC 100 CFI_STARTPROC
92 xor %edx,%edx 101 xor %edx,%edx
93 mov $(-EFAULT),%_ASM_AX 102 mov $(-EFAULT),%_ASM_AX
103 ASM_CLAC
94 ret 104 ret
95 CFI_ENDPROC 105 CFI_ENDPROC
96END(bad_get_user) 106END(bad_get_user)
diff --git a/arch/x86/lib/insn.c b/arch/x86/lib/insn.c
index b1e6c4b2e8eb..54fcffed28ed 100644
--- a/arch/x86/lib/insn.c
+++ b/arch/x86/lib/insn.c
@@ -18,7 +18,11 @@
18 * Copyright (C) IBM Corporation, 2002, 2004, 2009 18 * Copyright (C) IBM Corporation, 2002, 2004, 2009
19 */ 19 */
20 20
21#ifdef __KERNEL__
21#include <linux/string.h> 22#include <linux/string.h>
23#else
24#include <string.h>
25#endif
22#include <asm/inat.h> 26#include <asm/inat.h>
23#include <asm/insn.h> 27#include <asm/insn.h>
24 28
diff --git a/arch/x86/lib/putuser.S b/arch/x86/lib/putuser.S
index 7f951c8f76c4..fc6ba17a7eec 100644
--- a/arch/x86/lib/putuser.S
+++ b/arch/x86/lib/putuser.S
@@ -15,6 +15,7 @@
15#include <asm/thread_info.h> 15#include <asm/thread_info.h>
16#include <asm/errno.h> 16#include <asm/errno.h>
17#include <asm/asm.h> 17#include <asm/asm.h>
18#include <asm/smap.h>
18 19
19 20
20/* 21/*
@@ -31,7 +32,8 @@
31 32
32#define ENTER CFI_STARTPROC ; \ 33#define ENTER CFI_STARTPROC ; \
33 GET_THREAD_INFO(%_ASM_BX) 34 GET_THREAD_INFO(%_ASM_BX)
34#define EXIT ret ; \ 35#define EXIT ASM_CLAC ; \
36 ret ; \
35 CFI_ENDPROC 37 CFI_ENDPROC
36 38
37.text 39.text
@@ -39,6 +41,7 @@ ENTRY(__put_user_1)
39 ENTER 41 ENTER
40 cmp TI_addr_limit(%_ASM_BX),%_ASM_CX 42 cmp TI_addr_limit(%_ASM_BX),%_ASM_CX
41 jae bad_put_user 43 jae bad_put_user
44 ASM_STAC
421: movb %al,(%_ASM_CX) 451: movb %al,(%_ASM_CX)
43 xor %eax,%eax 46 xor %eax,%eax
44 EXIT 47 EXIT
@@ -50,6 +53,7 @@ ENTRY(__put_user_2)
50 sub $1,%_ASM_BX 53 sub $1,%_ASM_BX
51 cmp %_ASM_BX,%_ASM_CX 54 cmp %_ASM_BX,%_ASM_CX
52 jae bad_put_user 55 jae bad_put_user
56 ASM_STAC
532: movw %ax,(%_ASM_CX) 572: movw %ax,(%_ASM_CX)
54 xor %eax,%eax 58 xor %eax,%eax
55 EXIT 59 EXIT
@@ -61,6 +65,7 @@ ENTRY(__put_user_4)
61 sub $3,%_ASM_BX 65 sub $3,%_ASM_BX
62 cmp %_ASM_BX,%_ASM_CX 66 cmp %_ASM_BX,%_ASM_CX
63 jae bad_put_user 67 jae bad_put_user
68 ASM_STAC
643: movl %eax,(%_ASM_CX) 693: movl %eax,(%_ASM_CX)
65 xor %eax,%eax 70 xor %eax,%eax
66 EXIT 71 EXIT
@@ -72,6 +77,7 @@ ENTRY(__put_user_8)
72 sub $7,%_ASM_BX 77 sub $7,%_ASM_BX
73 cmp %_ASM_BX,%_ASM_CX 78 cmp %_ASM_BX,%_ASM_CX
74 jae bad_put_user 79 jae bad_put_user
80 ASM_STAC
754: mov %_ASM_AX,(%_ASM_CX) 814: mov %_ASM_AX,(%_ASM_CX)
76#ifdef CONFIG_X86_32 82#ifdef CONFIG_X86_32
775: movl %edx,4(%_ASM_CX) 835: movl %edx,4(%_ASM_CX)
diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 1781b2f950e2..98f6d6b68f5a 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -42,10 +42,11 @@ do { \
42 int __d0; \ 42 int __d0; \
43 might_fault(); \ 43 might_fault(); \
44 __asm__ __volatile__( \ 44 __asm__ __volatile__( \
45 ASM_STAC "\n" \
45 "0: rep; stosl\n" \ 46 "0: rep; stosl\n" \
46 " movl %2,%0\n" \ 47 " movl %2,%0\n" \
47 "1: rep; stosb\n" \ 48 "1: rep; stosb\n" \
48 "2:\n" \ 49 "2: " ASM_CLAC "\n" \
49 ".section .fixup,\"ax\"\n" \ 50 ".section .fixup,\"ax\"\n" \
50 "3: lea 0(%2,%0,4),%0\n" \ 51 "3: lea 0(%2,%0,4),%0\n" \
51 " jmp 2b\n" \ 52 " jmp 2b\n" \
@@ -626,10 +627,12 @@ survive:
626 return n; 627 return n;
627 } 628 }
628#endif 629#endif
630 stac();
629 if (movsl_is_ok(to, from, n)) 631 if (movsl_is_ok(to, from, n))
630 __copy_user(to, from, n); 632 __copy_user(to, from, n);
631 else 633 else
632 n = __copy_user_intel(to, from, n); 634 n = __copy_user_intel(to, from, n);
635 clac();
633 return n; 636 return n;
634} 637}
635EXPORT_SYMBOL(__copy_to_user_ll); 638EXPORT_SYMBOL(__copy_to_user_ll);
@@ -637,10 +640,12 @@ EXPORT_SYMBOL(__copy_to_user_ll);
637unsigned long __copy_from_user_ll(void *to, const void __user *from, 640unsigned long __copy_from_user_ll(void *to, const void __user *from,
638 unsigned long n) 641 unsigned long n)
639{ 642{
643 stac();
640 if (movsl_is_ok(to, from, n)) 644 if (movsl_is_ok(to, from, n))
641 __copy_user_zeroing(to, from, n); 645 __copy_user_zeroing(to, from, n);
642 else 646 else
643 n = __copy_user_zeroing_intel(to, from, n); 647 n = __copy_user_zeroing_intel(to, from, n);
648 clac();
644 return n; 649 return n;
645} 650}
646EXPORT_SYMBOL(__copy_from_user_ll); 651EXPORT_SYMBOL(__copy_from_user_ll);
@@ -648,11 +653,13 @@ EXPORT_SYMBOL(__copy_from_user_ll);
648unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, 653unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from,
649 unsigned long n) 654 unsigned long n)
650{ 655{
656 stac();
651 if (movsl_is_ok(to, from, n)) 657 if (movsl_is_ok(to, from, n))
652 __copy_user(to, from, n); 658 __copy_user(to, from, n);
653 else 659 else
654 n = __copy_user_intel((void __user *)to, 660 n = __copy_user_intel((void __user *)to,
655 (const void *)from, n); 661 (const void *)from, n);
662 clac();
656 return n; 663 return n;
657} 664}
658EXPORT_SYMBOL(__copy_from_user_ll_nozero); 665EXPORT_SYMBOL(__copy_from_user_ll_nozero);
@@ -660,6 +667,7 @@ EXPORT_SYMBOL(__copy_from_user_ll_nozero);
660unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, 667unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from,
661 unsigned long n) 668 unsigned long n)
662{ 669{
670 stac();
663#ifdef CONFIG_X86_INTEL_USERCOPY 671#ifdef CONFIG_X86_INTEL_USERCOPY
664 if (n > 64 && cpu_has_xmm2) 672 if (n > 64 && cpu_has_xmm2)
665 n = __copy_user_zeroing_intel_nocache(to, from, n); 673 n = __copy_user_zeroing_intel_nocache(to, from, n);
@@ -668,6 +676,7 @@ unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from,
668#else 676#else
669 __copy_user_zeroing(to, from, n); 677 __copy_user_zeroing(to, from, n);
670#endif 678#endif
679 clac();
671 return n; 680 return n;
672} 681}
673EXPORT_SYMBOL(__copy_from_user_ll_nocache); 682EXPORT_SYMBOL(__copy_from_user_ll_nocache);
@@ -675,6 +684,7 @@ EXPORT_SYMBOL(__copy_from_user_ll_nocache);
675unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, 684unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from,
676 unsigned long n) 685 unsigned long n)
677{ 686{
687 stac();
678#ifdef CONFIG_X86_INTEL_USERCOPY 688#ifdef CONFIG_X86_INTEL_USERCOPY
679 if (n > 64 && cpu_has_xmm2) 689 if (n > 64 && cpu_has_xmm2)
680 n = __copy_user_intel_nocache(to, from, n); 690 n = __copy_user_intel_nocache(to, from, n);
@@ -683,6 +693,7 @@ unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *fr
683#else 693#else
684 __copy_user(to, from, n); 694 __copy_user(to, from, n);
685#endif 695#endif
696 clac();
686 return n; 697 return n;
687} 698}
688EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); 699EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero);
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index e5b130bc2d0e..05928aae911e 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -18,6 +18,7 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
18 might_fault(); 18 might_fault();
19 /* no memory constraint because it doesn't change any memory gcc knows 19 /* no memory constraint because it doesn't change any memory gcc knows
20 about */ 20 about */
21 stac();
21 asm volatile( 22 asm volatile(
22 " testq %[size8],%[size8]\n" 23 " testq %[size8],%[size8]\n"
23 " jz 4f\n" 24 " jz 4f\n"
@@ -40,6 +41,7 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
40 : [size8] "=&c"(size), [dst] "=&D" (__d0) 41 : [size8] "=&c"(size), [dst] "=&D" (__d0)
41 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr), 42 : [size1] "r"(size & 7), "[size8]" (size / 8), "[dst]"(addr),
42 [zero] "r" (0UL), [eight] "r" (8UL)); 43 [zero] "r" (0UL), [eight] "r" (8UL));
44 clac();
43 return size; 45 return size;
44} 46}
45EXPORT_SYMBOL(__clear_user); 47EXPORT_SYMBOL(__clear_user);
@@ -82,5 +84,6 @@ copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest)
82 for (c = 0, zero_len = len; zerorest && zero_len; --zero_len) 84 for (c = 0, zero_len = len; zerorest && zero_len; --zero_len)
83 if (__put_user_nocheck(c, to++, sizeof(char))) 85 if (__put_user_nocheck(c, to++, sizeof(char)))
84 break; 86 break;
87 clac();
85 return len; 88 return len;
86} 89}
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 76dcd9d8e0bc..8e13ecb41bee 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -18,6 +18,7 @@
18#include <asm/pgalloc.h> /* pgd_*(), ... */ 18#include <asm/pgalloc.h> /* pgd_*(), ... */
19#include <asm/kmemcheck.h> /* kmemcheck_*(), ... */ 19#include <asm/kmemcheck.h> /* kmemcheck_*(), ... */
20#include <asm/fixmap.h> /* VSYSCALL_START */ 20#include <asm/fixmap.h> /* VSYSCALL_START */
21#include <asm/rcu.h> /* exception_enter(), ... */
21 22
22/* 23/*
23 * Page fault error code bits: 24 * Page fault error code bits:
@@ -995,13 +996,24 @@ static int fault_in_kernel_space(unsigned long address)
995 return address >= TASK_SIZE_MAX; 996 return address >= TASK_SIZE_MAX;
996} 997}
997 998
999static inline bool smap_violation(int error_code, struct pt_regs *regs)
1000{
1001 if (error_code & PF_USER)
1002 return false;
1003
1004 if (!user_mode_vm(regs) && (regs->flags & X86_EFLAGS_AC))
1005 return false;
1006
1007 return true;
1008}
1009
998/* 1010/*
999 * This routine handles page faults. It determines the address, 1011 * This routine handles page faults. It determines the address,
1000 * and the problem, and then passes it off to one of the appropriate 1012 * and the problem, and then passes it off to one of the appropriate
1001 * routines. 1013 * routines.
1002 */ 1014 */
1003dotraplinkage void __kprobes 1015static void __kprobes
1004do_page_fault(struct pt_regs *regs, unsigned long error_code) 1016__do_page_fault(struct pt_regs *regs, unsigned long error_code)
1005{ 1017{
1006 struct vm_area_struct *vma; 1018 struct vm_area_struct *vma;
1007 struct task_struct *tsk; 1019 struct task_struct *tsk;
@@ -1088,6 +1100,13 @@ do_page_fault(struct pt_regs *regs, unsigned long error_code)
1088 if (unlikely(error_code & PF_RSVD)) 1100 if (unlikely(error_code & PF_RSVD))
1089 pgtable_bad(regs, error_code, address); 1101 pgtable_bad(regs, error_code, address);
1090 1102
1103 if (static_cpu_has(X86_FEATURE_SMAP)) {
1104 if (unlikely(smap_violation(error_code, regs))) {
1105 bad_area_nosemaphore(regs, error_code, address);
1106 return;
1107 }
1108 }
1109
1091 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); 1110 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
1092 1111
1093 /* 1112 /*
@@ -1201,6 +1220,7 @@ good_area:
1201 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk 1220 /* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
1202 * of starvation. */ 1221 * of starvation. */
1203 flags &= ~FAULT_FLAG_ALLOW_RETRY; 1222 flags &= ~FAULT_FLAG_ALLOW_RETRY;
1223 flags |= FAULT_FLAG_TRIED;
1204 goto retry; 1224 goto retry;
1205 } 1225 }
1206 } 1226 }
@@ -1209,3 +1229,11 @@ good_area:
1209 1229
1210 up_read(&mm->mmap_sem); 1230 up_read(&mm->mmap_sem);
1211} 1231}
1232
1233dotraplinkage void __kprobes
1234do_page_fault(struct pt_regs *regs, unsigned long error_code)
1235{
1236 exception_enter(regs);
1237 __do_page_fault(regs, error_code);
1238 exception_exit(regs);
1239}
diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c
index b91e48512425..937bff5cdaa7 100644
--- a/arch/x86/mm/hugetlbpage.c
+++ b/arch/x86/mm/hugetlbpage.c
@@ -71,7 +71,6 @@ huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
71 struct address_space *mapping = vma->vm_file->f_mapping; 71 struct address_space *mapping = vma->vm_file->f_mapping;
72 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + 72 pgoff_t idx = ((addr - vma->vm_start) >> PAGE_SHIFT) +
73 vma->vm_pgoff; 73 vma->vm_pgoff;
74 struct prio_tree_iter iter;
75 struct vm_area_struct *svma; 74 struct vm_area_struct *svma;
76 unsigned long saddr; 75 unsigned long saddr;
77 pte_t *spte = NULL; 76 pte_t *spte = NULL;
@@ -81,7 +80,7 @@ huge_pmd_share(struct mm_struct *mm, unsigned long addr, pud_t *pud)
81 return (pte_t *)pmd_alloc(mm, pud, addr); 80 return (pte_t *)pmd_alloc(mm, pud, addr);
82 81
83 mutex_lock(&mapping->i_mmap_mutex); 82 mutex_lock(&mapping->i_mmap_mutex);
84 vma_prio_tree_foreach(svma, &iter, &mapping->i_mmap, idx, idx) { 83 vma_interval_tree_foreach(svma, &mapping->i_mmap, idx, idx) {
85 if (svma == vma) 84 if (svma == vma)
86 continue; 85 continue;
87 86
diff --git a/arch/x86/mm/init.c b/arch/x86/mm/init.c
index e0e6990723e9..ab1f6a93b527 100644
--- a/arch/x86/mm/init.c
+++ b/arch/x86/mm/init.c
@@ -319,7 +319,7 @@ unsigned long __init_refok init_memory_mapping(unsigned long start,
319 */ 319 */
320int devmem_is_allowed(unsigned long pagenr) 320int devmem_is_allowed(unsigned long pagenr)
321{ 321{
322 if (pagenr <= 256) 322 if (pagenr < 256)
323 return 1; 323 return 1;
324 if (iomem_is_exclusive(pagenr << PAGE_SHIFT)) 324 if (iomem_is_exclusive(pagenr << PAGE_SHIFT))
325 return 0; 325 return 0;
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 4f04db150027..11a58001b4ce 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -709,7 +709,7 @@ static void __init test_wp_bit(void)
709 "Checking if this processor honours the WP bit even in supervisor mode..."); 709 "Checking if this processor honours the WP bit even in supervisor mode...");
710 710
711 /* Any page-aligned address will do, the test is non-destructive */ 711 /* Any page-aligned address will do, the test is non-destructive */
712 __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_READONLY); 712 __set_fixmap(FIX_WP_TEST, __pa(&swapper_pg_dir), PAGE_KERNEL_RO);
713 boot_cpu_data.wp_works_ok = do_test_wp_bit(); 713 boot_cpu_data.wp_works_ok = do_test_wp_bit();
714 clear_fixmap(FIX_WP_TEST); 714 clear_fixmap(FIX_WP_TEST);
715 715
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 3d68ef6d2266..0eb572eda406 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -664,20 +664,20 @@ static void free_pfn_range(u64 paddr, unsigned long size)
664} 664}
665 665
666/* 666/*
667 * track_pfn_vma_copy is called when vma that is covering the pfnmap gets 667 * track_pfn_copy is called when vma that is covering the pfnmap gets
668 * copied through copy_page_range(). 668 * copied through copy_page_range().
669 * 669 *
670 * If the vma has a linear pfn mapping for the entire range, we get the prot 670 * If the vma has a linear pfn mapping for the entire range, we get the prot
671 * from pte and reserve the entire vma range with single reserve_pfn_range call. 671 * from pte and reserve the entire vma range with single reserve_pfn_range call.
672 */ 672 */
673int track_pfn_vma_copy(struct vm_area_struct *vma) 673int track_pfn_copy(struct vm_area_struct *vma)
674{ 674{
675 resource_size_t paddr; 675 resource_size_t paddr;
676 unsigned long prot; 676 unsigned long prot;
677 unsigned long vma_size = vma->vm_end - vma->vm_start; 677 unsigned long vma_size = vma->vm_end - vma->vm_start;
678 pgprot_t pgprot; 678 pgprot_t pgprot;
679 679
680 if (is_linear_pfn_mapping(vma)) { 680 if (vma->vm_flags & VM_PAT) {
681 /* 681 /*
682 * reserve the whole chunk covered by vma. We need the 682 * reserve the whole chunk covered by vma. We need the
683 * starting address and protection from pte. 683 * starting address and protection from pte.
@@ -694,31 +694,59 @@ int track_pfn_vma_copy(struct vm_area_struct *vma)
694} 694}
695 695
696/* 696/*
697 * track_pfn_vma_new is called when a _new_ pfn mapping is being established
698 * for physical range indicated by pfn and size.
699 *
700 * prot is passed in as a parameter for the new mapping. If the vma has a 697 * prot is passed in as a parameter for the new mapping. If the vma has a
701 * linear pfn mapping for the entire range reserve the entire vma range with 698 * linear pfn mapping for the entire range reserve the entire vma range with
702 * single reserve_pfn_range call. 699 * single reserve_pfn_range call.
703 */ 700 */
704int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t *prot, 701int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
705 unsigned long pfn, unsigned long size) 702 unsigned long pfn, unsigned long addr, unsigned long size)
706{ 703{
704 resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
707 unsigned long flags; 705 unsigned long flags;
708 resource_size_t paddr;
709 unsigned long vma_size = vma->vm_end - vma->vm_start;
710 706
711 if (is_linear_pfn_mapping(vma)) { 707 /* reserve the whole chunk starting from paddr */
712 /* reserve the whole chunk starting from vm_pgoff */ 708 if (addr == vma->vm_start && size == (vma->vm_end - vma->vm_start)) {
713 paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT; 709 int ret;
714 return reserve_pfn_range(paddr, vma_size, prot, 0); 710
711 ret = reserve_pfn_range(paddr, size, prot, 0);
712 if (!ret)
713 vma->vm_flags |= VM_PAT;
714 return ret;
715 } 715 }
716 716
717 if (!pat_enabled) 717 if (!pat_enabled)
718 return 0; 718 return 0;
719 719
720 /* for vm_insert_pfn and friends, we set prot based on lookup */ 720 /*
721 flags = lookup_memtype(pfn << PAGE_SHIFT); 721 * For anything smaller than the vma size we set prot based on the
722 * lookup.
723 */
724 flags = lookup_memtype(paddr);
725
726 /* Check memtype for the remaining pages */
727 while (size > PAGE_SIZE) {
728 size -= PAGE_SIZE;
729 paddr += PAGE_SIZE;
730 if (flags != lookup_memtype(paddr))
731 return -EINVAL;
732 }
733
734 *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
735 flags);
736
737 return 0;
738}
739
740int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
741 unsigned long pfn)
742{
743 unsigned long flags;
744
745 if (!pat_enabled)
746 return 0;
747
748 /* Set prot based on lookup */
749 flags = lookup_memtype((resource_size_t)pfn << PAGE_SHIFT);
722 *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) | 750 *prot = __pgprot((pgprot_val(vma->vm_page_prot) & (~_PAGE_CACHE_MASK)) |
723 flags); 751 flags);
724 752
@@ -726,22 +754,31 @@ int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t *prot,
726} 754}
727 755
728/* 756/*
729 * untrack_pfn_vma is called while unmapping a pfnmap for a region. 757 * untrack_pfn is called while unmapping a pfnmap for a region.
730 * untrack can be called for a specific region indicated by pfn and size or 758 * untrack can be called for a specific region indicated by pfn and size or
731 * can be for the entire vma (in which case size can be zero). 759 * can be for the entire vma (in which case pfn, size are zero).
732 */ 760 */
733void untrack_pfn_vma(struct vm_area_struct *vma, unsigned long pfn, 761void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
734 unsigned long size) 762 unsigned long size)
735{ 763{
736 resource_size_t paddr; 764 resource_size_t paddr;
737 unsigned long vma_size = vma->vm_end - vma->vm_start; 765 unsigned long prot;
738 766
739 if (is_linear_pfn_mapping(vma)) { 767 if (!(vma->vm_flags & VM_PAT))
740 /* free the whole chunk starting from vm_pgoff */
741 paddr = (resource_size_t)vma->vm_pgoff << PAGE_SHIFT;
742 free_pfn_range(paddr, vma_size);
743 return; 768 return;
769
770 /* free the chunk starting from pfn or the whole chunk */
771 paddr = (resource_size_t)pfn << PAGE_SHIFT;
772 if (!paddr && !size) {
773 if (follow_phys(vma, vma->vm_start, 0, &prot, &paddr)) {
774 WARN_ON_ONCE(1);
775 return;
776 }
777
778 size = vma->vm_end - vma->vm_start;
744 } 779 }
780 free_pfn_range(paddr, size);
781 vma->vm_flags &= ~VM_PAT;
745} 782}
746 783
747pgprot_t pgprot_writecombine(pgprot_t prot) 784pgprot_t pgprot_writecombine(pgprot_t prot)
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 8acaddd0fb21..415f6c4ced36 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -12,7 +12,7 @@
12#include <linux/debugfs.h> 12#include <linux/debugfs.h>
13#include <linux/kernel.h> 13#include <linux/kernel.h>
14#include <linux/module.h> 14#include <linux/module.h>
15#include <linux/rbtree.h> 15#include <linux/rbtree_augmented.h>
16#include <linux/sched.h> 16#include <linux/sched.h>
17#include <linux/gfp.h> 17#include <linux/gfp.h>
18 18
@@ -54,29 +54,24 @@ static u64 get_subtree_max_end(struct rb_node *node)
54 return ret; 54 return ret;
55} 55}
56 56
57/* Update 'subtree_max_end' for a node, based on node and its children */ 57static u64 compute_subtree_max_end(struct memtype *data)
58static void memtype_rb_augment_cb(struct rb_node *node, void *__unused)
59{ 58{
60 struct memtype *data; 59 u64 max_end = data->end, child_max_end;
61 u64 max_end, child_max_end;
62
63 if (!node)
64 return;
65 60
66 data = container_of(node, struct memtype, rb); 61 child_max_end = get_subtree_max_end(data->rb.rb_right);
67 max_end = data->end;
68
69 child_max_end = get_subtree_max_end(node->rb_right);
70 if (child_max_end > max_end) 62 if (child_max_end > max_end)
71 max_end = child_max_end; 63 max_end = child_max_end;
72 64
73 child_max_end = get_subtree_max_end(node->rb_left); 65 child_max_end = get_subtree_max_end(data->rb.rb_left);
74 if (child_max_end > max_end) 66 if (child_max_end > max_end)
75 max_end = child_max_end; 67 max_end = child_max_end;
76 68
77 data->subtree_max_end = max_end; 69 return max_end;
78} 70}
79 71
72RB_DECLARE_CALLBACKS(static, memtype_rb_augment_cb, struct memtype, rb,
73 u64, subtree_max_end, compute_subtree_max_end)
74
80/* Find the first (lowest start addr) overlapping range from rb tree */ 75/* Find the first (lowest start addr) overlapping range from rb tree */
81static struct memtype *memtype_rb_lowest_match(struct rb_root *root, 76static struct memtype *memtype_rb_lowest_match(struct rb_root *root,
82 u64 start, u64 end) 77 u64 start, u64 end)
@@ -179,15 +174,17 @@ static void memtype_rb_insert(struct rb_root *root, struct memtype *newdata)
179 struct memtype *data = container_of(*node, struct memtype, rb); 174 struct memtype *data = container_of(*node, struct memtype, rb);
180 175
181 parent = *node; 176 parent = *node;
177 if (data->subtree_max_end < newdata->end)
178 data->subtree_max_end = newdata->end;
182 if (newdata->start <= data->start) 179 if (newdata->start <= data->start)
183 node = &((*node)->rb_left); 180 node = &((*node)->rb_left);
184 else if (newdata->start > data->start) 181 else if (newdata->start > data->start)
185 node = &((*node)->rb_right); 182 node = &((*node)->rb_right);
186 } 183 }
187 184
185 newdata->subtree_max_end = newdata->end;
188 rb_link_node(&newdata->rb, parent, node); 186 rb_link_node(&newdata->rb, parent, node);
189 rb_insert_color(&newdata->rb, root); 187 rb_insert_augmented(&newdata->rb, root, &memtype_rb_augment_cb);
190 rb_augment_insert(&newdata->rb, memtype_rb_augment_cb, NULL);
191} 188}
192 189
193int rbt_memtype_check_insert(struct memtype *new, unsigned long *ret_type) 190int rbt_memtype_check_insert(struct memtype *new, unsigned long *ret_type)
@@ -209,16 +206,13 @@ int rbt_memtype_check_insert(struct memtype *new, unsigned long *ret_type)
209 206
210struct memtype *rbt_memtype_erase(u64 start, u64 end) 207struct memtype *rbt_memtype_erase(u64 start, u64 end)
211{ 208{
212 struct rb_node *deepest;
213 struct memtype *data; 209 struct memtype *data;
214 210
215 data = memtype_rb_exact_match(&memtype_rbroot, start, end); 211 data = memtype_rb_exact_match(&memtype_rbroot, start, end);
216 if (!data) 212 if (!data)
217 goto out; 213 goto out;
218 214
219 deepest = rb_augment_erase_begin(&data->rb); 215 rb_erase_augmented(&data->rb, &memtype_rbroot, &memtype_rb_augment_cb);
220 rb_erase(&data->rb, &memtype_rbroot);
221 rb_augment_erase_end(deepest, memtype_rb_augment_cb, NULL);
222out: 216out:
223 return data; 217 return data;
224} 218}
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 613cd83e8c0c..0777f042e400 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -98,6 +98,8 @@ static void flush_tlb_func(void *info)
98{ 98{
99 struct flush_tlb_info *f = info; 99 struct flush_tlb_info *f = info;
100 100
101 inc_irq_stat(irq_tlb_count);
102
101 if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) 103 if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
102 return; 104 return;
103 105
@@ -320,7 +322,7 @@ static ssize_t tlbflush_write_file(struct file *file,
320 if (kstrtos8(buf, 0, &shift)) 322 if (kstrtos8(buf, 0, &shift))
321 return -EINVAL; 323 return -EINVAL;
322 324
323 if (shift > 64) 325 if (shift < -1 || shift >= BITS_PER_LONG)
324 return -EINVAL; 326 return -EINVAL;
325 327
326 tlb_flushall_shift = shift; 328 tlb_flushall_shift = shift;
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 33643a8bcbbb..520d2bd0b9c5 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -280,6 +280,31 @@ void bpf_jit_compile(struct sk_filter *fp)
280 } 280 }
281 EMIT4(0x31, 0xd2, 0xf7, 0xf3); /* xor %edx,%edx; div %ebx */ 281 EMIT4(0x31, 0xd2, 0xf7, 0xf3); /* xor %edx,%edx; div %ebx */
282 break; 282 break;
283 case BPF_S_ALU_MOD_X: /* A %= X; */
284 seen |= SEEN_XREG;
285 EMIT2(0x85, 0xdb); /* test %ebx,%ebx */
286 if (pc_ret0 > 0) {
287 /* addrs[pc_ret0 - 1] is start address of target
288 * (addrs[i] - 6) is the address following this jmp
289 * ("xor %edx,%edx; div %ebx;mov %edx,%eax" being 6 bytes long)
290 */
291 EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
292 (addrs[i] - 6));
293 } else {
294 EMIT_COND_JMP(X86_JNE, 2 + 5);
295 CLEAR_A();
296 EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 6)); /* jmp .+off32 */
297 }
298 EMIT2(0x31, 0xd2); /* xor %edx,%edx */
299 EMIT2(0xf7, 0xf3); /* div %ebx */
300 EMIT2(0x89, 0xd0); /* mov %edx,%eax */
301 break;
302 case BPF_S_ALU_MOD_K: /* A %= K; */
303 EMIT2(0x31, 0xd2); /* xor %edx,%edx */
304 EMIT1(0xb9);EMIT(K, 4); /* mov imm32,%ecx */
305 EMIT2(0xf7, 0xf1); /* div %ecx */
306 EMIT2(0x89, 0xd0); /* mov %edx,%eax */
307 break;
283 case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */ 308 case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */
284 EMIT3(0x48, 0x69, 0xc0); /* imul imm32,%rax,%rax */ 309 EMIT3(0x48, 0x69, 0xc0); /* imul imm32,%rax,%rax */
285 EMIT(K, 4); 310 EMIT(K, 4);
@@ -310,9 +335,18 @@ void bpf_jit_compile(struct sk_filter *fp)
310 EMIT1_off32(0x0d, K); /* or imm32,%eax */ 335 EMIT1_off32(0x0d, K); /* or imm32,%eax */
311 break; 336 break;
312 case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ 337 case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */
338 case BPF_S_ALU_XOR_X:
313 seen |= SEEN_XREG; 339 seen |= SEEN_XREG;
314 EMIT2(0x31, 0xd8); /* xor %ebx,%eax */ 340 EMIT2(0x31, 0xd8); /* xor %ebx,%eax */
315 break; 341 break;
342 case BPF_S_ALU_XOR_K: /* A ^= K; */
343 if (K == 0)
344 break;
345 if (is_imm8(K))
346 EMIT3(0x83, 0xf0, K); /* xor imm8,%eax */
347 else
348 EMIT1_off32(0x35, K); /* xor imm32,%eax */
349 break;
316 case BPF_S_ALU_LSH_X: /* A <<= X; */ 350 case BPF_S_ALU_LSH_X: /* A <<= X; */
317 seen |= SEEN_XREG; 351 seen |= SEEN_XREG;
318 EMIT4(0x89, 0xd9, 0xd3, 0xe0); /* mov %ebx,%ecx; shl %cl,%eax */ 352 EMIT4(0x89, 0xd9, 0xd3, 0xe0); /* mov %ebx,%ecx; shl %cl,%eax */
diff --git a/arch/x86/pci/acpi.c b/arch/x86/pci/acpi.c
index 505acdd6d600..192397c98606 100644
--- a/arch/x86/pci/acpi.c
+++ b/arch/x86/pci/acpi.c
@@ -305,7 +305,6 @@ setup_resource(struct acpi_resource *acpi_res, void *data)
305 res->flags = flags; 305 res->flags = flags;
306 res->start = start; 306 res->start = start;
307 res->end = end; 307 res->end = end;
308 res->child = NULL;
309 308
310 if (!pci_use_crs) { 309 if (!pci_use_crs) {
311 dev_printk(KERN_DEBUG, &info->bridge->dev, 310 dev_printk(KERN_DEBUG, &info->bridge->dev,
@@ -434,7 +433,7 @@ probe_pci_root_info(struct pci_root_info *info, struct acpi_device *device,
434 433
435 size = sizeof(*info->res) * info->res_num; 434 size = sizeof(*info->res) * info->res_num;
436 info->res_num = 0; 435 info->res_num = 0;
437 info->res = kmalloc(size, GFP_KERNEL); 436 info->res = kzalloc(size, GFP_KERNEL);
438 if (!info->res) 437 if (!info->res)
439 return; 438 return;
440 439
diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c
index 937bcece7006..704b9ec043d7 100644
--- a/arch/x86/pci/mmconfig-shared.c
+++ b/arch/x86/pci/mmconfig-shared.c
@@ -585,7 +585,7 @@ static int __init pci_parse_mcfg(struct acpi_table_header *header)
585 while (i >= sizeof(struct acpi_mcfg_allocation)) { 585 while (i >= sizeof(struct acpi_mcfg_allocation)) {
586 entries++; 586 entries++;
587 i -= sizeof(struct acpi_mcfg_allocation); 587 i -= sizeof(struct acpi_mcfg_allocation);
588 }; 588 }
589 if (entries == 0) { 589 if (entries == 0) {
590 pr_err(PREFIX "MMCONFIG has no entries\n"); 590 pr_err(PREFIX "MMCONFIG has no entries\n");
591 return -ENODEV; 591 return -ENODEV;
diff --git a/arch/x86/pci/visws.c b/arch/x86/pci/visws.c
index 6f2f8eeed171..3e6d2a6db866 100644
--- a/arch/x86/pci/visws.c
+++ b/arch/x86/pci/visws.c
@@ -62,11 +62,6 @@ out:
62 return irq; 62 return irq;
63} 63}
64 64
65void __init pcibios_update_irq(struct pci_dev *dev, int irq)
66{
67 pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq);
68}
69
70int __init pci_visws_init(void) 65int __init pci_visws_init(void)
71{ 66{
72 pcibios_enable_irq = &pci_visws_enable_irq; 67 pcibios_enable_irq = &pci_visws_enable_irq;
diff --git a/arch/x86/platform/efi/Makefile b/arch/x86/platform/efi/Makefile
index 73b8be0f3675..6db1cc4c7534 100644
--- a/arch/x86/platform/efi/Makefile
+++ b/arch/x86/platform/efi/Makefile
@@ -1 +1,2 @@
1obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o 1obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o
2obj-$(CONFIG_ACPI_BGRT) += efi-bgrt.o
diff --git a/arch/x86/platform/efi/efi-bgrt.c b/arch/x86/platform/efi/efi-bgrt.c
new file mode 100644
index 000000000000..f6a0c1b8e518
--- /dev/null
+++ b/arch/x86/platform/efi/efi-bgrt.c
@@ -0,0 +1,76 @@
1/*
2 * Copyright 2012 Intel Corporation
3 * Author: Josh Triplett <josh@joshtriplett.org>
4 *
5 * Based on the bgrt driver:
6 * Copyright 2012 Red Hat, Inc <mjg@redhat.com>
7 * Author: Matthew Garrett
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13#include <linux/kernel.h>
14#include <linux/acpi.h>
15#include <linux/efi.h>
16#include <linux/efi-bgrt.h>
17
18struct acpi_table_bgrt *bgrt_tab;
19void *bgrt_image;
20size_t bgrt_image_size;
21
22struct bmp_header {
23 u16 id;
24 u32 size;
25} __packed;
26
27void efi_bgrt_init(void)
28{
29 acpi_status status;
30 void __iomem *image;
31 bool ioremapped = false;
32 struct bmp_header bmp_header;
33
34 if (acpi_disabled)
35 return;
36
37 status = acpi_get_table("BGRT", 0,
38 (struct acpi_table_header **)&bgrt_tab);
39 if (ACPI_FAILURE(status))
40 return;
41
42 if (bgrt_tab->version != 1)
43 return;
44 if (bgrt_tab->image_type != 0 || !bgrt_tab->image_address)
45 return;
46
47 image = efi_lookup_mapped_addr(bgrt_tab->image_address);
48 if (!image) {
49 image = ioremap(bgrt_tab->image_address, sizeof(bmp_header));
50 ioremapped = true;
51 if (!image)
52 return;
53 }
54
55 memcpy_fromio(&bmp_header, image, sizeof(bmp_header));
56 if (ioremapped)
57 iounmap(image);
58 bgrt_image_size = bmp_header.size;
59
60 bgrt_image = kmalloc(bgrt_image_size, GFP_KERNEL);
61 if (!bgrt_image)
62 return;
63
64 if (ioremapped) {
65 image = ioremap(bgrt_tab->image_address, bmp_header.size);
66 if (!image) {
67 kfree(bgrt_image);
68 bgrt_image = NULL;
69 return;
70 }
71 }
72
73 memcpy_fromio(bgrt_image, image, bgrt_image_size);
74 if (ioremapped)
75 iounmap(image);
76}
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 92660edaa1e7..aded2a91162a 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -31,6 +31,7 @@
31#include <linux/kernel.h> 31#include <linux/kernel.h>
32#include <linux/init.h> 32#include <linux/init.h>
33#include <linux/efi.h> 33#include <linux/efi.h>
34#include <linux/efi-bgrt.h>
34#include <linux/export.h> 35#include <linux/export.h>
35#include <linux/bootmem.h> 36#include <linux/bootmem.h>
36#include <linux/memblock.h> 37#include <linux/memblock.h>
@@ -419,10 +420,21 @@ void __init efi_reserve_boot_services(void)
419 } 420 }
420} 421}
421 422
422static void __init efi_free_boot_services(void) 423static void __init efi_unmap_memmap(void)
424{
425 if (memmap.map) {
426 early_iounmap(memmap.map, memmap.nr_map * memmap.desc_size);
427 memmap.map = NULL;
428 }
429}
430
431void __init efi_free_boot_services(void)
423{ 432{
424 void *p; 433 void *p;
425 434
435 if (!efi_native)
436 return;
437
426 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 438 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
427 efi_memory_desc_t *md = p; 439 efi_memory_desc_t *md = p;
428 unsigned long long start = md->phys_addr; 440 unsigned long long start = md->phys_addr;
@@ -438,6 +450,8 @@ static void __init efi_free_boot_services(void)
438 450
439 free_bootmem_late(start, size); 451 free_bootmem_late(start, size);
440 } 452 }
453
454 efi_unmap_memmap();
441} 455}
442 456
443static int __init efi_systab_init(void *phys) 457static int __init efi_systab_init(void *phys)
@@ -732,6 +746,11 @@ void __init efi_init(void)
732#endif 746#endif
733} 747}
734 748
749void __init efi_late_init(void)
750{
751 efi_bgrt_init();
752}
753
735void __init efi_set_executable(efi_memory_desc_t *md, bool executable) 754void __init efi_set_executable(efi_memory_desc_t *md, bool executable)
736{ 755{
737 u64 addr, npages; 756 u64 addr, npages;
@@ -764,6 +783,34 @@ static void __init runtime_code_page_mkexec(void)
764} 783}
765 784
766/* 785/*
786 * We can't ioremap data in EFI boot services RAM, because we've already mapped
787 * it as RAM. So, look it up in the existing EFI memory map instead. Only
788 * callable after efi_enter_virtual_mode and before efi_free_boot_services.
789 */
790void __iomem *efi_lookup_mapped_addr(u64 phys_addr)
791{
792 void *p;
793 if (WARN_ON(!memmap.map))
794 return NULL;
795 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
796 efi_memory_desc_t *md = p;
797 u64 size = md->num_pages << EFI_PAGE_SHIFT;
798 u64 end = md->phys_addr + size;
799 if (!(md->attribute & EFI_MEMORY_RUNTIME) &&
800 md->type != EFI_BOOT_SERVICES_CODE &&
801 md->type != EFI_BOOT_SERVICES_DATA)
802 continue;
803 if (!md->virt_addr)
804 continue;
805 if (phys_addr >= md->phys_addr && phys_addr < end) {
806 phys_addr += md->virt_addr - md->phys_addr;
807 return (__force void __iomem *)(unsigned long)phys_addr;
808 }
809 }
810 return NULL;
811}
812
813/*
767 * This function will switch the EFI runtime services to virtual mode. 814 * This function will switch the EFI runtime services to virtual mode.
768 * Essentially, look through the EFI memmap and map every region that 815 * Essentially, look through the EFI memmap and map every region that
769 * has the runtime attribute bit set in its memory descriptor and update 816 * has the runtime attribute bit set in its memory descriptor and update
@@ -787,8 +834,10 @@ void __init efi_enter_virtual_mode(void)
787 * non-native EFI 834 * non-native EFI
788 */ 835 */
789 836
790 if (!efi_native) 837 if (!efi_native) {
791 goto out; 838 efi_unmap_memmap();
839 return;
840 }
792 841
793 /* Merge contiguous regions of the same type and attribute */ 842 /* Merge contiguous regions of the same type and attribute */
794 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) { 843 for (p = memmap.map; p < memmap.map_end; p += memmap.desc_size) {
@@ -878,18 +927,12 @@ void __init efi_enter_virtual_mode(void)
878 } 927 }
879 928
880 /* 929 /*
881 * Thankfully, it does seem that no runtime services other than
882 * SetVirtualAddressMap() will touch boot services code, so we can
883 * get rid of it all at this point
884 */
885 efi_free_boot_services();
886
887 /*
888 * Now that EFI is in virtual mode, update the function 930 * Now that EFI is in virtual mode, update the function
889 * pointers in the runtime service table to the new virtual addresses. 931 * pointers in the runtime service table to the new virtual addresses.
890 * 932 *
891 * Call EFI services through wrapper functions. 933 * Call EFI services through wrapper functions.
892 */ 934 */
935 efi.runtime_version = efi_systab.fw_revision;
893 efi.get_time = virt_efi_get_time; 936 efi.get_time = virt_efi_get_time;
894 efi.set_time = virt_efi_set_time; 937 efi.set_time = virt_efi_set_time;
895 efi.get_wakeup_time = virt_efi_get_wakeup_time; 938 efi.get_wakeup_time = virt_efi_get_wakeup_time;
@@ -906,9 +949,6 @@ void __init efi_enter_virtual_mode(void)
906 if (__supported_pte_mask & _PAGE_NX) 949 if (__supported_pte_mask & _PAGE_NX)
907 runtime_code_page_mkexec(); 950 runtime_code_page_mkexec();
908 951
909out:
910 early_iounmap(memmap.map, memmap.nr_map * memmap.desc_size);
911 memmap.map = NULL;
912 kfree(new_memmap); 952 kfree(new_memmap);
913} 953}
914 954
diff --git a/arch/x86/realmode/rm/wakeup.h b/arch/x86/realmode/rm/wakeup.h
index 9317e0042f24..7dd86a419f5d 100644
--- a/arch/x86/realmode/rm/wakeup.h
+++ b/arch/x86/realmode/rm/wakeup.h
@@ -36,5 +36,7 @@ extern struct wakeup_header wakeup_header;
36 36
37/* Wakeup behavior bits */ 37/* Wakeup behavior bits */
38#define WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE 0 38#define WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE 0
39#define WAKEUP_BEHAVIOR_RESTORE_CR4 1
40#define WAKEUP_BEHAVIOR_RESTORE_EFER 2
39 41
40#endif /* ARCH_X86_KERNEL_ACPI_RM_WAKEUP_H */ 42#endif /* ARCH_X86_KERNEL_ACPI_RM_WAKEUP_H */
diff --git a/arch/x86/realmode/rm/wakeup_asm.S b/arch/x86/realmode/rm/wakeup_asm.S
index 8905166b0bbb..e56479e58053 100644
--- a/arch/x86/realmode/rm/wakeup_asm.S
+++ b/arch/x86/realmode/rm/wakeup_asm.S
@@ -74,9 +74,18 @@ ENTRY(wakeup_start)
74 74
75 lidtl wakeup_idt 75 lidtl wakeup_idt
76 76
77 /* Clear the EFLAGS */ 77 /* Clear the EFLAGS but remember if we have EFLAGS.ID */
78 pushl $0 78 movl $X86_EFLAGS_ID, %ecx
79 pushl %ecx
79 popfl 80 popfl
81 pushfl
82 popl %edi
83 pushl $0
84 popfl
85 pushfl
86 popl %edx
87 xorl %edx, %edi
88 andl %ecx, %edi /* %edi is zero iff CPUID & %cr4 are missing */
80 89
81 /* Check header signature... */ 90 /* Check header signature... */
82 movl signature, %eax 91 movl signature, %eax
@@ -93,8 +102,8 @@ ENTRY(wakeup_start)
93 102
94 /* Restore MISC_ENABLE before entering protected mode, in case 103 /* Restore MISC_ENABLE before entering protected mode, in case
95 BIOS decided to clear XD_DISABLE during S3. */ 104 BIOS decided to clear XD_DISABLE during S3. */
96 movl pmode_behavior, %eax 105 movl pmode_behavior, %edi
97 btl $WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE, %eax 106 btl $WAKEUP_BEHAVIOR_RESTORE_MISC_ENABLE, %edi
98 jnc 1f 107 jnc 1f
99 108
100 movl pmode_misc_en, %eax 109 movl pmode_misc_en, %eax
@@ -110,15 +119,15 @@ ENTRY(wakeup_start)
110 movl pmode_cr3, %eax 119 movl pmode_cr3, %eax
111 movl %eax, %cr3 120 movl %eax, %cr3
112 121
113 movl pmode_cr4, %ecx 122 btl $WAKEUP_BEHAVIOR_RESTORE_CR4, %edi
114 jecxz 1f 123 jz 1f
115 movl %ecx, %cr4 124 movl pmode_cr4, %eax
125 movl %eax, %cr4
1161: 1261:
127 btl $WAKEUP_BEHAVIOR_RESTORE_EFER, %edi
128 jz 1f
117 movl pmode_efer, %eax 129 movl pmode_efer, %eax
118 movl pmode_efer + 4, %edx 130 movl pmode_efer + 4, %edx
119 movl %eax, %ecx
120 orl %edx, %ecx
121 jz 1f
122 movl $MSR_EFER, %ecx 131 movl $MSR_EFER, %ecx
123 wrmsr 132 wrmsr
1241: 1331:
diff --git a/arch/x86/syscalls/Makefile b/arch/x86/syscalls/Makefile
index 3236aebc828d..f325af26107c 100644
--- a/arch/x86/syscalls/Makefile
+++ b/arch/x86/syscalls/Makefile
@@ -1,7 +1,9 @@
1out := $(obj)/../include/generated/asm 1out := $(obj)/../include/generated/asm
2uapi := $(obj)/../include/generated/uapi/asm
2 3
3# Create output directory if not already present 4# Create output directory if not already present
4_dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') 5_dummy := $(shell [ -d '$(out)' ] || mkdir -p '$(out)') \
6 $(shell [ -d '$(uapi)' ] || mkdir -p '$(uapi)')
5 7
6syscall32 := $(srctree)/$(src)/syscall_32.tbl 8syscall32 := $(srctree)/$(src)/syscall_32.tbl
7syscall64 := $(srctree)/$(src)/syscall_64.tbl 9syscall64 := $(srctree)/$(src)/syscall_64.tbl
@@ -18,7 +20,7 @@ quiet_cmd_systbl = SYSTBL $@
18 cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@ 20 cmd_systbl = $(CONFIG_SHELL) '$(systbl)' $< $@
19 21
20syshdr_abi_unistd_32 := i386 22syshdr_abi_unistd_32 := i386
21$(out)/unistd_32.h: $(syscall32) $(syshdr) 23$(uapi)/unistd_32.h: $(syscall32) $(syshdr)
22 $(call if_changed,syshdr) 24 $(call if_changed,syshdr)
23 25
24syshdr_abi_unistd_32_ia32 := i386 26syshdr_abi_unistd_32_ia32 := i386
@@ -28,11 +30,11 @@ $(out)/unistd_32_ia32.h: $(syscall32) $(syshdr)
28 30
29syshdr_abi_unistd_x32 := common,x32 31syshdr_abi_unistd_x32 := common,x32
30syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT 32syshdr_offset_unistd_x32 := __X32_SYSCALL_BIT
31$(out)/unistd_x32.h: $(syscall64) $(syshdr) 33$(uapi)/unistd_x32.h: $(syscall64) $(syshdr)
32 $(call if_changed,syshdr) 34 $(call if_changed,syshdr)
33 35
34syshdr_abi_unistd_64 := common,64 36syshdr_abi_unistd_64 := common,64
35$(out)/unistd_64.h: $(syscall64) $(syshdr) 37$(uapi)/unistd_64.h: $(syscall64) $(syshdr)
36 $(call if_changed,syshdr) 38 $(call if_changed,syshdr)
37 39
38syshdr_abi_unistd_64_x32 := x32 40syshdr_abi_unistd_64_x32 := x32
@@ -45,11 +47,12 @@ $(out)/syscalls_32.h: $(syscall32) $(systbl)
45$(out)/syscalls_64.h: $(syscall64) $(systbl) 47$(out)/syscalls_64.h: $(syscall64) $(systbl)
46 $(call if_changed,systbl) 48 $(call if_changed,systbl)
47 49
48syshdr-y += unistd_32.h unistd_64.h unistd_x32.h 50uapisyshdr-y += unistd_32.h unistd_64.h unistd_x32.h
49syshdr-y += syscalls_32.h 51syshdr-y += syscalls_32.h
50syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h 52syshdr-$(CONFIG_X86_64) += unistd_32_ia32.h unistd_64_x32.h
51syshdr-$(CONFIG_X86_64) += syscalls_64.h 53syshdr-$(CONFIG_X86_64) += syscalls_64.h
52 54
53targets += $(syshdr-y) 55targets += $(uapisyshdr-y) $(syshdr-y)
54 56
55all: $(addprefix $(out)/,$(targets)) 57all: $(addprefix $(uapi)/,$(uapisyshdr-y))
58all: $(addprefix $(out)/,$(syshdr-y))
diff --git a/arch/x86/syscalls/syscall_32.tbl b/arch/x86/syscalls/syscall_32.tbl
index 7a35a6e71d44..a47103fbc692 100644
--- a/arch/x86/syscalls/syscall_32.tbl
+++ b/arch/x86/syscalls/syscall_32.tbl
@@ -17,7 +17,7 @@
178 i386 creat sys_creat 178 i386 creat sys_creat
189 i386 link sys_link 189 i386 link sys_link
1910 i386 unlink sys_unlink 1910 i386 unlink sys_unlink
2011 i386 execve ptregs_execve stub32_execve 2011 i386 execve sys_execve stub32_execve
2112 i386 chdir sys_chdir 2112 i386 chdir sys_chdir
2213 i386 time sys_time compat_sys_time 2213 i386 time sys_time compat_sys_time
2314 i386 mknod sys_mknod 2314 i386 mknod sys_mknod
diff --git a/arch/x86/tools/Makefile b/arch/x86/tools/Makefile
index 733057b435b0..bae601f900ef 100644
--- a/arch/x86/tools/Makefile
+++ b/arch/x86/tools/Makefile
@@ -28,7 +28,7 @@ posttest: $(obj)/test_get_len vmlinux $(obj)/insn_sanity
28hostprogs-y += test_get_len insn_sanity 28hostprogs-y += test_get_len insn_sanity
29 29
30# -I needed for generated C source and C source which in the kernel tree. 30# -I needed for generated C source and C source which in the kernel tree.
31HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ 31HOSTCFLAGS_test_get_len.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/uapi/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/uapi/
32 32
33HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/ 33HOSTCFLAGS_insn_sanity.o := -Wall -I$(objtree)/arch/x86/lib/ -I$(srctree)/arch/x86/include/ -I$(srctree)/arch/x86/lib/ -I$(srctree)/include/
34 34
diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig
index 9926e11a772d..07611759ce35 100644
--- a/arch/x86/um/Kconfig
+++ b/arch/x86/um/Kconfig
@@ -13,6 +13,8 @@ endmenu
13config UML_X86 13config UML_X86
14 def_bool y 14 def_bool y
15 select GENERIC_FIND_FIRST_BIT 15 select GENERIC_FIND_FIRST_BIT
16 select GENERIC_KERNEL_THREAD
17 select GENERIC_KERNEL_EXECVE
16 18
17config 64BIT 19config 64BIT
18 bool "64-bit kernel" if SUBARCH = "x86" 20 bool "64-bit kernel" if SUBARCH = "x86"
@@ -21,9 +23,12 @@ config 64BIT
21config X86_32 23config X86_32
22 def_bool !64BIT 24 def_bool !64BIT
23 select HAVE_AOUT 25 select HAVE_AOUT
26 select ARCH_WANT_IPC_PARSE_VERSION
27 select MODULES_USE_ELF_REL
24 28
25config X86_64 29config X86_64
26 def_bool 64BIT 30 def_bool 64BIT
31 select MODULES_USE_ELF_RELA
27 32
28config RWSEM_XCHGADD_ALGORITHM 33config RWSEM_XCHGADD_ALGORITHM
29 def_bool X86_XADD && 64BIT 34 def_bool X86_XADD && 64BIT
diff --git a/arch/x86/um/asm/checksum.h b/arch/x86/um/asm/checksum.h
index b6efe2381b5d..4b181b74454f 100644
--- a/arch/x86/um/asm/checksum.h
+++ b/arch/x86/um/asm/checksum.h
@@ -1,6 +1,150 @@
1#ifndef __UM_CHECKSUM_H 1#ifndef __UM_CHECKSUM_H
2#define __UM_CHECKSUM_H 2#define __UM_CHECKSUM_H
3 3
4#include <linux/string.h>
5#include <linux/in6.h>
6
7/*
8 * computes the checksum of a memory block at buff, length len,
9 * and adds in "sum" (32-bit)
10 *
11 * returns a 32-bit number suitable for feeding into itself
12 * or csum_tcpudp_magic
13 *
14 * this function must be called with even lengths, except
15 * for the last fragment, which may be odd
16 *
17 * it's best to have buff aligned on a 32-bit boundary
18 */
19extern __wsum csum_partial(const void *buff, int len, __wsum sum);
20
21/*
22 * Note: when you get a NULL pointer exception here this means someone
23 * passed in an incorrect kernel address to one of these functions.
24 *
25 * If you use these functions directly please don't forget the
26 * access_ok().
27 */
28
29static __inline__
30__wsum csum_partial_copy_nocheck(const void *src, void *dst,
31 int len, __wsum sum)
32{
33 memcpy(dst, src, len);
34 return csum_partial(dst, len, sum);
35}
36
37/*
38 * the same as csum_partial, but copies from src while it
39 * checksums, and handles user-space pointer exceptions correctly, when needed.
40 *
41 * here even more important to align src and dst on a 32-bit (or even
42 * better 64-bit) boundary
43 */
44
45static __inline__
46__wsum csum_partial_copy_from_user(const void __user *src, void *dst,
47 int len, __wsum sum, int *err_ptr)
48{
49 if (copy_from_user(dst, src, len)) {
50 *err_ptr = -EFAULT;
51 return (__force __wsum)-1;
52 }
53
54 return csum_partial(dst, len, sum);
55}
56
57/**
58 * csum_fold - Fold and invert a 32bit checksum.
59 * sum: 32bit unfolded sum
60 *
61 * Fold a 32bit running checksum to 16bit and invert it. This is usually
62 * the last step before putting a checksum into a packet.
63 * Make sure not to mix with 64bit checksums.
64 */
65static inline __sum16 csum_fold(__wsum sum)
66{
67 __asm__(
68 " addl %1,%0\n"
69 " adcl $0xffff,%0"
70 : "=r" (sum)
71 : "r" ((__force u32)sum << 16),
72 "0" ((__force u32)sum & 0xffff0000)
73 );
74 return (__force __sum16)(~(__force u32)sum >> 16);
75}
76
77/**
78 * csum_tcpup_nofold - Compute an IPv4 pseudo header checksum.
79 * @saddr: source address
80 * @daddr: destination address
81 * @len: length of packet
82 * @proto: ip protocol of packet
83 * @sum: initial sum to be added in (32bit unfolded)
84 *
85 * Returns the pseudo header checksum the input data. Result is
86 * 32bit unfolded.
87 */
88static inline __wsum
89csum_tcpudp_nofold(__be32 saddr, __be32 daddr, unsigned short len,
90 unsigned short proto, __wsum sum)
91{
92 asm(" addl %1, %0\n"
93 " adcl %2, %0\n"
94 " adcl %3, %0\n"
95 " adcl $0, %0\n"
96 : "=r" (sum)
97 : "g" (daddr), "g" (saddr), "g" ((len + proto) << 8), "0" (sum));
98 return sum;
99}
100
101/*
102 * computes the checksum of the TCP/UDP pseudo-header
103 * returns a 16-bit checksum, already complemented
104 */
105static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
106 unsigned short len,
107 unsigned short proto,
108 __wsum sum)
109{
110 return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum));
111}
112
113/**
114 * ip_fast_csum - Compute the IPv4 header checksum efficiently.
115 * iph: ipv4 header
116 * ihl: length of header / 4
117 */
118static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
119{
120 unsigned int sum;
121
122 asm( " movl (%1), %0\n"
123 " subl $4, %2\n"
124 " jbe 2f\n"
125 " addl 4(%1), %0\n"
126 " adcl 8(%1), %0\n"
127 " adcl 12(%1), %0\n"
128 "1: adcl 16(%1), %0\n"
129 " lea 4(%1), %1\n"
130 " decl %2\n"
131 " jne 1b\n"
132 " adcl $0, %0\n"
133 " movl %0, %2\n"
134 " shrl $16, %0\n"
135 " addw %w2, %w0\n"
136 " adcl $0, %0\n"
137 " notl %0\n"
138 "2:"
139 /* Since the input registers which are loaded with iph and ipl
140 are modified, we must also specify them as outputs, or gcc
141 will assume they contain their original values. */
142 : "=r" (sum), "=r" (iph), "=r" (ihl)
143 : "1" (iph), "2" (ihl)
144 : "memory");
145 return (__force __sum16)sum;
146}
147
4#ifdef CONFIG_X86_32 148#ifdef CONFIG_X86_32
5# include "checksum_32.h" 149# include "checksum_32.h"
6#else 150#else
diff --git a/arch/x86/um/asm/checksum_32.h b/arch/x86/um/asm/checksum_32.h
index caab74252e27..ab77b6f9a4bf 100644
--- a/arch/x86/um/asm/checksum_32.h
+++ b/arch/x86/um/asm/checksum_32.h
@@ -5,145 +5,6 @@
5#ifndef __UM_SYSDEP_CHECKSUM_H 5#ifndef __UM_SYSDEP_CHECKSUM_H
6#define __UM_SYSDEP_CHECKSUM_H 6#define __UM_SYSDEP_CHECKSUM_H
7 7
8#include "linux/in6.h"
9#include "linux/string.h"
10
11/*
12 * computes the checksum of a memory block at buff, length len,
13 * and adds in "sum" (32-bit)
14 *
15 * returns a 32-bit number suitable for feeding into itself
16 * or csum_tcpudp_magic
17 *
18 * this function must be called with even lengths, except
19 * for the last fragment, which may be odd
20 *
21 * it's best to have buff aligned on a 32-bit boundary
22 */
23__wsum csum_partial(const void *buff, int len, __wsum sum);
24
25/*
26 * Note: when you get a NULL pointer exception here this means someone
27 * passed in an incorrect kernel address to one of these functions.
28 *
29 * If you use these functions directly please don't forget the
30 * access_ok().
31 */
32
33static __inline__
34__wsum csum_partial_copy_nocheck(const void *src, void *dst,
35 int len, __wsum sum)
36{
37 memcpy(dst, src, len);
38 return csum_partial(dst, len, sum);
39}
40
41/*
42 * the same as csum_partial, but copies from src while it
43 * checksums, and handles user-space pointer exceptions correctly, when needed.
44 *
45 * here even more important to align src and dst on a 32-bit (or even
46 * better 64-bit) boundary
47 */
48
49static __inline__
50__wsum csum_partial_copy_from_user(const void __user *src, void *dst,
51 int len, __wsum sum, int *err_ptr)
52{
53 if (copy_from_user(dst, src, len)) {
54 *err_ptr = -EFAULT;
55 return (__force __wsum)-1;
56 }
57
58 return csum_partial(dst, len, sum);
59}
60
61/*
62 * This is a version of ip_compute_csum() optimized for IP headers,
63 * which always checksum on 4 octet boundaries.
64 *
65 * By Jorge Cwik <jorge@laser.satlink.net>, adapted for linux by
66 * Arnt Gulbrandsen.
67 */
68static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
69{
70 unsigned int sum;
71
72 __asm__ __volatile__(
73 "movl (%1), %0 ;\n"
74 "subl $4, %2 ;\n"
75 "jbe 2f ;\n"
76 "addl 4(%1), %0 ;\n"
77 "adcl 8(%1), %0 ;\n"
78 "adcl 12(%1), %0 ;\n"
79"1: adcl 16(%1), %0 ;\n"
80 "lea 4(%1), %1 ;\n"
81 "decl %2 ;\n"
82 "jne 1b ;\n"
83 "adcl $0, %0 ;\n"
84 "movl %0, %2 ;\n"
85 "shrl $16, %0 ;\n"
86 "addw %w2, %w0 ;\n"
87 "adcl $0, %0 ;\n"
88 "notl %0 ;\n"
89"2: ;\n"
90 /* Since the input registers which are loaded with iph and ipl
91 are modified, we must also specify them as outputs, or gcc
92 will assume they contain their original values. */
93 : "=r" (sum), "=r" (iph), "=r" (ihl)
94 : "1" (iph), "2" (ihl)
95 : "memory");
96 return (__force __sum16)sum;
97}
98
99/*
100 * Fold a partial checksum
101 */
102
103static inline __sum16 csum_fold(__wsum sum)
104{
105 __asm__(
106 "addl %1, %0 ;\n"
107 "adcl $0xffff, %0 ;\n"
108 : "=r" (sum)
109 : "r" ((__force u32)sum << 16),
110 "0" ((__force u32)sum & 0xffff0000)
111 );
112 return (__force __sum16)(~(__force u32)sum >> 16);
113}
114
115static inline __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr,
116 unsigned short len,
117 unsigned short proto,
118 __wsum sum)
119{
120 __asm__(
121 "addl %1, %0 ;\n"
122 "adcl %2, %0 ;\n"
123 "adcl %3, %0 ;\n"
124 "adcl $0, %0 ;\n"
125 : "=r" (sum)
126 : "g" (daddr), "g"(saddr), "g"((len + proto) << 8), "0"(sum));
127 return sum;
128}
129
130/*
131 * computes the checksum of the TCP/UDP pseudo-header
132 * returns a 16-bit checksum, already complemented
133 */
134static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
135 unsigned short len,
136 unsigned short proto,
137 __wsum sum)
138{
139 return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum));
140}
141
142/*
143 * this routine is used for miscellaneous IP-like checksums, mainly
144 * in icmp.c
145 */
146
147static inline __sum16 ip_compute_csum(const void *buff, int len) 8static inline __sum16 ip_compute_csum(const void *buff, int len)
148{ 9{
149 return csum_fold (csum_partial(buff, len, 0)); 10 return csum_fold (csum_partial(buff, len, 0));
@@ -198,4 +59,3 @@ static __inline__ __wsum csum_and_copy_to_user(const void *src,
198} 59}
199 60
200#endif 61#endif
201
diff --git a/arch/x86/um/asm/checksum_64.h b/arch/x86/um/asm/checksum_64.h
index a5be9031ea85..7b6cd1921573 100644
--- a/arch/x86/um/asm/checksum_64.h
+++ b/arch/x86/um/asm/checksum_64.h
@@ -5,131 +5,6 @@
5#ifndef __UM_SYSDEP_CHECKSUM_H 5#ifndef __UM_SYSDEP_CHECKSUM_H
6#define __UM_SYSDEP_CHECKSUM_H 6#define __UM_SYSDEP_CHECKSUM_H
7 7
8#include "linux/string.h"
9#include "linux/in6.h"
10#include "asm/uaccess.h"
11
12extern __wsum csum_partial(const void *buff, int len, __wsum sum);
13
14/*
15 * Note: when you get a NULL pointer exception here this means someone
16 * passed in an incorrect kernel address to one of these functions.
17 *
18 * If you use these functions directly please don't forget the
19 * access_ok().
20 */
21
22static __inline__
23__wsum csum_partial_copy_nocheck(const void *src, void *dst,
24 int len, __wsum sum)
25{
26 memcpy(dst, src, len);
27 return(csum_partial(dst, len, sum));
28}
29
30static __inline__
31__wsum csum_partial_copy_from_user(const void __user *src,
32 void *dst, int len, __wsum sum,
33 int *err_ptr)
34{
35 if (copy_from_user(dst, src, len)) {
36 *err_ptr = -EFAULT;
37 return (__force __wsum)-1;
38 }
39 return csum_partial(dst, len, sum);
40}
41
42/**
43 * csum_fold - Fold and invert a 32bit checksum.
44 * sum: 32bit unfolded sum
45 *
46 * Fold a 32bit running checksum to 16bit and invert it. This is usually
47 * the last step before putting a checksum into a packet.
48 * Make sure not to mix with 64bit checksums.
49 */
50static inline __sum16 csum_fold(__wsum sum)
51{
52 __asm__(
53 " addl %1,%0\n"
54 " adcl $0xffff,%0"
55 : "=r" (sum)
56 : "r" ((__force u32)sum << 16),
57 "0" ((__force u32)sum & 0xffff0000)
58 );
59 return (__force __sum16)(~(__force u32)sum >> 16);
60}
61
62/**
63 * csum_tcpup_nofold - Compute an IPv4 pseudo header checksum.
64 * @saddr: source address
65 * @daddr: destination address
66 * @len: length of packet
67 * @proto: ip protocol of packet
68 * @sum: initial sum to be added in (32bit unfolded)
69 *
70 * Returns the pseudo header checksum the input data. Result is
71 * 32bit unfolded.
72 */
73static inline __wsum
74csum_tcpudp_nofold(__be32 saddr, __be32 daddr, unsigned short len,
75 unsigned short proto, __wsum sum)
76{
77 asm(" addl %1, %0\n"
78 " adcl %2, %0\n"
79 " adcl %3, %0\n"
80 " adcl $0, %0\n"
81 : "=r" (sum)
82 : "g" (daddr), "g" (saddr), "g" ((len + proto) << 8), "0" (sum));
83 return sum;
84}
85
86/*
87 * computes the checksum of the TCP/UDP pseudo-header
88 * returns a 16-bit checksum, already complemented
89 */
90static inline __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
91 unsigned short len,
92 unsigned short proto,
93 __wsum sum)
94{
95 return csum_fold(csum_tcpudp_nofold(saddr,daddr,len,proto,sum));
96}
97
98/**
99 * ip_fast_csum - Compute the IPv4 header checksum efficiently.
100 * iph: ipv4 header
101 * ihl: length of header / 4
102 */
103static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
104{
105 unsigned int sum;
106
107 asm( " movl (%1), %0\n"
108 " subl $4, %2\n"
109 " jbe 2f\n"
110 " addl 4(%1), %0\n"
111 " adcl 8(%1), %0\n"
112 " adcl 12(%1), %0\n"
113 "1: adcl 16(%1), %0\n"
114 " lea 4(%1), %1\n"
115 " decl %2\n"
116 " jne 1b\n"
117 " adcl $0, %0\n"
118 " movl %0, %2\n"
119 " shrl $16, %0\n"
120 " addw %w2, %w0\n"
121 " adcl $0, %0\n"
122 " notl %0\n"
123 "2:"
124 /* Since the input registers which are loaded with iph and ipl
125 are modified, we must also specify them as outputs, or gcc
126 will assume they contain their original values. */
127 : "=r" (sum), "=r" (iph), "=r" (ihl)
128 : "1" (iph), "2" (ihl)
129 : "memory");
130 return (__force __sum16)sum;
131}
132
133static inline unsigned add32_with_carry(unsigned a, unsigned b) 8static inline unsigned add32_with_carry(unsigned a, unsigned b)
134{ 9{
135 asm("addl %2,%0\n\t" 10 asm("addl %2,%0\n\t"
diff --git a/arch/x86/um/asm/elf.h b/arch/x86/um/asm/elf.h
index 0e07adc8cbe4..0feee2fd5077 100644
--- a/arch/x86/um/asm/elf.h
+++ b/arch/x86/um/asm/elf.h
@@ -6,7 +6,7 @@
6#define __UM_ELF_X86_H 6#define __UM_ELF_X86_H
7 7
8#include <asm/user.h> 8#include <asm/user.h>
9#include "skas.h" 9#include <skas.h>
10 10
11#ifdef CONFIG_X86_32 11#ifdef CONFIG_X86_32
12 12
diff --git a/arch/x86/um/asm/ptrace.h b/arch/x86/um/asm/ptrace.h
index e72cd0df5ba3..755133258c45 100644
--- a/arch/x86/um/asm/ptrace.h
+++ b/arch/x86/um/asm/ptrace.h
@@ -1,11 +1,13 @@
1#ifndef __UM_X86_PTRACE_H 1#ifndef __UM_X86_PTRACE_H
2#define __UM_X86_PTRACE_H 2#define __UM_X86_PTRACE_H
3 3
4#ifdef CONFIG_X86_32 4#include <linux/compiler.h>
5# include "ptrace_32.h" 5#ifndef CONFIG_X86_32
6#else 6#define __FRAME_OFFSETS /* Needed to get the R* macros */
7# include "ptrace_64.h"
8#endif 7#endif
8#include <asm/ptrace-generic.h>
9
10#define user_mode(r) UPT_IS_USER(&(r)->regs)
9 11
10#define PT_REGS_AX(r) UPT_AX(&(r)->regs) 12#define PT_REGS_AX(r) UPT_AX(&(r)->regs)
11#define PT_REGS_BX(r) UPT_BX(&(r)->regs) 13#define PT_REGS_BX(r) UPT_BX(&(r)->regs)
@@ -36,4 +38,52 @@ static inline long regs_return_value(struct pt_regs *regs)
36{ 38{
37 return PT_REGS_AX(regs); 39 return PT_REGS_AX(regs);
38} 40}
41
42/*
43 * Forward declaration to avoid including sysdep/tls.h, which causes a
44 * circular include, and compilation failures.
45 */
46struct user_desc;
47
48#ifdef CONFIG_X86_32
49
50#define HOST_AUDIT_ARCH AUDIT_ARCH_I386
51
52extern int ptrace_get_thread_area(struct task_struct *child, int idx,
53 struct user_desc __user *user_desc);
54
55extern int ptrace_set_thread_area(struct task_struct *child, int idx,
56 struct user_desc __user *user_desc);
57
58#else
59
60#define HOST_AUDIT_ARCH AUDIT_ARCH_X86_64
61
62#define PT_REGS_R8(r) UPT_R8(&(r)->regs)
63#define PT_REGS_R9(r) UPT_R9(&(r)->regs)
64#define PT_REGS_R10(r) UPT_R10(&(r)->regs)
65#define PT_REGS_R11(r) UPT_R11(&(r)->regs)
66#define PT_REGS_R12(r) UPT_R12(&(r)->regs)
67#define PT_REGS_R13(r) UPT_R13(&(r)->regs)
68#define PT_REGS_R14(r) UPT_R14(&(r)->regs)
69#define PT_REGS_R15(r) UPT_R15(&(r)->regs)
70
71#include <asm/errno.h>
72
73static inline int ptrace_get_thread_area(struct task_struct *child, int idx,
74 struct user_desc __user *user_desc)
75{
76 return -ENOSYS;
77}
78
79static inline int ptrace_set_thread_area(struct task_struct *child, int idx,
80 struct user_desc __user *user_desc)
81{
82 return -ENOSYS;
83}
84
85extern long arch_prctl(struct task_struct *task, int code,
86 unsigned long __user *addr);
87
88#endif
39#endif /* __UM_X86_PTRACE_H */ 89#endif /* __UM_X86_PTRACE_H */
diff --git a/arch/x86/um/asm/ptrace_32.h b/arch/x86/um/asm/ptrace_32.h
deleted file mode 100644
index 2cf225351b65..000000000000
--- a/arch/x86/um/asm/ptrace_32.h
+++ /dev/null
@@ -1,28 +0,0 @@
1/*
2 * Copyright (C) 2000 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com)
3 * Licensed under the GPL
4 */
5
6#ifndef __UM_PTRACE_I386_H
7#define __UM_PTRACE_I386_H
8
9#define HOST_AUDIT_ARCH AUDIT_ARCH_I386
10
11#include "linux/compiler.h"
12#include "asm/ptrace-generic.h"
13
14#define user_mode(r) UPT_IS_USER(&(r)->regs)
15
16/*
17 * Forward declaration to avoid including sysdep/tls.h, which causes a
18 * circular include, and compilation failures.
19 */
20struct user_desc;
21
22extern int ptrace_get_thread_area(struct task_struct *child, int idx,
23 struct user_desc __user *user_desc);
24
25extern int ptrace_set_thread_area(struct task_struct *child, int idx,
26 struct user_desc __user *user_desc);
27
28#endif
diff --git a/arch/x86/um/asm/ptrace_64.h b/arch/x86/um/asm/ptrace_64.h
deleted file mode 100644
index ea7bff394320..000000000000
--- a/arch/x86/um/asm/ptrace_64.h
+++ /dev/null
@@ -1,46 +0,0 @@
1/*
2 * Copyright 2003 PathScale, Inc.
3 *
4 * Licensed under the GPL
5 */
6
7#ifndef __UM_PTRACE_X86_64_H
8#define __UM_PTRACE_X86_64_H
9
10#include "linux/compiler.h"
11#include "asm/errno.h"
12
13#define __FRAME_OFFSETS /* Needed to get the R* macros */
14#include "asm/ptrace-generic.h"
15
16#define HOST_AUDIT_ARCH AUDIT_ARCH_X86_64
17
18#define PT_REGS_R8(r) UPT_R8(&(r)->regs)
19#define PT_REGS_R9(r) UPT_R9(&(r)->regs)
20#define PT_REGS_R10(r) UPT_R10(&(r)->regs)
21#define PT_REGS_R11(r) UPT_R11(&(r)->regs)
22#define PT_REGS_R12(r) UPT_R12(&(r)->regs)
23#define PT_REGS_R13(r) UPT_R13(&(r)->regs)
24#define PT_REGS_R14(r) UPT_R14(&(r)->regs)
25#define PT_REGS_R15(r) UPT_R15(&(r)->regs)
26
27/* XXX */
28#define user_mode(r) UPT_IS_USER(&(r)->regs)
29
30struct user_desc;
31
32static inline int ptrace_get_thread_area(struct task_struct *child, int idx,
33 struct user_desc __user *user_desc)
34{
35 return -ENOSYS;
36}
37
38static inline int ptrace_set_thread_area(struct task_struct *child, int idx,
39 struct user_desc __user *user_desc)
40{
41 return -ENOSYS;
42}
43
44extern long arch_prctl(struct task_struct *task, int code,
45 unsigned long __user *addr);
46#endif
diff --git a/arch/x86/um/bugs_32.c b/arch/x86/um/bugs_32.c
index 17d88cf2c6c4..33daff4dade4 100644
--- a/arch/x86/um/bugs_32.c
+++ b/arch/x86/um/bugs_32.c
@@ -4,9 +4,9 @@
4 */ 4 */
5 5
6#include <signal.h> 6#include <signal.h>
7#include "kern_util.h" 7#include <kern_util.h>
8#include "longjmp.h" 8#include <longjmp.h>
9#include "sysdep/ptrace.h" 9#include <sysdep/ptrace.h>
10#include <generated/asm-offsets.h> 10#include <generated/asm-offsets.h>
11 11
12/* Set during early boot */ 12/* Set during early boot */
diff --git a/arch/x86/um/bugs_64.c b/arch/x86/um/bugs_64.c
index 44e02ba2a265..8cc8256c698d 100644
--- a/arch/x86/um/bugs_64.c
+++ b/arch/x86/um/bugs_64.c
@@ -4,7 +4,7 @@
4 * Licensed under the GPL 4 * Licensed under the GPL
5 */ 5 */
6 6
7#include "sysdep/ptrace.h" 7#include <sysdep/ptrace.h>
8 8
9void arch_check_bugs(void) 9void arch_check_bugs(void)
10{ 10{
diff --git a/arch/x86/um/fault.c b/arch/x86/um/fault.c
index d670f68532f4..8784ab30d91b 100644
--- a/arch/x86/um/fault.c
+++ b/arch/x86/um/fault.c
@@ -3,7 +3,7 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "sysdep/ptrace.h" 6#include <sysdep/ptrace.h>
7 7
8/* These two are from asm-um/uaccess.h and linux/module.h, check them. */ 8/* These two are from asm-um/uaccess.h and linux/module.h, check them. */
9struct exception_table_entry 9struct exception_table_entry
diff --git a/arch/x86/um/ldt.c b/arch/x86/um/ldt.c
index 26b0e39d2ce9..8e08176f0bcb 100644
--- a/arch/x86/um/ldt.c
+++ b/arch/x86/um/ldt.c
@@ -7,11 +7,11 @@
7#include <linux/sched.h> 7#include <linux/sched.h>
8#include <linux/slab.h> 8#include <linux/slab.h>
9#include <asm/unistd.h> 9#include <asm/unistd.h>
10#include "os.h" 10#include <os.h>
11#include "proc_mm.h" 11#include <proc_mm.h>
12#include "skas.h" 12#include <skas.h>
13#include "skas_ptrace.h" 13#include <skas_ptrace.h>
14#include "sysdep/tls.h" 14#include <sysdep/tls.h>
15 15
16extern int modify_ldt(int func, void *ptr, unsigned long bytecount); 16extern int modify_ldt(int func, void *ptr, unsigned long bytecount);
17 17
diff --git a/arch/x86/um/mem_64.c b/arch/x86/um/mem_64.c
index 546518727a73..c6492e75797b 100644
--- a/arch/x86/um/mem_64.c
+++ b/arch/x86/um/mem_64.c
@@ -1,6 +1,6 @@
1#include "linux/mm.h" 1#include <linux/mm.h>
2#include "asm/page.h" 2#include <asm/page.h>
3#include "asm/mman.h" 3#include <asm/mman.h>
4 4
5const char *arch_vma_name(struct vm_area_struct *vma) 5const char *arch_vma_name(struct vm_area_struct *vma)
6{ 6{
diff --git a/arch/x86/um/os-Linux/registers.c b/arch/x86/um/os-Linux/registers.c
index 0cdbb86b012b..41bfe84e11ab 100644
--- a/arch/x86/um/os-Linux/registers.c
+++ b/arch/x86/um/os-Linux/registers.c
@@ -9,8 +9,8 @@
9#ifdef __i386__ 9#ifdef __i386__
10#include <sys/user.h> 10#include <sys/user.h>
11#endif 11#endif
12#include "longjmp.h" 12#include <longjmp.h>
13#include "sysdep/ptrace_user.h" 13#include <sysdep/ptrace_user.h>
14 14
15int save_fp_registers(int pid, unsigned long *fp_regs) 15int save_fp_registers(int pid, unsigned long *fp_regs)
16{ 16{
diff --git a/arch/x86/um/os-Linux/task_size.c b/arch/x86/um/os-Linux/task_size.c
index efb16c5c9bcf..8502ad30e61b 100644
--- a/arch/x86/um/os-Linux/task_size.c
+++ b/arch/x86/um/os-Linux/task_size.c
@@ -2,7 +2,7 @@
2#include <stdlib.h> 2#include <stdlib.h>
3#include <signal.h> 3#include <signal.h>
4#include <sys/mman.h> 4#include <sys/mman.h>
5#include "longjmp.h" 5#include <longjmp.h>
6 6
7#ifdef __i386__ 7#ifdef __i386__
8 8
diff --git a/arch/x86/um/os-Linux/tls.c b/arch/x86/um/os-Linux/tls.c
index 82276b6071af..9d94b3b76c74 100644
--- a/arch/x86/um/os-Linux/tls.c
+++ b/arch/x86/um/os-Linux/tls.c
@@ -5,7 +5,7 @@
5#include <sys/syscall.h> 5#include <sys/syscall.h>
6#include <unistd.h> 6#include <unistd.h>
7 7
8#include "sysdep/tls.h" 8#include <sysdep/tls.h>
9 9
10#ifndef PTRACE_GET_THREAD_AREA 10#ifndef PTRACE_GET_THREAD_AREA
11#define PTRACE_GET_THREAD_AREA 25 11#define PTRACE_GET_THREAD_AREA 25
diff --git a/arch/x86/um/ptrace_32.c b/arch/x86/um/ptrace_32.c
index 3b949daa095c..ce3dd4f36f3f 100644
--- a/arch/x86/um/ptrace_32.c
+++ b/arch/x86/um/ptrace_32.c
@@ -3,10 +3,10 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "linux/mm.h" 6#include <linux/mm.h>
7#include "linux/sched.h" 7#include <linux/sched.h>
8#include "asm/uaccess.h" 8#include <asm/uaccess.h>
9#include "skas.h" 9#include <skas.h>
10 10
11extern int arch_switch_tls(struct task_struct *to); 11extern int arch_switch_tls(struct task_struct *to);
12 12
diff --git a/arch/x86/um/ptrace_user.c b/arch/x86/um/ptrace_user.c
index 3960ca1dd35a..617885b18992 100644
--- a/arch/x86/um/ptrace_user.c
+++ b/arch/x86/um/ptrace_user.c
@@ -4,7 +4,7 @@
4 */ 4 */
5 5
6#include <errno.h> 6#include <errno.h>
7#include "ptrace_user.h" 7#include <ptrace_user.h>
8 8
9int ptrace_getregs(long pid, unsigned long *regs_out) 9int ptrace_getregs(long pid, unsigned long *regs_out)
10{ 10{
diff --git a/arch/x86/um/shared/sysdep/kernel-offsets.h b/arch/x86/um/shared/sysdep/kernel-offsets.h
index 5868526b5eef..46a9df99f3c5 100644
--- a/arch/x86/um/shared/sysdep/kernel-offsets.h
+++ b/arch/x86/um/shared/sysdep/kernel-offsets.h
@@ -7,9 +7,6 @@
7#define DEFINE(sym, val) \ 7#define DEFINE(sym, val) \
8 asm volatile("\n->" #sym " %0 " #val : : "i" (val)) 8 asm volatile("\n->" #sym " %0 " #val : : "i" (val))
9 9
10#define STR(x) #x
11#define DEFINE_STR(sym, val) asm volatile("\n->" #sym " " STR(val) " " #val: : )
12
13#define BLANK() asm volatile("\n->" : : ) 10#define BLANK() asm volatile("\n->" : : )
14 11
15#define OFFSET(sym, str, mem) \ 12#define OFFSET(sym, str, mem) \
diff --git a/arch/x86/um/shared/sysdep/ptrace.h b/arch/x86/um/shared/sysdep/ptrace.h
index 6ce2d76eb908..eb9356904ad3 100644
--- a/arch/x86/um/shared/sysdep/ptrace.h
+++ b/arch/x86/um/shared/sysdep/ptrace.h
@@ -2,7 +2,7 @@
2#define __SYSDEP_X86_PTRACE_H 2#define __SYSDEP_X86_PTRACE_H
3 3
4#include <generated/user_constants.h> 4#include <generated/user_constants.h>
5#include "sysdep/faultinfo.h" 5#include <sysdep/faultinfo.h>
6 6
7#define MAX_REG_OFFSET (UM_FRAME_SIZE) 7#define MAX_REG_OFFSET (UM_FRAME_SIZE)
8#define MAX_REG_NR ((MAX_REG_OFFSET) / sizeof(unsigned long)) 8#define MAX_REG_NR ((MAX_REG_OFFSET) / sizeof(unsigned long))
diff --git a/arch/x86/um/shared/sysdep/stub.h b/arch/x86/um/shared/sysdep/stub.h
index bd161e300102..3f55e5bd3cec 100644
--- a/arch/x86/um/shared/sysdep/stub.h
+++ b/arch/x86/um/shared/sysdep/stub.h
@@ -1,8 +1,8 @@
1#include <asm/unistd.h> 1#include <asm/unistd.h>
2#include <sys/mman.h> 2#include <sys/mman.h>
3#include <signal.h> 3#include <signal.h>
4#include "as-layout.h" 4#include <as-layout.h>
5#include "stub-data.h" 5#include <stub-data.h>
6 6
7#ifdef __i386__ 7#ifdef __i386__
8#include "stub_32.h" 8#include "stub_32.h"
diff --git a/arch/x86/um/shared/sysdep/syscalls.h b/arch/x86/um/shared/sysdep/syscalls.h
index bd9a89b67e41..ca255a805ed9 100644
--- a/arch/x86/um/shared/sysdep/syscalls.h
+++ b/arch/x86/um/shared/sysdep/syscalls.h
@@ -1,3 +1,5 @@
1extern long sys_clone(unsigned long clone_flags, unsigned long newsp,
2 void __user *parent_tid, void __user *child_tid);
1#ifdef __i386__ 3#ifdef __i386__
2#include "syscalls_32.h" 4#include "syscalls_32.h"
3#else 5#else
diff --git a/arch/x86/um/shared/sysdep/syscalls_32.h b/arch/x86/um/shared/sysdep/syscalls_32.h
index 05cb796aecb5..8436079be914 100644
--- a/arch/x86/um/shared/sysdep/syscalls_32.h
+++ b/arch/x86/um/shared/sysdep/syscalls_32.h
@@ -3,8 +3,8 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "asm/unistd.h" 6#include <asm/unistd.h>
7#include "sysdep/ptrace.h" 7#include <sysdep/ptrace.h>
8 8
9typedef long syscall_handler_t(struct pt_regs); 9typedef long syscall_handler_t(struct pt_regs);
10 10
diff --git a/arch/x86/um/signal.c b/arch/x86/um/signal.c
index a508cea13503..bdaa08cfbcf4 100644
--- a/arch/x86/um/signal.c
+++ b/arch/x86/um/signal.c
@@ -11,8 +11,8 @@
11#include <asm/unistd.h> 11#include <asm/unistd.h>
12#include <asm/uaccess.h> 12#include <asm/uaccess.h>
13#include <asm/ucontext.h> 13#include <asm/ucontext.h>
14#include "frame_kern.h" 14#include <frame_kern.h>
15#include "skas.h" 15#include <skas.h>
16 16
17#ifdef CONFIG_X86_32 17#ifdef CONFIG_X86_32
18 18
@@ -416,9 +416,6 @@ int setup_signal_stack_sc(unsigned long stack_top, int sig,
416 PT_REGS_AX(regs) = (unsigned long) sig; 416 PT_REGS_AX(regs) = (unsigned long) sig;
417 PT_REGS_DX(regs) = (unsigned long) 0; 417 PT_REGS_DX(regs) = (unsigned long) 0;
418 PT_REGS_CX(regs) = (unsigned long) 0; 418 PT_REGS_CX(regs) = (unsigned long) 0;
419
420 if ((current->ptrace & PT_DTRACE) && (current->ptrace & PT_PTRACED))
421 ptrace_notify(SIGTRAP);
422 return 0; 419 return 0;
423} 420}
424 421
@@ -466,9 +463,6 @@ int setup_signal_stack_si(unsigned long stack_top, int sig,
466 PT_REGS_AX(regs) = (unsigned long) sig; 463 PT_REGS_AX(regs) = (unsigned long) sig;
467 PT_REGS_DX(regs) = (unsigned long) &frame->info; 464 PT_REGS_DX(regs) = (unsigned long) &frame->info;
468 PT_REGS_CX(regs) = (unsigned long) &frame->uc; 465 PT_REGS_CX(regs) = (unsigned long) &frame->uc;
469
470 if ((current->ptrace & PT_DTRACE) && (current->ptrace & PT_PTRACED))
471 ptrace_notify(SIGTRAP);
472 return 0; 466 return 0;
473} 467}
474 468
diff --git a/arch/x86/um/stub_32.S b/arch/x86/um/stub_32.S
index 54a36ec20cb7..b972649d3a18 100644
--- a/arch/x86/um/stub_32.S
+++ b/arch/x86/um/stub_32.S
@@ -1,4 +1,4 @@
1#include "as-layout.h" 1#include <as-layout.h>
2 2
3 .globl syscall_stub 3 .globl syscall_stub
4.section .__syscall_stub, "ax" 4.section .__syscall_stub, "ax"
diff --git a/arch/x86/um/stub_64.S b/arch/x86/um/stub_64.S
index 20e4a96a6dcb..7160b20172d0 100644
--- a/arch/x86/um/stub_64.S
+++ b/arch/x86/um/stub_64.S
@@ -1,4 +1,4 @@
1#include "as-layout.h" 1#include <as-layout.h>
2 2
3 .globl syscall_stub 3 .globl syscall_stub
4.section .__syscall_stub, "ax" 4.section .__syscall_stub, "ax"
diff --git a/arch/x86/um/stub_segv.c b/arch/x86/um/stub_segv.c
index b7450bd22e7d..1518d2805ae8 100644
--- a/arch/x86/um/stub_segv.c
+++ b/arch/x86/um/stub_segv.c
@@ -3,9 +3,9 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "sysdep/stub.h" 6#include <sysdep/stub.h>
7#include "sysdep/faultinfo.h" 7#include <sysdep/faultinfo.h>
8#include "sysdep/mcontext.h" 8#include <sysdep/mcontext.h>
9 9
10void __attribute__ ((__section__ (".__syscall_stub"))) 10void __attribute__ ((__section__ (".__syscall_stub")))
11stub_segv_handler(int sig, siginfo_t *info, void *p) 11stub_segv_handler(int sig, siginfo_t *info, void *p)
diff --git a/arch/x86/um/sys_call_table_32.c b/arch/x86/um/sys_call_table_32.c
index 68d1dc91b37b..232e60504b3a 100644
--- a/arch/x86/um/sys_call_table_32.c
+++ b/arch/x86/um/sys_call_table_32.c
@@ -25,10 +25,9 @@
25#define old_mmap sys_old_mmap 25#define old_mmap sys_old_mmap
26 26
27#define ptregs_fork sys_fork 27#define ptregs_fork sys_fork
28#define ptregs_execve sys_execve
29#define ptregs_iopl sys_iopl 28#define ptregs_iopl sys_iopl
30#define ptregs_vm86old sys_vm86old 29#define ptregs_vm86old sys_vm86old
31#define ptregs_clone sys_clone 30#define ptregs_clone i386_clone
32#define ptregs_vm86 sys_vm86 31#define ptregs_vm86 sys_vm86
33#define ptregs_sigaltstack sys_sigaltstack 32#define ptregs_sigaltstack sys_sigaltstack
34#define ptregs_vfork sys_vfork 33#define ptregs_vfork sys_vfork
diff --git a/arch/x86/um/syscalls_32.c b/arch/x86/um/syscalls_32.c
index b853e8600b9d..db444c7218fe 100644
--- a/arch/x86/um/syscalls_32.c
+++ b/arch/x86/um/syscalls_32.c
@@ -3,37 +3,24 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "linux/sched.h" 6#include <linux/syscalls.h>
7#include "linux/shm.h" 7#include <sysdep/syscalls.h>
8#include "linux/ipc.h"
9#include "linux/syscalls.h"
10#include "asm/mman.h"
11#include "asm/uaccess.h"
12#include "asm/unistd.h"
13 8
14/* 9/*
15 * The prototype on i386 is: 10 * The prototype on i386 is:
16 * 11 *
17 * int clone(int flags, void * child_stack, int * parent_tidptr, struct user_desc * newtls, int * child_tidptr) 12 * int clone(int flags, void * child_stack, int * parent_tidptr, struct user_desc * newtls
18 * 13 *
19 * and the "newtls" arg. on i386 is read by copy_thread directly from the 14 * and the "newtls" arg. on i386 is read by copy_thread directly from the
20 * register saved on the stack. 15 * register saved on the stack.
21 */ 16 */
22long sys_clone(unsigned long clone_flags, unsigned long newsp, 17long i386_clone(unsigned long clone_flags, unsigned long newsp,
23 int __user *parent_tid, void *newtls, int __user *child_tid) 18 int __user *parent_tid, void *newtls, int __user *child_tid)
24{ 19{
25 long ret; 20 return sys_clone(clone_flags, newsp, parent_tid, child_tid);
26
27 if (!newsp)
28 newsp = UPT_SP(&current->thread.regs.regs);
29
30 current->thread.forking = 1;
31 ret = do_fork(clone_flags, newsp, &current->thread.regs, 0, parent_tid,
32 child_tid);
33 current->thread.forking = 0;
34 return ret;
35} 21}
36 22
23
37long sys_sigaction(int sig, const struct old_sigaction __user *act, 24long sys_sigaction(int sig, const struct old_sigaction __user *act,
38 struct old_sigaction __user *oact) 25 struct old_sigaction __user *oact)
39{ 26{
diff --git a/arch/x86/um/syscalls_64.c b/arch/x86/um/syscalls_64.c
index f3d82bb6e15a..adb08eb5c22a 100644
--- a/arch/x86/um/syscalls_64.c
+++ b/arch/x86/um/syscalls_64.c
@@ -5,12 +5,9 @@
5 * Licensed under the GPL 5 * Licensed under the GPL
6 */ 6 */
7 7
8#include "linux/linkage.h" 8#include <linux/sched.h>
9#include "linux/personality.h" 9#include <asm/prctl.h> /* XXX This should get the constants from libc */
10#include "linux/utsname.h" 10#include <os.h>
11#include "asm/prctl.h" /* XXX This should get the constants from libc */
12#include "asm/uaccess.h"
13#include "os.h"
14 11
15long arch_prctl(struct task_struct *task, int code, unsigned long __user *addr) 12long arch_prctl(struct task_struct *task, int code, unsigned long __user *addr)
16{ 13{
@@ -79,20 +76,6 @@ long sys_arch_prctl(int code, unsigned long addr)
79 return arch_prctl(current, code, (unsigned long __user *) addr); 76 return arch_prctl(current, code, (unsigned long __user *) addr);
80} 77}
81 78
82long sys_clone(unsigned long clone_flags, unsigned long newsp,
83 void __user *parent_tid, void __user *child_tid)
84{
85 long ret;
86
87 if (!newsp)
88 newsp = UPT_SP(&current->thread.regs.regs);
89 current->thread.forking = 1;
90 ret = do_fork(clone_flags, newsp, &current->thread.regs, 0, parent_tid,
91 child_tid);
92 current->thread.forking = 0;
93 return ret;
94}
95
96void arch_switch_to(struct task_struct *to) 79void arch_switch_to(struct task_struct *to)
97{ 80{
98 if ((to->thread.arch.fs == 0) || (to->mm == NULL)) 81 if ((to->thread.arch.fs == 0) || (to->mm == NULL))
diff --git a/arch/x86/um/sysrq_32.c b/arch/x86/um/sysrq_32.c
index 2d5cc51e9bef..c9bee5b8c0d3 100644
--- a/arch/x86/um/sysrq_32.c
+++ b/arch/x86/um/sysrq_32.c
@@ -3,12 +3,12 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "linux/kernel.h" 6#include <linux/kernel.h>
7#include "linux/smp.h" 7#include <linux/smp.h>
8#include "linux/sched.h" 8#include <linux/sched.h>
9#include "linux/kallsyms.h" 9#include <linux/kallsyms.h>
10#include "asm/ptrace.h" 10#include <asm/ptrace.h>
11#include "sysrq.h" 11#include <asm/sysrq.h>
12 12
13/* This is declared by <linux/sched.h> */ 13/* This is declared by <linux/sched.h> */
14void show_regs(struct pt_regs *regs) 14void show_regs(struct pt_regs *regs)
diff --git a/arch/x86/um/sysrq_64.c b/arch/x86/um/sysrq_64.c
index 08258f179969..a0e7fb1134a0 100644
--- a/arch/x86/um/sysrq_64.c
+++ b/arch/x86/um/sysrq_64.c
@@ -10,7 +10,7 @@
10#include <linux/utsname.h> 10#include <linux/utsname.h>
11#include <asm/current.h> 11#include <asm/current.h>
12#include <asm/ptrace.h> 12#include <asm/ptrace.h>
13#include "sysrq.h" 13#include <asm/sysrq.h>
14 14
15void __show_regs(struct pt_regs *regs) 15void __show_regs(struct pt_regs *regs)
16{ 16{
diff --git a/arch/x86/um/tls_32.c b/arch/x86/um/tls_32.c
index baba84f8ecb8..5f5feff3d24c 100644
--- a/arch/x86/um/tls_32.c
+++ b/arch/x86/um/tls_32.c
@@ -3,12 +3,12 @@
3 * Licensed under the GPL 3 * Licensed under the GPL
4 */ 4 */
5 5
6#include "linux/percpu.h" 6#include <linux/percpu.h>
7#include "linux/sched.h" 7#include <linux/sched.h>
8#include "asm/uaccess.h" 8#include <asm/uaccess.h>
9#include "os.h" 9#include <os.h>
10#include "skas.h" 10#include <skas.h>
11#include "sysdep/tls.h" 11#include <sysdep/tls.h>
12 12
13/* 13/*
14 * If needed we can detect when it's uninitialized. 14 * If needed we can detect when it's uninitialized.
diff --git a/arch/x86/um/tls_64.c b/arch/x86/um/tls_64.c
index f7ba46200ecd..d22363cb854e 100644
--- a/arch/x86/um/tls_64.c
+++ b/arch/x86/um/tls_64.c
@@ -1,4 +1,4 @@
1#include "linux/sched.h" 1#include <linux/sched.h>
2 2
3void clear_flushed_tls(struct task_struct *task) 3void clear_flushed_tls(struct task_struct *task)
4{ 4{
diff --git a/arch/x86/vdso/vclock_gettime.c b/arch/x86/vdso/vclock_gettime.c
index 885eff49d6ab..4df6c373421a 100644
--- a/arch/x86/vdso/vclock_gettime.c
+++ b/arch/x86/vdso/vclock_gettime.c
@@ -80,7 +80,7 @@ notrace static long vdso_fallback_gtod(struct timeval *tv, struct timezone *tz)
80} 80}
81 81
82 82
83notrace static inline long vgetns(void) 83notrace static inline u64 vgetsns(void)
84{ 84{
85 long v; 85 long v;
86 cycles_t cycles; 86 cycles_t cycles;
@@ -91,21 +91,24 @@ notrace static inline long vgetns(void)
91 else 91 else
92 return 0; 92 return 0;
93 v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask; 93 v = (cycles - gtod->clock.cycle_last) & gtod->clock.mask;
94 return (v * gtod->clock.mult) >> gtod->clock.shift; 94 return v * gtod->clock.mult;
95} 95}
96 96
97/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */ 97/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
98notrace static int __always_inline do_realtime(struct timespec *ts) 98notrace static int __always_inline do_realtime(struct timespec *ts)
99{ 99{
100 unsigned long seq, ns; 100 unsigned long seq;
101 u64 ns;
101 int mode; 102 int mode;
102 103
104 ts->tv_nsec = 0;
103 do { 105 do {
104 seq = read_seqcount_begin(&gtod->seq); 106 seq = read_seqcount_begin(&gtod->seq);
105 mode = gtod->clock.vclock_mode; 107 mode = gtod->clock.vclock_mode;
106 ts->tv_sec = gtod->wall_time_sec; 108 ts->tv_sec = gtod->wall_time_sec;
107 ts->tv_nsec = gtod->wall_time_nsec; 109 ns = gtod->wall_time_snsec;
108 ns = vgetns(); 110 ns += vgetsns();
111 ns >>= gtod->clock.shift;
109 } while (unlikely(read_seqcount_retry(&gtod->seq, seq))); 112 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
110 113
111 timespec_add_ns(ts, ns); 114 timespec_add_ns(ts, ns);
@@ -114,15 +117,18 @@ notrace static int __always_inline do_realtime(struct timespec *ts)
114 117
115notrace static int do_monotonic(struct timespec *ts) 118notrace static int do_monotonic(struct timespec *ts)
116{ 119{
117 unsigned long seq, ns; 120 unsigned long seq;
121 u64 ns;
118 int mode; 122 int mode;
119 123
124 ts->tv_nsec = 0;
120 do { 125 do {
121 seq = read_seqcount_begin(&gtod->seq); 126 seq = read_seqcount_begin(&gtod->seq);
122 mode = gtod->clock.vclock_mode; 127 mode = gtod->clock.vclock_mode;
123 ts->tv_sec = gtod->monotonic_time_sec; 128 ts->tv_sec = gtod->monotonic_time_sec;
124 ts->tv_nsec = gtod->monotonic_time_nsec; 129 ns = gtod->monotonic_time_snsec;
125 ns = vgetns(); 130 ns += vgetsns();
131 ns >>= gtod->clock.shift;
126 } while (unlikely(read_seqcount_retry(&gtod->seq, seq))); 132 } while (unlikely(read_seqcount_retry(&gtod->seq, seq)));
127 timespec_add_ns(ts, ns); 133 timespec_add_ns(ts, ns);
128 134
diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c
index 783522b66b0f..586d83812b67 100644
--- a/arch/x86/xen/enlighten.c
+++ b/arch/x86/xen/enlighten.c
@@ -1474,6 +1474,10 @@ asmlinkage void __init xen_start_kernel(void)
1474 pci_request_acs(); 1474 pci_request_acs();
1475 1475
1476 xen_acpi_sleep_register(); 1476 xen_acpi_sleep_register();
1477
1478 /* Avoid searching for BIOS MP tables */
1479 x86_init.mpparse.find_smp_config = x86_init_noop;
1480 x86_init.mpparse.get_smp_config = x86_init_uint_noop;
1477 } 1481 }
1478#ifdef CONFIG_PCI 1482#ifdef CONFIG_PCI
1479 /* PCI BIOS service won't work from a PV guest. */ 1483 /* PCI BIOS service won't work from a PV guest. */
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 963cb2df636a..6226c99729b9 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -2492,8 +2492,7 @@ int xen_remap_domain_mfn_range(struct vm_area_struct *vma,
2492 2492
2493 prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP); 2493 prot = __pgprot(pgprot_val(prot) | _PAGE_IOMAP);
2494 2494
2495 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_RESERVED | VM_IO)) == 2495 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO)));
2496 (VM_PFNMAP | VM_RESERVED | VM_IO)));
2497 2496
2498 rmd.mfn = mfn; 2497 rmd.mfn = mfn;
2499 rmd.prot = prot; 2498 rmd.prot = prot;
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index b5e4d302a067..95fb2aa5927e 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -906,9 +906,6 @@ int m2p_add_override(unsigned long mfn, struct page *page,
906 906
907 xen_mc_issue(PARAVIRT_LAZY_MMU); 907 xen_mc_issue(PARAVIRT_LAZY_MMU);
908 } 908 }
909 /* let's use dev_bus_addr to record the old mfn instead */
910 kmap_op->dev_bus_addr = page->index;
911 page->index = (unsigned long) kmap_op;
912 } 909 }
913 spin_lock_irqsave(&m2p_override_lock, flags); 910 spin_lock_irqsave(&m2p_override_lock, flags);
914 list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]); 911 list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]);
@@ -935,7 +932,8 @@ int m2p_add_override(unsigned long mfn, struct page *page,
935 return 0; 932 return 0;
936} 933}
937EXPORT_SYMBOL_GPL(m2p_add_override); 934EXPORT_SYMBOL_GPL(m2p_add_override);
938int m2p_remove_override(struct page *page, bool clear_pte) 935int m2p_remove_override(struct page *page,
936 struct gnttab_map_grant_ref *kmap_op)
939{ 937{
940 unsigned long flags; 938 unsigned long flags;
941 unsigned long mfn; 939 unsigned long mfn;
@@ -965,10 +963,8 @@ int m2p_remove_override(struct page *page, bool clear_pte)
965 WARN_ON(!PagePrivate(page)); 963 WARN_ON(!PagePrivate(page));
966 ClearPagePrivate(page); 964 ClearPagePrivate(page);
967 965
968 if (clear_pte) { 966 set_phys_to_machine(pfn, page->index);
969 struct gnttab_map_grant_ref *map_op = 967 if (kmap_op != NULL) {
970 (struct gnttab_map_grant_ref *) page->index;
971 set_phys_to_machine(pfn, map_op->dev_bus_addr);
972 if (!PageHighMem(page)) { 968 if (!PageHighMem(page)) {
973 struct multicall_space mcs; 969 struct multicall_space mcs;
974 struct gnttab_unmap_grant_ref *unmap_op; 970 struct gnttab_unmap_grant_ref *unmap_op;
@@ -980,13 +976,13 @@ int m2p_remove_override(struct page *page, bool clear_pte)
980 * issued. In this case handle is going to -1 because 976 * issued. In this case handle is going to -1 because
981 * it hasn't been modified yet. 977 * it hasn't been modified yet.
982 */ 978 */
983 if (map_op->handle == -1) 979 if (kmap_op->handle == -1)
984 xen_mc_flush(); 980 xen_mc_flush();
985 /* 981 /*
986 * Now if map_op->handle is negative it means that the 982 * Now if kmap_op->handle is negative it means that the
987 * hypercall actually returned an error. 983 * hypercall actually returned an error.
988 */ 984 */
989 if (map_op->handle == GNTST_general_error) { 985 if (kmap_op->handle == GNTST_general_error) {
990 printk(KERN_WARNING "m2p_remove_override: " 986 printk(KERN_WARNING "m2p_remove_override: "
991 "pfn %lx mfn %lx, failed to modify kernel mappings", 987 "pfn %lx mfn %lx, failed to modify kernel mappings",
992 pfn, mfn); 988 pfn, mfn);
@@ -996,8 +992,8 @@ int m2p_remove_override(struct page *page, bool clear_pte)
996 mcs = xen_mc_entry( 992 mcs = xen_mc_entry(
997 sizeof(struct gnttab_unmap_grant_ref)); 993 sizeof(struct gnttab_unmap_grant_ref));
998 unmap_op = mcs.args; 994 unmap_op = mcs.args;
999 unmap_op->host_addr = map_op->host_addr; 995 unmap_op->host_addr = kmap_op->host_addr;
1000 unmap_op->handle = map_op->handle; 996 unmap_op->handle = kmap_op->handle;
1001 unmap_op->dev_bus_addr = 0; 997 unmap_op->dev_bus_addr = 0;
1002 998
1003 MULTI_grant_table_op(mcs.mc, 999 MULTI_grant_table_op(mcs.mc,
@@ -1008,10 +1004,9 @@ int m2p_remove_override(struct page *page, bool clear_pte)
1008 set_pte_at(&init_mm, address, ptep, 1004 set_pte_at(&init_mm, address, ptep,
1009 pfn_pte(pfn, PAGE_KERNEL)); 1005 pfn_pte(pfn, PAGE_KERNEL));
1010 __flush_tlb_single(address); 1006 __flush_tlb_single(address);
1011 map_op->host_addr = 0; 1007 kmap_op->host_addr = 0;
1012 } 1008 }
1013 } else 1009 }
1014 set_phys_to_machine(pfn, page->index);
1015 1010
1016 /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present 1011 /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present
1017 * somewhere in this domain, even before being added to the 1012 * somewhere in this domain, even before being added to the
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 3edb320d508f..8971a26d21ab 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -17,6 +17,7 @@
17#include <asm/e820.h> 17#include <asm/e820.h>
18#include <asm/setup.h> 18#include <asm/setup.h>
19#include <asm/acpi.h> 19#include <asm/acpi.h>
20#include <asm/numa.h>
20#include <asm/xen/hypervisor.h> 21#include <asm/xen/hypervisor.h>
21#include <asm/xen/hypercall.h> 22#include <asm/xen/hypercall.h>
22 23
@@ -562,4 +563,7 @@ void __init xen_arch_setup(void)
562 disable_cpufreq(); 563 disable_cpufreq();
563 WARN_ON(set_pm_idle_to_default()); 564 WARN_ON(set_pm_idle_to_default());
564 fiddle_vdso(); 565 fiddle_vdso();
566#ifdef CONFIG_NUMA
567 numa_off = 1;
568#endif
565} 569}
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index f58dca7a6e52..353c50f18702 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -377,7 +377,8 @@ static int __cpuinit xen_cpu_up(unsigned int cpu, struct task_struct *idle)
377 return rc; 377 return rc;
378 378
379 if (num_online_cpus() == 1) 379 if (num_online_cpus() == 1)
380 alternatives_smp_switch(1); 380 /* Just in case we booted with a single CPU. */
381 alternatives_enable_smp();
381 382
382 rc = xen_smp_intr_init(cpu); 383 rc = xen_smp_intr_init(cpu);
383 if (rc) 384 if (rc)
@@ -424,9 +425,6 @@ static void xen_cpu_die(unsigned int cpu)
424 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu), NULL); 425 unbind_from_irqhandler(per_cpu(xen_irq_work, cpu), NULL);
425 xen_uninit_lock_cpu(cpu); 426 xen_uninit_lock_cpu(cpu);
426 xen_teardown_timer(cpu); 427 xen_teardown_timer(cpu);
427
428 if (num_online_cpus() == 1)
429 alternatives_smp_switch(0);
430} 428}
431 429
432static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */ 430static void __cpuinit xen_play_dead(void) /* used only with HOTPLUG_CPU */
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-23 20:07:04 -0500