diff options
Diffstat (limited to 'arch/x86')
118 files changed, 5043 insertions, 2297 deletions
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 34bc3a89228b..748e50a1a152 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig | |||
@@ -40,6 +40,7 @@ config X86 | |||
40 | select HAVE_GENERIC_DMA_COHERENT if X86_32 | 40 | select HAVE_GENERIC_DMA_COHERENT if X86_32 |
41 | select HAVE_EFFICIENT_UNALIGNED_ACCESS | 41 | select HAVE_EFFICIENT_UNALIGNED_ACCESS |
42 | select USER_STACKTRACE_SUPPORT | 42 | select USER_STACKTRACE_SUPPORT |
43 | select HAVE_DMA_API_DEBUG | ||
43 | select HAVE_KERNEL_GZIP | 44 | select HAVE_KERNEL_GZIP |
44 | select HAVE_KERNEL_BZIP2 | 45 | select HAVE_KERNEL_BZIP2 |
45 | select HAVE_KERNEL_LZMA | 46 | select HAVE_KERNEL_LZMA |
@@ -164,6 +165,9 @@ config AUDIT_ARCH | |||
164 | config ARCH_SUPPORTS_OPTIMIZED_INLINING | 165 | config ARCH_SUPPORTS_OPTIMIZED_INLINING |
165 | def_bool y | 166 | def_bool y |
166 | 167 | ||
168 | config ARCH_SUPPORTS_DEBUG_PAGEALLOC | ||
169 | def_bool y | ||
170 | |||
167 | # Use the generic interrupt handling code in kernel/irq/: | 171 | # Use the generic interrupt handling code in kernel/irq/: |
168 | config GENERIC_HARDIRQS | 172 | config GENERIC_HARDIRQS |
169 | bool | 173 | bool |
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index fdb45df608b6..a345cb5447a8 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug | |||
@@ -75,6 +75,7 @@ config DEBUG_STACK_USAGE | |||
75 | config DEBUG_PAGEALLOC | 75 | config DEBUG_PAGEALLOC |
76 | bool "Debug page memory allocations" | 76 | bool "Debug page memory allocations" |
77 | depends on DEBUG_KERNEL | 77 | depends on DEBUG_KERNEL |
78 | depends on ARCH_SUPPORTS_DEBUG_PAGEALLOC | ||
78 | ---help--- | 79 | ---help--- |
79 | Unmap pages from the kernel linear mapping after free_pages(). | 80 | Unmap pages from the kernel linear mapping after free_pages(). |
80 | This results in a large slowdown, but helps to find certain types | 81 | This results in a large slowdown, but helps to find certain types |
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index 903de4aa5094..ebe7deedd5b4 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile | |||
@@ -9,6 +9,7 @@ obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o | |||
9 | obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o | 9 | obj-$(CONFIG_CRYPTO_AES_X86_64) += aes-x86_64.o |
10 | obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o | 10 | obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o |
11 | obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o | 11 | obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o |
12 | obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o | ||
12 | 13 | ||
13 | obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o | 14 | obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o |
14 | 15 | ||
@@ -19,3 +20,5 @@ salsa20-i586-y := salsa20-i586-asm_32.o salsa20_glue.o | |||
19 | aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o | 20 | aes-x86_64-y := aes-x86_64-asm_64.o aes_glue.o |
20 | twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o | 21 | twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o |
21 | salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o | 22 | salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o |
23 | |||
24 | aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o | ||
diff --git a/arch/x86/crypto/aes-i586-asm_32.S b/arch/x86/crypto/aes-i586-asm_32.S index e41b147f4509..b949ec2f9af4 100644 --- a/arch/x86/crypto/aes-i586-asm_32.S +++ b/arch/x86/crypto/aes-i586-asm_32.S | |||
@@ -41,14 +41,14 @@ | |||
41 | #define tlen 1024 // length of each of 4 'xor' arrays (256 32-bit words) | 41 | #define tlen 1024 // length of each of 4 'xor' arrays (256 32-bit words) |
42 | 42 | ||
43 | /* offsets to parameters with one register pushed onto stack */ | 43 | /* offsets to parameters with one register pushed onto stack */ |
44 | #define tfm 8 | 44 | #define ctx 8 |
45 | #define out_blk 12 | 45 | #define out_blk 12 |
46 | #define in_blk 16 | 46 | #define in_blk 16 |
47 | 47 | ||
48 | /* offsets in crypto_tfm structure */ | 48 | /* offsets in crypto_aes_ctx structure */ |
49 | #define klen (crypto_tfm_ctx_offset + 0) | 49 | #define klen (480) |
50 | #define ekey (crypto_tfm_ctx_offset + 4) | 50 | #define ekey (0) |
51 | #define dkey (crypto_tfm_ctx_offset + 244) | 51 | #define dkey (240) |
52 | 52 | ||
53 | // register mapping for encrypt and decrypt subroutines | 53 | // register mapping for encrypt and decrypt subroutines |
54 | 54 | ||
@@ -217,7 +217,7 @@ | |||
217 | do_col (table, r5,r0,r1,r4, r2,r3); /* idx=r5 */ | 217 | do_col (table, r5,r0,r1,r4, r2,r3); /* idx=r5 */ |
218 | 218 | ||
219 | // AES (Rijndael) Encryption Subroutine | 219 | // AES (Rijndael) Encryption Subroutine |
220 | /* void aes_enc_blk(struct crypto_tfm *tfm, u8 *out_blk, const u8 *in_blk) */ | 220 | /* void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */ |
221 | 221 | ||
222 | .global aes_enc_blk | 222 | .global aes_enc_blk |
223 | 223 | ||
@@ -228,7 +228,7 @@ | |||
228 | 228 | ||
229 | aes_enc_blk: | 229 | aes_enc_blk: |
230 | push %ebp | 230 | push %ebp |
231 | mov tfm(%esp),%ebp | 231 | mov ctx(%esp),%ebp |
232 | 232 | ||
233 | // CAUTION: the order and the values used in these assigns | 233 | // CAUTION: the order and the values used in these assigns |
234 | // rely on the register mappings | 234 | // rely on the register mappings |
@@ -292,7 +292,7 @@ aes_enc_blk: | |||
292 | ret | 292 | ret |
293 | 293 | ||
294 | // AES (Rijndael) Decryption Subroutine | 294 | // AES (Rijndael) Decryption Subroutine |
295 | /* void aes_dec_blk(struct crypto_tfm *tfm, u8 *out_blk, const u8 *in_blk) */ | 295 | /* void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out_blk, const u8 *in_blk) */ |
296 | 296 | ||
297 | .global aes_dec_blk | 297 | .global aes_dec_blk |
298 | 298 | ||
@@ -303,7 +303,7 @@ aes_enc_blk: | |||
303 | 303 | ||
304 | aes_dec_blk: | 304 | aes_dec_blk: |
305 | push %ebp | 305 | push %ebp |
306 | mov tfm(%esp),%ebp | 306 | mov ctx(%esp),%ebp |
307 | 307 | ||
308 | // CAUTION: the order and the values used in these assigns | 308 | // CAUTION: the order and the values used in these assigns |
309 | // rely on the register mappings | 309 | // rely on the register mappings |
diff --git a/arch/x86/crypto/aes-x86_64-asm_64.S b/arch/x86/crypto/aes-x86_64-asm_64.S index a120f526c3df..5b577d5a059b 100644 --- a/arch/x86/crypto/aes-x86_64-asm_64.S +++ b/arch/x86/crypto/aes-x86_64-asm_64.S | |||
@@ -17,8 +17,6 @@ | |||
17 | 17 | ||
18 | #include <asm/asm-offsets.h> | 18 | #include <asm/asm-offsets.h> |
19 | 19 | ||
20 | #define BASE crypto_tfm_ctx_offset | ||
21 | |||
22 | #define R1 %rax | 20 | #define R1 %rax |
23 | #define R1E %eax | 21 | #define R1E %eax |
24 | #define R1X %ax | 22 | #define R1X %ax |
@@ -56,13 +54,13 @@ | |||
56 | .align 8; \ | 54 | .align 8; \ |
57 | FUNC: movq r1,r2; \ | 55 | FUNC: movq r1,r2; \ |
58 | movq r3,r4; \ | 56 | movq r3,r4; \ |
59 | leaq BASE+KEY+48+4(r8),r9; \ | 57 | leaq KEY+48(r8),r9; \ |
60 | movq r10,r11; \ | 58 | movq r10,r11; \ |
61 | movl (r7),r5 ## E; \ | 59 | movl (r7),r5 ## E; \ |
62 | movl 4(r7),r1 ## E; \ | 60 | movl 4(r7),r1 ## E; \ |
63 | movl 8(r7),r6 ## E; \ | 61 | movl 8(r7),r6 ## E; \ |
64 | movl 12(r7),r7 ## E; \ | 62 | movl 12(r7),r7 ## E; \ |
65 | movl BASE+0(r8),r10 ## E; \ | 63 | movl 480(r8),r10 ## E; \ |
66 | xorl -48(r9),r5 ## E; \ | 64 | xorl -48(r9),r5 ## E; \ |
67 | xorl -44(r9),r1 ## E; \ | 65 | xorl -44(r9),r1 ## E; \ |
68 | xorl -40(r9),r6 ## E; \ | 66 | xorl -40(r9),r6 ## E; \ |
diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c index 71f457827116..49ae9fe32b22 100644 --- a/arch/x86/crypto/aes_glue.c +++ b/arch/x86/crypto/aes_glue.c | |||
@@ -5,17 +5,29 @@ | |||
5 | 5 | ||
6 | #include <crypto/aes.h> | 6 | #include <crypto/aes.h> |
7 | 7 | ||
8 | asmlinkage void aes_enc_blk(struct crypto_tfm *tfm, u8 *out, const u8 *in); | 8 | asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); |
9 | asmlinkage void aes_dec_blk(struct crypto_tfm *tfm, u8 *out, const u8 *in); | 9 | asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); |
10 | |||
11 | void crypto_aes_encrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) | ||
12 | { | ||
13 | aes_enc_blk(ctx, dst, src); | ||
14 | } | ||
15 | EXPORT_SYMBOL_GPL(crypto_aes_encrypt_x86); | ||
16 | |||
17 | void crypto_aes_decrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) | ||
18 | { | ||
19 | aes_dec_blk(ctx, dst, src); | ||
20 | } | ||
21 | EXPORT_SYMBOL_GPL(crypto_aes_decrypt_x86); | ||
10 | 22 | ||
11 | static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | 23 | static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) |
12 | { | 24 | { |
13 | aes_enc_blk(tfm, dst, src); | 25 | aes_enc_blk(crypto_tfm_ctx(tfm), dst, src); |
14 | } | 26 | } |
15 | 27 | ||
16 | static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | 28 | static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) |
17 | { | 29 | { |
18 | aes_dec_blk(tfm, dst, src); | 30 | aes_dec_blk(crypto_tfm_ctx(tfm), dst, src); |
19 | } | 31 | } |
20 | 32 | ||
21 | static struct crypto_alg aes_alg = { | 33 | static struct crypto_alg aes_alg = { |
diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S new file mode 100644 index 000000000000..caba99601703 --- /dev/null +++ b/arch/x86/crypto/aesni-intel_asm.S | |||
@@ -0,0 +1,896 @@ | |||
1 | /* | ||
2 | * Implement AES algorithm in Intel AES-NI instructions. | ||
3 | * | ||
4 | * The white paper of AES-NI instructions can be downloaded from: | ||
5 | * http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf | ||
6 | * | ||
7 | * Copyright (C) 2008, Intel Corp. | ||
8 | * Author: Huang Ying <ying.huang@intel.com> | ||
9 | * Vinodh Gopal <vinodh.gopal@intel.com> | ||
10 | * Kahraman Akdemir | ||
11 | * | ||
12 | * This program is free software; you can redistribute it and/or modify | ||
13 | * it under the terms of the GNU General Public License as published by | ||
14 | * the Free Software Foundation; either version 2 of the License, or | ||
15 | * (at your option) any later version. | ||
16 | */ | ||
17 | |||
18 | #include <linux/linkage.h> | ||
19 | |||
20 | .text | ||
21 | |||
22 | #define STATE1 %xmm0 | ||
23 | #define STATE2 %xmm4 | ||
24 | #define STATE3 %xmm5 | ||
25 | #define STATE4 %xmm6 | ||
26 | #define STATE STATE1 | ||
27 | #define IN1 %xmm1 | ||
28 | #define IN2 %xmm7 | ||
29 | #define IN3 %xmm8 | ||
30 | #define IN4 %xmm9 | ||
31 | #define IN IN1 | ||
32 | #define KEY %xmm2 | ||
33 | #define IV %xmm3 | ||
34 | |||
35 | #define KEYP %rdi | ||
36 | #define OUTP %rsi | ||
37 | #define INP %rdx | ||
38 | #define LEN %rcx | ||
39 | #define IVP %r8 | ||
40 | #define KLEN %r9d | ||
41 | #define T1 %r10 | ||
42 | #define TKEYP T1 | ||
43 | #define T2 %r11 | ||
44 | |||
45 | _key_expansion_128: | ||
46 | _key_expansion_256a: | ||
47 | pshufd $0b11111111, %xmm1, %xmm1 | ||
48 | shufps $0b00010000, %xmm0, %xmm4 | ||
49 | pxor %xmm4, %xmm0 | ||
50 | shufps $0b10001100, %xmm0, %xmm4 | ||
51 | pxor %xmm4, %xmm0 | ||
52 | pxor %xmm1, %xmm0 | ||
53 | movaps %xmm0, (%rcx) | ||
54 | add $0x10, %rcx | ||
55 | ret | ||
56 | |||
57 | _key_expansion_192a: | ||
58 | pshufd $0b01010101, %xmm1, %xmm1 | ||
59 | shufps $0b00010000, %xmm0, %xmm4 | ||
60 | pxor %xmm4, %xmm0 | ||
61 | shufps $0b10001100, %xmm0, %xmm4 | ||
62 | pxor %xmm4, %xmm0 | ||
63 | pxor %xmm1, %xmm0 | ||
64 | |||
65 | movaps %xmm2, %xmm5 | ||
66 | movaps %xmm2, %xmm6 | ||
67 | pslldq $4, %xmm5 | ||
68 | pshufd $0b11111111, %xmm0, %xmm3 | ||
69 | pxor %xmm3, %xmm2 | ||
70 | pxor %xmm5, %xmm2 | ||
71 | |||
72 | movaps %xmm0, %xmm1 | ||
73 | shufps $0b01000100, %xmm0, %xmm6 | ||
74 | movaps %xmm6, (%rcx) | ||
75 | shufps $0b01001110, %xmm2, %xmm1 | ||
76 | movaps %xmm1, 16(%rcx) | ||
77 | add $0x20, %rcx | ||
78 | ret | ||
79 | |||
80 | _key_expansion_192b: | ||
81 | pshufd $0b01010101, %xmm1, %xmm1 | ||
82 | shufps $0b00010000, %xmm0, %xmm4 | ||
83 | pxor %xmm4, %xmm0 | ||
84 | shufps $0b10001100, %xmm0, %xmm4 | ||
85 | pxor %xmm4, %xmm0 | ||
86 | pxor %xmm1, %xmm0 | ||
87 | |||
88 | movaps %xmm2, %xmm5 | ||
89 | pslldq $4, %xmm5 | ||
90 | pshufd $0b11111111, %xmm0, %xmm3 | ||
91 | pxor %xmm3, %xmm2 | ||
92 | pxor %xmm5, %xmm2 | ||
93 | |||
94 | movaps %xmm0, (%rcx) | ||
95 | add $0x10, %rcx | ||
96 | ret | ||
97 | |||
98 | _key_expansion_256b: | ||
99 | pshufd $0b10101010, %xmm1, %xmm1 | ||
100 | shufps $0b00010000, %xmm2, %xmm4 | ||
101 | pxor %xmm4, %xmm2 | ||
102 | shufps $0b10001100, %xmm2, %xmm4 | ||
103 | pxor %xmm4, %xmm2 | ||
104 | pxor %xmm1, %xmm2 | ||
105 | movaps %xmm2, (%rcx) | ||
106 | add $0x10, %rcx | ||
107 | ret | ||
108 | |||
109 | /* | ||
110 | * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, | ||
111 | * unsigned int key_len) | ||
112 | */ | ||
113 | ENTRY(aesni_set_key) | ||
114 | movups (%rsi), %xmm0 # user key (first 16 bytes) | ||
115 | movaps %xmm0, (%rdi) | ||
116 | lea 0x10(%rdi), %rcx # key addr | ||
117 | movl %edx, 480(%rdi) | ||
118 | pxor %xmm4, %xmm4 # xmm4 is assumed 0 in _key_expansion_x | ||
119 | cmp $24, %dl | ||
120 | jb .Lenc_key128 | ||
121 | je .Lenc_key192 | ||
122 | movups 0x10(%rsi), %xmm2 # other user key | ||
123 | movaps %xmm2, (%rcx) | ||
124 | add $0x10, %rcx | ||
125 | # aeskeygenassist $0x1, %xmm2, %xmm1 # round 1 | ||
126 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x01 | ||
127 | call _key_expansion_256a | ||
128 | # aeskeygenassist $0x1, %xmm0, %xmm1 | ||
129 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x01 | ||
130 | call _key_expansion_256b | ||
131 | # aeskeygenassist $0x2, %xmm2, %xmm1 # round 2 | ||
132 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x02 | ||
133 | call _key_expansion_256a | ||
134 | # aeskeygenassist $0x2, %xmm0, %xmm1 | ||
135 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x02 | ||
136 | call _key_expansion_256b | ||
137 | # aeskeygenassist $0x4, %xmm2, %xmm1 # round 3 | ||
138 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x04 | ||
139 | call _key_expansion_256a | ||
140 | # aeskeygenassist $0x4, %xmm0, %xmm1 | ||
141 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x04 | ||
142 | call _key_expansion_256b | ||
143 | # aeskeygenassist $0x8, %xmm2, %xmm1 # round 4 | ||
144 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x08 | ||
145 | call _key_expansion_256a | ||
146 | # aeskeygenassist $0x8, %xmm0, %xmm1 | ||
147 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x08 | ||
148 | call _key_expansion_256b | ||
149 | # aeskeygenassist $0x10, %xmm2, %xmm1 # round 5 | ||
150 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x10 | ||
151 | call _key_expansion_256a | ||
152 | # aeskeygenassist $0x10, %xmm0, %xmm1 | ||
153 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x10 | ||
154 | call _key_expansion_256b | ||
155 | # aeskeygenassist $0x20, %xmm2, %xmm1 # round 6 | ||
156 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x20 | ||
157 | call _key_expansion_256a | ||
158 | # aeskeygenassist $0x20, %xmm0, %xmm1 | ||
159 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x20 | ||
160 | call _key_expansion_256b | ||
161 | # aeskeygenassist $0x40, %xmm2, %xmm1 # round 7 | ||
162 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x40 | ||
163 | call _key_expansion_256a | ||
164 | jmp .Ldec_key | ||
165 | .Lenc_key192: | ||
166 | movq 0x10(%rsi), %xmm2 # other user key | ||
167 | # aeskeygenassist $0x1, %xmm2, %xmm1 # round 1 | ||
168 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x01 | ||
169 | call _key_expansion_192a | ||
170 | # aeskeygenassist $0x2, %xmm2, %xmm1 # round 2 | ||
171 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x02 | ||
172 | call _key_expansion_192b | ||
173 | # aeskeygenassist $0x4, %xmm2, %xmm1 # round 3 | ||
174 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x04 | ||
175 | call _key_expansion_192a | ||
176 | # aeskeygenassist $0x8, %xmm2, %xmm1 # round 4 | ||
177 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x08 | ||
178 | call _key_expansion_192b | ||
179 | # aeskeygenassist $0x10, %xmm2, %xmm1 # round 5 | ||
180 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x10 | ||
181 | call _key_expansion_192a | ||
182 | # aeskeygenassist $0x20, %xmm2, %xmm1 # round 6 | ||
183 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x20 | ||
184 | call _key_expansion_192b | ||
185 | # aeskeygenassist $0x40, %xmm2, %xmm1 # round 7 | ||
186 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x40 | ||
187 | call _key_expansion_192a | ||
188 | # aeskeygenassist $0x80, %xmm2, %xmm1 # round 8 | ||
189 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xca, 0x80 | ||
190 | call _key_expansion_192b | ||
191 | jmp .Ldec_key | ||
192 | .Lenc_key128: | ||
193 | # aeskeygenassist $0x1, %xmm0, %xmm1 # round 1 | ||
194 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x01 | ||
195 | call _key_expansion_128 | ||
196 | # aeskeygenassist $0x2, %xmm0, %xmm1 # round 2 | ||
197 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x02 | ||
198 | call _key_expansion_128 | ||
199 | # aeskeygenassist $0x4, %xmm0, %xmm1 # round 3 | ||
200 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x04 | ||
201 | call _key_expansion_128 | ||
202 | # aeskeygenassist $0x8, %xmm0, %xmm1 # round 4 | ||
203 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x08 | ||
204 | call _key_expansion_128 | ||
205 | # aeskeygenassist $0x10, %xmm0, %xmm1 # round 5 | ||
206 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x10 | ||
207 | call _key_expansion_128 | ||
208 | # aeskeygenassist $0x20, %xmm0, %xmm1 # round 6 | ||
209 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x20 | ||
210 | call _key_expansion_128 | ||
211 | # aeskeygenassist $0x40, %xmm0, %xmm1 # round 7 | ||
212 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x40 | ||
213 | call _key_expansion_128 | ||
214 | # aeskeygenassist $0x80, %xmm0, %xmm1 # round 8 | ||
215 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x80 | ||
216 | call _key_expansion_128 | ||
217 | # aeskeygenassist $0x1b, %xmm0, %xmm1 # round 9 | ||
218 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x1b | ||
219 | call _key_expansion_128 | ||
220 | # aeskeygenassist $0x36, %xmm0, %xmm1 # round 10 | ||
221 | .byte 0x66, 0x0f, 0x3a, 0xdf, 0xc8, 0x36 | ||
222 | call _key_expansion_128 | ||
223 | .Ldec_key: | ||
224 | sub $0x10, %rcx | ||
225 | movaps (%rdi), %xmm0 | ||
226 | movaps (%rcx), %xmm1 | ||
227 | movaps %xmm0, 240(%rcx) | ||
228 | movaps %xmm1, 240(%rdi) | ||
229 | add $0x10, %rdi | ||
230 | lea 240-16(%rcx), %rsi | ||
231 | .align 4 | ||
232 | .Ldec_key_loop: | ||
233 | movaps (%rdi), %xmm0 | ||
234 | # aesimc %xmm0, %xmm1 | ||
235 | .byte 0x66, 0x0f, 0x38, 0xdb, 0xc8 | ||
236 | movaps %xmm1, (%rsi) | ||
237 | add $0x10, %rdi | ||
238 | sub $0x10, %rsi | ||
239 | cmp %rcx, %rdi | ||
240 | jb .Ldec_key_loop | ||
241 | xor %rax, %rax | ||
242 | ret | ||
243 | |||
244 | /* | ||
245 | * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) | ||
246 | */ | ||
247 | ENTRY(aesni_enc) | ||
248 | movl 480(KEYP), KLEN # key length | ||
249 | movups (INP), STATE # input | ||
250 | call _aesni_enc1 | ||
251 | movups STATE, (OUTP) # output | ||
252 | ret | ||
253 | |||
254 | /* | ||
255 | * _aesni_enc1: internal ABI | ||
256 | * input: | ||
257 | * KEYP: key struct pointer | ||
258 | * KLEN: round count | ||
259 | * STATE: initial state (input) | ||
260 | * output: | ||
261 | * STATE: finial state (output) | ||
262 | * changed: | ||
263 | * KEY | ||
264 | * TKEYP (T1) | ||
265 | */ | ||
266 | _aesni_enc1: | ||
267 | movaps (KEYP), KEY # key | ||
268 | mov KEYP, TKEYP | ||
269 | pxor KEY, STATE # round 0 | ||
270 | add $0x30, TKEYP | ||
271 | cmp $24, KLEN | ||
272 | jb .Lenc128 | ||
273 | lea 0x20(TKEYP), TKEYP | ||
274 | je .Lenc192 | ||
275 | add $0x20, TKEYP | ||
276 | movaps -0x60(TKEYP), KEY | ||
277 | # aesenc KEY, STATE | ||
278 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
279 | movaps -0x50(TKEYP), KEY | ||
280 | # aesenc KEY, STATE | ||
281 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
282 | .align 4 | ||
283 | .Lenc192: | ||
284 | movaps -0x40(TKEYP), KEY | ||
285 | # aesenc KEY, STATE | ||
286 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
287 | movaps -0x30(TKEYP), KEY | ||
288 | # aesenc KEY, STATE | ||
289 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
290 | .align 4 | ||
291 | .Lenc128: | ||
292 | movaps -0x20(TKEYP), KEY | ||
293 | # aesenc KEY, STATE | ||
294 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
295 | movaps -0x10(TKEYP), KEY | ||
296 | # aesenc KEY, STATE | ||
297 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
298 | movaps (TKEYP), KEY | ||
299 | # aesenc KEY, STATE | ||
300 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
301 | movaps 0x10(TKEYP), KEY | ||
302 | # aesenc KEY, STATE | ||
303 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
304 | movaps 0x20(TKEYP), KEY | ||
305 | # aesenc KEY, STATE | ||
306 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
307 | movaps 0x30(TKEYP), KEY | ||
308 | # aesenc KEY, STATE | ||
309 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
310 | movaps 0x40(TKEYP), KEY | ||
311 | # aesenc KEY, STATE | ||
312 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
313 | movaps 0x50(TKEYP), KEY | ||
314 | # aesenc KEY, STATE | ||
315 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
316 | movaps 0x60(TKEYP), KEY | ||
317 | # aesenc KEY, STATE | ||
318 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
319 | movaps 0x70(TKEYP), KEY | ||
320 | # aesenclast KEY, STATE # last round | ||
321 | .byte 0x66, 0x0f, 0x38, 0xdd, 0xc2 | ||
322 | ret | ||
323 | |||
324 | /* | ||
325 | * _aesni_enc4: internal ABI | ||
326 | * input: | ||
327 | * KEYP: key struct pointer | ||
328 | * KLEN: round count | ||
329 | * STATE1: initial state (input) | ||
330 | * STATE2 | ||
331 | * STATE3 | ||
332 | * STATE4 | ||
333 | * output: | ||
334 | * STATE1: finial state (output) | ||
335 | * STATE2 | ||
336 | * STATE3 | ||
337 | * STATE4 | ||
338 | * changed: | ||
339 | * KEY | ||
340 | * TKEYP (T1) | ||
341 | */ | ||
342 | _aesni_enc4: | ||
343 | movaps (KEYP), KEY # key | ||
344 | mov KEYP, TKEYP | ||
345 | pxor KEY, STATE1 # round 0 | ||
346 | pxor KEY, STATE2 | ||
347 | pxor KEY, STATE3 | ||
348 | pxor KEY, STATE4 | ||
349 | add $0x30, TKEYP | ||
350 | cmp $24, KLEN | ||
351 | jb .L4enc128 | ||
352 | lea 0x20(TKEYP), TKEYP | ||
353 | je .L4enc192 | ||
354 | add $0x20, TKEYP | ||
355 | movaps -0x60(TKEYP), KEY | ||
356 | # aesenc KEY, STATE1 | ||
357 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
358 | # aesenc KEY, STATE2 | ||
359 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
360 | # aesenc KEY, STATE3 | ||
361 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
362 | # aesenc KEY, STATE4 | ||
363 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
364 | movaps -0x50(TKEYP), KEY | ||
365 | # aesenc KEY, STATE1 | ||
366 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
367 | # aesenc KEY, STATE2 | ||
368 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
369 | # aesenc KEY, STATE3 | ||
370 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
371 | # aesenc KEY, STATE4 | ||
372 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
373 | #.align 4 | ||
374 | .L4enc192: | ||
375 | movaps -0x40(TKEYP), KEY | ||
376 | # aesenc KEY, STATE1 | ||
377 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
378 | # aesenc KEY, STATE2 | ||
379 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
380 | # aesenc KEY, STATE3 | ||
381 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
382 | # aesenc KEY, STATE4 | ||
383 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
384 | movaps -0x30(TKEYP), KEY | ||
385 | # aesenc KEY, STATE1 | ||
386 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
387 | # aesenc KEY, STATE2 | ||
388 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
389 | # aesenc KEY, STATE3 | ||
390 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
391 | # aesenc KEY, STATE4 | ||
392 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
393 | #.align 4 | ||
394 | .L4enc128: | ||
395 | movaps -0x20(TKEYP), KEY | ||
396 | # aesenc KEY, STATE1 | ||
397 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
398 | # aesenc KEY, STATE2 | ||
399 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
400 | # aesenc KEY, STATE3 | ||
401 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
402 | # aesenc KEY, STATE4 | ||
403 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
404 | movaps -0x10(TKEYP), KEY | ||
405 | # aesenc KEY, STATE1 | ||
406 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
407 | # aesenc KEY, STATE2 | ||
408 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
409 | # aesenc KEY, STATE3 | ||
410 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
411 | # aesenc KEY, STATE4 | ||
412 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
413 | movaps (TKEYP), KEY | ||
414 | # aesenc KEY, STATE1 | ||
415 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
416 | # aesenc KEY, STATE2 | ||
417 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
418 | # aesenc KEY, STATE3 | ||
419 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
420 | # aesenc KEY, STATE4 | ||
421 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
422 | movaps 0x10(TKEYP), KEY | ||
423 | # aesenc KEY, STATE1 | ||
424 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
425 | # aesenc KEY, STATE2 | ||
426 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
427 | # aesenc KEY, STATE3 | ||
428 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
429 | # aesenc KEY, STATE4 | ||
430 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
431 | movaps 0x20(TKEYP), KEY | ||
432 | # aesenc KEY, STATE1 | ||
433 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
434 | # aesenc KEY, STATE2 | ||
435 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
436 | # aesenc KEY, STATE3 | ||
437 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
438 | # aesenc KEY, STATE4 | ||
439 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
440 | movaps 0x30(TKEYP), KEY | ||
441 | # aesenc KEY, STATE1 | ||
442 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
443 | # aesenc KEY, STATE2 | ||
444 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
445 | # aesenc KEY, STATE3 | ||
446 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
447 | # aesenc KEY, STATE4 | ||
448 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
449 | movaps 0x40(TKEYP), KEY | ||
450 | # aesenc KEY, STATE1 | ||
451 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
452 | # aesenc KEY, STATE2 | ||
453 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
454 | # aesenc KEY, STATE3 | ||
455 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
456 | # aesenc KEY, STATE4 | ||
457 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
458 | movaps 0x50(TKEYP), KEY | ||
459 | # aesenc KEY, STATE1 | ||
460 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
461 | # aesenc KEY, STATE2 | ||
462 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
463 | # aesenc KEY, STATE3 | ||
464 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
465 | # aesenc KEY, STATE4 | ||
466 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
467 | movaps 0x60(TKEYP), KEY | ||
468 | # aesenc KEY, STATE1 | ||
469 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xc2 | ||
470 | # aesenc KEY, STATE2 | ||
471 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xe2 | ||
472 | # aesenc KEY, STATE3 | ||
473 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xea | ||
474 | # aesenc KEY, STATE4 | ||
475 | .byte 0x66, 0x0f, 0x38, 0xdc, 0xf2 | ||
476 | movaps 0x70(TKEYP), KEY | ||
477 | # aesenclast KEY, STATE1 # last round | ||
478 | .byte 0x66, 0x0f, 0x38, 0xdd, 0xc2 | ||
479 | # aesenclast KEY, STATE2 | ||
480 | .byte 0x66, 0x0f, 0x38, 0xdd, 0xe2 | ||
481 | # aesenclast KEY, STATE3 | ||
482 | .byte 0x66, 0x0f, 0x38, 0xdd, 0xea | ||
483 | # aesenclast KEY, STATE4 | ||
484 | .byte 0x66, 0x0f, 0x38, 0xdd, 0xf2 | ||
485 | ret | ||
486 | |||
487 | /* | ||
488 | * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src) | ||
489 | */ | ||
490 | ENTRY(aesni_dec) | ||
491 | mov 480(KEYP), KLEN # key length | ||
492 | add $240, KEYP | ||
493 | movups (INP), STATE # input | ||
494 | call _aesni_dec1 | ||
495 | movups STATE, (OUTP) #output | ||
496 | ret | ||
497 | |||
498 | /* | ||
499 | * _aesni_dec1: internal ABI | ||
500 | * input: | ||
501 | * KEYP: key struct pointer | ||
502 | * KLEN: key length | ||
503 | * STATE: initial state (input) | ||
504 | * output: | ||
505 | * STATE: finial state (output) | ||
506 | * changed: | ||
507 | * KEY | ||
508 | * TKEYP (T1) | ||
509 | */ | ||
510 | _aesni_dec1: | ||
511 | movaps (KEYP), KEY # key | ||
512 | mov KEYP, TKEYP | ||
513 | pxor KEY, STATE # round 0 | ||
514 | add $0x30, TKEYP | ||
515 | cmp $24, KLEN | ||
516 | jb .Ldec128 | ||
517 | lea 0x20(TKEYP), TKEYP | ||
518 | je .Ldec192 | ||
519 | add $0x20, TKEYP | ||
520 | movaps -0x60(TKEYP), KEY | ||
521 | # aesdec KEY, STATE | ||
522 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
523 | movaps -0x50(TKEYP), KEY | ||
524 | # aesdec KEY, STATE | ||
525 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
526 | .align 4 | ||
527 | .Ldec192: | ||
528 | movaps -0x40(TKEYP), KEY | ||
529 | # aesdec KEY, STATE | ||
530 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
531 | movaps -0x30(TKEYP), KEY | ||
532 | # aesdec KEY, STATE | ||
533 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
534 | .align 4 | ||
535 | .Ldec128: | ||
536 | movaps -0x20(TKEYP), KEY | ||
537 | # aesdec KEY, STATE | ||
538 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
539 | movaps -0x10(TKEYP), KEY | ||
540 | # aesdec KEY, STATE | ||
541 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
542 | movaps (TKEYP), KEY | ||
543 | # aesdec KEY, STATE | ||
544 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
545 | movaps 0x10(TKEYP), KEY | ||
546 | # aesdec KEY, STATE | ||
547 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
548 | movaps 0x20(TKEYP), KEY | ||
549 | # aesdec KEY, STATE | ||
550 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
551 | movaps 0x30(TKEYP), KEY | ||
552 | # aesdec KEY, STATE | ||
553 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
554 | movaps 0x40(TKEYP), KEY | ||
555 | # aesdec KEY, STATE | ||
556 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
557 | movaps 0x50(TKEYP), KEY | ||
558 | # aesdec KEY, STATE | ||
559 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
560 | movaps 0x60(TKEYP), KEY | ||
561 | # aesdec KEY, STATE | ||
562 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
563 | movaps 0x70(TKEYP), KEY | ||
564 | # aesdeclast KEY, STATE # last round | ||
565 | .byte 0x66, 0x0f, 0x38, 0xdf, 0xc2 | ||
566 | ret | ||
567 | |||
568 | /* | ||
569 | * _aesni_dec4: internal ABI | ||
570 | * input: | ||
571 | * KEYP: key struct pointer | ||
572 | * KLEN: key length | ||
573 | * STATE1: initial state (input) | ||
574 | * STATE2 | ||
575 | * STATE3 | ||
576 | * STATE4 | ||
577 | * output: | ||
578 | * STATE1: finial state (output) | ||
579 | * STATE2 | ||
580 | * STATE3 | ||
581 | * STATE4 | ||
582 | * changed: | ||
583 | * KEY | ||
584 | * TKEYP (T1) | ||
585 | */ | ||
586 | _aesni_dec4: | ||
587 | movaps (KEYP), KEY # key | ||
588 | mov KEYP, TKEYP | ||
589 | pxor KEY, STATE1 # round 0 | ||
590 | pxor KEY, STATE2 | ||
591 | pxor KEY, STATE3 | ||
592 | pxor KEY, STATE4 | ||
593 | add $0x30, TKEYP | ||
594 | cmp $24, KLEN | ||
595 | jb .L4dec128 | ||
596 | lea 0x20(TKEYP), TKEYP | ||
597 | je .L4dec192 | ||
598 | add $0x20, TKEYP | ||
599 | movaps -0x60(TKEYP), KEY | ||
600 | # aesdec KEY, STATE1 | ||
601 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
602 | # aesdec KEY, STATE2 | ||
603 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
604 | # aesdec KEY, STATE3 | ||
605 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
606 | # aesdec KEY, STATE4 | ||
607 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
608 | movaps -0x50(TKEYP), KEY | ||
609 | # aesdec KEY, STATE1 | ||
610 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
611 | # aesdec KEY, STATE2 | ||
612 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
613 | # aesdec KEY, STATE3 | ||
614 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
615 | # aesdec KEY, STATE4 | ||
616 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
617 | .align 4 | ||
618 | .L4dec192: | ||
619 | movaps -0x40(TKEYP), KEY | ||
620 | # aesdec KEY, STATE1 | ||
621 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
622 | # aesdec KEY, STATE2 | ||
623 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
624 | # aesdec KEY, STATE3 | ||
625 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
626 | # aesdec KEY, STATE4 | ||
627 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
628 | movaps -0x30(TKEYP), KEY | ||
629 | # aesdec KEY, STATE1 | ||
630 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
631 | # aesdec KEY, STATE2 | ||
632 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
633 | # aesdec KEY, STATE3 | ||
634 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
635 | # aesdec KEY, STATE4 | ||
636 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
637 | .align 4 | ||
638 | .L4dec128: | ||
639 | movaps -0x20(TKEYP), KEY | ||
640 | # aesdec KEY, STATE1 | ||
641 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
642 | # aesdec KEY, STATE2 | ||
643 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
644 | # aesdec KEY, STATE3 | ||
645 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
646 | # aesdec KEY, STATE4 | ||
647 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
648 | movaps -0x10(TKEYP), KEY | ||
649 | # aesdec KEY, STATE1 | ||
650 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
651 | # aesdec KEY, STATE2 | ||
652 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
653 | # aesdec KEY, STATE3 | ||
654 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
655 | # aesdec KEY, STATE4 | ||
656 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
657 | movaps (TKEYP), KEY | ||
658 | # aesdec KEY, STATE1 | ||
659 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
660 | # aesdec KEY, STATE2 | ||
661 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
662 | # aesdec KEY, STATE3 | ||
663 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
664 | # aesdec KEY, STATE4 | ||
665 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
666 | movaps 0x10(TKEYP), KEY | ||
667 | # aesdec KEY, STATE1 | ||
668 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
669 | # aesdec KEY, STATE2 | ||
670 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
671 | # aesdec KEY, STATE3 | ||
672 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
673 | # aesdec KEY, STATE4 | ||
674 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
675 | movaps 0x20(TKEYP), KEY | ||
676 | # aesdec KEY, STATE1 | ||
677 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
678 | # aesdec KEY, STATE2 | ||
679 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
680 | # aesdec KEY, STATE3 | ||
681 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
682 | # aesdec KEY, STATE4 | ||
683 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
684 | movaps 0x30(TKEYP), KEY | ||
685 | # aesdec KEY, STATE1 | ||
686 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
687 | # aesdec KEY, STATE2 | ||
688 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
689 | # aesdec KEY, STATE3 | ||
690 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
691 | # aesdec KEY, STATE4 | ||
692 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
693 | movaps 0x40(TKEYP), KEY | ||
694 | # aesdec KEY, STATE1 | ||
695 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
696 | # aesdec KEY, STATE2 | ||
697 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
698 | # aesdec KEY, STATE3 | ||
699 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
700 | # aesdec KEY, STATE4 | ||
701 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
702 | movaps 0x50(TKEYP), KEY | ||
703 | # aesdec KEY, STATE1 | ||
704 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
705 | # aesdec KEY, STATE2 | ||
706 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
707 | # aesdec KEY, STATE3 | ||
708 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
709 | # aesdec KEY, STATE4 | ||
710 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
711 | movaps 0x60(TKEYP), KEY | ||
712 | # aesdec KEY, STATE1 | ||
713 | .byte 0x66, 0x0f, 0x38, 0xde, 0xc2 | ||
714 | # aesdec KEY, STATE2 | ||
715 | .byte 0x66, 0x0f, 0x38, 0xde, 0xe2 | ||
716 | # aesdec KEY, STATE3 | ||
717 | .byte 0x66, 0x0f, 0x38, 0xde, 0xea | ||
718 | # aesdec KEY, STATE4 | ||
719 | .byte 0x66, 0x0f, 0x38, 0xde, 0xf2 | ||
720 | movaps 0x70(TKEYP), KEY | ||
721 | # aesdeclast KEY, STATE1 # last round | ||
722 | .byte 0x66, 0x0f, 0x38, 0xdf, 0xc2 | ||
723 | # aesdeclast KEY, STATE2 | ||
724 | .byte 0x66, 0x0f, 0x38, 0xdf, 0xe2 | ||
725 | # aesdeclast KEY, STATE3 | ||
726 | .byte 0x66, 0x0f, 0x38, 0xdf, 0xea | ||
727 | # aesdeclast KEY, STATE4 | ||
728 | .byte 0x66, 0x0f, 0x38, 0xdf, 0xf2 | ||
729 | ret | ||
730 | |||
731 | /* | ||
732 | * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, | ||
733 | * size_t len) | ||
734 | */ | ||
735 | ENTRY(aesni_ecb_enc) | ||
736 | test LEN, LEN # check length | ||
737 | jz .Lecb_enc_ret | ||
738 | mov 480(KEYP), KLEN | ||
739 | cmp $16, LEN | ||
740 | jb .Lecb_enc_ret | ||
741 | cmp $64, LEN | ||
742 | jb .Lecb_enc_loop1 | ||
743 | .align 4 | ||
744 | .Lecb_enc_loop4: | ||
745 | movups (INP), STATE1 | ||
746 | movups 0x10(INP), STATE2 | ||
747 | movups 0x20(INP), STATE3 | ||
748 | movups 0x30(INP), STATE4 | ||
749 | call _aesni_enc4 | ||
750 | movups STATE1, (OUTP) | ||
751 | movups STATE2, 0x10(OUTP) | ||
752 | movups STATE3, 0x20(OUTP) | ||
753 | movups STATE4, 0x30(OUTP) | ||
754 | sub $64, LEN | ||
755 | add $64, INP | ||
756 | add $64, OUTP | ||
757 | cmp $64, LEN | ||
758 | jge .Lecb_enc_loop4 | ||
759 | cmp $16, LEN | ||
760 | jb .Lecb_enc_ret | ||
761 | .align 4 | ||
762 | .Lecb_enc_loop1: | ||
763 | movups (INP), STATE1 | ||
764 | call _aesni_enc1 | ||
765 | movups STATE1, (OUTP) | ||
766 | sub $16, LEN | ||
767 | add $16, INP | ||
768 | add $16, OUTP | ||
769 | cmp $16, LEN | ||
770 | jge .Lecb_enc_loop1 | ||
771 | .Lecb_enc_ret: | ||
772 | ret | ||
773 | |||
774 | /* | ||
775 | * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, | ||
776 | * size_t len); | ||
777 | */ | ||
778 | ENTRY(aesni_ecb_dec) | ||
779 | test LEN, LEN | ||
780 | jz .Lecb_dec_ret | ||
781 | mov 480(KEYP), KLEN | ||
782 | add $240, KEYP | ||
783 | cmp $16, LEN | ||
784 | jb .Lecb_dec_ret | ||
785 | cmp $64, LEN | ||
786 | jb .Lecb_dec_loop1 | ||
787 | .align 4 | ||
788 | .Lecb_dec_loop4: | ||
789 | movups (INP), STATE1 | ||
790 | movups 0x10(INP), STATE2 | ||
791 | movups 0x20(INP), STATE3 | ||
792 | movups 0x30(INP), STATE4 | ||
793 | call _aesni_dec4 | ||
794 | movups STATE1, (OUTP) | ||
795 | movups STATE2, 0x10(OUTP) | ||
796 | movups STATE3, 0x20(OUTP) | ||
797 | movups STATE4, 0x30(OUTP) | ||
798 | sub $64, LEN | ||
799 | add $64, INP | ||
800 | add $64, OUTP | ||
801 | cmp $64, LEN | ||
802 | jge .Lecb_dec_loop4 | ||
803 | cmp $16, LEN | ||
804 | jb .Lecb_dec_ret | ||
805 | .align 4 | ||
806 | .Lecb_dec_loop1: | ||
807 | movups (INP), STATE1 | ||
808 | call _aesni_dec1 | ||
809 | movups STATE1, (OUTP) | ||
810 | sub $16, LEN | ||
811 | add $16, INP | ||
812 | add $16, OUTP | ||
813 | cmp $16, LEN | ||
814 | jge .Lecb_dec_loop1 | ||
815 | .Lecb_dec_ret: | ||
816 | ret | ||
817 | |||
818 | /* | ||
819 | * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, | ||
820 | * size_t len, u8 *iv) | ||
821 | */ | ||
822 | ENTRY(aesni_cbc_enc) | ||
823 | cmp $16, LEN | ||
824 | jb .Lcbc_enc_ret | ||
825 | mov 480(KEYP), KLEN | ||
826 | movups (IVP), STATE # load iv as initial state | ||
827 | .align 4 | ||
828 | .Lcbc_enc_loop: | ||
829 | movups (INP), IN # load input | ||
830 | pxor IN, STATE | ||
831 | call _aesni_enc1 | ||
832 | movups STATE, (OUTP) # store output | ||
833 | sub $16, LEN | ||
834 | add $16, INP | ||
835 | add $16, OUTP | ||
836 | cmp $16, LEN | ||
837 | jge .Lcbc_enc_loop | ||
838 | movups STATE, (IVP) | ||
839 | .Lcbc_enc_ret: | ||
840 | ret | ||
841 | |||
842 | /* | ||
843 | * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src, | ||
844 | * size_t len, u8 *iv) | ||
845 | */ | ||
846 | ENTRY(aesni_cbc_dec) | ||
847 | cmp $16, LEN | ||
848 | jb .Lcbc_dec_ret | ||
849 | mov 480(KEYP), KLEN | ||
850 | add $240, KEYP | ||
851 | movups (IVP), IV | ||
852 | cmp $64, LEN | ||
853 | jb .Lcbc_dec_loop1 | ||
854 | .align 4 | ||
855 | .Lcbc_dec_loop4: | ||
856 | movups (INP), IN1 | ||
857 | movaps IN1, STATE1 | ||
858 | movups 0x10(INP), IN2 | ||
859 | movaps IN2, STATE2 | ||
860 | movups 0x20(INP), IN3 | ||
861 | movaps IN3, STATE3 | ||
862 | movups 0x30(INP), IN4 | ||
863 | movaps IN4, STATE4 | ||
864 | call _aesni_dec4 | ||
865 | pxor IV, STATE1 | ||
866 | pxor IN1, STATE2 | ||
867 | pxor IN2, STATE3 | ||
868 | pxor IN3, STATE4 | ||
869 | movaps IN4, IV | ||
870 | movups STATE1, (OUTP) | ||
871 | movups STATE2, 0x10(OUTP) | ||
872 | movups STATE3, 0x20(OUTP) | ||
873 | movups STATE4, 0x30(OUTP) | ||
874 | sub $64, LEN | ||
875 | add $64, INP | ||
876 | add $64, OUTP | ||
877 | cmp $64, LEN | ||
878 | jge .Lcbc_dec_loop4 | ||
879 | cmp $16, LEN | ||
880 | jb .Lcbc_dec_ret | ||
881 | .align 4 | ||
882 | .Lcbc_dec_loop1: | ||
883 | movups (INP), IN | ||
884 | movaps IN, STATE | ||
885 | call _aesni_dec1 | ||
886 | pxor IV, STATE | ||
887 | movups STATE, (OUTP) | ||
888 | movaps IN, IV | ||
889 | sub $16, LEN | ||
890 | add $16, INP | ||
891 | add $16, OUTP | ||
892 | cmp $16, LEN | ||
893 | jge .Lcbc_dec_loop1 | ||
894 | movups IV, (IVP) | ||
895 | .Lcbc_dec_ret: | ||
896 | ret | ||
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c new file mode 100644 index 000000000000..02af0af65497 --- /dev/null +++ b/arch/x86/crypto/aesni-intel_glue.c | |||
@@ -0,0 +1,461 @@ | |||
1 | /* | ||
2 | * Support for Intel AES-NI instructions. This file contains glue | ||
3 | * code, the real AES implementation is in intel-aes_asm.S. | ||
4 | * | ||
5 | * Copyright (C) 2008, Intel Corp. | ||
6 | * Author: Huang Ying <ying.huang@intel.com> | ||
7 | * | ||
8 | * This program is free software; you can redistribute it and/or modify | ||
9 | * it under the terms of the GNU General Public License as published by | ||
10 | * the Free Software Foundation; either version 2 of the License, or | ||
11 | * (at your option) any later version. | ||
12 | */ | ||
13 | |||
14 | #include <linux/hardirq.h> | ||
15 | #include <linux/types.h> | ||
16 | #include <linux/crypto.h> | ||
17 | #include <linux/err.h> | ||
18 | #include <crypto/algapi.h> | ||
19 | #include <crypto/aes.h> | ||
20 | #include <crypto/cryptd.h> | ||
21 | #include <asm/i387.h> | ||
22 | #include <asm/aes.h> | ||
23 | |||
24 | struct async_aes_ctx { | ||
25 | struct cryptd_ablkcipher *cryptd_tfm; | ||
26 | }; | ||
27 | |||
28 | #define AESNI_ALIGN 16 | ||
29 | #define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) | ||
30 | |||
31 | asmlinkage int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, | ||
32 | unsigned int key_len); | ||
33 | asmlinkage void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, | ||
34 | const u8 *in); | ||
35 | asmlinkage void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, | ||
36 | const u8 *in); | ||
37 | asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, | ||
38 | const u8 *in, unsigned int len); | ||
39 | asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, | ||
40 | const u8 *in, unsigned int len); | ||
41 | asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, | ||
42 | const u8 *in, unsigned int len, u8 *iv); | ||
43 | asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, | ||
44 | const u8 *in, unsigned int len, u8 *iv); | ||
45 | |||
46 | static inline int kernel_fpu_using(void) | ||
47 | { | ||
48 | if (in_interrupt() && !(read_cr0() & X86_CR0_TS)) | ||
49 | return 1; | ||
50 | return 0; | ||
51 | } | ||
52 | |||
53 | static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx) | ||
54 | { | ||
55 | unsigned long addr = (unsigned long)raw_ctx; | ||
56 | unsigned long align = AESNI_ALIGN; | ||
57 | |||
58 | if (align <= crypto_tfm_ctx_alignment()) | ||
59 | align = 1; | ||
60 | return (struct crypto_aes_ctx *)ALIGN(addr, align); | ||
61 | } | ||
62 | |||
63 | static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, | ||
64 | const u8 *in_key, unsigned int key_len) | ||
65 | { | ||
66 | struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); | ||
67 | u32 *flags = &tfm->crt_flags; | ||
68 | int err; | ||
69 | |||
70 | if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && | ||
71 | key_len != AES_KEYSIZE_256) { | ||
72 | *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; | ||
73 | return -EINVAL; | ||
74 | } | ||
75 | |||
76 | if (kernel_fpu_using()) | ||
77 | err = crypto_aes_expand_key(ctx, in_key, key_len); | ||
78 | else { | ||
79 | kernel_fpu_begin(); | ||
80 | err = aesni_set_key(ctx, in_key, key_len); | ||
81 | kernel_fpu_end(); | ||
82 | } | ||
83 | |||
84 | return err; | ||
85 | } | ||
86 | |||
87 | static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, | ||
88 | unsigned int key_len) | ||
89 | { | ||
90 | return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); | ||
91 | } | ||
92 | |||
93 | static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | ||
94 | { | ||
95 | struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); | ||
96 | |||
97 | if (kernel_fpu_using()) | ||
98 | crypto_aes_encrypt_x86(ctx, dst, src); | ||
99 | else { | ||
100 | kernel_fpu_begin(); | ||
101 | aesni_enc(ctx, dst, src); | ||
102 | kernel_fpu_end(); | ||
103 | } | ||
104 | } | ||
105 | |||
106 | static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) | ||
107 | { | ||
108 | struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); | ||
109 | |||
110 | if (kernel_fpu_using()) | ||
111 | crypto_aes_decrypt_x86(ctx, dst, src); | ||
112 | else { | ||
113 | kernel_fpu_begin(); | ||
114 | aesni_dec(ctx, dst, src); | ||
115 | kernel_fpu_end(); | ||
116 | } | ||
117 | } | ||
118 | |||
119 | static struct crypto_alg aesni_alg = { | ||
120 | .cra_name = "aes", | ||
121 | .cra_driver_name = "aes-aesni", | ||
122 | .cra_priority = 300, | ||
123 | .cra_flags = CRYPTO_ALG_TYPE_CIPHER, | ||
124 | .cra_blocksize = AES_BLOCK_SIZE, | ||
125 | .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, | ||
126 | .cra_alignmask = 0, | ||
127 | .cra_module = THIS_MODULE, | ||
128 | .cra_list = LIST_HEAD_INIT(aesni_alg.cra_list), | ||
129 | .cra_u = { | ||
130 | .cipher = { | ||
131 | .cia_min_keysize = AES_MIN_KEY_SIZE, | ||
132 | .cia_max_keysize = AES_MAX_KEY_SIZE, | ||
133 | .cia_setkey = aes_set_key, | ||
134 | .cia_encrypt = aes_encrypt, | ||
135 | .cia_decrypt = aes_decrypt | ||
136 | } | ||
137 | } | ||
138 | }; | ||
139 | |||
140 | static int ecb_encrypt(struct blkcipher_desc *desc, | ||
141 | struct scatterlist *dst, struct scatterlist *src, | ||
142 | unsigned int nbytes) | ||
143 | { | ||
144 | struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); | ||
145 | struct blkcipher_walk walk; | ||
146 | int err; | ||
147 | |||
148 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
149 | err = blkcipher_walk_virt(desc, &walk); | ||
150 | |||
151 | kernel_fpu_begin(); | ||
152 | while ((nbytes = walk.nbytes)) { | ||
153 | aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, | ||
154 | nbytes & AES_BLOCK_MASK); | ||
155 | nbytes &= AES_BLOCK_SIZE - 1; | ||
156 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
157 | } | ||
158 | kernel_fpu_end(); | ||
159 | |||
160 | return err; | ||
161 | } | ||
162 | |||
163 | static int ecb_decrypt(struct blkcipher_desc *desc, | ||
164 | struct scatterlist *dst, struct scatterlist *src, | ||
165 | unsigned int nbytes) | ||
166 | { | ||
167 | struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); | ||
168 | struct blkcipher_walk walk; | ||
169 | int err; | ||
170 | |||
171 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
172 | err = blkcipher_walk_virt(desc, &walk); | ||
173 | |||
174 | kernel_fpu_begin(); | ||
175 | while ((nbytes = walk.nbytes)) { | ||
176 | aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, | ||
177 | nbytes & AES_BLOCK_MASK); | ||
178 | nbytes &= AES_BLOCK_SIZE - 1; | ||
179 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
180 | } | ||
181 | kernel_fpu_end(); | ||
182 | |||
183 | return err; | ||
184 | } | ||
185 | |||
186 | static struct crypto_alg blk_ecb_alg = { | ||
187 | .cra_name = "__ecb-aes-aesni", | ||
188 | .cra_driver_name = "__driver-ecb-aes-aesni", | ||
189 | .cra_priority = 0, | ||
190 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
191 | .cra_blocksize = AES_BLOCK_SIZE, | ||
192 | .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, | ||
193 | .cra_alignmask = 0, | ||
194 | .cra_type = &crypto_blkcipher_type, | ||
195 | .cra_module = THIS_MODULE, | ||
196 | .cra_list = LIST_HEAD_INIT(blk_ecb_alg.cra_list), | ||
197 | .cra_u = { | ||
198 | .blkcipher = { | ||
199 | .min_keysize = AES_MIN_KEY_SIZE, | ||
200 | .max_keysize = AES_MAX_KEY_SIZE, | ||
201 | .setkey = aes_set_key, | ||
202 | .encrypt = ecb_encrypt, | ||
203 | .decrypt = ecb_decrypt, | ||
204 | }, | ||
205 | }, | ||
206 | }; | ||
207 | |||
208 | static int cbc_encrypt(struct blkcipher_desc *desc, | ||
209 | struct scatterlist *dst, struct scatterlist *src, | ||
210 | unsigned int nbytes) | ||
211 | { | ||
212 | struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); | ||
213 | struct blkcipher_walk walk; | ||
214 | int err; | ||
215 | |||
216 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
217 | err = blkcipher_walk_virt(desc, &walk); | ||
218 | |||
219 | kernel_fpu_begin(); | ||
220 | while ((nbytes = walk.nbytes)) { | ||
221 | aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, | ||
222 | nbytes & AES_BLOCK_MASK, walk.iv); | ||
223 | nbytes &= AES_BLOCK_SIZE - 1; | ||
224 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
225 | } | ||
226 | kernel_fpu_end(); | ||
227 | |||
228 | return err; | ||
229 | } | ||
230 | |||
231 | static int cbc_decrypt(struct blkcipher_desc *desc, | ||
232 | struct scatterlist *dst, struct scatterlist *src, | ||
233 | unsigned int nbytes) | ||
234 | { | ||
235 | struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); | ||
236 | struct blkcipher_walk walk; | ||
237 | int err; | ||
238 | |||
239 | blkcipher_walk_init(&walk, dst, src, nbytes); | ||
240 | err = blkcipher_walk_virt(desc, &walk); | ||
241 | |||
242 | kernel_fpu_begin(); | ||
243 | while ((nbytes = walk.nbytes)) { | ||
244 | aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, | ||
245 | nbytes & AES_BLOCK_MASK, walk.iv); | ||
246 | nbytes &= AES_BLOCK_SIZE - 1; | ||
247 | err = blkcipher_walk_done(desc, &walk, nbytes); | ||
248 | } | ||
249 | kernel_fpu_end(); | ||
250 | |||
251 | return err; | ||
252 | } | ||
253 | |||
254 | static struct crypto_alg blk_cbc_alg = { | ||
255 | .cra_name = "__cbc-aes-aesni", | ||
256 | .cra_driver_name = "__driver-cbc-aes-aesni", | ||
257 | .cra_priority = 0, | ||
258 | .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, | ||
259 | .cra_blocksize = AES_BLOCK_SIZE, | ||
260 | .cra_ctxsize = sizeof(struct crypto_aes_ctx)+AESNI_ALIGN-1, | ||
261 | .cra_alignmask = 0, | ||
262 | .cra_type = &crypto_blkcipher_type, | ||
263 | .cra_module = THIS_MODULE, | ||
264 | .cra_list = LIST_HEAD_INIT(blk_cbc_alg.cra_list), | ||
265 | .cra_u = { | ||
266 | .blkcipher = { | ||
267 | .min_keysize = AES_MIN_KEY_SIZE, | ||
268 | .max_keysize = AES_MAX_KEY_SIZE, | ||
269 | .setkey = aes_set_key, | ||
270 | .encrypt = cbc_encrypt, | ||
271 | .decrypt = cbc_decrypt, | ||
272 | }, | ||
273 | }, | ||
274 | }; | ||
275 | |||
276 | static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, | ||
277 | unsigned int key_len) | ||
278 | { | ||
279 | struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); | ||
280 | |||
281 | return crypto_ablkcipher_setkey(&ctx->cryptd_tfm->base, key, key_len); | ||
282 | } | ||
283 | |||
284 | static int ablk_encrypt(struct ablkcipher_request *req) | ||
285 | { | ||
286 | struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); | ||
287 | struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); | ||
288 | |||
289 | if (kernel_fpu_using()) { | ||
290 | struct ablkcipher_request *cryptd_req = | ||
291 | ablkcipher_request_ctx(req); | ||
292 | memcpy(cryptd_req, req, sizeof(*req)); | ||
293 | ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); | ||
294 | return crypto_ablkcipher_encrypt(cryptd_req); | ||
295 | } else { | ||
296 | struct blkcipher_desc desc; | ||
297 | desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); | ||
298 | desc.info = req->info; | ||
299 | desc.flags = 0; | ||
300 | return crypto_blkcipher_crt(desc.tfm)->encrypt( | ||
301 | &desc, req->dst, req->src, req->nbytes); | ||
302 | } | ||
303 | } | ||
304 | |||
305 | static int ablk_decrypt(struct ablkcipher_request *req) | ||
306 | { | ||
307 | struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); | ||
308 | struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); | ||
309 | |||
310 | if (kernel_fpu_using()) { | ||
311 | struct ablkcipher_request *cryptd_req = | ||
312 | ablkcipher_request_ctx(req); | ||
313 | memcpy(cryptd_req, req, sizeof(*req)); | ||
314 | ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); | ||
315 | return crypto_ablkcipher_decrypt(cryptd_req); | ||
316 | } else { | ||
317 | struct blkcipher_desc desc; | ||
318 | desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); | ||
319 | desc.info = req->info; | ||
320 | desc.flags = 0; | ||
321 | return crypto_blkcipher_crt(desc.tfm)->decrypt( | ||
322 | &desc, req->dst, req->src, req->nbytes); | ||
323 | } | ||
324 | } | ||
325 | |||
326 | static void ablk_exit(struct crypto_tfm *tfm) | ||
327 | { | ||
328 | struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); | ||
329 | |||
330 | cryptd_free_ablkcipher(ctx->cryptd_tfm); | ||
331 | } | ||
332 | |||
333 | static void ablk_init_common(struct crypto_tfm *tfm, | ||
334 | struct cryptd_ablkcipher *cryptd_tfm) | ||
335 | { | ||
336 | struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); | ||
337 | |||
338 | ctx->cryptd_tfm = cryptd_tfm; | ||
339 | tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + | ||
340 | crypto_ablkcipher_reqsize(&cryptd_tfm->base); | ||
341 | } | ||
342 | |||
343 | static int ablk_ecb_init(struct crypto_tfm *tfm) | ||
344 | { | ||
345 | struct cryptd_ablkcipher *cryptd_tfm; | ||
346 | |||
347 | cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); | ||
348 | if (IS_ERR(cryptd_tfm)) | ||
349 | return PTR_ERR(cryptd_tfm); | ||
350 | ablk_init_common(tfm, cryptd_tfm); | ||
351 | return 0; | ||
352 | } | ||
353 | |||
354 | static struct crypto_alg ablk_ecb_alg = { | ||
355 | .cra_name = "ecb(aes)", | ||
356 | .cra_driver_name = "ecb-aes-aesni", | ||
357 | .cra_priority = 400, | ||
358 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, | ||
359 | .cra_blocksize = AES_BLOCK_SIZE, | ||
360 | .cra_ctxsize = sizeof(struct async_aes_ctx), | ||
361 | .cra_alignmask = 0, | ||
362 | .cra_type = &crypto_ablkcipher_type, | ||
363 | .cra_module = THIS_MODULE, | ||
364 | .cra_list = LIST_HEAD_INIT(ablk_ecb_alg.cra_list), | ||
365 | .cra_init = ablk_ecb_init, | ||
366 | .cra_exit = ablk_exit, | ||
367 | .cra_u = { | ||
368 | .ablkcipher = { | ||
369 | .min_keysize = AES_MIN_KEY_SIZE, | ||
370 | .max_keysize = AES_MAX_KEY_SIZE, | ||
371 | .setkey = ablk_set_key, | ||
372 | .encrypt = ablk_encrypt, | ||
373 | .decrypt = ablk_decrypt, | ||
374 | }, | ||
375 | }, | ||
376 | }; | ||
377 | |||
378 | static int ablk_cbc_init(struct crypto_tfm *tfm) | ||
379 | { | ||
380 | struct cryptd_ablkcipher *cryptd_tfm; | ||
381 | |||
382 | cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); | ||
383 | if (IS_ERR(cryptd_tfm)) | ||
384 | return PTR_ERR(cryptd_tfm); | ||
385 | ablk_init_common(tfm, cryptd_tfm); | ||
386 | return 0; | ||
387 | } | ||
388 | |||
389 | static struct crypto_alg ablk_cbc_alg = { | ||
390 | .cra_name = "cbc(aes)", | ||
391 | .cra_driver_name = "cbc-aes-aesni", | ||
392 | .cra_priority = 400, | ||
393 | .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC, | ||
394 | .cra_blocksize = AES_BLOCK_SIZE, | ||
395 | .cra_ctxsize = sizeof(struct async_aes_ctx), | ||
396 | .cra_alignmask = 0, | ||
397 | .cra_type = &crypto_ablkcipher_type, | ||
398 | .cra_module = THIS_MODULE, | ||
399 | .cra_list = LIST_HEAD_INIT(ablk_cbc_alg.cra_list), | ||
400 | .cra_init = ablk_cbc_init, | ||
401 | .cra_exit = ablk_exit, | ||
402 | .cra_u = { | ||
403 | .ablkcipher = { | ||
404 | .min_keysize = AES_MIN_KEY_SIZE, | ||
405 | .max_keysize = AES_MAX_KEY_SIZE, | ||
406 | .ivsize = AES_BLOCK_SIZE, | ||
407 | .setkey = ablk_set_key, | ||
408 | .encrypt = ablk_encrypt, | ||
409 | .decrypt = ablk_decrypt, | ||
410 | }, | ||
411 | }, | ||
412 | }; | ||
413 | |||
414 | static int __init aesni_init(void) | ||
415 | { | ||
416 | int err; | ||
417 | |||
418 | if (!cpu_has_aes) { | ||
419 | printk(KERN_ERR "Intel AES-NI instructions are not detected.\n"); | ||
420 | return -ENODEV; | ||
421 | } | ||
422 | if ((err = crypto_register_alg(&aesni_alg))) | ||
423 | goto aes_err; | ||
424 | if ((err = crypto_register_alg(&blk_ecb_alg))) | ||
425 | goto blk_ecb_err; | ||
426 | if ((err = crypto_register_alg(&blk_cbc_alg))) | ||
427 | goto blk_cbc_err; | ||
428 | if ((err = crypto_register_alg(&ablk_ecb_alg))) | ||
429 | goto ablk_ecb_err; | ||
430 | if ((err = crypto_register_alg(&ablk_cbc_alg))) | ||
431 | goto ablk_cbc_err; | ||
432 | |||
433 | return err; | ||
434 | |||
435 | ablk_cbc_err: | ||
436 | crypto_unregister_alg(&ablk_ecb_alg); | ||
437 | ablk_ecb_err: | ||
438 | crypto_unregister_alg(&blk_cbc_alg); | ||
439 | blk_cbc_err: | ||
440 | crypto_unregister_alg(&blk_ecb_alg); | ||
441 | blk_ecb_err: | ||
442 | crypto_unregister_alg(&aesni_alg); | ||
443 | aes_err: | ||
444 | return err; | ||
445 | } | ||
446 | |||
447 | static void __exit aesni_exit(void) | ||
448 | { | ||
449 | crypto_unregister_alg(&ablk_cbc_alg); | ||
450 | crypto_unregister_alg(&ablk_ecb_alg); | ||
451 | crypto_unregister_alg(&blk_cbc_alg); | ||
452 | crypto_unregister_alg(&blk_ecb_alg); | ||
453 | crypto_unregister_alg(&aesni_alg); | ||
454 | } | ||
455 | |||
456 | module_init(aesni_init); | ||
457 | module_exit(aesni_exit); | ||
458 | |||
459 | MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"); | ||
460 | MODULE_LICENSE("GPL"); | ||
461 | MODULE_ALIAS("aes"); | ||
diff --git a/arch/x86/ia32/ia32entry.S b/arch/x86/ia32/ia32entry.S index 097a6b64c24d..db0c803170ab 100644 --- a/arch/x86/ia32/ia32entry.S +++ b/arch/x86/ia32/ia32entry.S | |||
@@ -557,7 +557,7 @@ ia32_sys_call_table: | |||
557 | .quad sys32_olduname | 557 | .quad sys32_olduname |
558 | .quad sys_umask /* 60 */ | 558 | .quad sys_umask /* 60 */ |
559 | .quad sys_chroot | 559 | .quad sys_chroot |
560 | .quad sys32_ustat | 560 | .quad compat_sys_ustat |
561 | .quad sys_dup2 | 561 | .quad sys_dup2 |
562 | .quad sys_getppid | 562 | .quad sys_getppid |
563 | .quad sys_getpgrp /* 65 */ | 563 | .quad sys_getpgrp /* 65 */ |
diff --git a/arch/x86/ia32/sys_ia32.c b/arch/x86/ia32/sys_ia32.c index 6c0d7f6231af..efac92fd1efb 100644 --- a/arch/x86/ia32/sys_ia32.c +++ b/arch/x86/ia32/sys_ia32.c | |||
@@ -638,28 +638,6 @@ long sys32_uname(struct old_utsname __user *name) | |||
638 | return err ? -EFAULT : 0; | 638 | return err ? -EFAULT : 0; |
639 | } | 639 | } |
640 | 640 | ||
641 | long sys32_ustat(unsigned dev, struct ustat32 __user *u32p) | ||
642 | { | ||
643 | struct ustat u; | ||
644 | mm_segment_t seg; | ||
645 | int ret; | ||
646 | |||
647 | seg = get_fs(); | ||
648 | set_fs(KERNEL_DS); | ||
649 | ret = sys_ustat(dev, (struct ustat __user *)&u); | ||
650 | set_fs(seg); | ||
651 | if (ret < 0) | ||
652 | return ret; | ||
653 | |||
654 | if (!access_ok(VERIFY_WRITE, u32p, sizeof(struct ustat32)) || | ||
655 | __put_user((__u32) u.f_tfree, &u32p->f_tfree) || | ||
656 | __put_user((__u32) u.f_tinode, &u32p->f_tfree) || | ||
657 | __copy_to_user(&u32p->f_fname, u.f_fname, sizeof(u.f_fname)) || | ||
658 | __copy_to_user(&u32p->f_fpack, u.f_fpack, sizeof(u.f_fpack))) | ||
659 | ret = -EFAULT; | ||
660 | return ret; | ||
661 | } | ||
662 | |||
663 | asmlinkage long sys32_execve(char __user *name, compat_uptr_t __user *argv, | 641 | asmlinkage long sys32_execve(char __user *name, compat_uptr_t __user *argv, |
664 | compat_uptr_t __user *envp, struct pt_regs *regs) | 642 | compat_uptr_t __user *envp, struct pt_regs *regs) |
665 | { | 643 | { |
diff --git a/arch/x86/include/asm/aes.h b/arch/x86/include/asm/aes.h new file mode 100644 index 000000000000..80545a1cbe39 --- /dev/null +++ b/arch/x86/include/asm/aes.h | |||
@@ -0,0 +1,11 @@ | |||
1 | #ifndef ASM_X86_AES_H | ||
2 | #define ASM_X86_AES_H | ||
3 | |||
4 | #include <linux/crypto.h> | ||
5 | #include <crypto/aes.h> | ||
6 | |||
7 | void crypto_aes_encrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, | ||
8 | const u8 *src); | ||
9 | void crypto_aes_decrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, | ||
10 | const u8 *src); | ||
11 | #endif | ||
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 00f5962d82d0..df8a300dfe6c 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h | |||
@@ -75,7 +75,7 @@ static inline void default_inquire_remote_apic(int apicid) | |||
75 | #define setup_secondary_clock setup_secondary_APIC_clock | 75 | #define setup_secondary_clock setup_secondary_APIC_clock |
76 | #endif | 76 | #endif |
77 | 77 | ||
78 | #ifdef CONFIG_X86_VSMP | 78 | #ifdef CONFIG_X86_64 |
79 | extern int is_vsmp_box(void); | 79 | extern int is_vsmp_box(void); |
80 | #else | 80 | #else |
81 | static inline int is_vsmp_box(void) | 81 | static inline int is_vsmp_box(void) |
@@ -489,10 +489,19 @@ static inline int default_apic_id_registered(void) | |||
489 | return physid_isset(read_apic_id(), phys_cpu_present_map); | 489 | return physid_isset(read_apic_id(), phys_cpu_present_map); |
490 | } | 490 | } |
491 | 491 | ||
492 | static inline int default_phys_pkg_id(int cpuid_apic, int index_msb) | ||
493 | { | ||
494 | return cpuid_apic >> index_msb; | ||
495 | } | ||
496 | |||
497 | extern int default_apicid_to_node(int logical_apicid); | ||
498 | |||
499 | #endif | ||
500 | |||
492 | static inline unsigned int | 501 | static inline unsigned int |
493 | default_cpu_mask_to_apicid(const struct cpumask *cpumask) | 502 | default_cpu_mask_to_apicid(const struct cpumask *cpumask) |
494 | { | 503 | { |
495 | return cpumask_bits(cpumask)[0]; | 504 | return cpumask_bits(cpumask)[0] & APIC_ALL_CPUS; |
496 | } | 505 | } |
497 | 506 | ||
498 | static inline unsigned int | 507 | static inline unsigned int |
@@ -506,15 +515,6 @@ default_cpu_mask_to_apicid_and(const struct cpumask *cpumask, | |||
506 | return (unsigned int)(mask1 & mask2 & mask3); | 515 | return (unsigned int)(mask1 & mask2 & mask3); |
507 | } | 516 | } |
508 | 517 | ||
509 | static inline int default_phys_pkg_id(int cpuid_apic, int index_msb) | ||
510 | { | ||
511 | return cpuid_apic >> index_msb; | ||
512 | } | ||
513 | |||
514 | extern int default_apicid_to_node(int logical_apicid); | ||
515 | |||
516 | #endif | ||
517 | |||
518 | static inline unsigned long default_check_apicid_used(physid_mask_t bitmap, int apicid) | 518 | static inline unsigned long default_check_apicid_used(physid_mask_t bitmap, int apicid) |
519 | { | 519 | { |
520 | return physid_isset(apicid, bitmap); | 520 | return physid_isset(apicid, bitmap); |
diff --git a/arch/x86/include/asm/cacheflush.h b/arch/x86/include/asm/cacheflush.h index 5b301b7ff5f4..b3894bf52fcd 100644 --- a/arch/x86/include/asm/cacheflush.h +++ b/arch/x86/include/asm/cacheflush.h | |||
@@ -90,6 +90,9 @@ int set_memory_4k(unsigned long addr, int numpages); | |||
90 | int set_memory_array_uc(unsigned long *addr, int addrinarray); | 90 | int set_memory_array_uc(unsigned long *addr, int addrinarray); |
91 | int set_memory_array_wb(unsigned long *addr, int addrinarray); | 91 | int set_memory_array_wb(unsigned long *addr, int addrinarray); |
92 | 92 | ||
93 | int set_pages_array_uc(struct page **pages, int addrinarray); | ||
94 | int set_pages_array_wb(struct page **pages, int addrinarray); | ||
95 | |||
93 | /* | 96 | /* |
94 | * For legacy compatibility with the old APIs, a few functions | 97 | * For legacy compatibility with the old APIs, a few functions |
95 | * are provided that work on a "struct page". | 98 | * are provided that work on a "struct page". |
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 7301e60dc4a8..0beba0d1468d 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h | |||
@@ -213,6 +213,7 @@ extern const char * const x86_power_flags[32]; | |||
213 | #define cpu_has_xmm boot_cpu_has(X86_FEATURE_XMM) | 213 | #define cpu_has_xmm boot_cpu_has(X86_FEATURE_XMM) |
214 | #define cpu_has_xmm2 boot_cpu_has(X86_FEATURE_XMM2) | 214 | #define cpu_has_xmm2 boot_cpu_has(X86_FEATURE_XMM2) |
215 | #define cpu_has_xmm3 boot_cpu_has(X86_FEATURE_XMM3) | 215 | #define cpu_has_xmm3 boot_cpu_has(X86_FEATURE_XMM3) |
216 | #define cpu_has_aes boot_cpu_has(X86_FEATURE_AES) | ||
216 | #define cpu_has_ht boot_cpu_has(X86_FEATURE_HT) | 217 | #define cpu_has_ht boot_cpu_has(X86_FEATURE_HT) |
217 | #define cpu_has_mp boot_cpu_has(X86_FEATURE_MP) | 218 | #define cpu_has_mp boot_cpu_has(X86_FEATURE_MP) |
218 | #define cpu_has_nx boot_cpu_has(X86_FEATURE_NX) | 219 | #define cpu_has_nx boot_cpu_has(X86_FEATURE_NX) |
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h index 3c034f48fdb0..4994a20acbcb 100644 --- a/arch/x86/include/asm/device.h +++ b/arch/x86/include/asm/device.h | |||
@@ -6,7 +6,7 @@ struct dev_archdata { | |||
6 | void *acpi_handle; | 6 | void *acpi_handle; |
7 | #endif | 7 | #endif |
8 | #ifdef CONFIG_X86_64 | 8 | #ifdef CONFIG_X86_64 |
9 | struct dma_mapping_ops *dma_ops; | 9 | struct dma_map_ops *dma_ops; |
10 | #endif | 10 | #endif |
11 | #ifdef CONFIG_DMAR | 11 | #ifdef CONFIG_DMAR |
12 | void *iommu; /* hook for IOMMU specific extension */ | 12 | void *iommu; /* hook for IOMMU specific extension */ |
diff --git a/arch/x86/include/asm/dma-mapping.h b/arch/x86/include/asm/dma-mapping.h index 132a134d12f2..cea7b74963e9 100644 --- a/arch/x86/include/asm/dma-mapping.h +++ b/arch/x86/include/asm/dma-mapping.h | |||
@@ -7,6 +7,8 @@ | |||
7 | */ | 7 | */ |
8 | 8 | ||
9 | #include <linux/scatterlist.h> | 9 | #include <linux/scatterlist.h> |
10 | #include <linux/dma-debug.h> | ||
11 | #include <linux/dma-attrs.h> | ||
10 | #include <asm/io.h> | 12 | #include <asm/io.h> |
11 | #include <asm/swiotlb.h> | 13 | #include <asm/swiotlb.h> |
12 | #include <asm-generic/dma-coherent.h> | 14 | #include <asm-generic/dma-coherent.h> |
@@ -16,47 +18,9 @@ extern int iommu_merge; | |||
16 | extern struct device x86_dma_fallback_dev; | 18 | extern struct device x86_dma_fallback_dev; |
17 | extern int panic_on_overflow; | 19 | extern int panic_on_overflow; |
18 | 20 | ||
19 | struct dma_mapping_ops { | 21 | extern struct dma_map_ops *dma_ops; |
20 | int (*mapping_error)(struct device *dev, | 22 | |
21 | dma_addr_t dma_addr); | 23 | static inline struct dma_map_ops *get_dma_ops(struct device *dev) |
22 | void* (*alloc_coherent)(struct device *dev, size_t size, | ||
23 | dma_addr_t *dma_handle, gfp_t gfp); | ||
24 | void (*free_coherent)(struct device *dev, size_t size, | ||
25 | void *vaddr, dma_addr_t dma_handle); | ||
26 | dma_addr_t (*map_single)(struct device *hwdev, phys_addr_t ptr, | ||
27 | size_t size, int direction); | ||
28 | void (*unmap_single)(struct device *dev, dma_addr_t addr, | ||
29 | size_t size, int direction); | ||
30 | void (*sync_single_for_cpu)(struct device *hwdev, | ||
31 | dma_addr_t dma_handle, size_t size, | ||
32 | int direction); | ||
33 | void (*sync_single_for_device)(struct device *hwdev, | ||
34 | dma_addr_t dma_handle, size_t size, | ||
35 | int direction); | ||
36 | void (*sync_single_range_for_cpu)(struct device *hwdev, | ||
37 | dma_addr_t dma_handle, unsigned long offset, | ||
38 | size_t size, int direction); | ||
39 | void (*sync_single_range_for_device)(struct device *hwdev, | ||
40 | dma_addr_t dma_handle, unsigned long offset, | ||
41 | size_t size, int direction); | ||
42 | void (*sync_sg_for_cpu)(struct device *hwdev, | ||
43 | struct scatterlist *sg, int nelems, | ||
44 | int direction); | ||
45 | void (*sync_sg_for_device)(struct device *hwdev, | ||
46 | struct scatterlist *sg, int nelems, | ||
47 | int direction); | ||
48 | int (*map_sg)(struct device *hwdev, struct scatterlist *sg, | ||
49 | int nents, int direction); | ||
50 | void (*unmap_sg)(struct device *hwdev, | ||
51 | struct scatterlist *sg, int nents, | ||
52 | int direction); | ||
53 | int (*dma_supported)(struct device *hwdev, u64 mask); | ||
54 | int is_phys; | ||
55 | }; | ||
56 | |||
57 | extern struct dma_mapping_ops *dma_ops; | ||
58 | |||
59 | static inline struct dma_mapping_ops *get_dma_ops(struct device *dev) | ||
60 | { | 24 | { |
61 | #ifdef CONFIG_X86_32 | 25 | #ifdef CONFIG_X86_32 |
62 | return dma_ops; | 26 | return dma_ops; |
@@ -71,7 +35,7 @@ static inline struct dma_mapping_ops *get_dma_ops(struct device *dev) | |||
71 | /* Make sure we keep the same behaviour */ | 35 | /* Make sure we keep the same behaviour */ |
72 | static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) | 36 | static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) |
73 | { | 37 | { |
74 | struct dma_mapping_ops *ops = get_dma_ops(dev); | 38 | struct dma_map_ops *ops = get_dma_ops(dev); |
75 | if (ops->mapping_error) | 39 | if (ops->mapping_error) |
76 | return ops->mapping_error(dev, dma_addr); | 40 | return ops->mapping_error(dev, dma_addr); |
77 | 41 | ||
@@ -90,137 +54,167 @@ extern void *dma_generic_alloc_coherent(struct device *dev, size_t size, | |||
90 | 54 | ||
91 | static inline dma_addr_t | 55 | static inline dma_addr_t |
92 | dma_map_single(struct device *hwdev, void *ptr, size_t size, | 56 | dma_map_single(struct device *hwdev, void *ptr, size_t size, |
93 | int direction) | 57 | enum dma_data_direction dir) |
94 | { | 58 | { |
95 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 59 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
96 | 60 | dma_addr_t addr; | |
97 | BUG_ON(!valid_dma_direction(direction)); | 61 | |
98 | return ops->map_single(hwdev, virt_to_phys(ptr), size, direction); | 62 | BUG_ON(!valid_dma_direction(dir)); |
63 | addr = ops->map_page(hwdev, virt_to_page(ptr), | ||
64 | (unsigned long)ptr & ~PAGE_MASK, size, | ||
65 | dir, NULL); | ||
66 | debug_dma_map_page(hwdev, virt_to_page(ptr), | ||
67 | (unsigned long)ptr & ~PAGE_MASK, size, | ||
68 | dir, addr, true); | ||
69 | return addr; | ||
99 | } | 70 | } |
100 | 71 | ||
101 | static inline void | 72 | static inline void |
102 | dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, | 73 | dma_unmap_single(struct device *dev, dma_addr_t addr, size_t size, |
103 | int direction) | 74 | enum dma_data_direction dir) |
104 | { | 75 | { |
105 | struct dma_mapping_ops *ops = get_dma_ops(dev); | 76 | struct dma_map_ops *ops = get_dma_ops(dev); |
106 | 77 | ||
107 | BUG_ON(!valid_dma_direction(direction)); | 78 | BUG_ON(!valid_dma_direction(dir)); |
108 | if (ops->unmap_single) | 79 | if (ops->unmap_page) |
109 | ops->unmap_single(dev, addr, size, direction); | 80 | ops->unmap_page(dev, addr, size, dir, NULL); |
81 | debug_dma_unmap_page(dev, addr, size, dir, true); | ||
110 | } | 82 | } |
111 | 83 | ||
112 | static inline int | 84 | static inline int |
113 | dma_map_sg(struct device *hwdev, struct scatterlist *sg, | 85 | dma_map_sg(struct device *hwdev, struct scatterlist *sg, |
114 | int nents, int direction) | 86 | int nents, enum dma_data_direction dir) |
115 | { | 87 | { |
116 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 88 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
89 | int ents; | ||
90 | |||
91 | BUG_ON(!valid_dma_direction(dir)); | ||
92 | ents = ops->map_sg(hwdev, sg, nents, dir, NULL); | ||
93 | debug_dma_map_sg(hwdev, sg, nents, ents, dir); | ||
117 | 94 | ||
118 | BUG_ON(!valid_dma_direction(direction)); | 95 | return ents; |
119 | return ops->map_sg(hwdev, sg, nents, direction); | ||
120 | } | 96 | } |
121 | 97 | ||
122 | static inline void | 98 | static inline void |
123 | dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, | 99 | dma_unmap_sg(struct device *hwdev, struct scatterlist *sg, int nents, |
124 | int direction) | 100 | enum dma_data_direction dir) |
125 | { | 101 | { |
126 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 102 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
127 | 103 | ||
128 | BUG_ON(!valid_dma_direction(direction)); | 104 | BUG_ON(!valid_dma_direction(dir)); |
105 | debug_dma_unmap_sg(hwdev, sg, nents, dir); | ||
129 | if (ops->unmap_sg) | 106 | if (ops->unmap_sg) |
130 | ops->unmap_sg(hwdev, sg, nents, direction); | 107 | ops->unmap_sg(hwdev, sg, nents, dir, NULL); |
131 | } | 108 | } |
132 | 109 | ||
133 | static inline void | 110 | static inline void |
134 | dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, | 111 | dma_sync_single_for_cpu(struct device *hwdev, dma_addr_t dma_handle, |
135 | size_t size, int direction) | 112 | size_t size, enum dma_data_direction dir) |
136 | { | 113 | { |
137 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 114 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
138 | 115 | ||
139 | BUG_ON(!valid_dma_direction(direction)); | 116 | BUG_ON(!valid_dma_direction(dir)); |
140 | if (ops->sync_single_for_cpu) | 117 | if (ops->sync_single_for_cpu) |
141 | ops->sync_single_for_cpu(hwdev, dma_handle, size, direction); | 118 | ops->sync_single_for_cpu(hwdev, dma_handle, size, dir); |
119 | debug_dma_sync_single_for_cpu(hwdev, dma_handle, size, dir); | ||
142 | flush_write_buffers(); | 120 | flush_write_buffers(); |
143 | } | 121 | } |
144 | 122 | ||
145 | static inline void | 123 | static inline void |
146 | dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, | 124 | dma_sync_single_for_device(struct device *hwdev, dma_addr_t dma_handle, |
147 | size_t size, int direction) | 125 | size_t size, enum dma_data_direction dir) |
148 | { | 126 | { |
149 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 127 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
150 | 128 | ||
151 | BUG_ON(!valid_dma_direction(direction)); | 129 | BUG_ON(!valid_dma_direction(dir)); |
152 | if (ops->sync_single_for_device) | 130 | if (ops->sync_single_for_device) |
153 | ops->sync_single_for_device(hwdev, dma_handle, size, direction); | 131 | ops->sync_single_for_device(hwdev, dma_handle, size, dir); |
132 | debug_dma_sync_single_for_device(hwdev, dma_handle, size, dir); | ||
154 | flush_write_buffers(); | 133 | flush_write_buffers(); |
155 | } | 134 | } |
156 | 135 | ||
157 | static inline void | 136 | static inline void |
158 | dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, | 137 | dma_sync_single_range_for_cpu(struct device *hwdev, dma_addr_t dma_handle, |
159 | unsigned long offset, size_t size, int direction) | 138 | unsigned long offset, size_t size, |
139 | enum dma_data_direction dir) | ||
160 | { | 140 | { |
161 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 141 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
162 | 142 | ||
163 | BUG_ON(!valid_dma_direction(direction)); | 143 | BUG_ON(!valid_dma_direction(dir)); |
164 | if (ops->sync_single_range_for_cpu) | 144 | if (ops->sync_single_range_for_cpu) |
165 | ops->sync_single_range_for_cpu(hwdev, dma_handle, offset, | 145 | ops->sync_single_range_for_cpu(hwdev, dma_handle, offset, |
166 | size, direction); | 146 | size, dir); |
147 | debug_dma_sync_single_range_for_cpu(hwdev, dma_handle, | ||
148 | offset, size, dir); | ||
167 | flush_write_buffers(); | 149 | flush_write_buffers(); |
168 | } | 150 | } |
169 | 151 | ||
170 | static inline void | 152 | static inline void |
171 | dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, | 153 | dma_sync_single_range_for_device(struct device *hwdev, dma_addr_t dma_handle, |
172 | unsigned long offset, size_t size, | 154 | unsigned long offset, size_t size, |
173 | int direction) | 155 | enum dma_data_direction dir) |
174 | { | 156 | { |
175 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 157 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
176 | 158 | ||
177 | BUG_ON(!valid_dma_direction(direction)); | 159 | BUG_ON(!valid_dma_direction(dir)); |
178 | if (ops->sync_single_range_for_device) | 160 | if (ops->sync_single_range_for_device) |
179 | ops->sync_single_range_for_device(hwdev, dma_handle, | 161 | ops->sync_single_range_for_device(hwdev, dma_handle, |
180 | offset, size, direction); | 162 | offset, size, dir); |
163 | debug_dma_sync_single_range_for_device(hwdev, dma_handle, | ||
164 | offset, size, dir); | ||
181 | flush_write_buffers(); | 165 | flush_write_buffers(); |
182 | } | 166 | } |
183 | 167 | ||
184 | static inline void | 168 | static inline void |
185 | dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, | 169 | dma_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg, |
186 | int nelems, int direction) | 170 | int nelems, enum dma_data_direction dir) |
187 | { | 171 | { |
188 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 172 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
189 | 173 | ||
190 | BUG_ON(!valid_dma_direction(direction)); | 174 | BUG_ON(!valid_dma_direction(dir)); |
191 | if (ops->sync_sg_for_cpu) | 175 | if (ops->sync_sg_for_cpu) |
192 | ops->sync_sg_for_cpu(hwdev, sg, nelems, direction); | 176 | ops->sync_sg_for_cpu(hwdev, sg, nelems, dir); |
177 | debug_dma_sync_sg_for_cpu(hwdev, sg, nelems, dir); | ||
193 | flush_write_buffers(); | 178 | flush_write_buffers(); |
194 | } | 179 | } |
195 | 180 | ||
196 | static inline void | 181 | static inline void |
197 | dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, | 182 | dma_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg, |
198 | int nelems, int direction) | 183 | int nelems, enum dma_data_direction dir) |
199 | { | 184 | { |
200 | struct dma_mapping_ops *ops = get_dma_ops(hwdev); | 185 | struct dma_map_ops *ops = get_dma_ops(hwdev); |
201 | 186 | ||
202 | BUG_ON(!valid_dma_direction(direction)); | 187 | BUG_ON(!valid_dma_direction(dir)); |
203 | if (ops->sync_sg_for_device) | 188 | if (ops->sync_sg_for_device) |
204 | ops->sync_sg_for_device(hwdev, sg, nelems, direction); | 189 | ops->sync_sg_for_device(hwdev, sg, nelems, dir); |
190 | debug_dma_sync_sg_for_device(hwdev, sg, nelems, dir); | ||
205 | 191 | ||
206 | flush_write_buffers(); | 192 | flush_write_buffers(); |
207 | } | 193 | } |
208 | 194 | ||
209 | static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, | 195 | static inline dma_addr_t dma_map_page(struct device *dev, struct page *page, |
210 | size_t offset, size_t size, | 196 | size_t offset, size_t size, |
211 | int direction) | 197 | enum dma_data_direction dir) |
212 | { | 198 | { |
213 | struct dma_mapping_ops *ops = get_dma_ops(dev); | 199 | struct dma_map_ops *ops = get_dma_ops(dev); |
200 | dma_addr_t addr; | ||
214 | 201 | ||
215 | BUG_ON(!valid_dma_direction(direction)); | 202 | BUG_ON(!valid_dma_direction(dir)); |
216 | return ops->map_single(dev, page_to_phys(page) + offset, | 203 | addr = ops->map_page(dev, page, offset, size, dir, NULL); |
217 | size, direction); | 204 | debug_dma_map_page(dev, page, offset, size, dir, addr, false); |
205 | |||
206 | return addr; | ||
218 | } | 207 | } |
219 | 208 | ||
220 | static inline void dma_unmap_page(struct device *dev, dma_addr_t addr, | 209 | static inline void dma_unmap_page(struct device *dev, dma_addr_t addr, |
221 | size_t size, int direction) | 210 | size_t size, enum dma_data_direction dir) |
222 | { | 211 | { |
223 | dma_unmap_single(dev, addr, size, direction); | 212 | struct dma_map_ops *ops = get_dma_ops(dev); |
213 | |||
214 | BUG_ON(!valid_dma_direction(dir)); | ||
215 | if (ops->unmap_page) | ||
216 | ops->unmap_page(dev, addr, size, dir, NULL); | ||
217 | debug_dma_unmap_page(dev, addr, size, dir, false); | ||
224 | } | 218 | } |
225 | 219 | ||
226 | static inline void | 220 | static inline void |
@@ -266,7 +260,7 @@ static inline void * | |||
266 | dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, | 260 | dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, |
267 | gfp_t gfp) | 261 | gfp_t gfp) |
268 | { | 262 | { |
269 | struct dma_mapping_ops *ops = get_dma_ops(dev); | 263 | struct dma_map_ops *ops = get_dma_ops(dev); |
270 | void *memory; | 264 | void *memory; |
271 | 265 | ||
272 | gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); | 266 | gfp &= ~(__GFP_DMA | __GFP_HIGHMEM | __GFP_DMA32); |
@@ -285,20 +279,24 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle, | |||
285 | if (!ops->alloc_coherent) | 279 | if (!ops->alloc_coherent) |
286 | return NULL; | 280 | return NULL; |
287 | 281 | ||
288 | return ops->alloc_coherent(dev, size, dma_handle, | 282 | memory = ops->alloc_coherent(dev, size, dma_handle, |
289 | dma_alloc_coherent_gfp_flags(dev, gfp)); | 283 | dma_alloc_coherent_gfp_flags(dev, gfp)); |
284 | debug_dma_alloc_coherent(dev, size, *dma_handle, memory); | ||
285 | |||
286 | return memory; | ||
290 | } | 287 | } |
291 | 288 | ||
292 | static inline void dma_free_coherent(struct device *dev, size_t size, | 289 | static inline void dma_free_coherent(struct device *dev, size_t size, |
293 | void *vaddr, dma_addr_t bus) | 290 | void *vaddr, dma_addr_t bus) |
294 | { | 291 | { |
295 | struct dma_mapping_ops *ops = get_dma_ops(dev); | 292 | struct dma_map_ops *ops = get_dma_ops(dev); |
296 | 293 | ||
297 | WARN_ON(irqs_disabled()); /* for portability */ | 294 | WARN_ON(irqs_disabled()); /* for portability */ |
298 | 295 | ||
299 | if (dma_release_from_coherent(dev, get_order(size), vaddr)) | 296 | if (dma_release_from_coherent(dev, get_order(size), vaddr)) |
300 | return; | 297 | return; |
301 | 298 | ||
299 | debug_dma_free_coherent(dev, size, vaddr, bus); | ||
302 | if (ops->free_coherent) | 300 | if (ops->free_coherent) |
303 | ops->free_coherent(dev, size, vaddr, bus); | 301 | ops->free_coherent(dev, size, vaddr, bus); |
304 | } | 302 | } |
diff --git a/arch/x86/include/asm/e820.h b/arch/x86/include/asm/e820.h index 00d41ce4c844..7ecba4d85089 100644 --- a/arch/x86/include/asm/e820.h +++ b/arch/x86/include/asm/e820.h | |||
@@ -72,7 +72,7 @@ extern int e820_all_mapped(u64 start, u64 end, unsigned type); | |||
72 | extern void e820_add_region(u64 start, u64 size, int type); | 72 | extern void e820_add_region(u64 start, u64 size, int type); |
73 | extern void e820_print_map(char *who); | 73 | extern void e820_print_map(char *who); |
74 | extern int | 74 | extern int |
75 | sanitize_e820_map(struct e820entry *biosmap, int max_nr_map, int *pnr_map); | 75 | sanitize_e820_map(struct e820entry *biosmap, int max_nr_map, u32 *pnr_map); |
76 | extern u64 e820_update_range(u64 start, u64 size, unsigned old_type, | 76 | extern u64 e820_update_range(u64 start, u64 size, unsigned old_type, |
77 | unsigned new_type); | 77 | unsigned new_type); |
78 | extern u64 e820_remove_range(u64 start, u64 size, unsigned old_type, | 78 | extern u64 e820_remove_range(u64 start, u64 size, unsigned old_type, |
diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h index b55b4a7fbefd..db24c2278be0 100644 --- a/arch/x86/include/asm/ftrace.h +++ b/arch/x86/include/asm/ftrace.h | |||
@@ -55,29 +55,4 @@ struct dyn_arch_ftrace { | |||
55 | #endif /* __ASSEMBLY__ */ | 55 | #endif /* __ASSEMBLY__ */ |
56 | #endif /* CONFIG_FUNCTION_TRACER */ | 56 | #endif /* CONFIG_FUNCTION_TRACER */ |
57 | 57 | ||
58 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER | ||
59 | |||
60 | #ifndef __ASSEMBLY__ | ||
61 | |||
62 | /* | ||
63 | * Stack of return addresses for functions | ||
64 | * of a thread. | ||
65 | * Used in struct thread_info | ||
66 | */ | ||
67 | struct ftrace_ret_stack { | ||
68 | unsigned long ret; | ||
69 | unsigned long func; | ||
70 | unsigned long long calltime; | ||
71 | }; | ||
72 | |||
73 | /* | ||
74 | * Primary handler of a function return. | ||
75 | * It relays on ftrace_return_to_handler. | ||
76 | * Defined in entry_32/64.S | ||
77 | */ | ||
78 | extern void return_to_handler(void); | ||
79 | |||
80 | #endif /* __ASSEMBLY__ */ | ||
81 | #endif /* CONFIG_FUNCTION_GRAPH_TRACER */ | ||
82 | |||
83 | #endif /* _ASM_X86_FTRACE_H */ | 58 | #endif /* _ASM_X86_FTRACE_H */ |
diff --git a/arch/x86/include/asm/ia32.h b/arch/x86/include/asm/ia32.h index 50ca486fd88c..1f7e62517284 100644 --- a/arch/x86/include/asm/ia32.h +++ b/arch/x86/include/asm/ia32.h | |||
@@ -129,13 +129,6 @@ typedef struct compat_siginfo { | |||
129 | } _sifields; | 129 | } _sifields; |
130 | } compat_siginfo_t; | 130 | } compat_siginfo_t; |
131 | 131 | ||
132 | struct ustat32 { | ||
133 | __u32 f_tfree; | ||
134 | compat_ino_t f_tinode; | ||
135 | char f_fname[6]; | ||
136 | char f_fpack[6]; | ||
137 | }; | ||
138 | |||
139 | #define IA32_STACK_TOP IA32_PAGE_OFFSET | 132 | #define IA32_STACK_TOP IA32_PAGE_OFFSET |
140 | 133 | ||
141 | #ifdef __KERNEL__ | 134 | #ifdef __KERNEL__ |
diff --git a/arch/x86/include/asm/iommu.h b/arch/x86/include/asm/iommu.h index a6ee9e6f530f..af326a2975b5 100644 --- a/arch/x86/include/asm/iommu.h +++ b/arch/x86/include/asm/iommu.h | |||
@@ -3,7 +3,7 @@ | |||
3 | 3 | ||
4 | extern void pci_iommu_shutdown(void); | 4 | extern void pci_iommu_shutdown(void); |
5 | extern void no_iommu_init(void); | 5 | extern void no_iommu_init(void); |
6 | extern struct dma_mapping_ops nommu_dma_ops; | 6 | extern struct dma_map_ops nommu_dma_ops; |
7 | extern int force_iommu, no_iommu; | 7 | extern int force_iommu, no_iommu; |
8 | extern int iommu_detected; | 8 | extern int iommu_detected; |
9 | 9 | ||
diff --git a/arch/x86/include/asm/kvm.h b/arch/x86/include/asm/kvm.h index 886c9402ec45..dc3f6cf11704 100644 --- a/arch/x86/include/asm/kvm.h +++ b/arch/x86/include/asm/kvm.h | |||
@@ -15,6 +15,7 @@ | |||
15 | #define __KVM_HAVE_DEVICE_ASSIGNMENT | 15 | #define __KVM_HAVE_DEVICE_ASSIGNMENT |
16 | #define __KVM_HAVE_MSI | 16 | #define __KVM_HAVE_MSI |
17 | #define __KVM_HAVE_USER_NMI | 17 | #define __KVM_HAVE_USER_NMI |
18 | #define __KVM_HAVE_GUEST_DEBUG | ||
18 | 19 | ||
19 | /* Architectural interrupt line count. */ | 20 | /* Architectural interrupt line count. */ |
20 | #define KVM_NR_INTERRUPTS 256 | 21 | #define KVM_NR_INTERRUPTS 256 |
@@ -212,7 +213,30 @@ struct kvm_pit_channel_state { | |||
212 | __s64 count_load_time; | 213 | __s64 count_load_time; |
213 | }; | 214 | }; |
214 | 215 | ||
216 | struct kvm_debug_exit_arch { | ||
217 | __u32 exception; | ||
218 | __u32 pad; | ||
219 | __u64 pc; | ||
220 | __u64 dr6; | ||
221 | __u64 dr7; | ||
222 | }; | ||
223 | |||
224 | #define KVM_GUESTDBG_USE_SW_BP 0x00010000 | ||
225 | #define KVM_GUESTDBG_USE_HW_BP 0x00020000 | ||
226 | #define KVM_GUESTDBG_INJECT_DB 0x00040000 | ||
227 | #define KVM_GUESTDBG_INJECT_BP 0x00080000 | ||
228 | |||
229 | /* for KVM_SET_GUEST_DEBUG */ | ||
230 | struct kvm_guest_debug_arch { | ||
231 | __u64 debugreg[8]; | ||
232 | }; | ||
233 | |||
215 | struct kvm_pit_state { | 234 | struct kvm_pit_state { |
216 | struct kvm_pit_channel_state channels[3]; | 235 | struct kvm_pit_channel_state channels[3]; |
217 | }; | 236 | }; |
237 | |||
238 | struct kvm_reinject_control { | ||
239 | __u8 pit_reinject; | ||
240 | __u8 reserved[31]; | ||
241 | }; | ||
218 | #endif /* _ASM_X86_KVM_H */ | 242 | #endif /* _ASM_X86_KVM_H */ |
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 730843d1d2fb..f0faf58044ff 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h | |||
@@ -22,6 +22,7 @@ | |||
22 | #include <asm/pvclock-abi.h> | 22 | #include <asm/pvclock-abi.h> |
23 | #include <asm/desc.h> | 23 | #include <asm/desc.h> |
24 | #include <asm/mtrr.h> | 24 | #include <asm/mtrr.h> |
25 | #include <asm/msr-index.h> | ||
25 | 26 | ||
26 | #define KVM_MAX_VCPUS 16 | 27 | #define KVM_MAX_VCPUS 16 |
27 | #define KVM_MEMORY_SLOTS 32 | 28 | #define KVM_MEMORY_SLOTS 32 |
@@ -134,11 +135,18 @@ enum { | |||
134 | 135 | ||
135 | #define KVM_NR_MEM_OBJS 40 | 136 | #define KVM_NR_MEM_OBJS 40 |
136 | 137 | ||
137 | struct kvm_guest_debug { | 138 | #define KVM_NR_DB_REGS 4 |
138 | int enabled; | 139 | |
139 | unsigned long bp[4]; | 140 | #define DR6_BD (1 << 13) |
140 | int singlestep; | 141 | #define DR6_BS (1 << 14) |
141 | }; | 142 | #define DR6_FIXED_1 0xffff0ff0 |
143 | #define DR6_VOLATILE 0x0000e00f | ||
144 | |||
145 | #define DR7_BP_EN_MASK 0x000000ff | ||
146 | #define DR7_GE (1 << 9) | ||
147 | #define DR7_GD (1 << 13) | ||
148 | #define DR7_FIXED_1 0x00000400 | ||
149 | #define DR7_VOLATILE 0xffff23ff | ||
142 | 150 | ||
143 | /* | 151 | /* |
144 | * We don't want allocation failures within the mmu code, so we preallocate | 152 | * We don't want allocation failures within the mmu code, so we preallocate |
@@ -162,7 +170,8 @@ struct kvm_pte_chain { | |||
162 | * bits 0:3 - total guest paging levels (2-4, or zero for real mode) | 170 | * bits 0:3 - total guest paging levels (2-4, or zero for real mode) |
163 | * bits 4:7 - page table level for this shadow (1-4) | 171 | * bits 4:7 - page table level for this shadow (1-4) |
164 | * bits 8:9 - page table quadrant for 2-level guests | 172 | * bits 8:9 - page table quadrant for 2-level guests |
165 | * bit 16 - "metaphysical" - gfn is not a real page (huge page/real mode) | 173 | * bit 16 - direct mapping of virtual to physical mapping at gfn |
174 | * used for real mode and two-dimensional paging | ||
166 | * bits 17:19 - common access permissions for all ptes in this shadow page | 175 | * bits 17:19 - common access permissions for all ptes in this shadow page |
167 | */ | 176 | */ |
168 | union kvm_mmu_page_role { | 177 | union kvm_mmu_page_role { |
@@ -172,9 +181,10 @@ union kvm_mmu_page_role { | |||
172 | unsigned level:4; | 181 | unsigned level:4; |
173 | unsigned quadrant:2; | 182 | unsigned quadrant:2; |
174 | unsigned pad_for_nice_hex_output:6; | 183 | unsigned pad_for_nice_hex_output:6; |
175 | unsigned metaphysical:1; | 184 | unsigned direct:1; |
176 | unsigned access:3; | 185 | unsigned access:3; |
177 | unsigned invalid:1; | 186 | unsigned invalid:1; |
187 | unsigned cr4_pge:1; | ||
178 | }; | 188 | }; |
179 | }; | 189 | }; |
180 | 190 | ||
@@ -218,6 +228,18 @@ struct kvm_pv_mmu_op_buffer { | |||
218 | char buf[512] __aligned(sizeof(long)); | 228 | char buf[512] __aligned(sizeof(long)); |
219 | }; | 229 | }; |
220 | 230 | ||
231 | struct kvm_pio_request { | ||
232 | unsigned long count; | ||
233 | int cur_count; | ||
234 | gva_t guest_gva; | ||
235 | int in; | ||
236 | int port; | ||
237 | int size; | ||
238 | int string; | ||
239 | int down; | ||
240 | int rep; | ||
241 | }; | ||
242 | |||
221 | /* | 243 | /* |
222 | * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level | 244 | * x86 supports 3 paging modes (4-level 64-bit, 3-level 64-bit, and 2-level |
223 | * 32-bit). The kvm_mmu structure abstracts the details of the current mmu | 245 | * 32-bit). The kvm_mmu structure abstracts the details of the current mmu |
@@ -236,6 +258,7 @@ struct kvm_mmu { | |||
236 | hpa_t root_hpa; | 258 | hpa_t root_hpa; |
237 | int root_level; | 259 | int root_level; |
238 | int shadow_root_level; | 260 | int shadow_root_level; |
261 | union kvm_mmu_page_role base_role; | ||
239 | 262 | ||
240 | u64 *pae_root; | 263 | u64 *pae_root; |
241 | }; | 264 | }; |
@@ -258,6 +281,7 @@ struct kvm_vcpu_arch { | |||
258 | unsigned long cr3; | 281 | unsigned long cr3; |
259 | unsigned long cr4; | 282 | unsigned long cr4; |
260 | unsigned long cr8; | 283 | unsigned long cr8; |
284 | u32 hflags; | ||
261 | u64 pdptrs[4]; /* pae */ | 285 | u64 pdptrs[4]; /* pae */ |
262 | u64 shadow_efer; | 286 | u64 shadow_efer; |
263 | u64 apic_base; | 287 | u64 apic_base; |
@@ -338,6 +362,15 @@ struct kvm_vcpu_arch { | |||
338 | 362 | ||
339 | struct mtrr_state_type mtrr_state; | 363 | struct mtrr_state_type mtrr_state; |
340 | u32 pat; | 364 | u32 pat; |
365 | |||
366 | int switch_db_regs; | ||
367 | unsigned long host_db[KVM_NR_DB_REGS]; | ||
368 | unsigned long host_dr6; | ||
369 | unsigned long host_dr7; | ||
370 | unsigned long db[KVM_NR_DB_REGS]; | ||
371 | unsigned long dr6; | ||
372 | unsigned long dr7; | ||
373 | unsigned long eff_db[KVM_NR_DB_REGS]; | ||
341 | }; | 374 | }; |
342 | 375 | ||
343 | struct kvm_mem_alias { | 376 | struct kvm_mem_alias { |
@@ -378,6 +411,7 @@ struct kvm_arch{ | |||
378 | 411 | ||
379 | unsigned long irq_sources_bitmap; | 412 | unsigned long irq_sources_bitmap; |
380 | unsigned long irq_states[KVM_IOAPIC_NUM_PINS]; | 413 | unsigned long irq_states[KVM_IOAPIC_NUM_PINS]; |
414 | u64 vm_init_tsc; | ||
381 | }; | 415 | }; |
382 | 416 | ||
383 | struct kvm_vm_stat { | 417 | struct kvm_vm_stat { |
@@ -446,8 +480,7 @@ struct kvm_x86_ops { | |||
446 | void (*vcpu_put)(struct kvm_vcpu *vcpu); | 480 | void (*vcpu_put)(struct kvm_vcpu *vcpu); |
447 | 481 | ||
448 | int (*set_guest_debug)(struct kvm_vcpu *vcpu, | 482 | int (*set_guest_debug)(struct kvm_vcpu *vcpu, |
449 | struct kvm_debug_guest *dbg); | 483 | struct kvm_guest_debug *dbg); |
450 | void (*guest_debug_pre)(struct kvm_vcpu *vcpu); | ||
451 | int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); | 484 | int (*get_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata); |
452 | int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); | 485 | int (*set_msr)(struct kvm_vcpu *vcpu, u32 msr_index, u64 data); |
453 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); | 486 | u64 (*get_segment_base)(struct kvm_vcpu *vcpu, int seg); |
@@ -583,16 +616,12 @@ void kvm_queue_exception_e(struct kvm_vcpu *vcpu, unsigned nr, u32 error_code); | |||
583 | void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, | 616 | void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long cr2, |
584 | u32 error_code); | 617 | u32 error_code); |
585 | 618 | ||
586 | void kvm_pic_set_irq(void *opaque, int irq, int level); | 619 | int kvm_pic_set_irq(void *opaque, int irq, int level); |
587 | 620 | ||
588 | void kvm_inject_nmi(struct kvm_vcpu *vcpu); | 621 | void kvm_inject_nmi(struct kvm_vcpu *vcpu); |
589 | 622 | ||
590 | void fx_init(struct kvm_vcpu *vcpu); | 623 | void fx_init(struct kvm_vcpu *vcpu); |
591 | 624 | ||
592 | int emulator_read_std(unsigned long addr, | ||
593 | void *val, | ||
594 | unsigned int bytes, | ||
595 | struct kvm_vcpu *vcpu); | ||
596 | int emulator_write_emulated(unsigned long addr, | 625 | int emulator_write_emulated(unsigned long addr, |
597 | const void *val, | 626 | const void *val, |
598 | unsigned int bytes, | 627 | unsigned int bytes, |
@@ -737,6 +766,10 @@ enum { | |||
737 | TASK_SWITCH_GATE = 3, | 766 | TASK_SWITCH_GATE = 3, |
738 | }; | 767 | }; |
739 | 768 | ||
769 | #define HF_GIF_MASK (1 << 0) | ||
770 | #define HF_HIF_MASK (1 << 1) | ||
771 | #define HF_VINTR_MASK (1 << 2) | ||
772 | |||
740 | /* | 773 | /* |
741 | * Hardware virtualization extension instructions may fault if a | 774 | * Hardware virtualization extension instructions may fault if a |
742 | * reboot turns off virtualization while processes are running. | 775 | * reboot turns off virtualization while processes are running. |
diff --git a/arch/x86/include/asm/lguest_hcall.h b/arch/x86/include/asm/lguest_hcall.h index 43894428c3c2..0f4ee7148afe 100644 --- a/arch/x86/include/asm/lguest_hcall.h +++ b/arch/x86/include/asm/lguest_hcall.h | |||
@@ -26,36 +26,20 @@ | |||
26 | 26 | ||
27 | #ifndef __ASSEMBLY__ | 27 | #ifndef __ASSEMBLY__ |
28 | #include <asm/hw_irq.h> | 28 | #include <asm/hw_irq.h> |
29 | #include <asm/kvm_para.h> | ||
29 | 30 | ||
30 | /*G:031 But first, how does our Guest contact the Host to ask for privileged | 31 | /*G:031 But first, how does our Guest contact the Host to ask for privileged |
31 | * operations? There are two ways: the direct way is to make a "hypercall", | 32 | * operations? There are two ways: the direct way is to make a "hypercall", |
32 | * to make requests of the Host Itself. | 33 | * to make requests of the Host Itself. |
33 | * | 34 | * |
34 | * Our hypercall mechanism uses the highest unused trap code (traps 32 and | 35 | * We use the KVM hypercall mechanism. Eighteen hypercalls are |
35 | * above are used by real hardware interrupts). Fifteen hypercalls are | ||
36 | * available: the hypercall number is put in the %eax register, and the | 36 | * available: the hypercall number is put in the %eax register, and the |
37 | * arguments (when required) are placed in %edx, %ebx and %ecx. If a return | 37 | * arguments (when required) are placed in %ebx, %ecx and %edx. If a return |
38 | * value makes sense, it's returned in %eax. | 38 | * value makes sense, it's returned in %eax. |
39 | * | 39 | * |
40 | * Grossly invalid calls result in Sudden Death at the hands of the vengeful | 40 | * Grossly invalid calls result in Sudden Death at the hands of the vengeful |
41 | * Host, rather than returning failure. This reflects Winston Churchill's | 41 | * Host, rather than returning failure. This reflects Winston Churchill's |
42 | * definition of a gentleman: "someone who is only rude intentionally". */ | 42 | * definition of a gentleman: "someone who is only rude intentionally". */ |
43 | static inline unsigned long | ||
44 | hcall(unsigned long call, | ||
45 | unsigned long arg1, unsigned long arg2, unsigned long arg3) | ||
46 | { | ||
47 | /* "int" is the Intel instruction to trigger a trap. */ | ||
48 | asm volatile("int $" __stringify(LGUEST_TRAP_ENTRY) | ||
49 | /* The call in %eax (aka "a") might be overwritten */ | ||
50 | : "=a"(call) | ||
51 | /* The arguments are in %eax, %edx, %ebx & %ecx */ | ||
52 | : "a"(call), "d"(arg1), "b"(arg2), "c"(arg3) | ||
53 | /* "memory" means this might write somewhere in memory. | ||
54 | * This isn't true for all calls, but it's safe to tell | ||
55 | * gcc that it might happen so it doesn't get clever. */ | ||
56 | : "memory"); | ||
57 | return call; | ||
58 | } | ||
59 | /*:*/ | 43 | /*:*/ |
60 | 44 | ||
61 | /* Can't use our min() macro here: needs to be a constant */ | 45 | /* Can't use our min() macro here: needs to be a constant */ |
@@ -64,7 +48,7 @@ hcall(unsigned long call, | |||
64 | #define LHCALL_RING_SIZE 64 | 48 | #define LHCALL_RING_SIZE 64 |
65 | struct hcall_args { | 49 | struct hcall_args { |
66 | /* These map directly onto eax, ebx, ecx, edx in struct lguest_regs */ | 50 | /* These map directly onto eax, ebx, ecx, edx in struct lguest_regs */ |
67 | unsigned long arg0, arg2, arg3, arg1; | 51 | unsigned long arg0, arg1, arg2, arg3; |
68 | }; | 52 | }; |
69 | 53 | ||
70 | #endif /* !__ASSEMBLY__ */ | 54 | #endif /* !__ASSEMBLY__ */ |
diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h index 2dbd2314139e..ec41fc16c167 100644 --- a/arch/x86/include/asm/msr-index.h +++ b/arch/x86/include/asm/msr-index.h | |||
@@ -18,11 +18,15 @@ | |||
18 | #define _EFER_LME 8 /* Long mode enable */ | 18 | #define _EFER_LME 8 /* Long mode enable */ |
19 | #define _EFER_LMA 10 /* Long mode active (read-only) */ | 19 | #define _EFER_LMA 10 /* Long mode active (read-only) */ |
20 | #define _EFER_NX 11 /* No execute enable */ | 20 | #define _EFER_NX 11 /* No execute enable */ |
21 | #define _EFER_SVME 12 /* Enable virtualization */ | ||
22 | #define _EFER_FFXSR 14 /* Enable Fast FXSAVE/FXRSTOR */ | ||
21 | 23 | ||
22 | #define EFER_SCE (1<<_EFER_SCE) | 24 | #define EFER_SCE (1<<_EFER_SCE) |
23 | #define EFER_LME (1<<_EFER_LME) | 25 | #define EFER_LME (1<<_EFER_LME) |
24 | #define EFER_LMA (1<<_EFER_LMA) | 26 | #define EFER_LMA (1<<_EFER_LMA) |
25 | #define EFER_NX (1<<_EFER_NX) | 27 | #define EFER_NX (1<<_EFER_NX) |
28 | #define EFER_SVME (1<<_EFER_SVME) | ||
29 | #define EFER_FFXSR (1<<_EFER_FFXSR) | ||
26 | 30 | ||
27 | /* Intel MSRs. Some also available on other CPUs */ | 31 | /* Intel MSRs. Some also available on other CPUs */ |
28 | #define MSR_IA32_PERFCTR0 0x000000c1 | 32 | #define MSR_IA32_PERFCTR0 0x000000c1 |
@@ -365,4 +369,9 @@ | |||
365 | #define MSR_IA32_VMX_PROCBASED_CTLS2 0x0000048b | 369 | #define MSR_IA32_VMX_PROCBASED_CTLS2 0x0000048b |
366 | #define MSR_IA32_VMX_EPT_VPID_CAP 0x0000048c | 370 | #define MSR_IA32_VMX_EPT_VPID_CAP 0x0000048c |
367 | 371 | ||
372 | /* AMD-V MSRs */ | ||
373 | |||
374 | #define MSR_VM_CR 0xc0010114 | ||
375 | #define MSR_VM_HSAVE_PA 0xc0010117 | ||
376 | |||
368 | #endif /* _ASM_X86_MSR_INDEX_H */ | 377 | #endif /* _ASM_X86_MSR_INDEX_H */ |
diff --git a/arch/x86/include/asm/pci.h b/arch/x86/include/asm/pci.h index a977de23cb4d..a0301bfeb954 100644 --- a/arch/x86/include/asm/pci.h +++ b/arch/x86/include/asm/pci.h | |||
@@ -86,6 +86,9 @@ static inline void early_quirks(void) { } | |||
86 | 86 | ||
87 | extern void pci_iommu_alloc(void); | 87 | extern void pci_iommu_alloc(void); |
88 | 88 | ||
89 | /* MSI arch hook */ | ||
90 | #define arch_setup_msi_irqs arch_setup_msi_irqs | ||
91 | |||
89 | #endif /* __KERNEL__ */ | 92 | #endif /* __KERNEL__ */ |
90 | 93 | ||
91 | #ifdef CONFIG_X86_32 | 94 | #ifdef CONFIG_X86_32 |
diff --git a/arch/x86/include/asm/setup.h b/arch/x86/include/asm/setup.h index fbf0521eeed8..bdc2ada05ae0 100644 --- a/arch/x86/include/asm/setup.h +++ b/arch/x86/include/asm/setup.h | |||
@@ -64,7 +64,7 @@ extern void x86_quirk_time_init(void); | |||
64 | #include <asm/bootparam.h> | 64 | #include <asm/bootparam.h> |
65 | 65 | ||
66 | /* Interrupt control for vSMPowered x86_64 systems */ | 66 | /* Interrupt control for vSMPowered x86_64 systems */ |
67 | #ifdef CONFIG_X86_VSMP | 67 | #ifdef CONFIG_X86_64 |
68 | void vsmp_init(void); | 68 | void vsmp_init(void); |
69 | #else | 69 | #else |
70 | static inline void vsmp_init(void) { } | 70 | static inline void vsmp_init(void) { } |
diff --git a/arch/x86/include/asm/socket.h b/arch/x86/include/asm/socket.h index 8ab9cc8b2ecc..ca8bf2cd0ba9 100644 --- a/arch/x86/include/asm/socket.h +++ b/arch/x86/include/asm/socket.h | |||
@@ -54,4 +54,7 @@ | |||
54 | 54 | ||
55 | #define SO_MARK 36 | 55 | #define SO_MARK 36 |
56 | 56 | ||
57 | #define SO_TIMESTAMPING 37 | ||
58 | #define SCM_TIMESTAMPING SO_TIMESTAMPING | ||
59 | |||
57 | #endif /* _ASM_X86_SOCKET_H */ | 60 | #endif /* _ASM_X86_SOCKET_H */ |
diff --git a/arch/x86/include/asm/suspend_32.h b/arch/x86/include/asm/suspend_32.h index a5074bd0f8be..48dcfa62ea07 100644 --- a/arch/x86/include/asm/suspend_32.h +++ b/arch/x86/include/asm/suspend_32.h | |||
@@ -24,28 +24,4 @@ struct saved_context { | |||
24 | unsigned long return_address; | 24 | unsigned long return_address; |
25 | } __attribute__((packed)); | 25 | } __attribute__((packed)); |
26 | 26 | ||
27 | #ifdef CONFIG_ACPI | ||
28 | extern unsigned long saved_eip; | ||
29 | extern unsigned long saved_esp; | ||
30 | extern unsigned long saved_ebp; | ||
31 | extern unsigned long saved_ebx; | ||
32 | extern unsigned long saved_esi; | ||
33 | extern unsigned long saved_edi; | ||
34 | |||
35 | static inline void acpi_save_register_state(unsigned long return_point) | ||
36 | { | ||
37 | saved_eip = return_point; | ||
38 | asm volatile("movl %%esp,%0" : "=m" (saved_esp)); | ||
39 | asm volatile("movl %%ebp,%0" : "=m" (saved_ebp)); | ||
40 | asm volatile("movl %%ebx,%0" : "=m" (saved_ebx)); | ||
41 | asm volatile("movl %%edi,%0" : "=m" (saved_edi)); | ||
42 | asm volatile("movl %%esi,%0" : "=m" (saved_esi)); | ||
43 | } | ||
44 | |||
45 | #define acpi_restore_register_state() do {} while (0) | ||
46 | |||
47 | /* routines for saving/restoring kernel state */ | ||
48 | extern int acpi_save_state_mem(void); | ||
49 | #endif | ||
50 | |||
51 | #endif /* _ASM_X86_SUSPEND_32_H */ | 27 | #endif /* _ASM_X86_SUSPEND_32_H */ |
diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 1b8afa78e869..82ada75f3ebf 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h | |||
@@ -174,10 +174,6 @@ struct __attribute__ ((__packed__)) vmcb { | |||
174 | #define SVM_CPUID_FEATURE_SHIFT 2 | 174 | #define SVM_CPUID_FEATURE_SHIFT 2 |
175 | #define SVM_CPUID_FUNC 0x8000000a | 175 | #define SVM_CPUID_FUNC 0x8000000a |
176 | 176 | ||
177 | #define MSR_EFER_SVME_MASK (1ULL << 12) | ||
178 | #define MSR_VM_CR 0xc0010114 | ||
179 | #define MSR_VM_HSAVE_PA 0xc0010117ULL | ||
180 | |||
181 | #define SVM_VM_CR_SVM_DISABLE 4 | 177 | #define SVM_VM_CR_SVM_DISABLE 4 |
182 | 178 | ||
183 | #define SVM_SELECTOR_S_SHIFT 4 | 179 | #define SVM_SELECTOR_S_SHIFT 4 |
diff --git a/arch/x86/include/asm/sys_ia32.h b/arch/x86/include/asm/sys_ia32.h index ffb08be2a530..72a6dcd1299b 100644 --- a/arch/x86/include/asm/sys_ia32.h +++ b/arch/x86/include/asm/sys_ia32.h | |||
@@ -70,8 +70,6 @@ struct old_utsname; | |||
70 | asmlinkage long sys32_olduname(struct oldold_utsname __user *); | 70 | asmlinkage long sys32_olduname(struct oldold_utsname __user *); |
71 | long sys32_uname(struct old_utsname __user *); | 71 | long sys32_uname(struct old_utsname __user *); |
72 | 72 | ||
73 | long sys32_ustat(unsigned, struct ustat32 __user *); | ||
74 | |||
75 | asmlinkage long sys32_execve(char __user *, compat_uptr_t __user *, | 73 | asmlinkage long sys32_execve(char __user *, compat_uptr_t __user *, |
76 | compat_uptr_t __user *, struct pt_regs *); | 74 | compat_uptr_t __user *, struct pt_regs *); |
77 | asmlinkage long sys32_clone(unsigned int, unsigned int, struct pt_regs *); | 75 | asmlinkage long sys32_clone(unsigned int, unsigned int, struct pt_regs *); |
diff --git a/arch/x86/include/asm/timer.h b/arch/x86/include/asm/timer.h index a81195eaa2b3..bd37ed444a21 100644 --- a/arch/x86/include/asm/timer.h +++ b/arch/x86/include/asm/timer.h | |||
@@ -12,9 +12,9 @@ unsigned long native_calibrate_tsc(void); | |||
12 | 12 | ||
13 | #ifdef CONFIG_X86_32 | 13 | #ifdef CONFIG_X86_32 |
14 | extern int timer_ack; | 14 | extern int timer_ack; |
15 | extern int recalibrate_cpu_khz(void); | ||
16 | extern irqreturn_t timer_interrupt(int irq, void *dev_id); | 15 | extern irqreturn_t timer_interrupt(int irq, void *dev_id); |
17 | #endif /* CONFIG_X86_32 */ | 16 | #endif /* CONFIG_X86_32 */ |
17 | extern int recalibrate_cpu_khz(void); | ||
18 | 18 | ||
19 | extern int no_timer_check; | 19 | extern int no_timer_check; |
20 | 20 | ||
diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h index 77cfb2cfb386..744299c0b774 100644 --- a/arch/x86/include/asm/topology.h +++ b/arch/x86/include/asm/topology.h | |||
@@ -217,10 +217,6 @@ static inline cpumask_t node_to_cpumask(int node) | |||
217 | { | 217 | { |
218 | return cpu_online_map; | 218 | return cpu_online_map; |
219 | } | 219 | } |
220 | static inline int node_to_first_cpu(int node) | ||
221 | { | ||
222 | return first_cpu(cpu_online_map); | ||
223 | } | ||
224 | 220 | ||
225 | static inline void setup_node_to_cpumask_map(void) { } | 221 | static inline void setup_node_to_cpumask_map(void) { } |
226 | 222 | ||
@@ -237,14 +233,6 @@ static inline void setup_node_to_cpumask_map(void) { } | |||
237 | 233 | ||
238 | #include <asm-generic/topology.h> | 234 | #include <asm-generic/topology.h> |
239 | 235 | ||
240 | #ifdef CONFIG_NUMA | ||
241 | /* Returns the number of the first CPU on Node 'node'. */ | ||
242 | static inline int node_to_first_cpu(int node) | ||
243 | { | ||
244 | return cpumask_first(cpumask_of_node(node)); | ||
245 | } | ||
246 | #endif | ||
247 | |||
248 | extern cpumask_t cpu_coregroup_map(int cpu); | 236 | extern cpumask_t cpu_coregroup_map(int cpu); |
249 | extern const struct cpumask *cpu_coregroup_mask(int cpu); | 237 | extern const struct cpumask *cpu_coregroup_mask(int cpu); |
250 | 238 | ||
diff --git a/arch/x86/include/asm/virtext.h b/arch/x86/include/asm/virtext.h index 593636275238..e0f9aa16358b 100644 --- a/arch/x86/include/asm/virtext.h +++ b/arch/x86/include/asm/virtext.h | |||
@@ -118,7 +118,7 @@ static inline void cpu_svm_disable(void) | |||
118 | 118 | ||
119 | wrmsrl(MSR_VM_HSAVE_PA, 0); | 119 | wrmsrl(MSR_VM_HSAVE_PA, 0); |
120 | rdmsrl(MSR_EFER, efer); | 120 | rdmsrl(MSR_EFER, efer); |
121 | wrmsrl(MSR_EFER, efer & ~MSR_EFER_SVME_MASK); | 121 | wrmsrl(MSR_EFER, efer & ~EFER_SVME); |
122 | } | 122 | } |
123 | 123 | ||
124 | /** Makes sure SVM is disabled, if it is supported on the CPU | 124 | /** Makes sure SVM is disabled, if it is supported on the CPU |
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index d0238e6151d8..498f944010b9 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h | |||
@@ -270,8 +270,9 @@ enum vmcs_field { | |||
270 | 270 | ||
271 | #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ | 271 | #define INTR_TYPE_EXT_INTR (0 << 8) /* external interrupt */ |
272 | #define INTR_TYPE_NMI_INTR (2 << 8) /* NMI */ | 272 | #define INTR_TYPE_NMI_INTR (2 << 8) /* NMI */ |
273 | #define INTR_TYPE_EXCEPTION (3 << 8) /* processor exception */ | 273 | #define INTR_TYPE_HARD_EXCEPTION (3 << 8) /* processor exception */ |
274 | #define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ | 274 | #define INTR_TYPE_SOFT_INTR (4 << 8) /* software interrupt */ |
275 | #define INTR_TYPE_SOFT_EXCEPTION (6 << 8) /* software exception */ | ||
275 | 276 | ||
276 | /* GUEST_INTERRUPTIBILITY_INFO flags. */ | 277 | /* GUEST_INTERRUPTIBILITY_INFO flags. */ |
277 | #define GUEST_INTR_STATE_STI 0x00000001 | 278 | #define GUEST_INTR_STATE_STI 0x00000001 |
@@ -311,7 +312,7 @@ enum vmcs_field { | |||
311 | #define DEBUG_REG_ACCESS_TYPE 0x10 /* 4, direction of access */ | 312 | #define DEBUG_REG_ACCESS_TYPE 0x10 /* 4, direction of access */ |
312 | #define TYPE_MOV_TO_DR (0 << 4) | 313 | #define TYPE_MOV_TO_DR (0 << 4) |
313 | #define TYPE_MOV_FROM_DR (1 << 4) | 314 | #define TYPE_MOV_FROM_DR (1 << 4) |
314 | #define DEBUG_REG_ACCESS_REG 0xf00 /* 11:8, general purpose reg. */ | 315 | #define DEBUG_REG_ACCESS_REG(eq) (((eq) >> 8) & 0xf) /* 11:8, general purpose reg. */ |
315 | 316 | ||
316 | 317 | ||
317 | /* segment AR */ | 318 | /* segment AR */ |
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 339ce35648e6..c611ad64137f 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile | |||
@@ -70,7 +70,6 @@ obj-$(CONFIG_FUNCTION_GRAPH_TRACER) += ftrace.o | |||
70 | obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o | 70 | obj-$(CONFIG_KEXEC) += machine_kexec_$(BITS).o |
71 | obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o | 71 | obj-$(CONFIG_KEXEC) += relocate_kernel_$(BITS).o crash.o |
72 | obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o | 72 | obj-$(CONFIG_CRASH_DUMP) += crash_dump_$(BITS).o |
73 | obj-$(CONFIG_X86_VSMP) += vsmp_64.o | ||
74 | obj-$(CONFIG_KPROBES) += kprobes.o | 73 | obj-$(CONFIG_KPROBES) += kprobes.o |
75 | obj-$(CONFIG_MODULES) += module_$(BITS).o | 74 | obj-$(CONFIG_MODULES) += module_$(BITS).o |
76 | obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o | 75 | obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o |
@@ -106,7 +105,7 @@ obj-$(CONFIG_MICROCODE) += microcode.o | |||
106 | 105 | ||
107 | obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o | 106 | obj-$(CONFIG_X86_CHECK_BIOS_CORRUPTION) += check.o |
108 | 107 | ||
109 | obj-$(CONFIG_SWIOTLB) += pci-swiotlb_64.o # NB rename without _64 | 108 | obj-$(CONFIG_SWIOTLB) += pci-swiotlb.o |
110 | 109 | ||
111 | ### | 110 | ### |
112 | # 64 bit specific files | 111 | # 64 bit specific files |
@@ -120,4 +119,5 @@ ifeq ($(CONFIG_X86_64),y) | |||
120 | obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o | 119 | obj-$(CONFIG_AMD_IOMMU) += amd_iommu_init.o amd_iommu.o |
121 | 120 | ||
122 | obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o | 121 | obj-$(CONFIG_PCI_MMCONFIG) += mmconf-fam10h_64.o |
122 | obj-y += vsmp_64.o | ||
123 | endif | 123 | endif |
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 5113c080f0c4..c5962fe3796f 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c | |||
@@ -22,10 +22,9 @@ | |||
22 | #include <linux/bitops.h> | 22 | #include <linux/bitops.h> |
23 | #include <linux/debugfs.h> | 23 | #include <linux/debugfs.h> |
24 | #include <linux/scatterlist.h> | 24 | #include <linux/scatterlist.h> |
25 | #include <linux/dma-mapping.h> | ||
25 | #include <linux/iommu-helper.h> | 26 | #include <linux/iommu-helper.h> |
26 | #ifdef CONFIG_IOMMU_API | ||
27 | #include <linux/iommu.h> | 27 | #include <linux/iommu.h> |
28 | #endif | ||
29 | #include <asm/proto.h> | 28 | #include <asm/proto.h> |
30 | #include <asm/iommu.h> | 29 | #include <asm/iommu.h> |
31 | #include <asm/gart.h> | 30 | #include <asm/gart.h> |
@@ -1297,8 +1296,10 @@ static void __unmap_single(struct amd_iommu *iommu, | |||
1297 | /* | 1296 | /* |
1298 | * The exported map_single function for dma_ops. | 1297 | * The exported map_single function for dma_ops. |
1299 | */ | 1298 | */ |
1300 | static dma_addr_t map_single(struct device *dev, phys_addr_t paddr, | 1299 | static dma_addr_t map_page(struct device *dev, struct page *page, |
1301 | size_t size, int dir) | 1300 | unsigned long offset, size_t size, |
1301 | enum dma_data_direction dir, | ||
1302 | struct dma_attrs *attrs) | ||
1302 | { | 1303 | { |
1303 | unsigned long flags; | 1304 | unsigned long flags; |
1304 | struct amd_iommu *iommu; | 1305 | struct amd_iommu *iommu; |
@@ -1306,6 +1307,7 @@ static dma_addr_t map_single(struct device *dev, phys_addr_t paddr, | |||
1306 | u16 devid; | 1307 | u16 devid; |
1307 | dma_addr_t addr; | 1308 | dma_addr_t addr; |
1308 | u64 dma_mask; | 1309 | u64 dma_mask; |
1310 | phys_addr_t paddr = page_to_phys(page) + offset; | ||
1309 | 1311 | ||
1310 | INC_STATS_COUNTER(cnt_map_single); | 1312 | INC_STATS_COUNTER(cnt_map_single); |
1311 | 1313 | ||
@@ -1340,8 +1342,8 @@ out: | |||
1340 | /* | 1342 | /* |
1341 | * The exported unmap_single function for dma_ops. | 1343 | * The exported unmap_single function for dma_ops. |
1342 | */ | 1344 | */ |
1343 | static void unmap_single(struct device *dev, dma_addr_t dma_addr, | 1345 | static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, |
1344 | size_t size, int dir) | 1346 | enum dma_data_direction dir, struct dma_attrs *attrs) |
1345 | { | 1347 | { |
1346 | unsigned long flags; | 1348 | unsigned long flags; |
1347 | struct amd_iommu *iommu; | 1349 | struct amd_iommu *iommu; |
@@ -1390,7 +1392,8 @@ static int map_sg_no_iommu(struct device *dev, struct scatterlist *sglist, | |||
1390 | * lists). | 1392 | * lists). |
1391 | */ | 1393 | */ |
1392 | static int map_sg(struct device *dev, struct scatterlist *sglist, | 1394 | static int map_sg(struct device *dev, struct scatterlist *sglist, |
1393 | int nelems, int dir) | 1395 | int nelems, enum dma_data_direction dir, |
1396 | struct dma_attrs *attrs) | ||
1394 | { | 1397 | { |
1395 | unsigned long flags; | 1398 | unsigned long flags; |
1396 | struct amd_iommu *iommu; | 1399 | struct amd_iommu *iommu; |
@@ -1457,7 +1460,8 @@ unmap: | |||
1457 | * lists). | 1460 | * lists). |
1458 | */ | 1461 | */ |
1459 | static void unmap_sg(struct device *dev, struct scatterlist *sglist, | 1462 | static void unmap_sg(struct device *dev, struct scatterlist *sglist, |
1460 | int nelems, int dir) | 1463 | int nelems, enum dma_data_direction dir, |
1464 | struct dma_attrs *attrs) | ||
1461 | { | 1465 | { |
1462 | unsigned long flags; | 1466 | unsigned long flags; |
1463 | struct amd_iommu *iommu; | 1467 | struct amd_iommu *iommu; |
@@ -1644,11 +1648,11 @@ static void prealloc_protection_domains(void) | |||
1644 | } | 1648 | } |
1645 | } | 1649 | } |
1646 | 1650 | ||
1647 | static struct dma_mapping_ops amd_iommu_dma_ops = { | 1651 | static struct dma_map_ops amd_iommu_dma_ops = { |
1648 | .alloc_coherent = alloc_coherent, | 1652 | .alloc_coherent = alloc_coherent, |
1649 | .free_coherent = free_coherent, | 1653 | .free_coherent = free_coherent, |
1650 | .map_single = map_single, | 1654 | .map_page = map_page, |
1651 | .unmap_single = unmap_single, | 1655 | .unmap_page = unmap_page, |
1652 | .map_sg = map_sg, | 1656 | .map_sg = map_sg, |
1653 | .unmap_sg = unmap_sg, | 1657 | .unmap_sg = unmap_sg, |
1654 | .dma_supported = amd_iommu_dma_supported, | 1658 | .dma_supported = amd_iommu_dma_supported, |
diff --git a/arch/x86/kernel/apic/apic_flat_64.c b/arch/x86/kernel/apic/apic_flat_64.c index f933822dba18..0014714ea97b 100644 --- a/arch/x86/kernel/apic/apic_flat_64.c +++ b/arch/x86/kernel/apic/apic_flat_64.c | |||
@@ -159,20 +159,6 @@ static int flat_apic_id_registered(void) | |||
159 | return physid_isset(read_xapic_id(), phys_cpu_present_map); | 159 | return physid_isset(read_xapic_id(), phys_cpu_present_map); |
160 | } | 160 | } |
161 | 161 | ||
162 | static unsigned int flat_cpu_mask_to_apicid(const struct cpumask *cpumask) | ||
163 | { | ||
164 | return cpumask_bits(cpumask)[0] & APIC_ALL_CPUS; | ||
165 | } | ||
166 | |||
167 | static unsigned int flat_cpu_mask_to_apicid_and(const struct cpumask *cpumask, | ||
168 | const struct cpumask *andmask) | ||
169 | { | ||
170 | unsigned long mask1 = cpumask_bits(cpumask)[0] & APIC_ALL_CPUS; | ||
171 | unsigned long mask2 = cpumask_bits(andmask)[0] & APIC_ALL_CPUS; | ||
172 | |||
173 | return mask1 & mask2; | ||
174 | } | ||
175 | |||
176 | static int flat_phys_pkg_id(int initial_apic_id, int index_msb) | 162 | static int flat_phys_pkg_id(int initial_apic_id, int index_msb) |
177 | { | 163 | { |
178 | return hard_smp_processor_id() >> index_msb; | 164 | return hard_smp_processor_id() >> index_msb; |
@@ -213,8 +199,8 @@ struct apic apic_flat = { | |||
213 | .set_apic_id = set_apic_id, | 199 | .set_apic_id = set_apic_id, |
214 | .apic_id_mask = 0xFFu << 24, | 200 | .apic_id_mask = 0xFFu << 24, |
215 | 201 | ||
216 | .cpu_mask_to_apicid = flat_cpu_mask_to_apicid, | 202 | .cpu_mask_to_apicid = default_cpu_mask_to_apicid, |
217 | .cpu_mask_to_apicid_and = flat_cpu_mask_to_apicid_and, | 203 | .cpu_mask_to_apicid_and = default_cpu_mask_to_apicid_and, |
218 | 204 | ||
219 | .send_IPI_mask = flat_send_IPI_mask, | 205 | .send_IPI_mask = flat_send_IPI_mask, |
220 | .send_IPI_mask_allbutself = flat_send_IPI_mask_allbutself, | 206 | .send_IPI_mask_allbutself = flat_send_IPI_mask_allbutself, |
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c index 42cdc78427a2..1bb5c6cee3eb 100644 --- a/arch/x86/kernel/apic/io_apic.c +++ b/arch/x86/kernel/apic/io_apic.c | |||
@@ -592,10 +592,12 @@ set_desc_affinity(struct irq_desc *desc, const struct cpumask *mask) | |||
592 | if (assign_irq_vector(irq, cfg, mask)) | 592 | if (assign_irq_vector(irq, cfg, mask)) |
593 | return BAD_APICID; | 593 | return BAD_APICID; |
594 | 594 | ||
595 | cpumask_and(desc->affinity, cfg->domain, mask); | 595 | /* check that before desc->addinity get updated */ |
596 | set_extra_move_desc(desc, mask); | 596 | set_extra_move_desc(desc, mask); |
597 | 597 | ||
598 | return apic->cpu_mask_to_apicid_and(desc->affinity, cpu_online_mask); | 598 | cpumask_copy(desc->affinity, mask); |
599 | |||
600 | return apic->cpu_mask_to_apicid_and(desc->affinity, cfg->domain); | ||
599 | } | 601 | } |
600 | 602 | ||
601 | static void | 603 | static void |
@@ -1428,7 +1430,6 @@ void __setup_vector_irq(int cpu) | |||
1428 | 1430 | ||
1429 | static struct irq_chip ioapic_chip; | 1431 | static struct irq_chip ioapic_chip; |
1430 | static struct irq_chip ir_ioapic_chip; | 1432 | static struct irq_chip ir_ioapic_chip; |
1431 | static struct irq_chip msi_ir_chip; | ||
1432 | 1433 | ||
1433 | #define IOAPIC_AUTO -1 | 1434 | #define IOAPIC_AUTO -1 |
1434 | #define IOAPIC_EDGE 0 | 1435 | #define IOAPIC_EDGE 0 |
@@ -2663,20 +2664,20 @@ static struct irq_chip ioapic_chip __read_mostly = { | |||
2663 | .retrigger = ioapic_retrigger_irq, | 2664 | .retrigger = ioapic_retrigger_irq, |
2664 | }; | 2665 | }; |
2665 | 2666 | ||
2666 | #ifdef CONFIG_INTR_REMAP | ||
2667 | static struct irq_chip ir_ioapic_chip __read_mostly = { | 2667 | static struct irq_chip ir_ioapic_chip __read_mostly = { |
2668 | .name = "IR-IO-APIC", | 2668 | .name = "IR-IO-APIC", |
2669 | .startup = startup_ioapic_irq, | 2669 | .startup = startup_ioapic_irq, |
2670 | .mask = mask_IO_APIC_irq, | 2670 | .mask = mask_IO_APIC_irq, |
2671 | .unmask = unmask_IO_APIC_irq, | 2671 | .unmask = unmask_IO_APIC_irq, |
2672 | #ifdef CONFIG_INTR_REMAP | ||
2672 | .ack = ack_x2apic_edge, | 2673 | .ack = ack_x2apic_edge, |
2673 | .eoi = ack_x2apic_level, | 2674 | .eoi = ack_x2apic_level, |
2674 | #ifdef CONFIG_SMP | 2675 | #ifdef CONFIG_SMP |
2675 | .set_affinity = set_ir_ioapic_affinity_irq, | 2676 | .set_affinity = set_ir_ioapic_affinity_irq, |
2676 | #endif | 2677 | #endif |
2678 | #endif | ||
2677 | .retrigger = ioapic_retrigger_irq, | 2679 | .retrigger = ioapic_retrigger_irq, |
2678 | }; | 2680 | }; |
2679 | #endif | ||
2680 | 2681 | ||
2681 | static inline void init_IO_APIC_traps(void) | 2682 | static inline void init_IO_APIC_traps(void) |
2682 | { | 2683 | { |
@@ -3391,18 +3392,18 @@ static struct irq_chip msi_chip = { | |||
3391 | .retrigger = ioapic_retrigger_irq, | 3392 | .retrigger = ioapic_retrigger_irq, |
3392 | }; | 3393 | }; |
3393 | 3394 | ||
3394 | #ifdef CONFIG_INTR_REMAP | ||
3395 | static struct irq_chip msi_ir_chip = { | 3395 | static struct irq_chip msi_ir_chip = { |
3396 | .name = "IR-PCI-MSI", | 3396 | .name = "IR-PCI-MSI", |
3397 | .unmask = unmask_msi_irq, | 3397 | .unmask = unmask_msi_irq, |
3398 | .mask = mask_msi_irq, | 3398 | .mask = mask_msi_irq, |
3399 | #ifdef CONFIG_INTR_REMAP | ||
3399 | .ack = ack_x2apic_edge, | 3400 | .ack = ack_x2apic_edge, |
3400 | #ifdef CONFIG_SMP | 3401 | #ifdef CONFIG_SMP |
3401 | .set_affinity = ir_set_msi_irq_affinity, | 3402 | .set_affinity = ir_set_msi_irq_affinity, |
3402 | #endif | 3403 | #endif |
3404 | #endif | ||
3403 | .retrigger = ioapic_retrigger_irq, | 3405 | .retrigger = ioapic_retrigger_irq, |
3404 | }; | 3406 | }; |
3405 | #endif | ||
3406 | 3407 | ||
3407 | /* | 3408 | /* |
3408 | * Map the PCI dev to the corresponding remapping hardware unit | 3409 | * Map the PCI dev to the corresponding remapping hardware unit |
@@ -3464,9 +3465,13 @@ int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type) | |||
3464 | int ret, sub_handle; | 3465 | int ret, sub_handle; |
3465 | struct msi_desc *msidesc; | 3466 | struct msi_desc *msidesc; |
3466 | unsigned int irq_want; | 3467 | unsigned int irq_want; |
3467 | struct intel_iommu *iommu = 0; | 3468 | struct intel_iommu *iommu = NULL; |
3468 | int index = 0; | 3469 | int index = 0; |
3469 | 3470 | ||
3471 | /* x86 doesn't support multiple MSI yet */ | ||
3472 | if (type == PCI_CAP_ID_MSI && nvec > 1) | ||
3473 | return 1; | ||
3474 | |||
3470 | irq_want = nr_irqs_gsi; | 3475 | irq_want = nr_irqs_gsi; |
3471 | sub_handle = 0; | 3476 | sub_handle = 0; |
3472 | list_for_each_entry(msidesc, &dev->msi_list, list) { | 3477 | list_for_each_entry(msidesc, &dev->msi_list, list) { |
@@ -3599,7 +3604,7 @@ static void hpet_msi_set_affinity(unsigned int irq, const struct cpumask *mask) | |||
3599 | 3604 | ||
3600 | #endif /* CONFIG_SMP */ | 3605 | #endif /* CONFIG_SMP */ |
3601 | 3606 | ||
3602 | struct irq_chip hpet_msi_type = { | 3607 | static struct irq_chip hpet_msi_type = { |
3603 | .name = "HPET_MSI", | 3608 | .name = "HPET_MSI", |
3604 | .unmask = hpet_msi_unmask, | 3609 | .unmask = hpet_msi_unmask, |
3605 | .mask = hpet_msi_mask, | 3610 | .mask = hpet_msi_mask, |
@@ -4130,9 +4135,12 @@ static int __init ioapic_insert_resources(void) | |||
4130 | struct resource *r = ioapic_resources; | 4135 | struct resource *r = ioapic_resources; |
4131 | 4136 | ||
4132 | if (!r) { | 4137 | if (!r) { |
4133 | printk(KERN_ERR | 4138 | if (nr_ioapics > 0) { |
4134 | "IO APIC resources could be not be allocated.\n"); | 4139 | printk(KERN_ERR |
4135 | return -1; | 4140 | "IO APIC resources couldn't be allocated.\n"); |
4141 | return -1; | ||
4142 | } | ||
4143 | return 0; | ||
4136 | } | 4144 | } |
4137 | 4145 | ||
4138 | for (i = 0; i < nr_ioapics; i++) { | 4146 | for (i = 0; i < nr_ioapics; i++) { |
diff --git a/arch/x86/kernel/apm_32.c b/arch/x86/kernel/apm_32.c index 10033fe718e0..ac7783a67432 100644 --- a/arch/x86/kernel/apm_32.c +++ b/arch/x86/kernel/apm_32.c | |||
@@ -1190,8 +1190,10 @@ static int suspend(int vetoable) | |||
1190 | struct apm_user *as; | 1190 | struct apm_user *as; |
1191 | 1191 | ||
1192 | device_suspend(PMSG_SUSPEND); | 1192 | device_suspend(PMSG_SUSPEND); |
1193 | local_irq_disable(); | 1193 | |
1194 | device_power_down(PMSG_SUSPEND); | 1194 | device_power_down(PMSG_SUSPEND); |
1195 | |||
1196 | local_irq_disable(); | ||
1195 | sysdev_suspend(PMSG_SUSPEND); | 1197 | sysdev_suspend(PMSG_SUSPEND); |
1196 | 1198 | ||
1197 | local_irq_enable(); | 1199 | local_irq_enable(); |
@@ -1209,9 +1211,12 @@ static int suspend(int vetoable) | |||
1209 | if (err != APM_SUCCESS) | 1211 | if (err != APM_SUCCESS) |
1210 | apm_error("suspend", err); | 1212 | apm_error("suspend", err); |
1211 | err = (err == APM_SUCCESS) ? 0 : -EIO; | 1213 | err = (err == APM_SUCCESS) ? 0 : -EIO; |
1214 | |||
1212 | sysdev_resume(); | 1215 | sysdev_resume(); |
1213 | device_power_up(PMSG_RESUME); | ||
1214 | local_irq_enable(); | 1216 | local_irq_enable(); |
1217 | |||
1218 | device_power_up(PMSG_RESUME); | ||
1219 | |||
1215 | device_resume(PMSG_RESUME); | 1220 | device_resume(PMSG_RESUME); |
1216 | queue_event(APM_NORMAL_RESUME, NULL); | 1221 | queue_event(APM_NORMAL_RESUME, NULL); |
1217 | spin_lock(&user_list_lock); | 1222 | spin_lock(&user_list_lock); |
@@ -1228,8 +1233,9 @@ static void standby(void) | |||
1228 | { | 1233 | { |
1229 | int err; | 1234 | int err; |
1230 | 1235 | ||
1231 | local_irq_disable(); | ||
1232 | device_power_down(PMSG_SUSPEND); | 1236 | device_power_down(PMSG_SUSPEND); |
1237 | |||
1238 | local_irq_disable(); | ||
1233 | sysdev_suspend(PMSG_SUSPEND); | 1239 | sysdev_suspend(PMSG_SUSPEND); |
1234 | local_irq_enable(); | 1240 | local_irq_enable(); |
1235 | 1241 | ||
@@ -1239,8 +1245,9 @@ static void standby(void) | |||
1239 | 1245 | ||
1240 | local_irq_disable(); | 1246 | local_irq_disable(); |
1241 | sysdev_resume(); | 1247 | sysdev_resume(); |
1242 | device_power_up(PMSG_RESUME); | ||
1243 | local_irq_enable(); | 1248 | local_irq_enable(); |
1249 | |||
1250 | device_power_up(PMSG_RESUME); | ||
1244 | } | 1251 | } |
1245 | 1252 | ||
1246 | static apm_event_t get_event(void) | 1253 | static apm_event_t get_event(void) |
diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c index fbf2f33e3080..5a6aa1c1162f 100644 --- a/arch/x86/kernel/asm-offsets_32.c +++ b/arch/x86/kernel/asm-offsets_32.c | |||
@@ -18,6 +18,7 @@ | |||
18 | #include <asm/thread_info.h> | 18 | #include <asm/thread_info.h> |
19 | #include <asm/bootparam.h> | 19 | #include <asm/bootparam.h> |
20 | #include <asm/elf.h> | 20 | #include <asm/elf.h> |
21 | #include <asm/suspend.h> | ||
21 | 22 | ||
22 | #include <xen/interface/xen.h> | 23 | #include <xen/interface/xen.h> |
23 | 24 | ||
diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index 8793ab33e2c1..e72f062fb4b5 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c | |||
@@ -16,6 +16,7 @@ | |||
16 | #include <asm/thread_info.h> | 16 | #include <asm/thread_info.h> |
17 | #include <asm/ia32.h> | 17 | #include <asm/ia32.h> |
18 | #include <asm/bootparam.h> | 18 | #include <asm/bootparam.h> |
19 | #include <asm/suspend.h> | ||
19 | 20 | ||
20 | #include <xen/interface/xen.h> | 21 | #include <xen/interface/xen.h> |
21 | 22 | ||
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 9469ecb5aeb8..6de9a908e400 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h | |||
@@ -3,25 +3,25 @@ | |||
3 | #define ARCH_X86_CPU_H | 3 | #define ARCH_X86_CPU_H |
4 | 4 | ||
5 | struct cpu_model_info { | 5 | struct cpu_model_info { |
6 | int vendor; | 6 | int vendor; |
7 | int family; | 7 | int family; |
8 | const char *model_names[16]; | 8 | const char *model_names[16]; |
9 | }; | 9 | }; |
10 | 10 | ||
11 | /* attempt to consolidate cpu attributes */ | 11 | /* attempt to consolidate cpu attributes */ |
12 | struct cpu_dev { | 12 | struct cpu_dev { |
13 | const char * c_vendor; | 13 | const char *c_vendor; |
14 | 14 | ||
15 | /* some have two possibilities for cpuid string */ | 15 | /* some have two possibilities for cpuid string */ |
16 | const char * c_ident[2]; | 16 | const char *c_ident[2]; |
17 | 17 | ||
18 | struct cpu_model_info c_models[4]; | 18 | struct cpu_model_info c_models[4]; |
19 | 19 | ||
20 | void (*c_early_init)(struct cpuinfo_x86 *c); | 20 | void (*c_early_init)(struct cpuinfo_x86 *); |
21 | void (*c_init)(struct cpuinfo_x86 * c); | 21 | void (*c_init)(struct cpuinfo_x86 *); |
22 | void (*c_identify)(struct cpuinfo_x86 * c); | 22 | void (*c_identify)(struct cpuinfo_x86 *); |
23 | unsigned int (*c_size_cache)(struct cpuinfo_x86 * c, unsigned int size); | 23 | unsigned int (*c_size_cache)(struct cpuinfo_x86 *, unsigned int); |
24 | int c_x86_vendor; | 24 | int c_x86_vendor; |
25 | }; | 25 | }; |
26 | 26 | ||
27 | #define cpu_dev_register(cpu_devX) \ | 27 | #define cpu_dev_register(cpu_devX) \ |
diff --git a/arch/x86/kernel/cpu/cpufreq/Kconfig b/arch/x86/kernel/cpu/cpufreq/Kconfig index 65792c2cc462..52c839875478 100644 --- a/arch/x86/kernel/cpu/cpufreq/Kconfig +++ b/arch/x86/kernel/cpu/cpufreq/Kconfig | |||
@@ -87,30 +87,15 @@ config X86_POWERNOW_K7_ACPI | |||
87 | config X86_POWERNOW_K8 | 87 | config X86_POWERNOW_K8 |
88 | tristate "AMD Opteron/Athlon64 PowerNow!" | 88 | tristate "AMD Opteron/Athlon64 PowerNow!" |
89 | select CPU_FREQ_TABLE | 89 | select CPU_FREQ_TABLE |
90 | depends on ACPI && ACPI_PROCESSOR | ||
90 | help | 91 | help |
91 | This adds the CPUFreq driver for mobile AMD Opteron/Athlon64 processors. | 92 | This adds the CPUFreq driver for K8/K10 Opteron/Athlon64 processors. |
92 | 93 | ||
93 | To compile this driver as a module, choose M here: the | 94 | To compile this driver as a module, choose M here: the |
94 | module will be called powernow-k8. | 95 | module will be called powernow-k8. |
95 | 96 | ||
96 | For details, take a look at <file:Documentation/cpu-freq/>. | 97 | For details, take a look at <file:Documentation/cpu-freq/>. |
97 | 98 | ||
98 | If in doubt, say N. | ||
99 | |||
100 | config X86_POWERNOW_K8_ACPI | ||
101 | bool | ||
102 | prompt "ACPI Support" if X86_32 | ||
103 | depends on ACPI && X86_POWERNOW_K8 && ACPI_PROCESSOR | ||
104 | depends on !(X86_POWERNOW_K8 = y && ACPI_PROCESSOR = m) | ||
105 | default y | ||
106 | help | ||
107 | This provides access to the K8s Processor Performance States via ACPI. | ||
108 | This driver is probably required for CPUFreq to work with multi-socket and | ||
109 | SMP systems. It is not required on at least some single-socket yet | ||
110 | multi-core systems, even if SMP is enabled. | ||
111 | |||
112 | It is safe to say Y here. | ||
113 | |||
114 | config X86_GX_SUSPMOD | 99 | config X86_GX_SUSPMOD |
115 | tristate "Cyrix MediaGX/NatSemi Geode Suspend Modulation" | 100 | tristate "Cyrix MediaGX/NatSemi Geode Suspend Modulation" |
116 | depends on X86_32 && PCI | 101 | depends on X86_32 && PCI |
diff --git a/arch/x86/kernel/cpu/cpufreq/Makefile b/arch/x86/kernel/cpu/cpufreq/Makefile index 560f7760dae5..509296df294d 100644 --- a/arch/x86/kernel/cpu/cpufreq/Makefile +++ b/arch/x86/kernel/cpu/cpufreq/Makefile | |||
@@ -1,6 +1,11 @@ | |||
1 | # Link order matters. K8 is preferred to ACPI because of firmware bugs in early | ||
2 | # K8 systems. ACPI is preferred to all other hardware-specific drivers. | ||
3 | # speedstep-* is preferred over p4-clockmod. | ||
4 | |||
5 | obj-$(CONFIG_X86_POWERNOW_K8) += powernow-k8.o | ||
6 | obj-$(CONFIG_X86_ACPI_CPUFREQ) += acpi-cpufreq.o | ||
1 | obj-$(CONFIG_X86_POWERNOW_K6) += powernow-k6.o | 7 | obj-$(CONFIG_X86_POWERNOW_K6) += powernow-k6.o |
2 | obj-$(CONFIG_X86_POWERNOW_K7) += powernow-k7.o | 8 | obj-$(CONFIG_X86_POWERNOW_K7) += powernow-k7.o |
3 | obj-$(CONFIG_X86_POWERNOW_K8) += powernow-k8.o | ||
4 | obj-$(CONFIG_X86_LONGHAUL) += longhaul.o | 9 | obj-$(CONFIG_X86_LONGHAUL) += longhaul.o |
5 | obj-$(CONFIG_X86_E_POWERSAVER) += e_powersaver.o | 10 | obj-$(CONFIG_X86_E_POWERSAVER) += e_powersaver.o |
6 | obj-$(CONFIG_ELAN_CPUFREQ) += elanfreq.o | 11 | obj-$(CONFIG_ELAN_CPUFREQ) += elanfreq.o |
@@ -10,7 +15,6 @@ obj-$(CONFIG_X86_GX_SUSPMOD) += gx-suspmod.o | |||
10 | obj-$(CONFIG_X86_SPEEDSTEP_ICH) += speedstep-ich.o | 15 | obj-$(CONFIG_X86_SPEEDSTEP_ICH) += speedstep-ich.o |
11 | obj-$(CONFIG_X86_SPEEDSTEP_LIB) += speedstep-lib.o | 16 | obj-$(CONFIG_X86_SPEEDSTEP_LIB) += speedstep-lib.o |
12 | obj-$(CONFIG_X86_SPEEDSTEP_SMI) += speedstep-smi.o | 17 | obj-$(CONFIG_X86_SPEEDSTEP_SMI) += speedstep-smi.o |
13 | obj-$(CONFIG_X86_ACPI_CPUFREQ) += acpi-cpufreq.o | ||
14 | obj-$(CONFIG_X86_SPEEDSTEP_CENTRINO) += speedstep-centrino.o | 18 | obj-$(CONFIG_X86_SPEEDSTEP_CENTRINO) += speedstep-centrino.o |
15 | obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o | 19 | obj-$(CONFIG_X86_P4_CLOCKMOD) += p4-clockmod.o |
16 | obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o | 20 | obj-$(CONFIG_X86_CPUFREQ_NFORCE2) += cpufreq-nforce2.o |
diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c index 22590cf688ae..23da96e57b17 100644 --- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c +++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c | |||
@@ -1,5 +1,5 @@ | |||
1 | /* | 1 | /* |
2 | * acpi-cpufreq.c - ACPI Processor P-States Driver ($Revision: 1.4 $) | 2 | * acpi-cpufreq.c - ACPI Processor P-States Driver |
3 | * | 3 | * |
4 | * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com> | 4 | * Copyright (C) 2001, 2002 Andy Grover <andrew.grover@intel.com> |
5 | * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> | 5 | * Copyright (C) 2001, 2002 Paul Diefenbaugh <paul.s.diefenbaugh@intel.com> |
@@ -36,16 +36,18 @@ | |||
36 | #include <linux/ftrace.h> | 36 | #include <linux/ftrace.h> |
37 | 37 | ||
38 | #include <linux/acpi.h> | 38 | #include <linux/acpi.h> |
39 | #include <linux/io.h> | ||
40 | #include <linux/delay.h> | ||
41 | #include <linux/uaccess.h> | ||
42 | |||
39 | #include <acpi/processor.h> | 43 | #include <acpi/processor.h> |
40 | 44 | ||
41 | #include <asm/io.h> | ||
42 | #include <asm/msr.h> | 45 | #include <asm/msr.h> |
43 | #include <asm/processor.h> | 46 | #include <asm/processor.h> |
44 | #include <asm/cpufeature.h> | 47 | #include <asm/cpufeature.h> |
45 | #include <asm/delay.h> | ||
46 | #include <asm/uaccess.h> | ||
47 | 48 | ||
48 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "acpi-cpufreq", msg) | 49 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
50 | "acpi-cpufreq", msg) | ||
49 | 51 | ||
50 | MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski"); | 52 | MODULE_AUTHOR("Paul Diefenbaugh, Dominik Brodowski"); |
51 | MODULE_DESCRIPTION("ACPI Processor P-States Driver"); | 53 | MODULE_DESCRIPTION("ACPI Processor P-States Driver"); |
@@ -95,7 +97,7 @@ static unsigned extract_io(u32 value, struct acpi_cpufreq_data *data) | |||
95 | 97 | ||
96 | perf = data->acpi_data; | 98 | perf = data->acpi_data; |
97 | 99 | ||
98 | for (i=0; i<perf->state_count; i++) { | 100 | for (i = 0; i < perf->state_count; i++) { |
99 | if (value == perf->states[i].status) | 101 | if (value == perf->states[i].status) |
100 | return data->freq_table[i].frequency; | 102 | return data->freq_table[i].frequency; |
101 | } | 103 | } |
@@ -110,7 +112,7 @@ static unsigned extract_msr(u32 msr, struct acpi_cpufreq_data *data) | |||
110 | msr &= INTEL_MSR_RANGE; | 112 | msr &= INTEL_MSR_RANGE; |
111 | perf = data->acpi_data; | 113 | perf = data->acpi_data; |
112 | 114 | ||
113 | for (i=0; data->freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { | 115 | for (i = 0; data->freq_table[i].frequency != CPUFREQ_TABLE_END; i++) { |
114 | if (msr == perf->states[data->freq_table[i].index].status) | 116 | if (msr == perf->states[data->freq_table[i].index].status) |
115 | return data->freq_table[i].frequency; | 117 | return data->freq_table[i].frequency; |
116 | } | 118 | } |
@@ -138,15 +140,13 @@ struct io_addr { | |||
138 | u8 bit_width; | 140 | u8 bit_width; |
139 | }; | 141 | }; |
140 | 142 | ||
141 | typedef union { | ||
142 | struct msr_addr msr; | ||
143 | struct io_addr io; | ||
144 | } drv_addr_union; | ||
145 | |||
146 | struct drv_cmd { | 143 | struct drv_cmd { |
147 | unsigned int type; | 144 | unsigned int type; |
148 | const struct cpumask *mask; | 145 | const struct cpumask *mask; |
149 | drv_addr_union addr; | 146 | union { |
147 | struct msr_addr msr; | ||
148 | struct io_addr io; | ||
149 | } addr; | ||
150 | u32 val; | 150 | u32 val; |
151 | }; | 151 | }; |
152 | 152 | ||
@@ -369,7 +369,7 @@ static unsigned int check_freqs(const struct cpumask *mask, unsigned int freq, | |||
369 | unsigned int cur_freq; | 369 | unsigned int cur_freq; |
370 | unsigned int i; | 370 | unsigned int i; |
371 | 371 | ||
372 | for (i=0; i<100; i++) { | 372 | for (i = 0; i < 100; i++) { |
373 | cur_freq = extract_freq(get_cur_val(mask), data); | 373 | cur_freq = extract_freq(get_cur_val(mask), data); |
374 | if (cur_freq == freq) | 374 | if (cur_freq == freq) |
375 | return 1; | 375 | return 1; |
@@ -494,7 +494,7 @@ acpi_cpufreq_guess_freq(struct acpi_cpufreq_data *data, unsigned int cpu) | |||
494 | unsigned long freq; | 494 | unsigned long freq; |
495 | unsigned long freqn = perf->states[0].core_frequency * 1000; | 495 | unsigned long freqn = perf->states[0].core_frequency * 1000; |
496 | 496 | ||
497 | for (i=0; i<(perf->state_count-1); i++) { | 497 | for (i = 0; i < (perf->state_count-1); i++) { |
498 | freq = freqn; | 498 | freq = freqn; |
499 | freqn = perf->states[i+1].core_frequency * 1000; | 499 | freqn = perf->states[i+1].core_frequency * 1000; |
500 | if ((2 * cpu_khz) > (freqn + freq)) { | 500 | if ((2 * cpu_khz) > (freqn + freq)) { |
@@ -673,7 +673,7 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) | |||
673 | 673 | ||
674 | /* detect transition latency */ | 674 | /* detect transition latency */ |
675 | policy->cpuinfo.transition_latency = 0; | 675 | policy->cpuinfo.transition_latency = 0; |
676 | for (i=0; i<perf->state_count; i++) { | 676 | for (i = 0; i < perf->state_count; i++) { |
677 | if ((perf->states[i].transition_latency * 1000) > | 677 | if ((perf->states[i].transition_latency * 1000) > |
678 | policy->cpuinfo.transition_latency) | 678 | policy->cpuinfo.transition_latency) |
679 | policy->cpuinfo.transition_latency = | 679 | policy->cpuinfo.transition_latency = |
@@ -682,8 +682,8 @@ static int acpi_cpufreq_cpu_init(struct cpufreq_policy *policy) | |||
682 | 682 | ||
683 | data->max_freq = perf->states[0].core_frequency * 1000; | 683 | data->max_freq = perf->states[0].core_frequency * 1000; |
684 | /* table init */ | 684 | /* table init */ |
685 | for (i=0; i<perf->state_count; i++) { | 685 | for (i = 0; i < perf->state_count; i++) { |
686 | if (i>0 && perf->states[i].core_frequency >= | 686 | if (i > 0 && perf->states[i].core_frequency >= |
687 | data->freq_table[valid_states-1].frequency / 1000) | 687 | data->freq_table[valid_states-1].frequency / 1000) |
688 | continue; | 688 | continue; |
689 | 689 | ||
diff --git a/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c b/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c index 965ea52767ac..733093d60436 100644 --- a/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c +++ b/arch/x86/kernel/cpu/cpufreq/cpufreq-nforce2.c | |||
@@ -32,7 +32,7 @@ | |||
32 | * nforce2_chipset: | 32 | * nforce2_chipset: |
33 | * FSB is changed using the chipset | 33 | * FSB is changed using the chipset |
34 | */ | 34 | */ |
35 | static struct pci_dev *nforce2_chipset_dev; | 35 | static struct pci_dev *nforce2_dev; |
36 | 36 | ||
37 | /* fid: | 37 | /* fid: |
38 | * multiplier * 10 | 38 | * multiplier * 10 |
@@ -56,7 +56,9 @@ MODULE_PARM_DESC(fid, "CPU multiplier to use (11.5 = 115)"); | |||
56 | MODULE_PARM_DESC(min_fsb, | 56 | MODULE_PARM_DESC(min_fsb, |
57 | "Minimum FSB to use, if not defined: current FSB - 50"); | 57 | "Minimum FSB to use, if not defined: current FSB - 50"); |
58 | 58 | ||
59 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "cpufreq-nforce2", msg) | 59 | #define PFX "cpufreq-nforce2: " |
60 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ | ||
61 | "cpufreq-nforce2", msg) | ||
60 | 62 | ||
61 | /** | 63 | /** |
62 | * nforce2_calc_fsb - calculate FSB | 64 | * nforce2_calc_fsb - calculate FSB |
@@ -118,11 +120,11 @@ static void nforce2_write_pll(int pll) | |||
118 | int temp; | 120 | int temp; |
119 | 121 | ||
120 | /* Set the pll addr. to 0x00 */ | 122 | /* Set the pll addr. to 0x00 */ |
121 | pci_write_config_dword(nforce2_chipset_dev, NFORCE2_PLLADR, 0); | 123 | pci_write_config_dword(nforce2_dev, NFORCE2_PLLADR, 0); |
122 | 124 | ||
123 | /* Now write the value in all 64 registers */ | 125 | /* Now write the value in all 64 registers */ |
124 | for (temp = 0; temp <= 0x3f; temp++) | 126 | for (temp = 0; temp <= 0x3f; temp++) |
125 | pci_write_config_dword(nforce2_chipset_dev, NFORCE2_PLLREG, pll); | 127 | pci_write_config_dword(nforce2_dev, NFORCE2_PLLREG, pll); |
126 | 128 | ||
127 | return; | 129 | return; |
128 | } | 130 | } |
@@ -139,8 +141,8 @@ static unsigned int nforce2_fsb_read(int bootfsb) | |||
139 | u32 fsb, temp = 0; | 141 | u32 fsb, temp = 0; |
140 | 142 | ||
141 | /* Get chipset boot FSB from subdevice 5 (FSB at boot-time) */ | 143 | /* Get chipset boot FSB from subdevice 5 (FSB at boot-time) */ |
142 | nforce2_sub5 = pci_get_subsys(PCI_VENDOR_ID_NVIDIA, | 144 | nforce2_sub5 = pci_get_subsys(PCI_VENDOR_ID_NVIDIA, 0x01EF, |
143 | 0x01EF, PCI_ANY_ID, PCI_ANY_ID, NULL); | 145 | PCI_ANY_ID, PCI_ANY_ID, NULL); |
144 | if (!nforce2_sub5) | 146 | if (!nforce2_sub5) |
145 | return 0; | 147 | return 0; |
146 | 148 | ||
@@ -148,13 +150,13 @@ static unsigned int nforce2_fsb_read(int bootfsb) | |||
148 | fsb /= 1000000; | 150 | fsb /= 1000000; |
149 | 151 | ||
150 | /* Check if PLL register is already set */ | 152 | /* Check if PLL register is already set */ |
151 | pci_read_config_byte(nforce2_chipset_dev, NFORCE2_PLLENABLE, (u8 *)&temp); | 153 | pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp); |
152 | 154 | ||
153 | if (bootfsb || !temp) | 155 | if (bootfsb || !temp) |
154 | return fsb; | 156 | return fsb; |
155 | 157 | ||
156 | /* Use PLL register FSB value */ | 158 | /* Use PLL register FSB value */ |
157 | pci_read_config_dword(nforce2_chipset_dev, NFORCE2_PLLREG, &temp); | 159 | pci_read_config_dword(nforce2_dev, NFORCE2_PLLREG, &temp); |
158 | fsb = nforce2_calc_fsb(temp); | 160 | fsb = nforce2_calc_fsb(temp); |
159 | 161 | ||
160 | return fsb; | 162 | return fsb; |
@@ -174,18 +176,18 @@ static int nforce2_set_fsb(unsigned int fsb) | |||
174 | int pll = 0; | 176 | int pll = 0; |
175 | 177 | ||
176 | if ((fsb > max_fsb) || (fsb < NFORCE2_MIN_FSB)) { | 178 | if ((fsb > max_fsb) || (fsb < NFORCE2_MIN_FSB)) { |
177 | printk(KERN_ERR "cpufreq: FSB %d is out of range!\n", fsb); | 179 | printk(KERN_ERR PFX "FSB %d is out of range!\n", fsb); |
178 | return -EINVAL; | 180 | return -EINVAL; |
179 | } | 181 | } |
180 | 182 | ||
181 | tfsb = nforce2_fsb_read(0); | 183 | tfsb = nforce2_fsb_read(0); |
182 | if (!tfsb) { | 184 | if (!tfsb) { |
183 | printk(KERN_ERR "cpufreq: Error while reading the FSB\n"); | 185 | printk(KERN_ERR PFX "Error while reading the FSB\n"); |
184 | return -EINVAL; | 186 | return -EINVAL; |
185 | } | 187 | } |
186 | 188 | ||
187 | /* First write? Then set actual value */ | 189 | /* First write? Then set actual value */ |
188 | pci_read_config_byte(nforce2_chipset_dev, NFORCE2_PLLENABLE, (u8 *)&temp); | 190 | pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp); |
189 | if (!temp) { | 191 | if (!temp) { |
190 | pll = nforce2_calc_pll(tfsb); | 192 | pll = nforce2_calc_pll(tfsb); |
191 | 193 | ||
@@ -197,7 +199,7 @@ static int nforce2_set_fsb(unsigned int fsb) | |||
197 | 199 | ||
198 | /* Enable write access */ | 200 | /* Enable write access */ |
199 | temp = 0x01; | 201 | temp = 0x01; |
200 | pci_write_config_byte(nforce2_chipset_dev, NFORCE2_PLLENABLE, (u8)temp); | 202 | pci_write_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8)temp); |
201 | 203 | ||
202 | diff = tfsb - fsb; | 204 | diff = tfsb - fsb; |
203 | 205 | ||
@@ -222,7 +224,7 @@ static int nforce2_set_fsb(unsigned int fsb) | |||
222 | } | 224 | } |
223 | 225 | ||
224 | temp = 0x40; | 226 | temp = 0x40; |
225 | pci_write_config_byte(nforce2_chipset_dev, NFORCE2_PLLADR, (u8)temp); | 227 | pci_write_config_byte(nforce2_dev, NFORCE2_PLLADR, (u8)temp); |
226 | 228 | ||
227 | return 0; | 229 | return 0; |
228 | } | 230 | } |
@@ -244,7 +246,8 @@ static unsigned int nforce2_get(unsigned int cpu) | |||
244 | * nforce2_target - set a new CPUFreq policy | 246 | * nforce2_target - set a new CPUFreq policy |
245 | * @policy: new policy | 247 | * @policy: new policy |
246 | * @target_freq: the target frequency | 248 | * @target_freq: the target frequency |
247 | * @relation: how that frequency relates to achieved frequency (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) | 249 | * @relation: how that frequency relates to achieved frequency |
250 | * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) | ||
248 | * | 251 | * |
249 | * Sets a new CPUFreq policy. | 252 | * Sets a new CPUFreq policy. |
250 | */ | 253 | */ |
@@ -276,7 +279,7 @@ static int nforce2_target(struct cpufreq_policy *policy, | |||
276 | /* local_irq_save(flags); */ | 279 | /* local_irq_save(flags); */ |
277 | 280 | ||
278 | if (nforce2_set_fsb(target_fsb) < 0) | 281 | if (nforce2_set_fsb(target_fsb) < 0) |
279 | printk(KERN_ERR "cpufreq: Changing FSB to %d failed\n", | 282 | printk(KERN_ERR PFX "Changing FSB to %d failed\n", |
280 | target_fsb); | 283 | target_fsb); |
281 | else | 284 | else |
282 | dprintk("Changed FSB successfully to %d\n", | 285 | dprintk("Changed FSB successfully to %d\n", |
@@ -327,8 +330,8 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) | |||
327 | /* FIX: Get FID from CPU */ | 330 | /* FIX: Get FID from CPU */ |
328 | if (!fid) { | 331 | if (!fid) { |
329 | if (!cpu_khz) { | 332 | if (!cpu_khz) { |
330 | printk(KERN_WARNING | 333 | printk(KERN_WARNING PFX |
331 | "cpufreq: cpu_khz not set, can't calculate multiplier!\n"); | 334 | "cpu_khz not set, can't calculate multiplier!\n"); |
332 | return -ENODEV; | 335 | return -ENODEV; |
333 | } | 336 | } |
334 | 337 | ||
@@ -343,7 +346,7 @@ static int nforce2_cpu_init(struct cpufreq_policy *policy) | |||
343 | } | 346 | } |
344 | } | 347 | } |
345 | 348 | ||
346 | printk(KERN_INFO "cpufreq: FSB currently at %i MHz, FID %d.%d\n", fsb, | 349 | printk(KERN_INFO PFX "FSB currently at %i MHz, FID %d.%d\n", fsb, |
347 | fid / 10, fid % 10); | 350 | fid / 10, fid % 10); |
348 | 351 | ||
349 | /* Set maximum FSB to FSB at boot time */ | 352 | /* Set maximum FSB to FSB at boot time */ |
@@ -392,17 +395,18 @@ static struct cpufreq_driver nforce2_driver = { | |||
392 | */ | 395 | */ |
393 | static unsigned int nforce2_detect_chipset(void) | 396 | static unsigned int nforce2_detect_chipset(void) |
394 | { | 397 | { |
395 | nforce2_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_NVIDIA, | 398 | nforce2_dev = pci_get_subsys(PCI_VENDOR_ID_NVIDIA, |
396 | PCI_DEVICE_ID_NVIDIA_NFORCE2, | 399 | PCI_DEVICE_ID_NVIDIA_NFORCE2, |
397 | PCI_ANY_ID, PCI_ANY_ID, NULL); | 400 | PCI_ANY_ID, PCI_ANY_ID, NULL); |
398 | 401 | ||
399 | if (nforce2_chipset_dev == NULL) | 402 | if (nforce2_dev == NULL) |
400 | return -ENODEV; | 403 | return -ENODEV; |
401 | 404 | ||
402 | printk(KERN_INFO "cpufreq: Detected nForce2 chipset revision %X\n", | 405 | printk(KERN_INFO PFX "Detected nForce2 chipset revision %X\n", |
403 | nforce2_chipset_dev->revision); | 406 | nforce2_dev->revision); |
404 | printk(KERN_INFO | 407 | printk(KERN_INFO PFX |
405 | "cpufreq: FSB changing is maybe unstable and can lead to crashes and data loss.\n"); | 408 | "FSB changing is maybe unstable and can lead to " |
409 | "crashes and data loss.\n"); | ||
406 | 410 | ||
407 | return 0; | 411 | return 0; |
408 | } | 412 | } |
@@ -420,7 +424,7 @@ static int __init nforce2_init(void) | |||
420 | 424 | ||
421 | /* detect chipset */ | 425 | /* detect chipset */ |
422 | if (nforce2_detect_chipset()) { | 426 | if (nforce2_detect_chipset()) { |
423 | printk(KERN_ERR "cpufreq: No nForce2 chipset.\n"); | 427 | printk(KERN_INFO PFX "No nForce2 chipset.\n"); |
424 | return -ENODEV; | 428 | return -ENODEV; |
425 | } | 429 | } |
426 | 430 | ||
diff --git a/arch/x86/kernel/cpu/cpufreq/e_powersaver.c b/arch/x86/kernel/cpu/cpufreq/e_powersaver.c index 41ab3f064cb1..35a257dd4bb7 100644 --- a/arch/x86/kernel/cpu/cpufreq/e_powersaver.c +++ b/arch/x86/kernel/cpu/cpufreq/e_powersaver.c | |||
@@ -12,12 +12,12 @@ | |||
12 | #include <linux/cpufreq.h> | 12 | #include <linux/cpufreq.h> |
13 | #include <linux/ioport.h> | 13 | #include <linux/ioport.h> |
14 | #include <linux/slab.h> | 14 | #include <linux/slab.h> |
15 | #include <linux/timex.h> | ||
16 | #include <linux/io.h> | ||
17 | #include <linux/delay.h> | ||
15 | 18 | ||
16 | #include <asm/msr.h> | 19 | #include <asm/msr.h> |
17 | #include <asm/tsc.h> | 20 | #include <asm/tsc.h> |
18 | #include <asm/timex.h> | ||
19 | #include <asm/io.h> | ||
20 | #include <asm/delay.h> | ||
21 | 21 | ||
22 | #define EPS_BRAND_C7M 0 | 22 | #define EPS_BRAND_C7M 0 |
23 | #define EPS_BRAND_C7 1 | 23 | #define EPS_BRAND_C7 1 |
@@ -184,7 +184,7 @@ static int eps_cpu_init(struct cpufreq_policy *policy) | |||
184 | break; | 184 | break; |
185 | } | 185 | } |
186 | 186 | ||
187 | switch(brand) { | 187 | switch (brand) { |
188 | case EPS_BRAND_C7M: | 188 | case EPS_BRAND_C7M: |
189 | printk(KERN_CONT "C7-M\n"); | 189 | printk(KERN_CONT "C7-M\n"); |
190 | break; | 190 | break; |
@@ -218,17 +218,20 @@ static int eps_cpu_init(struct cpufreq_policy *policy) | |||
218 | /* Print voltage and multiplier */ | 218 | /* Print voltage and multiplier */ |
219 | rdmsr(MSR_IA32_PERF_STATUS, lo, hi); | 219 | rdmsr(MSR_IA32_PERF_STATUS, lo, hi); |
220 | current_voltage = lo & 0xff; | 220 | current_voltage = lo & 0xff; |
221 | printk(KERN_INFO "eps: Current voltage = %dmV\n", current_voltage * 16 + 700); | 221 | printk(KERN_INFO "eps: Current voltage = %dmV\n", |
222 | current_voltage * 16 + 700); | ||
222 | current_multiplier = (lo >> 8) & 0xff; | 223 | current_multiplier = (lo >> 8) & 0xff; |
223 | printk(KERN_INFO "eps: Current multiplier = %d\n", current_multiplier); | 224 | printk(KERN_INFO "eps: Current multiplier = %d\n", current_multiplier); |
224 | 225 | ||
225 | /* Print limits */ | 226 | /* Print limits */ |
226 | max_voltage = hi & 0xff; | 227 | max_voltage = hi & 0xff; |
227 | printk(KERN_INFO "eps: Highest voltage = %dmV\n", max_voltage * 16 + 700); | 228 | printk(KERN_INFO "eps: Highest voltage = %dmV\n", |
229 | max_voltage * 16 + 700); | ||
228 | max_multiplier = (hi >> 8) & 0xff; | 230 | max_multiplier = (hi >> 8) & 0xff; |
229 | printk(KERN_INFO "eps: Highest multiplier = %d\n", max_multiplier); | 231 | printk(KERN_INFO "eps: Highest multiplier = %d\n", max_multiplier); |
230 | min_voltage = (hi >> 16) & 0xff; | 232 | min_voltage = (hi >> 16) & 0xff; |
231 | printk(KERN_INFO "eps: Lowest voltage = %dmV\n", min_voltage * 16 + 700); | 233 | printk(KERN_INFO "eps: Lowest voltage = %dmV\n", |
234 | min_voltage * 16 + 700); | ||
232 | min_multiplier = (hi >> 24) & 0xff; | 235 | min_multiplier = (hi >> 24) & 0xff; |
233 | printk(KERN_INFO "eps: Lowest multiplier = %d\n", min_multiplier); | 236 | printk(KERN_INFO "eps: Lowest multiplier = %d\n", min_multiplier); |
234 | 237 | ||
@@ -318,7 +321,7 @@ static int eps_cpu_exit(struct cpufreq_policy *policy) | |||
318 | return 0; | 321 | return 0; |
319 | } | 322 | } |
320 | 323 | ||
321 | static struct freq_attr* eps_attr[] = { | 324 | static struct freq_attr *eps_attr[] = { |
322 | &cpufreq_freq_attr_scaling_available_freqs, | 325 | &cpufreq_freq_attr_scaling_available_freqs, |
323 | NULL, | 326 | NULL, |
324 | }; | 327 | }; |
@@ -356,7 +359,7 @@ static void __exit eps_exit(void) | |||
356 | cpufreq_unregister_driver(&eps_driver); | 359 | cpufreq_unregister_driver(&eps_driver); |
357 | } | 360 | } |
358 | 361 | ||
359 | MODULE_AUTHOR("Rafa³ Bilski <rafalbilski@interia.pl>"); | 362 | MODULE_AUTHOR("Rafal Bilski <rafalbilski@interia.pl>"); |
360 | MODULE_DESCRIPTION("Enhanced PowerSaver driver for VIA C7 CPU's."); | 363 | MODULE_DESCRIPTION("Enhanced PowerSaver driver for VIA C7 CPU's."); |
361 | MODULE_LICENSE("GPL"); | 364 | MODULE_LICENSE("GPL"); |
362 | 365 | ||
diff --git a/arch/x86/kernel/cpu/cpufreq/elanfreq.c b/arch/x86/kernel/cpu/cpufreq/elanfreq.c index fe613c93b366..006b278b0d5d 100644 --- a/arch/x86/kernel/cpu/cpufreq/elanfreq.c +++ b/arch/x86/kernel/cpu/cpufreq/elanfreq.c | |||
@@ -184,7 +184,8 @@ static int elanfreq_target(struct cpufreq_policy *policy, | |||
184 | { | 184 | { |
185 | unsigned int newstate = 0; | 185 | unsigned int newstate = 0; |
186 | 186 | ||
187 | if (cpufreq_frequency_table_target(policy, &elanfreq_table[0], target_freq, relation, &newstate)) | 187 | if (cpufreq_frequency_table_target(policy, &elanfreq_table[0], |
188 | target_freq, relation, &newstate)) | ||
188 | return -EINVAL; | 189 | return -EINVAL; |
189 | 190 | ||
190 | elanfreq_set_cpu_state(newstate); | 191 | elanfreq_set_cpu_state(newstate); |
@@ -301,7 +302,8 @@ static void __exit elanfreq_exit(void) | |||
301 | module_param(max_freq, int, 0444); | 302 | module_param(max_freq, int, 0444); |
302 | 303 | ||
303 | MODULE_LICENSE("GPL"); | 304 | MODULE_LICENSE("GPL"); |
304 | MODULE_AUTHOR("Robert Schwebel <r.schwebel@pengutronix.de>, Sven Geggus <sven@geggus.net>"); | 305 | MODULE_AUTHOR("Robert Schwebel <r.schwebel@pengutronix.de>, " |
306 | "Sven Geggus <sven@geggus.net>"); | ||
305 | MODULE_DESCRIPTION("cpufreq driver for AMD's Elan CPUs"); | 307 | MODULE_DESCRIPTION("cpufreq driver for AMD's Elan CPUs"); |
306 | 308 | ||
307 | module_init(elanfreq_init); | 309 | module_init(elanfreq_init); |
diff --git a/arch/x86/kernel/cpu/cpufreq/gx-suspmod.c b/arch/x86/kernel/cpu/cpufreq/gx-suspmod.c index 9d9eae82e60f..ac27ec2264d5 100644 --- a/arch/x86/kernel/cpu/cpufreq/gx-suspmod.c +++ b/arch/x86/kernel/cpu/cpufreq/gx-suspmod.c | |||
@@ -79,8 +79,9 @@ | |||
79 | #include <linux/smp.h> | 79 | #include <linux/smp.h> |
80 | #include <linux/cpufreq.h> | 80 | #include <linux/cpufreq.h> |
81 | #include <linux/pci.h> | 81 | #include <linux/pci.h> |
82 | #include <linux/errno.h> | ||
83 | |||
82 | #include <asm/processor-cyrix.h> | 84 | #include <asm/processor-cyrix.h> |
83 | #include <asm/errno.h> | ||
84 | 85 | ||
85 | /* PCI config registers, all at F0 */ | 86 | /* PCI config registers, all at F0 */ |
86 | #define PCI_PMER1 0x80 /* power management enable register 1 */ | 87 | #define PCI_PMER1 0x80 /* power management enable register 1 */ |
@@ -122,8 +123,8 @@ static struct gxfreq_params *gx_params; | |||
122 | static int stock_freq; | 123 | static int stock_freq; |
123 | 124 | ||
124 | /* PCI bus clock - defaults to 30.000 if cpu_khz is not available */ | 125 | /* PCI bus clock - defaults to 30.000 if cpu_khz is not available */ |
125 | static int pci_busclk = 0; | 126 | static int pci_busclk; |
126 | module_param (pci_busclk, int, 0444); | 127 | module_param(pci_busclk, int, 0444); |
127 | 128 | ||
128 | /* maximum duration for which the cpu may be suspended | 129 | /* maximum duration for which the cpu may be suspended |
129 | * (32us * MAX_DURATION). If no parameter is given, this defaults | 130 | * (32us * MAX_DURATION). If no parameter is given, this defaults |
@@ -132,7 +133,7 @@ module_param (pci_busclk, int, 0444); | |||
132 | * is suspended -- processing power is just 0.39% of what it used to be, | 133 | * is suspended -- processing power is just 0.39% of what it used to be, |
133 | * though. 781.25 kHz(!) for a 200 MHz processor -- wow. */ | 134 | * though. 781.25 kHz(!) for a 200 MHz processor -- wow. */ |
134 | static int max_duration = 255; | 135 | static int max_duration = 255; |
135 | module_param (max_duration, int, 0444); | 136 | module_param(max_duration, int, 0444); |
136 | 137 | ||
137 | /* For the default policy, we want at least some processing power | 138 | /* For the default policy, we want at least some processing power |
138 | * - let's say 5%. (min = maxfreq / POLICY_MIN_DIV) | 139 | * - let's say 5%. (min = maxfreq / POLICY_MIN_DIV) |
@@ -140,7 +141,8 @@ module_param (max_duration, int, 0444); | |||
140 | #define POLICY_MIN_DIV 20 | 141 | #define POLICY_MIN_DIV 20 |
141 | 142 | ||
142 | 143 | ||
143 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "gx-suspmod", msg) | 144 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
145 | "gx-suspmod", msg) | ||
144 | 146 | ||
145 | /** | 147 | /** |
146 | * we can detect a core multipiler from dir0_lsb | 148 | * we can detect a core multipiler from dir0_lsb |
@@ -166,12 +168,20 @@ static int gx_freq_mult[16] = { | |||
166 | * Low Level chipset interface * | 168 | * Low Level chipset interface * |
167 | ****************************************************************/ | 169 | ****************************************************************/ |
168 | static struct pci_device_id gx_chipset_tbl[] __initdata = { | 170 | static struct pci_device_id gx_chipset_tbl[] __initdata = { |
169 | { PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5530_LEGACY, PCI_ANY_ID, PCI_ANY_ID }, | 171 | { PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5530_LEGACY, |
170 | { PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5520, PCI_ANY_ID, PCI_ANY_ID }, | 172 | PCI_ANY_ID, PCI_ANY_ID }, |
171 | { PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5510, PCI_ANY_ID, PCI_ANY_ID }, | 173 | { PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5520, |
174 | PCI_ANY_ID, PCI_ANY_ID }, | ||
175 | { PCI_VENDOR_ID_CYRIX, PCI_DEVICE_ID_CYRIX_5510, | ||
176 | PCI_ANY_ID, PCI_ANY_ID }, | ||
172 | { 0, }, | 177 | { 0, }, |
173 | }; | 178 | }; |
174 | 179 | ||
180 | static void gx_write_byte(int reg, int value) | ||
181 | { | ||
182 | pci_write_config_byte(gx_params->cs55x0, reg, value); | ||
183 | } | ||
184 | |||
175 | /** | 185 | /** |
176 | * gx_detect_chipset: | 186 | * gx_detect_chipset: |
177 | * | 187 | * |
@@ -200,7 +210,8 @@ static __init struct pci_dev *gx_detect_chipset(void) | |||
200 | /** | 210 | /** |
201 | * gx_get_cpuspeed: | 211 | * gx_get_cpuspeed: |
202 | * | 212 | * |
203 | * Finds out at which efficient frequency the Cyrix MediaGX/NatSemi Geode CPU runs. | 213 | * Finds out at which efficient frequency the Cyrix MediaGX/NatSemi |
214 | * Geode CPU runs. | ||
204 | */ | 215 | */ |
205 | static unsigned int gx_get_cpuspeed(unsigned int cpu) | 216 | static unsigned int gx_get_cpuspeed(unsigned int cpu) |
206 | { | 217 | { |
@@ -217,17 +228,18 @@ static unsigned int gx_get_cpuspeed(unsigned int cpu) | |||
217 | * | 228 | * |
218 | **/ | 229 | **/ |
219 | 230 | ||
220 | static unsigned int gx_validate_speed(unsigned int khz, u8 *on_duration, u8 *off_duration) | 231 | static unsigned int gx_validate_speed(unsigned int khz, u8 *on_duration, |
232 | u8 *off_duration) | ||
221 | { | 233 | { |
222 | unsigned int i; | 234 | unsigned int i; |
223 | u8 tmp_on, tmp_off; | 235 | u8 tmp_on, tmp_off; |
224 | int old_tmp_freq = stock_freq; | 236 | int old_tmp_freq = stock_freq; |
225 | int tmp_freq; | 237 | int tmp_freq; |
226 | 238 | ||
227 | *off_duration=1; | 239 | *off_duration = 1; |
228 | *on_duration=0; | 240 | *on_duration = 0; |
229 | 241 | ||
230 | for (i=max_duration; i>0; i--) { | 242 | for (i = max_duration; i > 0; i--) { |
231 | tmp_off = ((khz * i) / stock_freq) & 0xff; | 243 | tmp_off = ((khz * i) / stock_freq) & 0xff; |
232 | tmp_on = i - tmp_off; | 244 | tmp_on = i - tmp_off; |
233 | tmp_freq = (stock_freq * tmp_off) / i; | 245 | tmp_freq = (stock_freq * tmp_off) / i; |
@@ -259,26 +271,34 @@ static void gx_set_cpuspeed(unsigned int khz) | |||
259 | freqs.cpu = 0; | 271 | freqs.cpu = 0; |
260 | freqs.old = gx_get_cpuspeed(0); | 272 | freqs.old = gx_get_cpuspeed(0); |
261 | 273 | ||
262 | new_khz = gx_validate_speed(khz, &gx_params->on_duration, &gx_params->off_duration); | 274 | new_khz = gx_validate_speed(khz, &gx_params->on_duration, |
275 | &gx_params->off_duration); | ||
263 | 276 | ||
264 | freqs.new = new_khz; | 277 | freqs.new = new_khz; |
265 | 278 | ||
266 | cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); | 279 | cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); |
267 | local_irq_save(flags); | 280 | local_irq_save(flags); |
268 | 281 | ||
269 | if (new_khz != stock_freq) { /* if new khz == 100% of CPU speed, it is special case */ | 282 | |
283 | |||
284 | if (new_khz != stock_freq) { | ||
285 | /* if new khz == 100% of CPU speed, it is special case */ | ||
270 | switch (gx_params->cs55x0->device) { | 286 | switch (gx_params->cs55x0->device) { |
271 | case PCI_DEVICE_ID_CYRIX_5530_LEGACY: | 287 | case PCI_DEVICE_ID_CYRIX_5530_LEGACY: |
272 | pmer1 = gx_params->pci_pmer1 | IRQ_SPDUP | VID_SPDUP; | 288 | pmer1 = gx_params->pci_pmer1 | IRQ_SPDUP | VID_SPDUP; |
273 | /* FIXME: need to test other values -- Zwane,Miura */ | 289 | /* FIXME: need to test other values -- Zwane,Miura */ |
274 | pci_write_config_byte(gx_params->cs55x0, PCI_IRQTC, 4); /* typical 2 to 4ms */ | 290 | /* typical 2 to 4ms */ |
275 | pci_write_config_byte(gx_params->cs55x0, PCI_VIDTC, 100);/* typical 50 to 100ms */ | 291 | gx_write_byte(PCI_IRQTC, 4); |
276 | pci_write_config_byte(gx_params->cs55x0, PCI_PMER1, pmer1); | 292 | /* typical 50 to 100ms */ |
277 | 293 | gx_write_byte(PCI_VIDTC, 100); | |
278 | if (gx_params->cs55x0->revision < 0x10) { /* CS5530(rev 1.2, 1.3) */ | 294 | gx_write_byte(PCI_PMER1, pmer1); |
279 | suscfg = gx_params->pci_suscfg | SUSMOD; | 295 | |
280 | } else { /* CS5530A,B.. */ | 296 | if (gx_params->cs55x0->revision < 0x10) { |
281 | suscfg = gx_params->pci_suscfg | SUSMOD | PWRSVE; | 297 | /* CS5530(rev 1.2, 1.3) */ |
298 | suscfg = gx_params->pci_suscfg|SUSMOD; | ||
299 | } else { | ||
300 | /* CS5530A,B.. */ | ||
301 | suscfg = gx_params->pci_suscfg|SUSMOD|PWRSVE; | ||
282 | } | 302 | } |
283 | break; | 303 | break; |
284 | case PCI_DEVICE_ID_CYRIX_5520: | 304 | case PCI_DEVICE_ID_CYRIX_5520: |
@@ -294,13 +314,13 @@ static void gx_set_cpuspeed(unsigned int khz) | |||
294 | suscfg = gx_params->pci_suscfg & ~(SUSMOD); | 314 | suscfg = gx_params->pci_suscfg & ~(SUSMOD); |
295 | gx_params->off_duration = 0; | 315 | gx_params->off_duration = 0; |
296 | gx_params->on_duration = 0; | 316 | gx_params->on_duration = 0; |
297 | dprintk("suspend modulation disabled: cpu runs 100 percent speed.\n"); | 317 | dprintk("suspend modulation disabled: cpu runs 100%% speed.\n"); |
298 | } | 318 | } |
299 | 319 | ||
300 | pci_write_config_byte(gx_params->cs55x0, PCI_MODOFF, gx_params->off_duration); | 320 | gx_write_byte(PCI_MODOFF, gx_params->off_duration); |
301 | pci_write_config_byte(gx_params->cs55x0, PCI_MODON, gx_params->on_duration); | 321 | gx_write_byte(PCI_MODON, gx_params->on_duration); |
302 | 322 | ||
303 | pci_write_config_byte(gx_params->cs55x0, PCI_SUSCFG, suscfg); | 323 | gx_write_byte(PCI_SUSCFG, suscfg); |
304 | pci_read_config_byte(gx_params->cs55x0, PCI_SUSCFG, &suscfg); | 324 | pci_read_config_byte(gx_params->cs55x0, PCI_SUSCFG, &suscfg); |
305 | 325 | ||
306 | local_irq_restore(flags); | 326 | local_irq_restore(flags); |
@@ -334,7 +354,8 @@ static int cpufreq_gx_verify(struct cpufreq_policy *policy) | |||
334 | return -EINVAL; | 354 | return -EINVAL; |
335 | 355 | ||
336 | policy->cpu = 0; | 356 | policy->cpu = 0; |
337 | cpufreq_verify_within_limits(policy, (stock_freq / max_duration), stock_freq); | 357 | cpufreq_verify_within_limits(policy, (stock_freq / max_duration), |
358 | stock_freq); | ||
338 | 359 | ||
339 | /* it needs to be assured that at least one supported frequency is | 360 | /* it needs to be assured that at least one supported frequency is |
340 | * within policy->min and policy->max. If it is not, policy->max | 361 | * within policy->min and policy->max. If it is not, policy->max |
@@ -354,7 +375,8 @@ static int cpufreq_gx_verify(struct cpufreq_policy *policy) | |||
354 | policy->max = tmp_freq; | 375 | policy->max = tmp_freq; |
355 | if (policy->max < policy->min) | 376 | if (policy->max < policy->min) |
356 | policy->max = policy->min; | 377 | policy->max = policy->min; |
357 | cpufreq_verify_within_limits(policy, (stock_freq / max_duration), stock_freq); | 378 | cpufreq_verify_within_limits(policy, (stock_freq / max_duration), |
379 | stock_freq); | ||
358 | 380 | ||
359 | return 0; | 381 | return 0; |
360 | } | 382 | } |
@@ -398,18 +420,18 @@ static int cpufreq_gx_cpu_init(struct cpufreq_policy *policy) | |||
398 | return -ENODEV; | 420 | return -ENODEV; |
399 | 421 | ||
400 | /* determine maximum frequency */ | 422 | /* determine maximum frequency */ |
401 | if (pci_busclk) { | 423 | if (pci_busclk) |
402 | maxfreq = pci_busclk * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f]; | 424 | maxfreq = pci_busclk * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f]; |
403 | } else if (cpu_khz) { | 425 | else if (cpu_khz) |
404 | maxfreq = cpu_khz; | 426 | maxfreq = cpu_khz; |
405 | } else { | 427 | else |
406 | maxfreq = 30000 * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f]; | 428 | maxfreq = 30000 * gx_freq_mult[getCx86(CX86_DIR1) & 0x0f]; |
407 | } | 429 | |
408 | stock_freq = maxfreq; | 430 | stock_freq = maxfreq; |
409 | curfreq = gx_get_cpuspeed(0); | 431 | curfreq = gx_get_cpuspeed(0); |
410 | 432 | ||
411 | dprintk("cpu max frequency is %d.\n", maxfreq); | 433 | dprintk("cpu max frequency is %d.\n", maxfreq); |
412 | dprintk("cpu current frequency is %dkHz.\n",curfreq); | 434 | dprintk("cpu current frequency is %dkHz.\n", curfreq); |
413 | 435 | ||
414 | /* setup basic struct for cpufreq API */ | 436 | /* setup basic struct for cpufreq API */ |
415 | policy->cpu = 0; | 437 | policy->cpu = 0; |
@@ -447,7 +469,8 @@ static int __init cpufreq_gx_init(void) | |||
447 | struct pci_dev *gx_pci; | 469 | struct pci_dev *gx_pci; |
448 | 470 | ||
449 | /* Test if we have the right hardware */ | 471 | /* Test if we have the right hardware */ |
450 | if ((gx_pci = gx_detect_chipset()) == NULL) | 472 | gx_pci = gx_detect_chipset(); |
473 | if (gx_pci == NULL) | ||
451 | return -ENODEV; | 474 | return -ENODEV; |
452 | 475 | ||
453 | /* check whether module parameters are sane */ | 476 | /* check whether module parameters are sane */ |
@@ -468,9 +491,11 @@ static int __init cpufreq_gx_init(void) | |||
468 | pci_read_config_byte(params->cs55x0, PCI_PMER1, &(params->pci_pmer1)); | 491 | pci_read_config_byte(params->cs55x0, PCI_PMER1, &(params->pci_pmer1)); |
469 | pci_read_config_byte(params->cs55x0, PCI_PMER2, &(params->pci_pmer2)); | 492 | pci_read_config_byte(params->cs55x0, PCI_PMER2, &(params->pci_pmer2)); |
470 | pci_read_config_byte(params->cs55x0, PCI_MODON, &(params->on_duration)); | 493 | pci_read_config_byte(params->cs55x0, PCI_MODON, &(params->on_duration)); |
471 | pci_read_config_byte(params->cs55x0, PCI_MODOFF, &(params->off_duration)); | 494 | pci_read_config_byte(params->cs55x0, PCI_MODOFF, |
495 | &(params->off_duration)); | ||
472 | 496 | ||
473 | if ((ret = cpufreq_register_driver(&gx_suspmod_driver))) { | 497 | ret = cpufreq_register_driver(&gx_suspmod_driver); |
498 | if (ret) { | ||
474 | kfree(params); | 499 | kfree(params); |
475 | return ret; /* register error! */ | 500 | return ret; /* register error! */ |
476 | } | 501 | } |
@@ -485,9 +510,9 @@ static void __exit cpufreq_gx_exit(void) | |||
485 | kfree(gx_params); | 510 | kfree(gx_params); |
486 | } | 511 | } |
487 | 512 | ||
488 | MODULE_AUTHOR ("Hiroshi Miura <miura@da-cha.org>"); | 513 | MODULE_AUTHOR("Hiroshi Miura <miura@da-cha.org>"); |
489 | MODULE_DESCRIPTION ("Cpufreq driver for Cyrix MediaGX and NatSemi Geode"); | 514 | MODULE_DESCRIPTION("Cpufreq driver for Cyrix MediaGX and NatSemi Geode"); |
490 | MODULE_LICENSE ("GPL"); | 515 | MODULE_LICENSE("GPL"); |
491 | 516 | ||
492 | module_init(cpufreq_gx_init); | 517 | module_init(cpufreq_gx_init); |
493 | module_exit(cpufreq_gx_exit); | 518 | module_exit(cpufreq_gx_exit); |
diff --git a/arch/x86/kernel/cpu/cpufreq/longhaul.c b/arch/x86/kernel/cpu/cpufreq/longhaul.c index a4cff5d6e380..f1c51aea064d 100644 --- a/arch/x86/kernel/cpu/cpufreq/longhaul.c +++ b/arch/x86/kernel/cpu/cpufreq/longhaul.c | |||
@@ -30,12 +30,12 @@ | |||
30 | #include <linux/slab.h> | 30 | #include <linux/slab.h> |
31 | #include <linux/string.h> | 31 | #include <linux/string.h> |
32 | #include <linux/delay.h> | 32 | #include <linux/delay.h> |
33 | #include <linux/timex.h> | ||
34 | #include <linux/io.h> | ||
35 | #include <linux/acpi.h> | ||
36 | #include <linux/kernel.h> | ||
33 | 37 | ||
34 | #include <asm/msr.h> | 38 | #include <asm/msr.h> |
35 | #include <asm/timex.h> | ||
36 | #include <asm/io.h> | ||
37 | #include <asm/acpi.h> | ||
38 | #include <linux/acpi.h> | ||
39 | #include <acpi/processor.h> | 39 | #include <acpi/processor.h> |
40 | 40 | ||
41 | #include "longhaul.h" | 41 | #include "longhaul.h" |
@@ -58,7 +58,7 @@ | |||
58 | #define USE_NORTHBRIDGE (1 << 2) | 58 | #define USE_NORTHBRIDGE (1 << 2) |
59 | 59 | ||
60 | static int cpu_model; | 60 | static int cpu_model; |
61 | static unsigned int numscales=16; | 61 | static unsigned int numscales = 16; |
62 | static unsigned int fsb; | 62 | static unsigned int fsb; |
63 | 63 | ||
64 | static const struct mV_pos *vrm_mV_table; | 64 | static const struct mV_pos *vrm_mV_table; |
@@ -67,8 +67,8 @@ static const unsigned char *mV_vrm_table; | |||
67 | static unsigned int highest_speed, lowest_speed; /* kHz */ | 67 | static unsigned int highest_speed, lowest_speed; /* kHz */ |
68 | static unsigned int minmult, maxmult; | 68 | static unsigned int minmult, maxmult; |
69 | static int can_scale_voltage; | 69 | static int can_scale_voltage; |
70 | static struct acpi_processor *pr = NULL; | 70 | static struct acpi_processor *pr; |
71 | static struct acpi_processor_cx *cx = NULL; | 71 | static struct acpi_processor_cx *cx; |
72 | static u32 acpi_regs_addr; | 72 | static u32 acpi_regs_addr; |
73 | static u8 longhaul_flags; | 73 | static u8 longhaul_flags; |
74 | static unsigned int longhaul_index; | 74 | static unsigned int longhaul_index; |
@@ -78,12 +78,13 @@ static int scale_voltage; | |||
78 | static int disable_acpi_c3; | 78 | static int disable_acpi_c3; |
79 | static int revid_errata; | 79 | static int revid_errata; |
80 | 80 | ||
81 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "longhaul", msg) | 81 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
82 | "longhaul", msg) | ||
82 | 83 | ||
83 | 84 | ||
84 | /* Clock ratios multiplied by 10 */ | 85 | /* Clock ratios multiplied by 10 */ |
85 | static int clock_ratio[32]; | 86 | static int mults[32]; |
86 | static int eblcr_table[32]; | 87 | static int eblcr[32]; |
87 | static int longhaul_version; | 88 | static int longhaul_version; |
88 | static struct cpufreq_frequency_table *longhaul_table; | 89 | static struct cpufreq_frequency_table *longhaul_table; |
89 | 90 | ||
@@ -93,7 +94,7 @@ static char speedbuffer[8]; | |||
93 | static char *print_speed(int speed) | 94 | static char *print_speed(int speed) |
94 | { | 95 | { |
95 | if (speed < 1000) { | 96 | if (speed < 1000) { |
96 | snprintf(speedbuffer, sizeof(speedbuffer),"%dMHz", speed); | 97 | snprintf(speedbuffer, sizeof(speedbuffer), "%dMHz", speed); |
97 | return speedbuffer; | 98 | return speedbuffer; |
98 | } | 99 | } |
99 | 100 | ||
@@ -122,27 +123,28 @@ static unsigned int calc_speed(int mult) | |||
122 | 123 | ||
123 | static int longhaul_get_cpu_mult(void) | 124 | static int longhaul_get_cpu_mult(void) |
124 | { | 125 | { |
125 | unsigned long invalue=0,lo, hi; | 126 | unsigned long invalue = 0, lo, hi; |
126 | 127 | ||
127 | rdmsr (MSR_IA32_EBL_CR_POWERON, lo, hi); | 128 | rdmsr(MSR_IA32_EBL_CR_POWERON, lo, hi); |
128 | invalue = (lo & (1<<22|1<<23|1<<24|1<<25)) >>22; | 129 | invalue = (lo & (1<<22|1<<23|1<<24|1<<25))>>22; |
129 | if (longhaul_version==TYPE_LONGHAUL_V2 || longhaul_version==TYPE_POWERSAVER) { | 130 | if (longhaul_version == TYPE_LONGHAUL_V2 || |
131 | longhaul_version == TYPE_POWERSAVER) { | ||
130 | if (lo & (1<<27)) | 132 | if (lo & (1<<27)) |
131 | invalue+=16; | 133 | invalue += 16; |
132 | } | 134 | } |
133 | return eblcr_table[invalue]; | 135 | return eblcr[invalue]; |
134 | } | 136 | } |
135 | 137 | ||
136 | /* For processor with BCR2 MSR */ | 138 | /* For processor with BCR2 MSR */ |
137 | 139 | ||
138 | static void do_longhaul1(unsigned int clock_ratio_index) | 140 | static void do_longhaul1(unsigned int mults_index) |
139 | { | 141 | { |
140 | union msr_bcr2 bcr2; | 142 | union msr_bcr2 bcr2; |
141 | 143 | ||
142 | rdmsrl(MSR_VIA_BCR2, bcr2.val); | 144 | rdmsrl(MSR_VIA_BCR2, bcr2.val); |
143 | /* Enable software clock multiplier */ | 145 | /* Enable software clock multiplier */ |
144 | bcr2.bits.ESOFTBF = 1; | 146 | bcr2.bits.ESOFTBF = 1; |
145 | bcr2.bits.CLOCKMUL = clock_ratio_index & 0xff; | 147 | bcr2.bits.CLOCKMUL = mults_index & 0xff; |
146 | 148 | ||
147 | /* Sync to timer tick */ | 149 | /* Sync to timer tick */ |
148 | safe_halt(); | 150 | safe_halt(); |
@@ -161,7 +163,7 @@ static void do_longhaul1(unsigned int clock_ratio_index) | |||
161 | 163 | ||
162 | /* For processor with Longhaul MSR */ | 164 | /* For processor with Longhaul MSR */ |
163 | 165 | ||
164 | static void do_powersaver(int cx_address, unsigned int clock_ratio_index, | 166 | static void do_powersaver(int cx_address, unsigned int mults_index, |
165 | unsigned int dir) | 167 | unsigned int dir) |
166 | { | 168 | { |
167 | union msr_longhaul longhaul; | 169 | union msr_longhaul longhaul; |
@@ -173,11 +175,11 @@ static void do_powersaver(int cx_address, unsigned int clock_ratio_index, | |||
173 | longhaul.bits.RevisionKey = longhaul.bits.RevisionID; | 175 | longhaul.bits.RevisionKey = longhaul.bits.RevisionID; |
174 | else | 176 | else |
175 | longhaul.bits.RevisionKey = 0; | 177 | longhaul.bits.RevisionKey = 0; |
176 | longhaul.bits.SoftBusRatio = clock_ratio_index & 0xf; | 178 | longhaul.bits.SoftBusRatio = mults_index & 0xf; |
177 | longhaul.bits.SoftBusRatio4 = (clock_ratio_index & 0x10) >> 4; | 179 | longhaul.bits.SoftBusRatio4 = (mults_index & 0x10) >> 4; |
178 | /* Setup new voltage */ | 180 | /* Setup new voltage */ |
179 | if (can_scale_voltage) | 181 | if (can_scale_voltage) |
180 | longhaul.bits.SoftVID = (clock_ratio_index >> 8) & 0x1f; | 182 | longhaul.bits.SoftVID = (mults_index >> 8) & 0x1f; |
181 | /* Sync to timer tick */ | 183 | /* Sync to timer tick */ |
182 | safe_halt(); | 184 | safe_halt(); |
183 | /* Raise voltage if necessary */ | 185 | /* Raise voltage if necessary */ |
@@ -240,14 +242,14 @@ static void do_powersaver(int cx_address, unsigned int clock_ratio_index, | |||
240 | 242 | ||
241 | /** | 243 | /** |
242 | * longhaul_set_cpu_frequency() | 244 | * longhaul_set_cpu_frequency() |
243 | * @clock_ratio_index : bitpattern of the new multiplier. | 245 | * @mults_index : bitpattern of the new multiplier. |
244 | * | 246 | * |
245 | * Sets a new clock ratio. | 247 | * Sets a new clock ratio. |
246 | */ | 248 | */ |
247 | 249 | ||
248 | static void longhaul_setstate(unsigned int table_index) | 250 | static void longhaul_setstate(unsigned int table_index) |
249 | { | 251 | { |
250 | unsigned int clock_ratio_index; | 252 | unsigned int mults_index; |
251 | int speed, mult; | 253 | int speed, mult; |
252 | struct cpufreq_freqs freqs; | 254 | struct cpufreq_freqs freqs; |
253 | unsigned long flags; | 255 | unsigned long flags; |
@@ -256,9 +258,9 @@ static void longhaul_setstate(unsigned int table_index) | |||
256 | u32 bm_timeout = 1000; | 258 | u32 bm_timeout = 1000; |
257 | unsigned int dir = 0; | 259 | unsigned int dir = 0; |
258 | 260 | ||
259 | clock_ratio_index = longhaul_table[table_index].index; | 261 | mults_index = longhaul_table[table_index].index; |
260 | /* Safety precautions */ | 262 | /* Safety precautions */ |
261 | mult = clock_ratio[clock_ratio_index & 0x1f]; | 263 | mult = mults[mults_index & 0x1f]; |
262 | if (mult == -1) | 264 | if (mult == -1) |
263 | return; | 265 | return; |
264 | speed = calc_speed(mult); | 266 | speed = calc_speed(mult); |
@@ -274,7 +276,7 @@ static void longhaul_setstate(unsigned int table_index) | |||
274 | 276 | ||
275 | cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); | 277 | cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); |
276 | 278 | ||
277 | dprintk ("Setting to FSB:%dMHz Mult:%d.%dx (%s)\n", | 279 | dprintk("Setting to FSB:%dMHz Mult:%d.%dx (%s)\n", |
278 | fsb, mult/10, mult%10, print_speed(speed/1000)); | 280 | fsb, mult/10, mult%10, print_speed(speed/1000)); |
279 | retry_loop: | 281 | retry_loop: |
280 | preempt_disable(); | 282 | preempt_disable(); |
@@ -282,8 +284,8 @@ retry_loop: | |||
282 | 284 | ||
283 | pic2_mask = inb(0xA1); | 285 | pic2_mask = inb(0xA1); |
284 | pic1_mask = inb(0x21); /* works on C3. save mask. */ | 286 | pic1_mask = inb(0x21); /* works on C3. save mask. */ |
285 | outb(0xFF,0xA1); /* Overkill */ | 287 | outb(0xFF, 0xA1); /* Overkill */ |
286 | outb(0xFE,0x21); /* TMR0 only */ | 288 | outb(0xFE, 0x21); /* TMR0 only */ |
287 | 289 | ||
288 | /* Wait while PCI bus is busy. */ | 290 | /* Wait while PCI bus is busy. */ |
289 | if (acpi_regs_addr && (longhaul_flags & USE_NORTHBRIDGE | 291 | if (acpi_regs_addr && (longhaul_flags & USE_NORTHBRIDGE |
@@ -312,7 +314,7 @@ retry_loop: | |||
312 | * Software controlled multipliers only. | 314 | * Software controlled multipliers only. |
313 | */ | 315 | */ |
314 | case TYPE_LONGHAUL_V1: | 316 | case TYPE_LONGHAUL_V1: |
315 | do_longhaul1(clock_ratio_index); | 317 | do_longhaul1(mults_index); |
316 | break; | 318 | break; |
317 | 319 | ||
318 | /* | 320 | /* |
@@ -327,9 +329,9 @@ retry_loop: | |||
327 | if (longhaul_flags & USE_ACPI_C3) { | 329 | if (longhaul_flags & USE_ACPI_C3) { |
328 | /* Don't allow wakeup */ | 330 | /* Don't allow wakeup */ |
329 | acpi_set_register(ACPI_BITREG_BUS_MASTER_RLD, 0); | 331 | acpi_set_register(ACPI_BITREG_BUS_MASTER_RLD, 0); |
330 | do_powersaver(cx->address, clock_ratio_index, dir); | 332 | do_powersaver(cx->address, mults_index, dir); |
331 | } else { | 333 | } else { |
332 | do_powersaver(0, clock_ratio_index, dir); | 334 | do_powersaver(0, mults_index, dir); |
333 | } | 335 | } |
334 | break; | 336 | break; |
335 | } | 337 | } |
@@ -341,8 +343,8 @@ retry_loop: | |||
341 | /* Enable bus master arbitration */ | 343 | /* Enable bus master arbitration */ |
342 | acpi_set_register(ACPI_BITREG_ARB_DISABLE, 0); | 344 | acpi_set_register(ACPI_BITREG_ARB_DISABLE, 0); |
343 | } | 345 | } |
344 | outb(pic2_mask,0xA1); /* restore mask */ | 346 | outb(pic2_mask, 0xA1); /* restore mask */ |
345 | outb(pic1_mask,0x21); | 347 | outb(pic1_mask, 0x21); |
346 | 348 | ||
347 | local_irq_restore(flags); | 349 | local_irq_restore(flags); |
348 | preempt_enable(); | 350 | preempt_enable(); |
@@ -392,7 +394,8 @@ retry_loop: | |||
392 | cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); | 394 | cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); |
393 | 395 | ||
394 | if (!bm_timeout) | 396 | if (!bm_timeout) |
395 | printk(KERN_INFO PFX "Warning: Timeout while waiting for idle PCI bus.\n"); | 397 | printk(KERN_INFO PFX "Warning: Timeout while waiting for " |
398 | "idle PCI bus.\n"); | ||
396 | } | 399 | } |
397 | 400 | ||
398 | /* | 401 | /* |
@@ -458,31 +461,32 @@ static int __init longhaul_get_ranges(void) | |||
458 | break; | 461 | break; |
459 | } | 462 | } |
460 | 463 | ||
461 | dprintk ("MinMult:%d.%dx MaxMult:%d.%dx\n", | 464 | dprintk("MinMult:%d.%dx MaxMult:%d.%dx\n", |
462 | minmult/10, minmult%10, maxmult/10, maxmult%10); | 465 | minmult/10, minmult%10, maxmult/10, maxmult%10); |
463 | 466 | ||
464 | highest_speed = calc_speed(maxmult); | 467 | highest_speed = calc_speed(maxmult); |
465 | lowest_speed = calc_speed(minmult); | 468 | lowest_speed = calc_speed(minmult); |
466 | dprintk ("FSB:%dMHz Lowest speed: %s Highest speed:%s\n", fsb, | 469 | dprintk("FSB:%dMHz Lowest speed: %s Highest speed:%s\n", fsb, |
467 | print_speed(lowest_speed/1000), | 470 | print_speed(lowest_speed/1000), |
468 | print_speed(highest_speed/1000)); | 471 | print_speed(highest_speed/1000)); |
469 | 472 | ||
470 | if (lowest_speed == highest_speed) { | 473 | if (lowest_speed == highest_speed) { |
471 | printk (KERN_INFO PFX "highestspeed == lowest, aborting.\n"); | 474 | printk(KERN_INFO PFX "highestspeed == lowest, aborting.\n"); |
472 | return -EINVAL; | 475 | return -EINVAL; |
473 | } | 476 | } |
474 | if (lowest_speed > highest_speed) { | 477 | if (lowest_speed > highest_speed) { |
475 | printk (KERN_INFO PFX "nonsense! lowest (%d > %d) !\n", | 478 | printk(KERN_INFO PFX "nonsense! lowest (%d > %d) !\n", |
476 | lowest_speed, highest_speed); | 479 | lowest_speed, highest_speed); |
477 | return -EINVAL; | 480 | return -EINVAL; |
478 | } | 481 | } |
479 | 482 | ||
480 | longhaul_table = kmalloc((numscales + 1) * sizeof(struct cpufreq_frequency_table), GFP_KERNEL); | 483 | longhaul_table = kmalloc((numscales + 1) * sizeof(*longhaul_table), |
481 | if(!longhaul_table) | 484 | GFP_KERNEL); |
485 | if (!longhaul_table) | ||
482 | return -ENOMEM; | 486 | return -ENOMEM; |
483 | 487 | ||
484 | for (j = 0; j < numscales; j++) { | 488 | for (j = 0; j < numscales; j++) { |
485 | ratio = clock_ratio[j]; | 489 | ratio = mults[j]; |
486 | if (ratio == -1) | 490 | if (ratio == -1) |
487 | continue; | 491 | continue; |
488 | if (ratio > maxmult || ratio < minmult) | 492 | if (ratio > maxmult || ratio < minmult) |
@@ -507,13 +511,10 @@ static int __init longhaul_get_ranges(void) | |||
507 | } | 511 | } |
508 | } | 512 | } |
509 | if (min_i != j) { | 513 | if (min_i != j) { |
510 | unsigned int temp; | 514 | swap(longhaul_table[j].frequency, |
511 | temp = longhaul_table[j].frequency; | 515 | longhaul_table[min_i].frequency); |
512 | longhaul_table[j].frequency = longhaul_table[min_i].frequency; | 516 | swap(longhaul_table[j].index, |
513 | longhaul_table[min_i].frequency = temp; | 517 | longhaul_table[min_i].index); |
514 | temp = longhaul_table[j].index; | ||
515 | longhaul_table[j].index = longhaul_table[min_i].index; | ||
516 | longhaul_table[min_i].index = temp; | ||
517 | } | 518 | } |
518 | } | 519 | } |
519 | 520 | ||
@@ -521,7 +522,7 @@ static int __init longhaul_get_ranges(void) | |||
521 | 522 | ||
522 | /* Find index we are running on */ | 523 | /* Find index we are running on */ |
523 | for (j = 0; j < k; j++) { | 524 | for (j = 0; j < k; j++) { |
524 | if (clock_ratio[longhaul_table[j].index & 0x1f] == mult) { | 525 | if (mults[longhaul_table[j].index & 0x1f] == mult) { |
525 | longhaul_index = j; | 526 | longhaul_index = j; |
526 | break; | 527 | break; |
527 | } | 528 | } |
@@ -559,20 +560,22 @@ static void __init longhaul_setup_voltagescaling(void) | |||
559 | maxvid = vrm_mV_table[longhaul.bits.MaximumVID]; | 560 | maxvid = vrm_mV_table[longhaul.bits.MaximumVID]; |
560 | 561 | ||
561 | if (minvid.mV == 0 || maxvid.mV == 0 || minvid.mV > maxvid.mV) { | 562 | if (minvid.mV == 0 || maxvid.mV == 0 || minvid.mV > maxvid.mV) { |
562 | printk (KERN_INFO PFX "Bogus values Min:%d.%03d Max:%d.%03d. " | 563 | printk(KERN_INFO PFX "Bogus values Min:%d.%03d Max:%d.%03d. " |
563 | "Voltage scaling disabled.\n", | 564 | "Voltage scaling disabled.\n", |
564 | minvid.mV/1000, minvid.mV%1000, maxvid.mV/1000, maxvid.mV%1000); | 565 | minvid.mV/1000, minvid.mV%1000, |
566 | maxvid.mV/1000, maxvid.mV%1000); | ||
565 | return; | 567 | return; |
566 | } | 568 | } |
567 | 569 | ||
568 | if (minvid.mV == maxvid.mV) { | 570 | if (minvid.mV == maxvid.mV) { |
569 | printk (KERN_INFO PFX "Claims to support voltage scaling but min & max are " | 571 | printk(KERN_INFO PFX "Claims to support voltage scaling but " |
570 | "both %d.%03d. Voltage scaling disabled\n", | 572 | "min & max are both %d.%03d. " |
573 | "Voltage scaling disabled\n", | ||
571 | maxvid.mV/1000, maxvid.mV%1000); | 574 | maxvid.mV/1000, maxvid.mV%1000); |
572 | return; | 575 | return; |
573 | } | 576 | } |
574 | 577 | ||
575 | /* How many voltage steps */ | 578 | /* How many voltage steps*/ |
576 | numvscales = maxvid.pos - minvid.pos + 1; | 579 | numvscales = maxvid.pos - minvid.pos + 1; |
577 | printk(KERN_INFO PFX | 580 | printk(KERN_INFO PFX |
578 | "Max VID=%d.%03d " | 581 | "Max VID=%d.%03d " |
@@ -586,7 +589,7 @@ static void __init longhaul_setup_voltagescaling(void) | |||
586 | j = longhaul.bits.MinMHzBR; | 589 | j = longhaul.bits.MinMHzBR; |
587 | if (longhaul.bits.MinMHzBR4) | 590 | if (longhaul.bits.MinMHzBR4) |
588 | j += 16; | 591 | j += 16; |
589 | min_vid_speed = eblcr_table[j]; | 592 | min_vid_speed = eblcr[j]; |
590 | if (min_vid_speed == -1) | 593 | if (min_vid_speed == -1) |
591 | return; | 594 | return; |
592 | switch (longhaul.bits.MinMHzFSB) { | 595 | switch (longhaul.bits.MinMHzFSB) { |
@@ -617,7 +620,8 @@ static void __init longhaul_setup_voltagescaling(void) | |||
617 | pos = minvid.pos; | 620 | pos = minvid.pos; |
618 | longhaul_table[j].index |= mV_vrm_table[pos] << 8; | 621 | longhaul_table[j].index |= mV_vrm_table[pos] << 8; |
619 | vid = vrm_mV_table[mV_vrm_table[pos]]; | 622 | vid = vrm_mV_table[mV_vrm_table[pos]]; |
620 | printk(KERN_INFO PFX "f: %d kHz, index: %d, vid: %d mV\n", speed, j, vid.mV); | 623 | printk(KERN_INFO PFX "f: %d kHz, index: %d, vid: %d mV\n", |
624 | speed, j, vid.mV); | ||
621 | j++; | 625 | j++; |
622 | } | 626 | } |
623 | 627 | ||
@@ -640,7 +644,8 @@ static int longhaul_target(struct cpufreq_policy *policy, | |||
640 | unsigned int dir = 0; | 644 | unsigned int dir = 0; |
641 | u8 vid, current_vid; | 645 | u8 vid, current_vid; |
642 | 646 | ||
643 | if (cpufreq_frequency_table_target(policy, longhaul_table, target_freq, relation, &table_index)) | 647 | if (cpufreq_frequency_table_target(policy, longhaul_table, target_freq, |
648 | relation, &table_index)) | ||
644 | return -EINVAL; | 649 | return -EINVAL; |
645 | 650 | ||
646 | /* Don't set same frequency again */ | 651 | /* Don't set same frequency again */ |
@@ -656,7 +661,8 @@ static int longhaul_target(struct cpufreq_policy *policy, | |||
656 | * this in hardware, C3 is old and we need to do this | 661 | * this in hardware, C3 is old and we need to do this |
657 | * in software. */ | 662 | * in software. */ |
658 | i = longhaul_index; | 663 | i = longhaul_index; |
659 | current_vid = (longhaul_table[longhaul_index].index >> 8) & 0x1f; | 664 | current_vid = (longhaul_table[longhaul_index].index >> 8); |
665 | current_vid &= 0x1f; | ||
660 | if (table_index > longhaul_index) | 666 | if (table_index > longhaul_index) |
661 | dir = 1; | 667 | dir = 1; |
662 | while (i != table_index) { | 668 | while (i != table_index) { |
@@ -691,9 +697,9 @@ static acpi_status longhaul_walk_callback(acpi_handle obj_handle, | |||
691 | { | 697 | { |
692 | struct acpi_device *d; | 698 | struct acpi_device *d; |
693 | 699 | ||
694 | if ( acpi_bus_get_device(obj_handle, &d) ) { | 700 | if (acpi_bus_get_device(obj_handle, &d)) |
695 | return 0; | 701 | return 0; |
696 | } | 702 | |
697 | *return_value = acpi_driver_data(d); | 703 | *return_value = acpi_driver_data(d); |
698 | return 1; | 704 | return 1; |
699 | } | 705 | } |
@@ -750,7 +756,7 @@ static int longhaul_setup_southbridge(void) | |||
750 | /* Find VT8235 southbridge */ | 756 | /* Find VT8235 southbridge */ |
751 | dev = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8235, NULL); | 757 | dev = pci_get_device(PCI_VENDOR_ID_VIA, PCI_DEVICE_ID_VIA_8235, NULL); |
752 | if (dev == NULL) | 758 | if (dev == NULL) |
753 | /* Find VT8237 southbridge */ | 759 | /* Find VT8237 southbridge */ |
754 | dev = pci_get_device(PCI_VENDOR_ID_VIA, | 760 | dev = pci_get_device(PCI_VENDOR_ID_VIA, |
755 | PCI_DEVICE_ID_VIA_8237, NULL); | 761 | PCI_DEVICE_ID_VIA_8237, NULL); |
756 | if (dev != NULL) { | 762 | if (dev != NULL) { |
@@ -769,7 +775,8 @@ static int longhaul_setup_southbridge(void) | |||
769 | if (pci_cmd & 1 << 7) { | 775 | if (pci_cmd & 1 << 7) { |
770 | pci_read_config_dword(dev, 0x88, &acpi_regs_addr); | 776 | pci_read_config_dword(dev, 0x88, &acpi_regs_addr); |
771 | acpi_regs_addr &= 0xff00; | 777 | acpi_regs_addr &= 0xff00; |
772 | printk(KERN_INFO PFX "ACPI I/O at 0x%x\n", acpi_regs_addr); | 778 | printk(KERN_INFO PFX "ACPI I/O at 0x%x\n", |
779 | acpi_regs_addr); | ||
773 | } | 780 | } |
774 | 781 | ||
775 | pci_dev_put(dev); | 782 | pci_dev_put(dev); |
@@ -781,7 +788,7 @@ static int longhaul_setup_southbridge(void) | |||
781 | static int __init longhaul_cpu_init(struct cpufreq_policy *policy) | 788 | static int __init longhaul_cpu_init(struct cpufreq_policy *policy) |
782 | { | 789 | { |
783 | struct cpuinfo_x86 *c = &cpu_data(0); | 790 | struct cpuinfo_x86 *c = &cpu_data(0); |
784 | char *cpuname=NULL; | 791 | char *cpuname = NULL; |
785 | int ret; | 792 | int ret; |
786 | u32 lo, hi; | 793 | u32 lo, hi; |
787 | 794 | ||
@@ -791,8 +798,8 @@ static int __init longhaul_cpu_init(struct cpufreq_policy *policy) | |||
791 | cpu_model = CPU_SAMUEL; | 798 | cpu_model = CPU_SAMUEL; |
792 | cpuname = "C3 'Samuel' [C5A]"; | 799 | cpuname = "C3 'Samuel' [C5A]"; |
793 | longhaul_version = TYPE_LONGHAUL_V1; | 800 | longhaul_version = TYPE_LONGHAUL_V1; |
794 | memcpy (clock_ratio, samuel1_clock_ratio, sizeof(samuel1_clock_ratio)); | 801 | memcpy(mults, samuel1_mults, sizeof(samuel1_mults)); |
795 | memcpy (eblcr_table, samuel1_eblcr, sizeof(samuel1_eblcr)); | 802 | memcpy(eblcr, samuel1_eblcr, sizeof(samuel1_eblcr)); |
796 | break; | 803 | break; |
797 | 804 | ||
798 | case 7: | 805 | case 7: |
@@ -803,10 +810,8 @@ static int __init longhaul_cpu_init(struct cpufreq_policy *policy) | |||
803 | cpuname = "C3 'Samuel 2' [C5B]"; | 810 | cpuname = "C3 'Samuel 2' [C5B]"; |
804 | /* Note, this is not a typo, early Samuel2's had | 811 | /* Note, this is not a typo, early Samuel2's had |
805 | * Samuel1 ratios. */ | 812 | * Samuel1 ratios. */ |
806 | memcpy(clock_ratio, samuel1_clock_ratio, | 813 | memcpy(mults, samuel1_mults, sizeof(samuel1_mults)); |
807 | sizeof(samuel1_clock_ratio)); | 814 | memcpy(eblcr, samuel2_eblcr, sizeof(samuel2_eblcr)); |
808 | memcpy(eblcr_table, samuel2_eblcr, | ||
809 | sizeof(samuel2_eblcr)); | ||
810 | break; | 815 | break; |
811 | case 1 ... 15: | 816 | case 1 ... 15: |
812 | longhaul_version = TYPE_LONGHAUL_V1; | 817 | longhaul_version = TYPE_LONGHAUL_V1; |
@@ -817,10 +822,8 @@ static int __init longhaul_cpu_init(struct cpufreq_policy *policy) | |||
817 | cpu_model = CPU_EZRA; | 822 | cpu_model = CPU_EZRA; |
818 | cpuname = "C3 'Ezra' [C5C]"; | 823 | cpuname = "C3 'Ezra' [C5C]"; |
819 | } | 824 | } |
820 | memcpy(clock_ratio, ezra_clock_ratio, | 825 | memcpy(mults, ezra_mults, sizeof(ezra_mults)); |
821 | sizeof(ezra_clock_ratio)); | 826 | memcpy(eblcr, ezra_eblcr, sizeof(ezra_eblcr)); |
822 | memcpy(eblcr_table, ezra_eblcr, | ||
823 | sizeof(ezra_eblcr)); | ||
824 | break; | 827 | break; |
825 | } | 828 | } |
826 | break; | 829 | break; |
@@ -829,18 +832,16 @@ static int __init longhaul_cpu_init(struct cpufreq_policy *policy) | |||
829 | cpu_model = CPU_EZRA_T; | 832 | cpu_model = CPU_EZRA_T; |
830 | cpuname = "C3 'Ezra-T' [C5M]"; | 833 | cpuname = "C3 'Ezra-T' [C5M]"; |
831 | longhaul_version = TYPE_POWERSAVER; | 834 | longhaul_version = TYPE_POWERSAVER; |
832 | numscales=32; | 835 | numscales = 32; |
833 | memcpy (clock_ratio, ezrat_clock_ratio, sizeof(ezrat_clock_ratio)); | 836 | memcpy(mults, ezrat_mults, sizeof(ezrat_mults)); |
834 | memcpy (eblcr_table, ezrat_eblcr, sizeof(ezrat_eblcr)); | 837 | memcpy(eblcr, ezrat_eblcr, sizeof(ezrat_eblcr)); |
835 | break; | 838 | break; |
836 | 839 | ||
837 | case 9: | 840 | case 9: |
838 | longhaul_version = TYPE_POWERSAVER; | 841 | longhaul_version = TYPE_POWERSAVER; |
839 | numscales = 32; | 842 | numscales = 32; |
840 | memcpy(clock_ratio, | 843 | memcpy(mults, nehemiah_mults, sizeof(nehemiah_mults)); |
841 | nehemiah_clock_ratio, | 844 | memcpy(eblcr, nehemiah_eblcr, sizeof(nehemiah_eblcr)); |
842 | sizeof(nehemiah_clock_ratio)); | ||
843 | memcpy(eblcr_table, nehemiah_eblcr, sizeof(nehemiah_eblcr)); | ||
844 | switch (c->x86_mask) { | 845 | switch (c->x86_mask) { |
845 | case 0 ... 1: | 846 | case 0 ... 1: |
846 | cpu_model = CPU_NEHEMIAH; | 847 | cpu_model = CPU_NEHEMIAH; |
@@ -869,14 +870,14 @@ static int __init longhaul_cpu_init(struct cpufreq_policy *policy) | |||
869 | longhaul_version = TYPE_LONGHAUL_V1; | 870 | longhaul_version = TYPE_LONGHAUL_V1; |
870 | } | 871 | } |
871 | 872 | ||
872 | printk (KERN_INFO PFX "VIA %s CPU detected. ", cpuname); | 873 | printk(KERN_INFO PFX "VIA %s CPU detected. ", cpuname); |
873 | switch (longhaul_version) { | 874 | switch (longhaul_version) { |
874 | case TYPE_LONGHAUL_V1: | 875 | case TYPE_LONGHAUL_V1: |
875 | case TYPE_LONGHAUL_V2: | 876 | case TYPE_LONGHAUL_V2: |
876 | printk ("Longhaul v%d supported.\n", longhaul_version); | 877 | printk(KERN_CONT "Longhaul v%d supported.\n", longhaul_version); |
877 | break; | 878 | break; |
878 | case TYPE_POWERSAVER: | 879 | case TYPE_POWERSAVER: |
879 | printk ("Powersaver supported.\n"); | 880 | printk(KERN_CONT "Powersaver supported.\n"); |
880 | break; | 881 | break; |
881 | }; | 882 | }; |
882 | 883 | ||
@@ -940,7 +941,7 @@ static int __devexit longhaul_cpu_exit(struct cpufreq_policy *policy) | |||
940 | return 0; | 941 | return 0; |
941 | } | 942 | } |
942 | 943 | ||
943 | static struct freq_attr* longhaul_attr[] = { | 944 | static struct freq_attr *longhaul_attr[] = { |
944 | &cpufreq_freq_attr_scaling_available_freqs, | 945 | &cpufreq_freq_attr_scaling_available_freqs, |
945 | NULL, | 946 | NULL, |
946 | }; | 947 | }; |
@@ -966,13 +967,15 @@ static int __init longhaul_init(void) | |||
966 | 967 | ||
967 | #ifdef CONFIG_SMP | 968 | #ifdef CONFIG_SMP |
968 | if (num_online_cpus() > 1) { | 969 | if (num_online_cpus() > 1) { |
969 | printk(KERN_ERR PFX "More than 1 CPU detected, longhaul disabled.\n"); | 970 | printk(KERN_ERR PFX "More than 1 CPU detected, " |
971 | "longhaul disabled.\n"); | ||
970 | return -ENODEV; | 972 | return -ENODEV; |
971 | } | 973 | } |
972 | #endif | 974 | #endif |
973 | #ifdef CONFIG_X86_IO_APIC | 975 | #ifdef CONFIG_X86_IO_APIC |
974 | if (cpu_has_apic) { | 976 | if (cpu_has_apic) { |
975 | printk(KERN_ERR PFX "APIC detected. Longhaul is currently broken in this configuration.\n"); | 977 | printk(KERN_ERR PFX "APIC detected. Longhaul is currently " |
978 | "broken in this configuration.\n"); | ||
976 | return -ENODEV; | 979 | return -ENODEV; |
977 | } | 980 | } |
978 | #endif | 981 | #endif |
@@ -993,8 +996,8 @@ static void __exit longhaul_exit(void) | |||
993 | { | 996 | { |
994 | int i; | 997 | int i; |
995 | 998 | ||
996 | for (i=0; i < numscales; i++) { | 999 | for (i = 0; i < numscales; i++) { |
997 | if (clock_ratio[i] == maxmult) { | 1000 | if (mults[i] == maxmult) { |
998 | longhaul_setstate(i); | 1001 | longhaul_setstate(i); |
999 | break; | 1002 | break; |
1000 | } | 1003 | } |
@@ -1007,11 +1010,11 @@ static void __exit longhaul_exit(void) | |||
1007 | /* Even if BIOS is exporting ACPI C3 state, and it is used | 1010 | /* Even if BIOS is exporting ACPI C3 state, and it is used |
1008 | * with success when CPU is idle, this state doesn't | 1011 | * with success when CPU is idle, this state doesn't |
1009 | * trigger frequency transition in some cases. */ | 1012 | * trigger frequency transition in some cases. */ |
1010 | module_param (disable_acpi_c3, int, 0644); | 1013 | module_param(disable_acpi_c3, int, 0644); |
1011 | MODULE_PARM_DESC(disable_acpi_c3, "Don't use ACPI C3 support"); | 1014 | MODULE_PARM_DESC(disable_acpi_c3, "Don't use ACPI C3 support"); |
1012 | /* Change CPU voltage with frequency. Very usefull to save | 1015 | /* Change CPU voltage with frequency. Very usefull to save |
1013 | * power, but most VIA C3 processors aren't supporting it. */ | 1016 | * power, but most VIA C3 processors aren't supporting it. */ |
1014 | module_param (scale_voltage, int, 0644); | 1017 | module_param(scale_voltage, int, 0644); |
1015 | MODULE_PARM_DESC(scale_voltage, "Scale voltage of processor"); | 1018 | MODULE_PARM_DESC(scale_voltage, "Scale voltage of processor"); |
1016 | /* Force revision key to 0 for processors which doesn't | 1019 | /* Force revision key to 0 for processors which doesn't |
1017 | * support voltage scaling, but are introducing itself as | 1020 | * support voltage scaling, but are introducing itself as |
@@ -1019,9 +1022,9 @@ MODULE_PARM_DESC(scale_voltage, "Scale voltage of processor"); | |||
1019 | module_param(revid_errata, int, 0644); | 1022 | module_param(revid_errata, int, 0644); |
1020 | MODULE_PARM_DESC(revid_errata, "Ignore CPU Revision ID"); | 1023 | MODULE_PARM_DESC(revid_errata, "Ignore CPU Revision ID"); |
1021 | 1024 | ||
1022 | MODULE_AUTHOR ("Dave Jones <davej@redhat.com>"); | 1025 | MODULE_AUTHOR("Dave Jones <davej@redhat.com>"); |
1023 | MODULE_DESCRIPTION ("Longhaul driver for VIA Cyrix processors."); | 1026 | MODULE_DESCRIPTION("Longhaul driver for VIA Cyrix processors."); |
1024 | MODULE_LICENSE ("GPL"); | 1027 | MODULE_LICENSE("GPL"); |
1025 | 1028 | ||
1026 | late_initcall(longhaul_init); | 1029 | late_initcall(longhaul_init); |
1027 | module_exit(longhaul_exit); | 1030 | module_exit(longhaul_exit); |
diff --git a/arch/x86/kernel/cpu/cpufreq/longhaul.h b/arch/x86/kernel/cpu/cpufreq/longhaul.h index 4fcc320997df..e2360a469f79 100644 --- a/arch/x86/kernel/cpu/cpufreq/longhaul.h +++ b/arch/x86/kernel/cpu/cpufreq/longhaul.h | |||
@@ -49,14 +49,14 @@ union msr_longhaul { | |||
49 | 49 | ||
50 | /* | 50 | /* |
51 | * Clock ratio tables. Div/Mod by 10 to get ratio. | 51 | * Clock ratio tables. Div/Mod by 10 to get ratio. |
52 | * The eblcr ones specify the ratio read from the CPU. | 52 | * The eblcr values specify the ratio read from the CPU. |
53 | * The clock_ratio ones specify what to write to the CPU. | 53 | * The mults values specify what to write to the CPU. |
54 | */ | 54 | */ |
55 | 55 | ||
56 | /* | 56 | /* |
57 | * VIA C3 Samuel 1 & Samuel 2 (stepping 0) | 57 | * VIA C3 Samuel 1 & Samuel 2 (stepping 0) |
58 | */ | 58 | */ |
59 | static const int __initdata samuel1_clock_ratio[16] = { | 59 | static const int __initdata samuel1_mults[16] = { |
60 | -1, /* 0000 -> RESERVED */ | 60 | -1, /* 0000 -> RESERVED */ |
61 | 30, /* 0001 -> 3.0x */ | 61 | 30, /* 0001 -> 3.0x */ |
62 | 40, /* 0010 -> 4.0x */ | 62 | 40, /* 0010 -> 4.0x */ |
@@ -119,7 +119,7 @@ static const int __initdata samuel2_eblcr[16] = { | |||
119 | /* | 119 | /* |
120 | * VIA C3 Ezra | 120 | * VIA C3 Ezra |
121 | */ | 121 | */ |
122 | static const int __initdata ezra_clock_ratio[16] = { | 122 | static const int __initdata ezra_mults[16] = { |
123 | 100, /* 0000 -> 10.0x */ | 123 | 100, /* 0000 -> 10.0x */ |
124 | 30, /* 0001 -> 3.0x */ | 124 | 30, /* 0001 -> 3.0x */ |
125 | 40, /* 0010 -> 4.0x */ | 125 | 40, /* 0010 -> 4.0x */ |
@@ -160,7 +160,7 @@ static const int __initdata ezra_eblcr[16] = { | |||
160 | /* | 160 | /* |
161 | * VIA C3 (Ezra-T) [C5M]. | 161 | * VIA C3 (Ezra-T) [C5M]. |
162 | */ | 162 | */ |
163 | static const int __initdata ezrat_clock_ratio[32] = { | 163 | static const int __initdata ezrat_mults[32] = { |
164 | 100, /* 0000 -> 10.0x */ | 164 | 100, /* 0000 -> 10.0x */ |
165 | 30, /* 0001 -> 3.0x */ | 165 | 30, /* 0001 -> 3.0x */ |
166 | 40, /* 0010 -> 4.0x */ | 166 | 40, /* 0010 -> 4.0x */ |
@@ -235,7 +235,7 @@ static const int __initdata ezrat_eblcr[32] = { | |||
235 | /* | 235 | /* |
236 | * VIA C3 Nehemiah */ | 236 | * VIA C3 Nehemiah */ |
237 | 237 | ||
238 | static const int __initdata nehemiah_clock_ratio[32] = { | 238 | static const int __initdata nehemiah_mults[32] = { |
239 | 100, /* 0000 -> 10.0x */ | 239 | 100, /* 0000 -> 10.0x */ |
240 | -1, /* 0001 -> 16.0x */ | 240 | -1, /* 0001 -> 16.0x */ |
241 | 40, /* 0010 -> 4.0x */ | 241 | 40, /* 0010 -> 4.0x */ |
diff --git a/arch/x86/kernel/cpu/cpufreq/longrun.c b/arch/x86/kernel/cpu/cpufreq/longrun.c index 777a7ff075de..da5f70fcb766 100644 --- a/arch/x86/kernel/cpu/cpufreq/longrun.c +++ b/arch/x86/kernel/cpu/cpufreq/longrun.c | |||
@@ -11,12 +11,13 @@ | |||
11 | #include <linux/init.h> | 11 | #include <linux/init.h> |
12 | #include <linux/slab.h> | 12 | #include <linux/slab.h> |
13 | #include <linux/cpufreq.h> | 13 | #include <linux/cpufreq.h> |
14 | #include <linux/timex.h> | ||
14 | 15 | ||
15 | #include <asm/msr.h> | 16 | #include <asm/msr.h> |
16 | #include <asm/processor.h> | 17 | #include <asm/processor.h> |
17 | #include <asm/timex.h> | ||
18 | 18 | ||
19 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "longrun", msg) | 19 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
20 | "longrun", msg) | ||
20 | 21 | ||
21 | static struct cpufreq_driver longrun_driver; | 22 | static struct cpufreq_driver longrun_driver; |
22 | 23 | ||
@@ -51,7 +52,7 @@ static void __init longrun_get_policy(struct cpufreq_policy *policy) | |||
51 | msr_lo &= 0x0000007F; | 52 | msr_lo &= 0x0000007F; |
52 | msr_hi &= 0x0000007F; | 53 | msr_hi &= 0x0000007F; |
53 | 54 | ||
54 | if ( longrun_high_freq <= longrun_low_freq ) { | 55 | if (longrun_high_freq <= longrun_low_freq) { |
55 | /* Assume degenerate Longrun table */ | 56 | /* Assume degenerate Longrun table */ |
56 | policy->min = policy->max = longrun_high_freq; | 57 | policy->min = policy->max = longrun_high_freq; |
57 | } else { | 58 | } else { |
@@ -79,7 +80,7 @@ static int longrun_set_policy(struct cpufreq_policy *policy) | |||
79 | if (!policy) | 80 | if (!policy) |
80 | return -EINVAL; | 81 | return -EINVAL; |
81 | 82 | ||
82 | if ( longrun_high_freq <= longrun_low_freq ) { | 83 | if (longrun_high_freq <= longrun_low_freq) { |
83 | /* Assume degenerate Longrun table */ | 84 | /* Assume degenerate Longrun table */ |
84 | pctg_lo = pctg_hi = 100; | 85 | pctg_lo = pctg_hi = 100; |
85 | } else { | 86 | } else { |
@@ -152,7 +153,7 @@ static unsigned int longrun_get(unsigned int cpu) | |||
152 | cpuid(0x80860007, &eax, &ebx, &ecx, &edx); | 153 | cpuid(0x80860007, &eax, &ebx, &ecx, &edx); |
153 | dprintk("cpuid eax is %u\n", eax); | 154 | dprintk("cpuid eax is %u\n", eax); |
154 | 155 | ||
155 | return (eax * 1000); | 156 | return eax * 1000; |
156 | } | 157 | } |
157 | 158 | ||
158 | /** | 159 | /** |
@@ -196,7 +197,8 @@ static unsigned int __init longrun_determine_freqs(unsigned int *low_freq, | |||
196 | rdmsr(MSR_TMTA_LRTI_VOLT_MHZ, msr_lo, msr_hi); | 197 | rdmsr(MSR_TMTA_LRTI_VOLT_MHZ, msr_lo, msr_hi); |
197 | *high_freq = msr_lo * 1000; /* to kHz */ | 198 | *high_freq = msr_lo * 1000; /* to kHz */ |
198 | 199 | ||
199 | dprintk("longrun table interface told %u - %u kHz\n", *low_freq, *high_freq); | 200 | dprintk("longrun table interface told %u - %u kHz\n", |
201 | *low_freq, *high_freq); | ||
200 | 202 | ||
201 | if (*low_freq > *high_freq) | 203 | if (*low_freq > *high_freq) |
202 | *low_freq = *high_freq; | 204 | *low_freq = *high_freq; |
@@ -219,7 +221,7 @@ static unsigned int __init longrun_determine_freqs(unsigned int *low_freq, | |||
219 | cpuid(0x80860007, &eax, &ebx, &ecx, &edx); | 221 | cpuid(0x80860007, &eax, &ebx, &ecx, &edx); |
220 | /* try decreasing in 10% steps, some processors react only | 222 | /* try decreasing in 10% steps, some processors react only |
221 | * on some barrier values */ | 223 | * on some barrier values */ |
222 | for (try_hi = 80; try_hi > 0 && ecx > 90; try_hi -=10) { | 224 | for (try_hi = 80; try_hi > 0 && ecx > 90; try_hi -= 10) { |
223 | /* set to 0 to try_hi perf_pctg */ | 225 | /* set to 0 to try_hi perf_pctg */ |
224 | msr_lo &= 0xFFFFFF80; | 226 | msr_lo &= 0xFFFFFF80; |
225 | msr_hi &= 0xFFFFFF80; | 227 | msr_hi &= 0xFFFFFF80; |
@@ -236,7 +238,7 @@ static unsigned int __init longrun_determine_freqs(unsigned int *low_freq, | |||
236 | 238 | ||
237 | /* performance_pctg = (current_freq - low_freq)/(high_freq - low_freq) | 239 | /* performance_pctg = (current_freq - low_freq)/(high_freq - low_freq) |
238 | * eqals | 240 | * eqals |
239 | * low_freq * ( 1 - perf_pctg) = (cur_freq - high_freq * perf_pctg) | 241 | * low_freq * (1 - perf_pctg) = (cur_freq - high_freq * perf_pctg) |
240 | * | 242 | * |
241 | * high_freq * perf_pctg is stored tempoarily into "ebx". | 243 | * high_freq * perf_pctg is stored tempoarily into "ebx". |
242 | */ | 244 | */ |
@@ -317,9 +319,10 @@ static void __exit longrun_exit(void) | |||
317 | } | 319 | } |
318 | 320 | ||
319 | 321 | ||
320 | MODULE_AUTHOR ("Dominik Brodowski <linux@brodo.de>"); | 322 | MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>"); |
321 | MODULE_DESCRIPTION ("LongRun driver for Transmeta Crusoe and Efficeon processors."); | 323 | MODULE_DESCRIPTION("LongRun driver for Transmeta Crusoe and " |
322 | MODULE_LICENSE ("GPL"); | 324 | "Efficeon processors."); |
325 | MODULE_LICENSE("GPL"); | ||
323 | 326 | ||
324 | module_init(longrun_init); | 327 | module_init(longrun_init); |
325 | module_exit(longrun_exit); | 328 | module_exit(longrun_exit); |
diff --git a/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c b/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c index 3178c3acd97e..41ed94915f97 100644 --- a/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c +++ b/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c | |||
@@ -27,15 +27,17 @@ | |||
27 | #include <linux/cpufreq.h> | 27 | #include <linux/cpufreq.h> |
28 | #include <linux/slab.h> | 28 | #include <linux/slab.h> |
29 | #include <linux/cpumask.h> | 29 | #include <linux/cpumask.h> |
30 | #include <linux/timex.h> | ||
30 | 31 | ||
31 | #include <asm/processor.h> | 32 | #include <asm/processor.h> |
32 | #include <asm/msr.h> | 33 | #include <asm/msr.h> |
33 | #include <asm/timex.h> | 34 | #include <asm/timer.h> |
34 | 35 | ||
35 | #include "speedstep-lib.h" | 36 | #include "speedstep-lib.h" |
36 | 37 | ||
37 | #define PFX "p4-clockmod: " | 38 | #define PFX "p4-clockmod: " |
38 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "p4-clockmod", msg) | 39 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
40 | "p4-clockmod", msg) | ||
39 | 41 | ||
40 | /* | 42 | /* |
41 | * Duty Cycle (3bits), note DC_DISABLE is not specified in | 43 | * Duty Cycle (3bits), note DC_DISABLE is not specified in |
@@ -58,7 +60,8 @@ static int cpufreq_p4_setdc(unsigned int cpu, unsigned int newstate) | |||
58 | { | 60 | { |
59 | u32 l, h; | 61 | u32 l, h; |
60 | 62 | ||
61 | if (!cpu_online(cpu) || (newstate > DC_DISABLE) || (newstate == DC_RESV)) | 63 | if (!cpu_online(cpu) || |
64 | (newstate > DC_DISABLE) || (newstate == DC_RESV)) | ||
62 | return -EINVAL; | 65 | return -EINVAL; |
63 | 66 | ||
64 | rdmsr_on_cpu(cpu, MSR_IA32_THERM_STATUS, &l, &h); | 67 | rdmsr_on_cpu(cpu, MSR_IA32_THERM_STATUS, &l, &h); |
@@ -66,7 +69,8 @@ static int cpufreq_p4_setdc(unsigned int cpu, unsigned int newstate) | |||
66 | if (l & 0x01) | 69 | if (l & 0x01) |
67 | dprintk("CPU#%d currently thermal throttled\n", cpu); | 70 | dprintk("CPU#%d currently thermal throttled\n", cpu); |
68 | 71 | ||
69 | if (has_N44_O17_errata[cpu] && (newstate == DC_25PT || newstate == DC_DFLT)) | 72 | if (has_N44_O17_errata[cpu] && |
73 | (newstate == DC_25PT || newstate == DC_DFLT)) | ||
70 | newstate = DC_38PT; | 74 | newstate = DC_38PT; |
71 | 75 | ||
72 | rdmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &l, &h); | 76 | rdmsr_on_cpu(cpu, MSR_IA32_THERM_CONTROL, &l, &h); |
@@ -112,7 +116,8 @@ static int cpufreq_p4_target(struct cpufreq_policy *policy, | |||
112 | struct cpufreq_freqs freqs; | 116 | struct cpufreq_freqs freqs; |
113 | int i; | 117 | int i; |
114 | 118 | ||
115 | if (cpufreq_frequency_table_target(policy, &p4clockmod_table[0], target_freq, relation, &newstate)) | 119 | if (cpufreq_frequency_table_target(policy, &p4clockmod_table[0], |
120 | target_freq, relation, &newstate)) | ||
116 | return -EINVAL; | 121 | return -EINVAL; |
117 | 122 | ||
118 | freqs.old = cpufreq_p4_get(policy->cpu); | 123 | freqs.old = cpufreq_p4_get(policy->cpu); |
@@ -127,7 +132,8 @@ static int cpufreq_p4_target(struct cpufreq_policy *policy, | |||
127 | cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); | 132 | cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); |
128 | } | 133 | } |
129 | 134 | ||
130 | /* run on each logical CPU, see section 13.15.3 of IA32 Intel Architecture Software | 135 | /* run on each logical CPU, |
136 | * see section 13.15.3 of IA32 Intel Architecture Software | ||
131 | * Developer's Manual, Volume 3 | 137 | * Developer's Manual, Volume 3 |
132 | */ | 138 | */ |
133 | for_each_cpu(i, policy->cpus) | 139 | for_each_cpu(i, policy->cpus) |
@@ -153,28 +159,30 @@ static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c) | |||
153 | { | 159 | { |
154 | if (c->x86 == 0x06) { | 160 | if (c->x86 == 0x06) { |
155 | if (cpu_has(c, X86_FEATURE_EST)) | 161 | if (cpu_has(c, X86_FEATURE_EST)) |
156 | printk(KERN_WARNING PFX "Warning: EST-capable CPU detected. " | 162 | printk(KERN_WARNING PFX "Warning: EST-capable CPU " |
157 | "The acpi-cpufreq module offers voltage scaling" | 163 | "detected. The acpi-cpufreq module offers " |
158 | " in addition of frequency scaling. You should use " | 164 | "voltage scaling in addition of frequency " |
159 | "that instead of p4-clockmod, if possible.\n"); | 165 | "scaling. You should use that instead of " |
166 | "p4-clockmod, if possible.\n"); | ||
160 | switch (c->x86_model) { | 167 | switch (c->x86_model) { |
161 | case 0x0E: /* Core */ | 168 | case 0x0E: /* Core */ |
162 | case 0x0F: /* Core Duo */ | 169 | case 0x0F: /* Core Duo */ |
163 | case 0x16: /* Celeron Core */ | 170 | case 0x16: /* Celeron Core */ |
164 | p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; | 171 | p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; |
165 | return speedstep_get_processor_frequency(SPEEDSTEP_PROCESSOR_PCORE); | 172 | return speedstep_get_frequency(SPEEDSTEP_CPU_PCORE); |
166 | case 0x0D: /* Pentium M (Dothan) */ | 173 | case 0x0D: /* Pentium M (Dothan) */ |
167 | p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; | 174 | p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; |
168 | /* fall through */ | 175 | /* fall through */ |
169 | case 0x09: /* Pentium M (Banias) */ | 176 | case 0x09: /* Pentium M (Banias) */ |
170 | return speedstep_get_processor_frequency(SPEEDSTEP_PROCESSOR_PM); | 177 | return speedstep_get_frequency(SPEEDSTEP_CPU_PM); |
171 | } | 178 | } |
172 | } | 179 | } |
173 | 180 | ||
174 | if (c->x86 != 0xF) { | 181 | if (c->x86 != 0xF) { |
175 | if (!cpu_has(c, X86_FEATURE_EST)) | 182 | if (!cpu_has(c, X86_FEATURE_EST)) |
176 | printk(KERN_WARNING PFX "Unknown p4-clockmod-capable CPU. " | 183 | printk(KERN_WARNING PFX "Unknown CPU. " |
177 | "Please send an e-mail to <cpufreq@vger.kernel.org>\n"); | 184 | "Please send an e-mail to " |
185 | "<cpufreq@vger.kernel.org>\n"); | ||
178 | return 0; | 186 | return 0; |
179 | } | 187 | } |
180 | 188 | ||
@@ -182,16 +190,16 @@ static unsigned int cpufreq_p4_get_frequency(struct cpuinfo_x86 *c) | |||
182 | * throttling is active or not. */ | 190 | * throttling is active or not. */ |
183 | p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; | 191 | p4clockmod_driver.flags |= CPUFREQ_CONST_LOOPS; |
184 | 192 | ||
185 | if (speedstep_detect_processor() == SPEEDSTEP_PROCESSOR_P4M) { | 193 | if (speedstep_detect_processor() == SPEEDSTEP_CPU_P4M) { |
186 | printk(KERN_WARNING PFX "Warning: Pentium 4-M detected. " | 194 | printk(KERN_WARNING PFX "Warning: Pentium 4-M detected. " |
187 | "The speedstep-ich or acpi cpufreq modules offer " | 195 | "The speedstep-ich or acpi cpufreq modules offer " |
188 | "voltage scaling in addition of frequency scaling. " | 196 | "voltage scaling in addition of frequency scaling. " |
189 | "You should use either one instead of p4-clockmod, " | 197 | "You should use either one instead of p4-clockmod, " |
190 | "if possible.\n"); | 198 | "if possible.\n"); |
191 | return speedstep_get_processor_frequency(SPEEDSTEP_PROCESSOR_P4M); | 199 | return speedstep_get_frequency(SPEEDSTEP_CPU_P4M); |
192 | } | 200 | } |
193 | 201 | ||
194 | return speedstep_get_processor_frequency(SPEEDSTEP_PROCESSOR_P4D); | 202 | return speedstep_get_frequency(SPEEDSTEP_CPU_P4D); |
195 | } | 203 | } |
196 | 204 | ||
197 | 205 | ||
@@ -217,14 +225,20 @@ static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy) | |||
217 | dprintk("has errata -- disabling low frequencies\n"); | 225 | dprintk("has errata -- disabling low frequencies\n"); |
218 | } | 226 | } |
219 | 227 | ||
228 | if (speedstep_detect_processor() == SPEEDSTEP_CPU_P4D && | ||
229 | c->x86_model < 2) { | ||
230 | /* switch to maximum frequency and measure result */ | ||
231 | cpufreq_p4_setdc(policy->cpu, DC_DISABLE); | ||
232 | recalibrate_cpu_khz(); | ||
233 | } | ||
220 | /* get max frequency */ | 234 | /* get max frequency */ |
221 | stock_freq = cpufreq_p4_get_frequency(c); | 235 | stock_freq = cpufreq_p4_get_frequency(c); |
222 | if (!stock_freq) | 236 | if (!stock_freq) |
223 | return -EINVAL; | 237 | return -EINVAL; |
224 | 238 | ||
225 | /* table init */ | 239 | /* table init */ |
226 | for (i=1; (p4clockmod_table[i].frequency != CPUFREQ_TABLE_END); i++) { | 240 | for (i = 1; (p4clockmod_table[i].frequency != CPUFREQ_TABLE_END); i++) { |
227 | if ((i<2) && (has_N44_O17_errata[policy->cpu])) | 241 | if ((i < 2) && (has_N44_O17_errata[policy->cpu])) |
228 | p4clockmod_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 242 | p4clockmod_table[i].frequency = CPUFREQ_ENTRY_INVALID; |
229 | else | 243 | else |
230 | p4clockmod_table[i].frequency = (stock_freq * i)/8; | 244 | p4clockmod_table[i].frequency = (stock_freq * i)/8; |
@@ -232,7 +246,10 @@ static int cpufreq_p4_cpu_init(struct cpufreq_policy *policy) | |||
232 | cpufreq_frequency_table_get_attr(p4clockmod_table, policy->cpu); | 246 | cpufreq_frequency_table_get_attr(p4clockmod_table, policy->cpu); |
233 | 247 | ||
234 | /* cpuinfo and default policy values */ | 248 | /* cpuinfo and default policy values */ |
235 | policy->cpuinfo.transition_latency = 1000000; /* assumed */ | 249 | |
250 | /* the transition latency is set to be 1 higher than the maximum | ||
251 | * transition latency of the ondemand governor */ | ||
252 | policy->cpuinfo.transition_latency = 10000001; | ||
236 | policy->cur = stock_freq; | 253 | policy->cur = stock_freq; |
237 | 254 | ||
238 | return cpufreq_frequency_table_cpuinfo(policy, &p4clockmod_table[0]); | 255 | return cpufreq_frequency_table_cpuinfo(policy, &p4clockmod_table[0]); |
@@ -258,12 +275,12 @@ static unsigned int cpufreq_p4_get(unsigned int cpu) | |||
258 | l = DC_DISABLE; | 275 | l = DC_DISABLE; |
259 | 276 | ||
260 | if (l != DC_DISABLE) | 277 | if (l != DC_DISABLE) |
261 | return (stock_freq * l / 8); | 278 | return stock_freq * l / 8; |
262 | 279 | ||
263 | return stock_freq; | 280 | return stock_freq; |
264 | } | 281 | } |
265 | 282 | ||
266 | static struct freq_attr* p4clockmod_attr[] = { | 283 | static struct freq_attr *p4clockmod_attr[] = { |
267 | &cpufreq_freq_attr_scaling_available_freqs, | 284 | &cpufreq_freq_attr_scaling_available_freqs, |
268 | NULL, | 285 | NULL, |
269 | }; | 286 | }; |
@@ -298,9 +315,10 @@ static int __init cpufreq_p4_init(void) | |||
298 | 315 | ||
299 | ret = cpufreq_register_driver(&p4clockmod_driver); | 316 | ret = cpufreq_register_driver(&p4clockmod_driver); |
300 | if (!ret) | 317 | if (!ret) |
301 | printk(KERN_INFO PFX "P4/Xeon(TM) CPU On-Demand Clock Modulation available\n"); | 318 | printk(KERN_INFO PFX "P4/Xeon(TM) CPU On-Demand Clock " |
319 | "Modulation available\n"); | ||
302 | 320 | ||
303 | return (ret); | 321 | return ret; |
304 | } | 322 | } |
305 | 323 | ||
306 | 324 | ||
@@ -310,9 +328,9 @@ static void __exit cpufreq_p4_exit(void) | |||
310 | } | 328 | } |
311 | 329 | ||
312 | 330 | ||
313 | MODULE_AUTHOR ("Zwane Mwaikambo <zwane@commfireservices.com>"); | 331 | MODULE_AUTHOR("Zwane Mwaikambo <zwane@commfireservices.com>"); |
314 | MODULE_DESCRIPTION ("cpufreq driver for Pentium(TM) 4/Xeon(TM)"); | 332 | MODULE_DESCRIPTION("cpufreq driver for Pentium(TM) 4/Xeon(TM)"); |
315 | MODULE_LICENSE ("GPL"); | 333 | MODULE_LICENSE("GPL"); |
316 | 334 | ||
317 | late_initcall(cpufreq_p4_init); | 335 | late_initcall(cpufreq_p4_init); |
318 | module_exit(cpufreq_p4_exit); | 336 | module_exit(cpufreq_p4_exit); |
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k6.c b/arch/x86/kernel/cpu/cpufreq/powernow-k6.c index c1ac5790c63e..f10dea409f40 100644 --- a/arch/x86/kernel/cpu/cpufreq/powernow-k6.c +++ b/arch/x86/kernel/cpu/cpufreq/powernow-k6.c | |||
@@ -1,6 +1,7 @@ | |||
1 | /* | 1 | /* |
2 | * This file was based upon code in Powertweak Linux (http://powertweak.sf.net) | 2 | * This file was based upon code in Powertweak Linux (http://powertweak.sf.net) |
3 | * (C) 2000-2003 Dave Jones, Arjan van de Ven, Janne Pänkälä, Dominik Brodowski. | 3 | * (C) 2000-2003 Dave Jones, Arjan van de Ven, Janne Pänkälä, |
4 | * Dominik Brodowski. | ||
4 | * | 5 | * |
5 | * Licensed under the terms of the GNU GPL License version 2. | 6 | * Licensed under the terms of the GNU GPL License version 2. |
6 | * | 7 | * |
@@ -13,14 +14,15 @@ | |||
13 | #include <linux/cpufreq.h> | 14 | #include <linux/cpufreq.h> |
14 | #include <linux/ioport.h> | 15 | #include <linux/ioport.h> |
15 | #include <linux/slab.h> | 16 | #include <linux/slab.h> |
16 | |||
17 | #include <asm/msr.h> | ||
18 | #include <linux/timex.h> | 17 | #include <linux/timex.h> |
19 | #include <linux/io.h> | 18 | #include <linux/io.h> |
20 | 19 | ||
20 | #include <asm/msr.h> | ||
21 | |||
21 | #define POWERNOW_IOPORT 0xfff0 /* it doesn't matter where, as long | 22 | #define POWERNOW_IOPORT 0xfff0 /* it doesn't matter where, as long |
22 | as it is unused */ | 23 | as it is unused */ |
23 | 24 | ||
25 | #define PFX "powernow-k6: " | ||
24 | static unsigned int busfreq; /* FSB, in 10 kHz */ | 26 | static unsigned int busfreq; /* FSB, in 10 kHz */ |
25 | static unsigned int max_multiplier; | 27 | static unsigned int max_multiplier; |
26 | 28 | ||
@@ -47,8 +49,8 @@ static struct cpufreq_frequency_table clock_ratio[] = { | |||
47 | */ | 49 | */ |
48 | static int powernow_k6_get_cpu_multiplier(void) | 50 | static int powernow_k6_get_cpu_multiplier(void) |
49 | { | 51 | { |
50 | u64 invalue = 0; | 52 | u64 invalue = 0; |
51 | u32 msrval; | 53 | u32 msrval; |
52 | 54 | ||
53 | msrval = POWERNOW_IOPORT + 0x1; | 55 | msrval = POWERNOW_IOPORT + 0x1; |
54 | wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */ | 56 | wrmsr(MSR_K6_EPMR, msrval, 0); /* enable the PowerNow port */ |
@@ -68,12 +70,12 @@ static int powernow_k6_get_cpu_multiplier(void) | |||
68 | */ | 70 | */ |
69 | static void powernow_k6_set_state(unsigned int best_i) | 71 | static void powernow_k6_set_state(unsigned int best_i) |
70 | { | 72 | { |
71 | unsigned long outvalue = 0, invalue = 0; | 73 | unsigned long outvalue = 0, invalue = 0; |
72 | unsigned long msrval; | 74 | unsigned long msrval; |
73 | struct cpufreq_freqs freqs; | 75 | struct cpufreq_freqs freqs; |
74 | 76 | ||
75 | if (clock_ratio[best_i].index > max_multiplier) { | 77 | if (clock_ratio[best_i].index > max_multiplier) { |
76 | printk(KERN_ERR "cpufreq: invalid target frequency\n"); | 78 | printk(KERN_ERR PFX "invalid target frequency\n"); |
77 | return; | 79 | return; |
78 | } | 80 | } |
79 | 81 | ||
@@ -119,7 +121,8 @@ static int powernow_k6_verify(struct cpufreq_policy *policy) | |||
119 | * powernow_k6_setpolicy - sets a new CPUFreq policy | 121 | * powernow_k6_setpolicy - sets a new CPUFreq policy |
120 | * @policy: new policy | 122 | * @policy: new policy |
121 | * @target_freq: the target frequency | 123 | * @target_freq: the target frequency |
122 | * @relation: how that frequency relates to achieved frequency (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) | 124 | * @relation: how that frequency relates to achieved frequency |
125 | * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) | ||
123 | * | 126 | * |
124 | * sets a new CPUFreq policy | 127 | * sets a new CPUFreq policy |
125 | */ | 128 | */ |
@@ -127,9 +130,10 @@ static int powernow_k6_target(struct cpufreq_policy *policy, | |||
127 | unsigned int target_freq, | 130 | unsigned int target_freq, |
128 | unsigned int relation) | 131 | unsigned int relation) |
129 | { | 132 | { |
130 | unsigned int newstate = 0; | 133 | unsigned int newstate = 0; |
131 | 134 | ||
132 | if (cpufreq_frequency_table_target(policy, &clock_ratio[0], target_freq, relation, &newstate)) | 135 | if (cpufreq_frequency_table_target(policy, &clock_ratio[0], |
136 | target_freq, relation, &newstate)) | ||
133 | return -EINVAL; | 137 | return -EINVAL; |
134 | 138 | ||
135 | powernow_k6_set_state(newstate); | 139 | powernow_k6_set_state(newstate); |
@@ -140,7 +144,7 @@ static int powernow_k6_target(struct cpufreq_policy *policy, | |||
140 | 144 | ||
141 | static int powernow_k6_cpu_init(struct cpufreq_policy *policy) | 145 | static int powernow_k6_cpu_init(struct cpufreq_policy *policy) |
142 | { | 146 | { |
143 | unsigned int i; | 147 | unsigned int i, f; |
144 | int result; | 148 | int result; |
145 | 149 | ||
146 | if (policy->cpu != 0) | 150 | if (policy->cpu != 0) |
@@ -152,10 +156,11 @@ static int powernow_k6_cpu_init(struct cpufreq_policy *policy) | |||
152 | 156 | ||
153 | /* table init */ | 157 | /* table init */ |
154 | for (i = 0; (clock_ratio[i].frequency != CPUFREQ_TABLE_END); i++) { | 158 | for (i = 0; (clock_ratio[i].frequency != CPUFREQ_TABLE_END); i++) { |
155 | if (clock_ratio[i].index > max_multiplier) | 159 | f = clock_ratio[i].index; |
160 | if (f > max_multiplier) | ||
156 | clock_ratio[i].frequency = CPUFREQ_ENTRY_INVALID; | 161 | clock_ratio[i].frequency = CPUFREQ_ENTRY_INVALID; |
157 | else | 162 | else |
158 | clock_ratio[i].frequency = busfreq * clock_ratio[i].index; | 163 | clock_ratio[i].frequency = busfreq * f; |
159 | } | 164 | } |
160 | 165 | ||
161 | /* cpuinfo and default policy values */ | 166 | /* cpuinfo and default policy values */ |
@@ -185,7 +190,9 @@ static int powernow_k6_cpu_exit(struct cpufreq_policy *policy) | |||
185 | 190 | ||
186 | static unsigned int powernow_k6_get(unsigned int cpu) | 191 | static unsigned int powernow_k6_get(unsigned int cpu) |
187 | { | 192 | { |
188 | return busfreq * powernow_k6_get_cpu_multiplier(); | 193 | unsigned int ret; |
194 | ret = (busfreq * powernow_k6_get_cpu_multiplier()); | ||
195 | return ret; | ||
189 | } | 196 | } |
190 | 197 | ||
191 | static struct freq_attr *powernow_k6_attr[] = { | 198 | static struct freq_attr *powernow_k6_attr[] = { |
@@ -221,7 +228,7 @@ static int __init powernow_k6_init(void) | |||
221 | return -ENODEV; | 228 | return -ENODEV; |
222 | 229 | ||
223 | if (!request_region(POWERNOW_IOPORT, 16, "PowerNow!")) { | 230 | if (!request_region(POWERNOW_IOPORT, 16, "PowerNow!")) { |
224 | printk("cpufreq: PowerNow IOPORT region already used.\n"); | 231 | printk(KERN_INFO PFX "PowerNow IOPORT region already used.\n"); |
225 | return -EIO; | 232 | return -EIO; |
226 | } | 233 | } |
227 | 234 | ||
@@ -246,7 +253,8 @@ static void __exit powernow_k6_exit(void) | |||
246 | } | 253 | } |
247 | 254 | ||
248 | 255 | ||
249 | MODULE_AUTHOR("Arjan van de Ven, Dave Jones <davej@redhat.com>, Dominik Brodowski <linux@brodo.de>"); | 256 | MODULE_AUTHOR("Arjan van de Ven, Dave Jones <davej@redhat.com>, " |
257 | "Dominik Brodowski <linux@brodo.de>"); | ||
250 | MODULE_DESCRIPTION("PowerNow! driver for AMD K6-2+ / K6-3+ processors."); | 258 | MODULE_DESCRIPTION("PowerNow! driver for AMD K6-2+ / K6-3+ processors."); |
251 | MODULE_LICENSE("GPL"); | 259 | MODULE_LICENSE("GPL"); |
252 | 260 | ||
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k7.c b/arch/x86/kernel/cpu/cpufreq/powernow-k7.c index 1b446d79a8fd..3c28ccd49742 100644 --- a/arch/x86/kernel/cpu/cpufreq/powernow-k7.c +++ b/arch/x86/kernel/cpu/cpufreq/powernow-k7.c | |||
@@ -6,10 +6,12 @@ | |||
6 | * Licensed under the terms of the GNU GPL License version 2. | 6 | * Licensed under the terms of the GNU GPL License version 2. |
7 | * Based upon datasheets & sample CPUs kindly provided by AMD. | 7 | * Based upon datasheets & sample CPUs kindly provided by AMD. |
8 | * | 8 | * |
9 | * Errata 5: Processor may fail to execute a FID/VID change in presence of interrupt. | 9 | * Errata 5: |
10 | * - We cli/sti on stepping A0 CPUs around the FID/VID transition. | 10 | * CPU may fail to execute a FID/VID change in presence of interrupt. |
11 | * Errata 15: Processors with half frequency multipliers may hang upon wakeup from disconnect. | 11 | * - We cli/sti on stepping A0 CPUs around the FID/VID transition. |
12 | * - We disable half multipliers if ACPI is used on A0 stepping CPUs. | 12 | * Errata 15: |
13 | * CPU with half frequency multipliers may hang upon wakeup from disconnect. | ||
14 | * - We disable half multipliers if ACPI is used on A0 stepping CPUs. | ||
13 | */ | 15 | */ |
14 | 16 | ||
15 | #include <linux/kernel.h> | 17 | #include <linux/kernel.h> |
@@ -20,11 +22,11 @@ | |||
20 | #include <linux/slab.h> | 22 | #include <linux/slab.h> |
21 | #include <linux/string.h> | 23 | #include <linux/string.h> |
22 | #include <linux/dmi.h> | 24 | #include <linux/dmi.h> |
25 | #include <linux/timex.h> | ||
26 | #include <linux/io.h> | ||
23 | 27 | ||
28 | #include <asm/timer.h> /* Needed for recalibrate_cpu_khz() */ | ||
24 | #include <asm/msr.h> | 29 | #include <asm/msr.h> |
25 | #include <asm/timer.h> | ||
26 | #include <asm/timex.h> | ||
27 | #include <asm/io.h> | ||
28 | #include <asm/system.h> | 30 | #include <asm/system.h> |
29 | 31 | ||
30 | #ifdef CONFIG_X86_POWERNOW_K7_ACPI | 32 | #ifdef CONFIG_X86_POWERNOW_K7_ACPI |
@@ -58,9 +60,9 @@ struct pst_s { | |||
58 | union powernow_acpi_control_t { | 60 | union powernow_acpi_control_t { |
59 | struct { | 61 | struct { |
60 | unsigned long fid:5, | 62 | unsigned long fid:5, |
61 | vid:5, | 63 | vid:5, |
62 | sgtc:20, | 64 | sgtc:20, |
63 | res1:2; | 65 | res1:2; |
64 | } bits; | 66 | } bits; |
65 | unsigned long val; | 67 | unsigned long val; |
66 | }; | 68 | }; |
@@ -94,14 +96,15 @@ static struct cpufreq_frequency_table *powernow_table; | |||
94 | 96 | ||
95 | static unsigned int can_scale_bus; | 97 | static unsigned int can_scale_bus; |
96 | static unsigned int can_scale_vid; | 98 | static unsigned int can_scale_vid; |
97 | static unsigned int minimum_speed=-1; | 99 | static unsigned int minimum_speed = -1; |
98 | static unsigned int maximum_speed; | 100 | static unsigned int maximum_speed; |
99 | static unsigned int number_scales; | 101 | static unsigned int number_scales; |
100 | static unsigned int fsb; | 102 | static unsigned int fsb; |
101 | static unsigned int latency; | 103 | static unsigned int latency; |
102 | static char have_a0; | 104 | static char have_a0; |
103 | 105 | ||
104 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "powernow-k7", msg) | 106 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
107 | "powernow-k7", msg) | ||
105 | 108 | ||
106 | static int check_fsb(unsigned int fsbspeed) | 109 | static int check_fsb(unsigned int fsbspeed) |
107 | { | 110 | { |
@@ -109,7 +112,7 @@ static int check_fsb(unsigned int fsbspeed) | |||
109 | unsigned int f = fsb / 1000; | 112 | unsigned int f = fsb / 1000; |
110 | 113 | ||
111 | delta = (fsbspeed > f) ? fsbspeed - f : f - fsbspeed; | 114 | delta = (fsbspeed > f) ? fsbspeed - f : f - fsbspeed; |
112 | return (delta < 5); | 115 | return delta < 5; |
113 | } | 116 | } |
114 | 117 | ||
115 | static int check_powernow(void) | 118 | static int check_powernow(void) |
@@ -117,24 +120,26 @@ static int check_powernow(void) | |||
117 | struct cpuinfo_x86 *c = &cpu_data(0); | 120 | struct cpuinfo_x86 *c = &cpu_data(0); |
118 | unsigned int maxei, eax, ebx, ecx, edx; | 121 | unsigned int maxei, eax, ebx, ecx, edx; |
119 | 122 | ||
120 | if ((c->x86_vendor != X86_VENDOR_AMD) || (c->x86 !=6)) { | 123 | if ((c->x86_vendor != X86_VENDOR_AMD) || (c->x86 != 6)) { |
121 | #ifdef MODULE | 124 | #ifdef MODULE |
122 | printk (KERN_INFO PFX "This module only works with AMD K7 CPUs\n"); | 125 | printk(KERN_INFO PFX "This module only works with " |
126 | "AMD K7 CPUs\n"); | ||
123 | #endif | 127 | #endif |
124 | return 0; | 128 | return 0; |
125 | } | 129 | } |
126 | 130 | ||
127 | /* Get maximum capabilities */ | 131 | /* Get maximum capabilities */ |
128 | maxei = cpuid_eax (0x80000000); | 132 | maxei = cpuid_eax(0x80000000); |
129 | if (maxei < 0x80000007) { /* Any powernow info ? */ | 133 | if (maxei < 0x80000007) { /* Any powernow info ? */ |
130 | #ifdef MODULE | 134 | #ifdef MODULE |
131 | printk (KERN_INFO PFX "No powernow capabilities detected\n"); | 135 | printk(KERN_INFO PFX "No powernow capabilities detected\n"); |
132 | #endif | 136 | #endif |
133 | return 0; | 137 | return 0; |
134 | } | 138 | } |
135 | 139 | ||
136 | if ((c->x86_model == 6) && (c->x86_mask == 0)) { | 140 | if ((c->x86_model == 6) && (c->x86_mask == 0)) { |
137 | printk (KERN_INFO PFX "K7 660[A0] core detected, enabling errata workarounds\n"); | 141 | printk(KERN_INFO PFX "K7 660[A0] core detected, " |
142 | "enabling errata workarounds\n"); | ||
138 | have_a0 = 1; | 143 | have_a0 = 1; |
139 | } | 144 | } |
140 | 145 | ||
@@ -144,37 +149,42 @@ static int check_powernow(void) | |||
144 | if (!(edx & (1 << 1 | 1 << 2))) | 149 | if (!(edx & (1 << 1 | 1 << 2))) |
145 | return 0; | 150 | return 0; |
146 | 151 | ||
147 | printk (KERN_INFO PFX "PowerNOW! Technology present. Can scale: "); | 152 | printk(KERN_INFO PFX "PowerNOW! Technology present. Can scale: "); |
148 | 153 | ||
149 | if (edx & 1 << 1) { | 154 | if (edx & 1 << 1) { |
150 | printk ("frequency"); | 155 | printk("frequency"); |
151 | can_scale_bus=1; | 156 | can_scale_bus = 1; |
152 | } | 157 | } |
153 | 158 | ||
154 | if ((edx & (1 << 1 | 1 << 2)) == 0x6) | 159 | if ((edx & (1 << 1 | 1 << 2)) == 0x6) |
155 | printk (" and "); | 160 | printk(" and "); |
156 | 161 | ||
157 | if (edx & 1 << 2) { | 162 | if (edx & 1 << 2) { |
158 | printk ("voltage"); | 163 | printk("voltage"); |
159 | can_scale_vid=1; | 164 | can_scale_vid = 1; |
160 | } | 165 | } |
161 | 166 | ||
162 | printk (".\n"); | 167 | printk(".\n"); |
163 | return 1; | 168 | return 1; |
164 | } | 169 | } |
165 | 170 | ||
171 | static void invalidate_entry(unsigned int entry) | ||
172 | { | ||
173 | powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID; | ||
174 | } | ||
166 | 175 | ||
167 | static int get_ranges (unsigned char *pst) | 176 | static int get_ranges(unsigned char *pst) |
168 | { | 177 | { |
169 | unsigned int j; | 178 | unsigned int j; |
170 | unsigned int speed; | 179 | unsigned int speed; |
171 | u8 fid, vid; | 180 | u8 fid, vid; |
172 | 181 | ||
173 | powernow_table = kzalloc((sizeof(struct cpufreq_frequency_table) * (number_scales + 1)), GFP_KERNEL); | 182 | powernow_table = kzalloc((sizeof(struct cpufreq_frequency_table) * |
183 | (number_scales + 1)), GFP_KERNEL); | ||
174 | if (!powernow_table) | 184 | if (!powernow_table) |
175 | return -ENOMEM; | 185 | return -ENOMEM; |
176 | 186 | ||
177 | for (j=0 ; j < number_scales; j++) { | 187 | for (j = 0 ; j < number_scales; j++) { |
178 | fid = *pst++; | 188 | fid = *pst++; |
179 | 189 | ||
180 | powernow_table[j].frequency = (fsb * fid_codes[fid]) / 10; | 190 | powernow_table[j].frequency = (fsb * fid_codes[fid]) / 10; |
@@ -182,10 +192,10 @@ static int get_ranges (unsigned char *pst) | |||
182 | 192 | ||
183 | speed = powernow_table[j].frequency; | 193 | speed = powernow_table[j].frequency; |
184 | 194 | ||
185 | if ((fid_codes[fid] % 10)==5) { | 195 | if ((fid_codes[fid] % 10) == 5) { |
186 | #ifdef CONFIG_X86_POWERNOW_K7_ACPI | 196 | #ifdef CONFIG_X86_POWERNOW_K7_ACPI |
187 | if (have_a0 == 1) | 197 | if (have_a0 == 1) |
188 | powernow_table[j].frequency = CPUFREQ_ENTRY_INVALID; | 198 | invalidate_entry(j); |
189 | #endif | 199 | #endif |
190 | } | 200 | } |
191 | 201 | ||
@@ -197,7 +207,7 @@ static int get_ranges (unsigned char *pst) | |||
197 | vid = *pst++; | 207 | vid = *pst++; |
198 | powernow_table[j].index |= (vid << 8); /* upper 8 bits */ | 208 | powernow_table[j].index |= (vid << 8); /* upper 8 bits */ |
199 | 209 | ||
200 | dprintk (" FID: 0x%x (%d.%dx [%dMHz]) " | 210 | dprintk(" FID: 0x%x (%d.%dx [%dMHz]) " |
201 | "VID: 0x%x (%d.%03dV)\n", fid, fid_codes[fid] / 10, | 211 | "VID: 0x%x (%d.%03dV)\n", fid, fid_codes[fid] / 10, |
202 | fid_codes[fid] % 10, speed/1000, vid, | 212 | fid_codes[fid] % 10, speed/1000, vid, |
203 | mobile_vid_table[vid]/1000, | 213 | mobile_vid_table[vid]/1000, |
@@ -214,13 +224,13 @@ static void change_FID(int fid) | |||
214 | { | 224 | { |
215 | union msr_fidvidctl fidvidctl; | 225 | union msr_fidvidctl fidvidctl; |
216 | 226 | ||
217 | rdmsrl (MSR_K7_FID_VID_CTL, fidvidctl.val); | 227 | rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); |
218 | if (fidvidctl.bits.FID != fid) { | 228 | if (fidvidctl.bits.FID != fid) { |
219 | fidvidctl.bits.SGTC = latency; | 229 | fidvidctl.bits.SGTC = latency; |
220 | fidvidctl.bits.FID = fid; | 230 | fidvidctl.bits.FID = fid; |
221 | fidvidctl.bits.VIDC = 0; | 231 | fidvidctl.bits.VIDC = 0; |
222 | fidvidctl.bits.FIDC = 1; | 232 | fidvidctl.bits.FIDC = 1; |
223 | wrmsrl (MSR_K7_FID_VID_CTL, fidvidctl.val); | 233 | wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); |
224 | } | 234 | } |
225 | } | 235 | } |
226 | 236 | ||
@@ -229,18 +239,18 @@ static void change_VID(int vid) | |||
229 | { | 239 | { |
230 | union msr_fidvidctl fidvidctl; | 240 | union msr_fidvidctl fidvidctl; |
231 | 241 | ||
232 | rdmsrl (MSR_K7_FID_VID_CTL, fidvidctl.val); | 242 | rdmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); |
233 | if (fidvidctl.bits.VID != vid) { | 243 | if (fidvidctl.bits.VID != vid) { |
234 | fidvidctl.bits.SGTC = latency; | 244 | fidvidctl.bits.SGTC = latency; |
235 | fidvidctl.bits.VID = vid; | 245 | fidvidctl.bits.VID = vid; |
236 | fidvidctl.bits.FIDC = 0; | 246 | fidvidctl.bits.FIDC = 0; |
237 | fidvidctl.bits.VIDC = 1; | 247 | fidvidctl.bits.VIDC = 1; |
238 | wrmsrl (MSR_K7_FID_VID_CTL, fidvidctl.val); | 248 | wrmsrl(MSR_K7_FID_VID_CTL, fidvidctl.val); |
239 | } | 249 | } |
240 | } | 250 | } |
241 | 251 | ||
242 | 252 | ||
243 | static void change_speed (unsigned int index) | 253 | static void change_speed(unsigned int index) |
244 | { | 254 | { |
245 | u8 fid, vid; | 255 | u8 fid, vid; |
246 | struct cpufreq_freqs freqs; | 256 | struct cpufreq_freqs freqs; |
@@ -257,7 +267,7 @@ static void change_speed (unsigned int index) | |||
257 | 267 | ||
258 | freqs.cpu = 0; | 268 | freqs.cpu = 0; |
259 | 269 | ||
260 | rdmsrl (MSR_K7_FID_VID_STATUS, fidvidstatus.val); | 270 | rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); |
261 | cfid = fidvidstatus.bits.CFID; | 271 | cfid = fidvidstatus.bits.CFID; |
262 | freqs.old = fsb * fid_codes[cfid] / 10; | 272 | freqs.old = fsb * fid_codes[cfid] / 10; |
263 | 273 | ||
@@ -321,12 +331,14 @@ static int powernow_acpi_init(void) | |||
321 | goto err1; | 331 | goto err1; |
322 | } | 332 | } |
323 | 333 | ||
324 | if (acpi_processor_perf->control_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) { | 334 | if (acpi_processor_perf->control_register.space_id != |
335 | ACPI_ADR_SPACE_FIXED_HARDWARE) { | ||
325 | retval = -ENODEV; | 336 | retval = -ENODEV; |
326 | goto err2; | 337 | goto err2; |
327 | } | 338 | } |
328 | 339 | ||
329 | if (acpi_processor_perf->status_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) { | 340 | if (acpi_processor_perf->status_register.space_id != |
341 | ACPI_ADR_SPACE_FIXED_HARDWARE) { | ||
330 | retval = -ENODEV; | 342 | retval = -ENODEV; |
331 | goto err2; | 343 | goto err2; |
332 | } | 344 | } |
@@ -338,7 +350,8 @@ static int powernow_acpi_init(void) | |||
338 | goto err2; | 350 | goto err2; |
339 | } | 351 | } |
340 | 352 | ||
341 | powernow_table = kzalloc((number_scales + 1) * (sizeof(struct cpufreq_frequency_table)), GFP_KERNEL); | 353 | powernow_table = kzalloc((sizeof(struct cpufreq_frequency_table) * |
354 | (number_scales + 1)), GFP_KERNEL); | ||
342 | if (!powernow_table) { | 355 | if (!powernow_table) { |
343 | retval = -ENOMEM; | 356 | retval = -ENOMEM; |
344 | goto err2; | 357 | goto err2; |
@@ -352,7 +365,7 @@ static int powernow_acpi_init(void) | |||
352 | unsigned int speed, speed_mhz; | 365 | unsigned int speed, speed_mhz; |
353 | 366 | ||
354 | pc.val = (unsigned long) state->control; | 367 | pc.val = (unsigned long) state->control; |
355 | dprintk ("acpi: P%d: %d MHz %d mW %d uS control %08x SGTC %d\n", | 368 | dprintk("acpi: P%d: %d MHz %d mW %d uS control %08x SGTC %d\n", |
356 | i, | 369 | i, |
357 | (u32) state->core_frequency, | 370 | (u32) state->core_frequency, |
358 | (u32) state->power, | 371 | (u32) state->power, |
@@ -381,12 +394,12 @@ static int powernow_acpi_init(void) | |||
381 | if (speed % 1000 > 0) | 394 | if (speed % 1000 > 0) |
382 | speed_mhz++; | 395 | speed_mhz++; |
383 | 396 | ||
384 | if ((fid_codes[fid] % 10)==5) { | 397 | if ((fid_codes[fid] % 10) == 5) { |
385 | if (have_a0 == 1) | 398 | if (have_a0 == 1) |
386 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 399 | invalidate_entry(i); |
387 | } | 400 | } |
388 | 401 | ||
389 | dprintk (" FID: 0x%x (%d.%dx [%dMHz]) " | 402 | dprintk(" FID: 0x%x (%d.%dx [%dMHz]) " |
390 | "VID: 0x%x (%d.%03dV)\n", fid, fid_codes[fid] / 10, | 403 | "VID: 0x%x (%d.%03dV)\n", fid, fid_codes[fid] / 10, |
391 | fid_codes[fid] % 10, speed_mhz, vid, | 404 | fid_codes[fid] % 10, speed_mhz, vid, |
392 | mobile_vid_table[vid]/1000, | 405 | mobile_vid_table[vid]/1000, |
@@ -422,7 +435,8 @@ err1: | |||
422 | err05: | 435 | err05: |
423 | kfree(acpi_processor_perf); | 436 | kfree(acpi_processor_perf); |
424 | err0: | 437 | err0: |
425 | printk(KERN_WARNING PFX "ACPI perflib can not be used in this platform\n"); | 438 | printk(KERN_WARNING PFX "ACPI perflib can not be used on " |
439 | "this platform\n"); | ||
426 | acpi_processor_perf = NULL; | 440 | acpi_processor_perf = NULL; |
427 | return retval; | 441 | return retval; |
428 | } | 442 | } |
@@ -435,7 +449,14 @@ static int powernow_acpi_init(void) | |||
435 | } | 449 | } |
436 | #endif | 450 | #endif |
437 | 451 | ||
438 | static int powernow_decode_bios (int maxfid, int startvid) | 452 | static void print_pst_entry(struct pst_s *pst, unsigned int j) |
453 | { | ||
454 | dprintk("PST:%d (@%p)\n", j, pst); | ||
455 | dprintk(" cpuid: 0x%x fsb: %d maxFID: 0x%x startvid: 0x%x\n", | ||
456 | pst->cpuid, pst->fsbspeed, pst->maxfid, pst->startvid); | ||
457 | } | ||
458 | |||
459 | static int powernow_decode_bios(int maxfid, int startvid) | ||
439 | { | 460 | { |
440 | struct psb_s *psb; | 461 | struct psb_s *psb; |
441 | struct pst_s *pst; | 462 | struct pst_s *pst; |
@@ -446,61 +467,67 @@ static int powernow_decode_bios (int maxfid, int startvid) | |||
446 | 467 | ||
447 | etuple = cpuid_eax(0x80000001); | 468 | etuple = cpuid_eax(0x80000001); |
448 | 469 | ||
449 | for (i=0xC0000; i < 0xffff0 ; i+=16) { | 470 | for (i = 0xC0000; i < 0xffff0 ; i += 16) { |
450 | 471 | ||
451 | p = phys_to_virt(i); | 472 | p = phys_to_virt(i); |
452 | 473 | ||
453 | if (memcmp(p, "AMDK7PNOW!", 10) == 0){ | 474 | if (memcmp(p, "AMDK7PNOW!", 10) == 0) { |
454 | dprintk ("Found PSB header at %p\n", p); | 475 | dprintk("Found PSB header at %p\n", p); |
455 | psb = (struct psb_s *) p; | 476 | psb = (struct psb_s *) p; |
456 | dprintk ("Table version: 0x%x\n", psb->tableversion); | 477 | dprintk("Table version: 0x%x\n", psb->tableversion); |
457 | if (psb->tableversion != 0x12) { | 478 | if (psb->tableversion != 0x12) { |
458 | printk (KERN_INFO PFX "Sorry, only v1.2 tables supported right now\n"); | 479 | printk(KERN_INFO PFX "Sorry, only v1.2 tables" |
480 | " supported right now\n"); | ||
459 | return -ENODEV; | 481 | return -ENODEV; |
460 | } | 482 | } |
461 | 483 | ||
462 | dprintk ("Flags: 0x%x\n", psb->flags); | 484 | dprintk("Flags: 0x%x\n", psb->flags); |
463 | if ((psb->flags & 1)==0) { | 485 | if ((psb->flags & 1) == 0) |
464 | dprintk ("Mobile voltage regulator\n"); | 486 | dprintk("Mobile voltage regulator\n"); |
465 | } else { | 487 | else |
466 | dprintk ("Desktop voltage regulator\n"); | 488 | dprintk("Desktop voltage regulator\n"); |
467 | } | ||
468 | 489 | ||
469 | latency = psb->settlingtime; | 490 | latency = psb->settlingtime; |
470 | if (latency < 100) { | 491 | if (latency < 100) { |
471 | printk(KERN_INFO PFX "BIOS set settling time to %d microseconds. " | 492 | printk(KERN_INFO PFX "BIOS set settling time " |
472 | "Should be at least 100. Correcting.\n", latency); | 493 | "to %d microseconds. " |
494 | "Should be at least 100. " | ||
495 | "Correcting.\n", latency); | ||
473 | latency = 100; | 496 | latency = 100; |
474 | } | 497 | } |
475 | dprintk ("Settling Time: %d microseconds.\n", psb->settlingtime); | 498 | dprintk("Settling Time: %d microseconds.\n", |
476 | dprintk ("Has %d PST tables. (Only dumping ones relevant to this CPU).\n", psb->numpst); | 499 | psb->settlingtime); |
500 | dprintk("Has %d PST tables. (Only dumping ones " | ||
501 | "relevant to this CPU).\n", | ||
502 | psb->numpst); | ||
477 | 503 | ||
478 | p += sizeof (struct psb_s); | 504 | p += sizeof(struct psb_s); |
479 | 505 | ||
480 | pst = (struct pst_s *) p; | 506 | pst = (struct pst_s *) p; |
481 | 507 | ||
482 | for (j=0; j<psb->numpst; j++) { | 508 | for (j = 0; j < psb->numpst; j++) { |
483 | pst = (struct pst_s *) p; | 509 | pst = (struct pst_s *) p; |
484 | number_scales = pst->numpstates; | 510 | number_scales = pst->numpstates; |
485 | 511 | ||
486 | if ((etuple == pst->cpuid) && check_fsb(pst->fsbspeed) && | 512 | if ((etuple == pst->cpuid) && |
487 | (maxfid==pst->maxfid) && (startvid==pst->startvid)) | 513 | check_fsb(pst->fsbspeed) && |
488 | { | 514 | (maxfid == pst->maxfid) && |
489 | dprintk ("PST:%d (@%p)\n", j, pst); | 515 | (startvid == pst->startvid)) { |
490 | dprintk (" cpuid: 0x%x fsb: %d maxFID: 0x%x startvid: 0x%x\n", | 516 | print_pst_entry(pst, j); |
491 | pst->cpuid, pst->fsbspeed, pst->maxfid, pst->startvid); | 517 | p = (char *)pst + sizeof(struct pst_s); |
492 | 518 | ret = get_ranges(p); | |
493 | ret = get_ranges ((char *) pst + sizeof (struct pst_s)); | ||
494 | return ret; | 519 | return ret; |
495 | } else { | 520 | } else { |
496 | unsigned int k; | 521 | unsigned int k; |
497 | p = (char *) pst + sizeof (struct pst_s); | 522 | p = (char *)pst + sizeof(struct pst_s); |
498 | for (k=0; k<number_scales; k++) | 523 | for (k = 0; k < number_scales; k++) |
499 | p+=2; | 524 | p += 2; |
500 | } | 525 | } |
501 | } | 526 | } |
502 | printk (KERN_INFO PFX "No PST tables match this cpuid (0x%x)\n", etuple); | 527 | printk(KERN_INFO PFX "No PST tables match this cpuid " |
503 | printk (KERN_INFO PFX "This is indicative of a broken BIOS.\n"); | 528 | "(0x%x)\n", etuple); |
529 | printk(KERN_INFO PFX "This is indicative of a broken " | ||
530 | "BIOS.\n"); | ||
504 | 531 | ||
505 | return -EINVAL; | 532 | return -EINVAL; |
506 | } | 533 | } |
@@ -511,13 +538,14 @@ static int powernow_decode_bios (int maxfid, int startvid) | |||
511 | } | 538 | } |
512 | 539 | ||
513 | 540 | ||
514 | static int powernow_target (struct cpufreq_policy *policy, | 541 | static int powernow_target(struct cpufreq_policy *policy, |
515 | unsigned int target_freq, | 542 | unsigned int target_freq, |
516 | unsigned int relation) | 543 | unsigned int relation) |
517 | { | 544 | { |
518 | unsigned int newstate; | 545 | unsigned int newstate; |
519 | 546 | ||
520 | if (cpufreq_frequency_table_target(policy, powernow_table, target_freq, relation, &newstate)) | 547 | if (cpufreq_frequency_table_target(policy, powernow_table, target_freq, |
548 | relation, &newstate)) | ||
521 | return -EINVAL; | 549 | return -EINVAL; |
522 | 550 | ||
523 | change_speed(newstate); | 551 | change_speed(newstate); |
@@ -526,7 +554,7 @@ static int powernow_target (struct cpufreq_policy *policy, | |||
526 | } | 554 | } |
527 | 555 | ||
528 | 556 | ||
529 | static int powernow_verify (struct cpufreq_policy *policy) | 557 | static int powernow_verify(struct cpufreq_policy *policy) |
530 | { | 558 | { |
531 | return cpufreq_frequency_table_verify(policy, powernow_table); | 559 | return cpufreq_frequency_table_verify(policy, powernow_table); |
532 | } | 560 | } |
@@ -566,18 +594,23 @@ static unsigned int powernow_get(unsigned int cpu) | |||
566 | 594 | ||
567 | if (cpu) | 595 | if (cpu) |
568 | return 0; | 596 | return 0; |
569 | rdmsrl (MSR_K7_FID_VID_STATUS, fidvidstatus.val); | 597 | rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); |
570 | cfid = fidvidstatus.bits.CFID; | 598 | cfid = fidvidstatus.bits.CFID; |
571 | 599 | ||
572 | return (fsb * fid_codes[cfid] / 10); | 600 | return fsb * fid_codes[cfid] / 10; |
573 | } | 601 | } |
574 | 602 | ||
575 | 603 | ||
576 | static int __init acer_cpufreq_pst(const struct dmi_system_id *d) | 604 | static int __init acer_cpufreq_pst(const struct dmi_system_id *d) |
577 | { | 605 | { |
578 | printk(KERN_WARNING "%s laptop with broken PST tables in BIOS detected.\n", d->ident); | 606 | printk(KERN_WARNING PFX |
579 | printk(KERN_WARNING "You need to downgrade to 3A21 (09/09/2002), or try a newer BIOS than 3A71 (01/20/2003)\n"); | 607 | "%s laptop with broken PST tables in BIOS detected.\n", |
580 | printk(KERN_WARNING "cpufreq scaling has been disabled as a result of this.\n"); | 608 | d->ident); |
609 | printk(KERN_WARNING PFX | ||
610 | "You need to downgrade to 3A21 (09/09/2002), or try a newer " | ||
611 | "BIOS than 3A71 (01/20/2003)\n"); | ||
612 | printk(KERN_WARNING PFX | ||
613 | "cpufreq scaling has been disabled as a result of this.\n"); | ||
581 | return 0; | 614 | return 0; |
582 | } | 615 | } |
583 | 616 | ||
@@ -598,7 +631,7 @@ static struct dmi_system_id __initdata powernow_dmi_table[] = { | |||
598 | { } | 631 | { } |
599 | }; | 632 | }; |
600 | 633 | ||
601 | static int __init powernow_cpu_init (struct cpufreq_policy *policy) | 634 | static int __init powernow_cpu_init(struct cpufreq_policy *policy) |
602 | { | 635 | { |
603 | union msr_fidvidstatus fidvidstatus; | 636 | union msr_fidvidstatus fidvidstatus; |
604 | int result; | 637 | int result; |
@@ -606,7 +639,7 @@ static int __init powernow_cpu_init (struct cpufreq_policy *policy) | |||
606 | if (policy->cpu != 0) | 639 | if (policy->cpu != 0) |
607 | return -ENODEV; | 640 | return -ENODEV; |
608 | 641 | ||
609 | rdmsrl (MSR_K7_FID_VID_STATUS, fidvidstatus.val); | 642 | rdmsrl(MSR_K7_FID_VID_STATUS, fidvidstatus.val); |
610 | 643 | ||
611 | recalibrate_cpu_khz(); | 644 | recalibrate_cpu_khz(); |
612 | 645 | ||
@@ -618,19 +651,21 @@ static int __init powernow_cpu_init (struct cpufreq_policy *policy) | |||
618 | dprintk("FSB: %3dMHz\n", fsb/1000); | 651 | dprintk("FSB: %3dMHz\n", fsb/1000); |
619 | 652 | ||
620 | if (dmi_check_system(powernow_dmi_table) || acpi_force) { | 653 | if (dmi_check_system(powernow_dmi_table) || acpi_force) { |
621 | printk (KERN_INFO PFX "PSB/PST known to be broken. Trying ACPI instead\n"); | 654 | printk(KERN_INFO PFX "PSB/PST known to be broken. " |
655 | "Trying ACPI instead\n"); | ||
622 | result = powernow_acpi_init(); | 656 | result = powernow_acpi_init(); |
623 | } else { | 657 | } else { |
624 | result = powernow_decode_bios(fidvidstatus.bits.MFID, fidvidstatus.bits.SVID); | 658 | result = powernow_decode_bios(fidvidstatus.bits.MFID, |
659 | fidvidstatus.bits.SVID); | ||
625 | if (result) { | 660 | if (result) { |
626 | printk (KERN_INFO PFX "Trying ACPI perflib\n"); | 661 | printk(KERN_INFO PFX "Trying ACPI perflib\n"); |
627 | maximum_speed = 0; | 662 | maximum_speed = 0; |
628 | minimum_speed = -1; | 663 | minimum_speed = -1; |
629 | latency = 0; | 664 | latency = 0; |
630 | result = powernow_acpi_init(); | 665 | result = powernow_acpi_init(); |
631 | if (result) { | 666 | if (result) { |
632 | printk (KERN_INFO PFX "ACPI and legacy methods failed\n"); | 667 | printk(KERN_INFO PFX |
633 | printk (KERN_INFO PFX "See http://www.codemonkey.org.uk/projects/cpufreq/powernow-k7.html\n"); | 668 | "ACPI and legacy methods failed\n"); |
634 | } | 669 | } |
635 | } else { | 670 | } else { |
636 | /* SGTC use the bus clock as timer */ | 671 | /* SGTC use the bus clock as timer */ |
@@ -642,10 +677,11 @@ static int __init powernow_cpu_init (struct cpufreq_policy *policy) | |||
642 | if (result) | 677 | if (result) |
643 | return result; | 678 | return result; |
644 | 679 | ||
645 | printk (KERN_INFO PFX "Minimum speed %d MHz. Maximum speed %d MHz.\n", | 680 | printk(KERN_INFO PFX "Minimum speed %d MHz. Maximum speed %d MHz.\n", |
646 | minimum_speed/1000, maximum_speed/1000); | 681 | minimum_speed/1000, maximum_speed/1000); |
647 | 682 | ||
648 | policy->cpuinfo.transition_latency = cpufreq_scale(2000000UL, fsb, latency); | 683 | policy->cpuinfo.transition_latency = |
684 | cpufreq_scale(2000000UL, fsb, latency); | ||
649 | 685 | ||
650 | policy->cur = powernow_get(0); | 686 | policy->cur = powernow_get(0); |
651 | 687 | ||
@@ -654,7 +690,8 @@ static int __init powernow_cpu_init (struct cpufreq_policy *policy) | |||
654 | return cpufreq_frequency_table_cpuinfo(policy, powernow_table); | 690 | return cpufreq_frequency_table_cpuinfo(policy, powernow_table); |
655 | } | 691 | } |
656 | 692 | ||
657 | static int powernow_cpu_exit (struct cpufreq_policy *policy) { | 693 | static int powernow_cpu_exit(struct cpufreq_policy *policy) |
694 | { | ||
658 | cpufreq_frequency_table_put_attr(policy->cpu); | 695 | cpufreq_frequency_table_put_attr(policy->cpu); |
659 | 696 | ||
660 | #ifdef CONFIG_X86_POWERNOW_K7_ACPI | 697 | #ifdef CONFIG_X86_POWERNOW_K7_ACPI |
@@ -669,7 +706,7 @@ static int powernow_cpu_exit (struct cpufreq_policy *policy) { | |||
669 | return 0; | 706 | return 0; |
670 | } | 707 | } |
671 | 708 | ||
672 | static struct freq_attr* powernow_table_attr[] = { | 709 | static struct freq_attr *powernow_table_attr[] = { |
673 | &cpufreq_freq_attr_scaling_available_freqs, | 710 | &cpufreq_freq_attr_scaling_available_freqs, |
674 | NULL, | 711 | NULL, |
675 | }; | 712 | }; |
@@ -685,15 +722,15 @@ static struct cpufreq_driver powernow_driver = { | |||
685 | .attr = powernow_table_attr, | 722 | .attr = powernow_table_attr, |
686 | }; | 723 | }; |
687 | 724 | ||
688 | static int __init powernow_init (void) | 725 | static int __init powernow_init(void) |
689 | { | 726 | { |
690 | if (check_powernow()==0) | 727 | if (check_powernow() == 0) |
691 | return -ENODEV; | 728 | return -ENODEV; |
692 | return cpufreq_register_driver(&powernow_driver); | 729 | return cpufreq_register_driver(&powernow_driver); |
693 | } | 730 | } |
694 | 731 | ||
695 | 732 | ||
696 | static void __exit powernow_exit (void) | 733 | static void __exit powernow_exit(void) |
697 | { | 734 | { |
698 | cpufreq_unregister_driver(&powernow_driver); | 735 | cpufreq_unregister_driver(&powernow_driver); |
699 | } | 736 | } |
@@ -701,9 +738,9 @@ static void __exit powernow_exit (void) | |||
701 | module_param(acpi_force, int, 0444); | 738 | module_param(acpi_force, int, 0444); |
702 | MODULE_PARM_DESC(acpi_force, "Force ACPI to be used."); | 739 | MODULE_PARM_DESC(acpi_force, "Force ACPI to be used."); |
703 | 740 | ||
704 | MODULE_AUTHOR ("Dave Jones <davej@redhat.com>"); | 741 | MODULE_AUTHOR("Dave Jones <davej@redhat.com>"); |
705 | MODULE_DESCRIPTION ("Powernow driver for AMD K7 processors."); | 742 | MODULE_DESCRIPTION("Powernow driver for AMD K7 processors."); |
706 | MODULE_LICENSE ("GPL"); | 743 | MODULE_LICENSE("GPL"); |
707 | 744 | ||
708 | late_initcall(powernow_init); | 745 | late_initcall(powernow_init); |
709 | module_exit(powernow_exit); | 746 | module_exit(powernow_exit); |
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c index 6428aa17b40e..a15ac94e0b9b 100644 --- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c +++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c | |||
@@ -33,16 +33,14 @@ | |||
33 | #include <linux/string.h> | 33 | #include <linux/string.h> |
34 | #include <linux/cpumask.h> | 34 | #include <linux/cpumask.h> |
35 | #include <linux/sched.h> /* for current / set_cpus_allowed() */ | 35 | #include <linux/sched.h> /* for current / set_cpus_allowed() */ |
36 | #include <linux/io.h> | ||
37 | #include <linux/delay.h> | ||
36 | 38 | ||
37 | #include <asm/msr.h> | 39 | #include <asm/msr.h> |
38 | #include <asm/io.h> | ||
39 | #include <asm/delay.h> | ||
40 | 40 | ||
41 | #ifdef CONFIG_X86_POWERNOW_K8_ACPI | ||
42 | #include <linux/acpi.h> | 41 | #include <linux/acpi.h> |
43 | #include <linux/mutex.h> | 42 | #include <linux/mutex.h> |
44 | #include <acpi/processor.h> | 43 | #include <acpi/processor.h> |
45 | #endif | ||
46 | 44 | ||
47 | #define PFX "powernow-k8: " | 45 | #define PFX "powernow-k8: " |
48 | #define VERSION "version 2.20.00" | 46 | #define VERSION "version 2.20.00" |
@@ -71,7 +69,8 @@ static u32 find_khz_freq_from_fid(u32 fid) | |||
71 | return 1000 * find_freq_from_fid(fid); | 69 | return 1000 * find_freq_from_fid(fid); |
72 | } | 70 | } |
73 | 71 | ||
74 | static u32 find_khz_freq_from_pstate(struct cpufreq_frequency_table *data, u32 pstate) | 72 | static u32 find_khz_freq_from_pstate(struct cpufreq_frequency_table *data, |
73 | u32 pstate) | ||
75 | { | 74 | { |
76 | return data[pstate].frequency; | 75 | return data[pstate].frequency; |
77 | } | 76 | } |
@@ -186,7 +185,9 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid) | |||
186 | return 1; | 185 | return 1; |
187 | } | 186 | } |
188 | 187 | ||
189 | lo = fid | (data->currvid << MSR_C_LO_VID_SHIFT) | MSR_C_LO_INIT_FID_VID; | 188 | lo = fid; |
189 | lo |= (data->currvid << MSR_C_LO_VID_SHIFT); | ||
190 | lo |= MSR_C_LO_INIT_FID_VID; | ||
190 | 191 | ||
191 | dprintk("writing fid 0x%x, lo 0x%x, hi 0x%x\n", | 192 | dprintk("writing fid 0x%x, lo 0x%x, hi 0x%x\n", |
192 | fid, lo, data->plllock * PLL_LOCK_CONVERSION); | 193 | fid, lo, data->plllock * PLL_LOCK_CONVERSION); |
@@ -194,7 +195,9 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid) | |||
194 | do { | 195 | do { |
195 | wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION); | 196 | wrmsr(MSR_FIDVID_CTL, lo, data->plllock * PLL_LOCK_CONVERSION); |
196 | if (i++ > 100) { | 197 | if (i++ > 100) { |
197 | printk(KERN_ERR PFX "Hardware error - pending bit very stuck - no further pstate changes possible\n"); | 198 | printk(KERN_ERR PFX |
199 | "Hardware error - pending bit very stuck - " | ||
200 | "no further pstate changes possible\n"); | ||
198 | return 1; | 201 | return 1; |
199 | } | 202 | } |
200 | } while (query_current_values_with_pending_wait(data)); | 203 | } while (query_current_values_with_pending_wait(data)); |
@@ -202,14 +205,16 @@ static int write_new_fid(struct powernow_k8_data *data, u32 fid) | |||
202 | count_off_irt(data); | 205 | count_off_irt(data); |
203 | 206 | ||
204 | if (savevid != data->currvid) { | 207 | if (savevid != data->currvid) { |
205 | printk(KERN_ERR PFX "vid change on fid trans, old 0x%x, new 0x%x\n", | 208 | printk(KERN_ERR PFX |
206 | savevid, data->currvid); | 209 | "vid change on fid trans, old 0x%x, new 0x%x\n", |
210 | savevid, data->currvid); | ||
207 | return 1; | 211 | return 1; |
208 | } | 212 | } |
209 | 213 | ||
210 | if (fid != data->currfid) { | 214 | if (fid != data->currfid) { |
211 | printk(KERN_ERR PFX "fid trans failed, fid 0x%x, curr 0x%x\n", fid, | 215 | printk(KERN_ERR PFX |
212 | data->currfid); | 216 | "fid trans failed, fid 0x%x, curr 0x%x\n", fid, |
217 | data->currfid); | ||
213 | return 1; | 218 | return 1; |
214 | } | 219 | } |
215 | 220 | ||
@@ -228,7 +233,9 @@ static int write_new_vid(struct powernow_k8_data *data, u32 vid) | |||
228 | return 1; | 233 | return 1; |
229 | } | 234 | } |
230 | 235 | ||
231 | lo = data->currfid | (vid << MSR_C_LO_VID_SHIFT) | MSR_C_LO_INIT_FID_VID; | 236 | lo = data->currfid; |
237 | lo |= (vid << MSR_C_LO_VID_SHIFT); | ||
238 | lo |= MSR_C_LO_INIT_FID_VID; | ||
232 | 239 | ||
233 | dprintk("writing vid 0x%x, lo 0x%x, hi 0x%x\n", | 240 | dprintk("writing vid 0x%x, lo 0x%x, hi 0x%x\n", |
234 | vid, lo, STOP_GRANT_5NS); | 241 | vid, lo, STOP_GRANT_5NS); |
@@ -236,20 +243,24 @@ static int write_new_vid(struct powernow_k8_data *data, u32 vid) | |||
236 | do { | 243 | do { |
237 | wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS); | 244 | wrmsr(MSR_FIDVID_CTL, lo, STOP_GRANT_5NS); |
238 | if (i++ > 100) { | 245 | if (i++ > 100) { |
239 | printk(KERN_ERR PFX "internal error - pending bit very stuck - no further pstate changes possible\n"); | 246 | printk(KERN_ERR PFX "internal error - pending bit " |
247 | "very stuck - no further pstate " | ||
248 | "changes possible\n"); | ||
240 | return 1; | 249 | return 1; |
241 | } | 250 | } |
242 | } while (query_current_values_with_pending_wait(data)); | 251 | } while (query_current_values_with_pending_wait(data)); |
243 | 252 | ||
244 | if (savefid != data->currfid) { | 253 | if (savefid != data->currfid) { |
245 | printk(KERN_ERR PFX "fid changed on vid trans, old 0x%x new 0x%x\n", | 254 | printk(KERN_ERR PFX "fid changed on vid trans, old " |
255 | "0x%x new 0x%x\n", | ||
246 | savefid, data->currfid); | 256 | savefid, data->currfid); |
247 | return 1; | 257 | return 1; |
248 | } | 258 | } |
249 | 259 | ||
250 | if (vid != data->currvid) { | 260 | if (vid != data->currvid) { |
251 | printk(KERN_ERR PFX "vid trans failed, vid 0x%x, curr 0x%x\n", vid, | 261 | printk(KERN_ERR PFX "vid trans failed, vid 0x%x, " |
252 | data->currvid); | 262 | "curr 0x%x\n", |
263 | vid, data->currvid); | ||
253 | return 1; | 264 | return 1; |
254 | } | 265 | } |
255 | 266 | ||
@@ -261,7 +272,8 @@ static int write_new_vid(struct powernow_k8_data *data, u32 vid) | |||
261 | * Decreasing vid codes represent increasing voltages: | 272 | * Decreasing vid codes represent increasing voltages: |
262 | * vid of 0 is 1.550V, vid of 0x1e is 0.800V, vid of VID_OFF is off. | 273 | * vid of 0 is 1.550V, vid of 0x1e is 0.800V, vid of VID_OFF is off. |
263 | */ | 274 | */ |
264 | static int decrease_vid_code_by_step(struct powernow_k8_data *data, u32 reqvid, u32 step) | 275 | static int decrease_vid_code_by_step(struct powernow_k8_data *data, |
276 | u32 reqvid, u32 step) | ||
265 | { | 277 | { |
266 | if ((data->currvid - reqvid) > step) | 278 | if ((data->currvid - reqvid) > step) |
267 | reqvid = data->currvid - step; | 279 | reqvid = data->currvid - step; |
@@ -283,7 +295,8 @@ static int transition_pstate(struct powernow_k8_data *data, u32 pstate) | |||
283 | } | 295 | } |
284 | 296 | ||
285 | /* Change Opteron/Athlon64 fid and vid, by the 3 phases. */ | 297 | /* Change Opteron/Athlon64 fid and vid, by the 3 phases. */ |
286 | static int transition_fid_vid(struct powernow_k8_data *data, u32 reqfid, u32 reqvid) | 298 | static int transition_fid_vid(struct powernow_k8_data *data, |
299 | u32 reqfid, u32 reqvid) | ||
287 | { | 300 | { |
288 | if (core_voltage_pre_transition(data, reqvid)) | 301 | if (core_voltage_pre_transition(data, reqvid)) |
289 | return 1; | 302 | return 1; |
@@ -298,7 +311,8 @@ static int transition_fid_vid(struct powernow_k8_data *data, u32 reqfid, u32 req | |||
298 | return 1; | 311 | return 1; |
299 | 312 | ||
300 | if ((reqfid != data->currfid) || (reqvid != data->currvid)) { | 313 | if ((reqfid != data->currfid) || (reqvid != data->currvid)) { |
301 | printk(KERN_ERR PFX "failed (cpu%d): req 0x%x 0x%x, curr 0x%x 0x%x\n", | 314 | printk(KERN_ERR PFX "failed (cpu%d): req 0x%x 0x%x, " |
315 | "curr 0x%x 0x%x\n", | ||
302 | smp_processor_id(), | 316 | smp_processor_id(), |
303 | reqfid, reqvid, data->currfid, data->currvid); | 317 | reqfid, reqvid, data->currfid, data->currvid); |
304 | return 1; | 318 | return 1; |
@@ -311,13 +325,15 @@ static int transition_fid_vid(struct powernow_k8_data *data, u32 reqfid, u32 req | |||
311 | } | 325 | } |
312 | 326 | ||
313 | /* Phase 1 - core voltage transition ... setup voltage */ | 327 | /* Phase 1 - core voltage transition ... setup voltage */ |
314 | static int core_voltage_pre_transition(struct powernow_k8_data *data, u32 reqvid) | 328 | static int core_voltage_pre_transition(struct powernow_k8_data *data, |
329 | u32 reqvid) | ||
315 | { | 330 | { |
316 | u32 rvosteps = data->rvo; | 331 | u32 rvosteps = data->rvo; |
317 | u32 savefid = data->currfid; | 332 | u32 savefid = data->currfid; |
318 | u32 maxvid, lo; | 333 | u32 maxvid, lo; |
319 | 334 | ||
320 | dprintk("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, reqvid 0x%x, rvo 0x%x\n", | 335 | dprintk("ph1 (cpu%d): start, currfid 0x%x, currvid 0x%x, " |
336 | "reqvid 0x%x, rvo 0x%x\n", | ||
321 | smp_processor_id(), | 337 | smp_processor_id(), |
322 | data->currfid, data->currvid, reqvid, data->rvo); | 338 | data->currfid, data->currvid, reqvid, data->rvo); |
323 | 339 | ||
@@ -340,7 +356,7 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data, u32 reqvid | |||
340 | } else { | 356 | } else { |
341 | dprintk("ph1: changing vid for rvo, req 0x%x\n", | 357 | dprintk("ph1: changing vid for rvo, req 0x%x\n", |
342 | data->currvid - 1); | 358 | data->currvid - 1); |
343 | if (decrease_vid_code_by_step(data, data->currvid - 1, 1)) | 359 | if (decrease_vid_code_by_step(data, data->currvid-1, 1)) |
344 | return 1; | 360 | return 1; |
345 | rvosteps--; | 361 | rvosteps--; |
346 | } | 362 | } |
@@ -350,7 +366,8 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data, u32 reqvid | |||
350 | return 1; | 366 | return 1; |
351 | 367 | ||
352 | if (savefid != data->currfid) { | 368 | if (savefid != data->currfid) { |
353 | printk(KERN_ERR PFX "ph1 err, currfid changed 0x%x\n", data->currfid); | 369 | printk(KERN_ERR PFX "ph1 err, currfid changed 0x%x\n", |
370 | data->currfid); | ||
354 | return 1; | 371 | return 1; |
355 | } | 372 | } |
356 | 373 | ||
@@ -363,20 +380,24 @@ static int core_voltage_pre_transition(struct powernow_k8_data *data, u32 reqvid | |||
363 | /* Phase 2 - core frequency transition */ | 380 | /* Phase 2 - core frequency transition */ |
364 | static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) | 381 | static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) |
365 | { | 382 | { |
366 | u32 vcoreqfid, vcocurrfid, vcofiddiff, fid_interval, savevid = data->currvid; | 383 | u32 vcoreqfid, vcocurrfid, vcofiddiff; |
384 | u32 fid_interval, savevid = data->currvid; | ||
367 | 385 | ||
368 | if ((reqfid < HI_FID_TABLE_BOTTOM) && (data->currfid < HI_FID_TABLE_BOTTOM)) { | 386 | if ((reqfid < HI_FID_TABLE_BOTTOM) && |
369 | printk(KERN_ERR PFX "ph2: illegal lo-lo transition 0x%x 0x%x\n", | 387 | (data->currfid < HI_FID_TABLE_BOTTOM)) { |
370 | reqfid, data->currfid); | 388 | printk(KERN_ERR PFX "ph2: illegal lo-lo transition " |
389 | "0x%x 0x%x\n", reqfid, data->currfid); | ||
371 | return 1; | 390 | return 1; |
372 | } | 391 | } |
373 | 392 | ||
374 | if (data->currfid == reqfid) { | 393 | if (data->currfid == reqfid) { |
375 | printk(KERN_ERR PFX "ph2 null fid transition 0x%x\n", data->currfid); | 394 | printk(KERN_ERR PFX "ph2 null fid transition 0x%x\n", |
395 | data->currfid); | ||
376 | return 0; | 396 | return 0; |
377 | } | 397 | } |
378 | 398 | ||
379 | dprintk("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, reqfid 0x%x\n", | 399 | dprintk("ph2 (cpu%d): starting, currfid 0x%x, currvid 0x%x, " |
400 | "reqfid 0x%x\n", | ||
380 | smp_processor_id(), | 401 | smp_processor_id(), |
381 | data->currfid, data->currvid, reqfid); | 402 | data->currfid, data->currvid, reqfid); |
382 | 403 | ||
@@ -390,14 +411,14 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) | |||
390 | 411 | ||
391 | if (reqfid > data->currfid) { | 412 | if (reqfid > data->currfid) { |
392 | if (data->currfid > LO_FID_TABLE_TOP) { | 413 | if (data->currfid > LO_FID_TABLE_TOP) { |
393 | if (write_new_fid(data, data->currfid + fid_interval)) { | 414 | if (write_new_fid(data, |
415 | data->currfid + fid_interval)) | ||
394 | return 1; | 416 | return 1; |
395 | } | ||
396 | } else { | 417 | } else { |
397 | if (write_new_fid | 418 | if (write_new_fid |
398 | (data, 2 + convert_fid_to_vco_fid(data->currfid))) { | 419 | (data, |
420 | 2 + convert_fid_to_vco_fid(data->currfid))) | ||
399 | return 1; | 421 | return 1; |
400 | } | ||
401 | } | 422 | } |
402 | } else { | 423 | } else { |
403 | if (write_new_fid(data, data->currfid - fid_interval)) | 424 | if (write_new_fid(data, data->currfid - fid_interval)) |
@@ -417,7 +438,8 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) | |||
417 | 438 | ||
418 | if (data->currfid != reqfid) { | 439 | if (data->currfid != reqfid) { |
419 | printk(KERN_ERR PFX | 440 | printk(KERN_ERR PFX |
420 | "ph2: mismatch, failed fid transition, curr 0x%x, req 0x%x\n", | 441 | "ph2: mismatch, failed fid transition, " |
442 | "curr 0x%x, req 0x%x\n", | ||
421 | data->currfid, reqfid); | 443 | data->currfid, reqfid); |
422 | return 1; | 444 | return 1; |
423 | } | 445 | } |
@@ -435,7 +457,8 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid) | |||
435 | } | 457 | } |
436 | 458 | ||
437 | /* Phase 3 - core voltage transition flow ... jump to the final vid. */ | 459 | /* Phase 3 - core voltage transition flow ... jump to the final vid. */ |
438 | static int core_voltage_post_transition(struct powernow_k8_data *data, u32 reqvid) | 460 | static int core_voltage_post_transition(struct powernow_k8_data *data, |
461 | u32 reqvid) | ||
439 | { | 462 | { |
440 | u32 savefid = data->currfid; | 463 | u32 savefid = data->currfid; |
441 | u32 savereqvid = reqvid; | 464 | u32 savereqvid = reqvid; |
@@ -457,7 +480,8 @@ static int core_voltage_post_transition(struct powernow_k8_data *data, u32 reqvi | |||
457 | 480 | ||
458 | if (data->currvid != reqvid) { | 481 | if (data->currvid != reqvid) { |
459 | printk(KERN_ERR PFX | 482 | printk(KERN_ERR PFX |
460 | "ph3: failed vid transition\n, req 0x%x, curr 0x%x", | 483 | "ph3: failed vid transition\n, " |
484 | "req 0x%x, curr 0x%x", | ||
461 | reqvid, data->currvid); | 485 | reqvid, data->currvid); |
462 | return 1; | 486 | return 1; |
463 | } | 487 | } |
@@ -508,7 +532,8 @@ static int check_supported_cpu(unsigned int cpu) | |||
508 | if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) { | 532 | if ((eax & CPUID_XFAM) == CPUID_XFAM_K8) { |
509 | if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) || | 533 | if (((eax & CPUID_USE_XFAM_XMOD) != CPUID_USE_XFAM_XMOD) || |
510 | ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) { | 534 | ((eax & CPUID_XMOD) > CPUID_XMOD_REV_MASK)) { |
511 | printk(KERN_INFO PFX "Processor cpuid %x not supported\n", eax); | 535 | printk(KERN_INFO PFX |
536 | "Processor cpuid %x not supported\n", eax); | ||
512 | goto out; | 537 | goto out; |
513 | } | 538 | } |
514 | 539 | ||
@@ -520,8 +545,10 @@ static int check_supported_cpu(unsigned int cpu) | |||
520 | } | 545 | } |
521 | 546 | ||
522 | cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx); | 547 | cpuid(CPUID_FREQ_VOLT_CAPABILITIES, &eax, &ebx, &ecx, &edx); |
523 | if ((edx & P_STATE_TRANSITION_CAPABLE) != P_STATE_TRANSITION_CAPABLE) { | 548 | if ((edx & P_STATE_TRANSITION_CAPABLE) |
524 | printk(KERN_INFO PFX "Power state transitions not supported\n"); | 549 | != P_STATE_TRANSITION_CAPABLE) { |
550 | printk(KERN_INFO PFX | ||
551 | "Power state transitions not supported\n"); | ||
525 | goto out; | 552 | goto out; |
526 | } | 553 | } |
527 | } else { /* must be a HW Pstate capable processor */ | 554 | } else { /* must be a HW Pstate capable processor */ |
@@ -539,7 +566,8 @@ out: | |||
539 | return rc; | 566 | return rc; |
540 | } | 567 | } |
541 | 568 | ||
542 | static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, u8 maxvid) | 569 | static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, |
570 | u8 maxvid) | ||
543 | { | 571 | { |
544 | unsigned int j; | 572 | unsigned int j; |
545 | u8 lastfid = 0xff; | 573 | u8 lastfid = 0xff; |
@@ -550,12 +578,14 @@ static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, u8 | |||
550 | j, pst[j].vid); | 578 | j, pst[j].vid); |
551 | return -EINVAL; | 579 | return -EINVAL; |
552 | } | 580 | } |
553 | if (pst[j].vid < data->rvo) { /* vid + rvo >= 0 */ | 581 | if (pst[j].vid < data->rvo) { |
582 | /* vid + rvo >= 0 */ | ||
554 | printk(KERN_ERR FW_BUG PFX "0 vid exceeded with pstate" | 583 | printk(KERN_ERR FW_BUG PFX "0 vid exceeded with pstate" |
555 | " %d\n", j); | 584 | " %d\n", j); |
556 | return -ENODEV; | 585 | return -ENODEV; |
557 | } | 586 | } |
558 | if (pst[j].vid < maxvid + data->rvo) { /* vid + rvo >= maxvid */ | 587 | if (pst[j].vid < maxvid + data->rvo) { |
588 | /* vid + rvo >= maxvid */ | ||
559 | printk(KERN_ERR FW_BUG PFX "maxvid exceeded with pstate" | 589 | printk(KERN_ERR FW_BUG PFX "maxvid exceeded with pstate" |
560 | " %d\n", j); | 590 | " %d\n", j); |
561 | return -ENODEV; | 591 | return -ENODEV; |
@@ -579,23 +609,31 @@ static int check_pst_table(struct powernow_k8_data *data, struct pst_s *pst, u8 | |||
579 | return -EINVAL; | 609 | return -EINVAL; |
580 | } | 610 | } |
581 | if (lastfid > LO_FID_TABLE_TOP) | 611 | if (lastfid > LO_FID_TABLE_TOP) |
582 | printk(KERN_INFO FW_BUG PFX "first fid not from lo freq table\n"); | 612 | printk(KERN_INFO FW_BUG PFX |
613 | "first fid not from lo freq table\n"); | ||
583 | 614 | ||
584 | return 0; | 615 | return 0; |
585 | } | 616 | } |
586 | 617 | ||
618 | static void invalidate_entry(struct powernow_k8_data *data, unsigned int entry) | ||
619 | { | ||
620 | data->powernow_table[entry].frequency = CPUFREQ_ENTRY_INVALID; | ||
621 | } | ||
622 | |||
587 | static void print_basics(struct powernow_k8_data *data) | 623 | static void print_basics(struct powernow_k8_data *data) |
588 | { | 624 | { |
589 | int j; | 625 | int j; |
590 | for (j = 0; j < data->numps; j++) { | 626 | for (j = 0; j < data->numps; j++) { |
591 | if (data->powernow_table[j].frequency != CPUFREQ_ENTRY_INVALID) { | 627 | if (data->powernow_table[j].frequency != |
628 | CPUFREQ_ENTRY_INVALID) { | ||
592 | if (cpu_family == CPU_HW_PSTATE) { | 629 | if (cpu_family == CPU_HW_PSTATE) { |
593 | printk(KERN_INFO PFX " %d : pstate %d (%d MHz)\n", | 630 | printk(KERN_INFO PFX |
594 | j, | 631 | " %d : pstate %d (%d MHz)\n", j, |
595 | data->powernow_table[j].index, | 632 | data->powernow_table[j].index, |
596 | data->powernow_table[j].frequency/1000); | 633 | data->powernow_table[j].frequency/1000); |
597 | } else { | 634 | } else { |
598 | printk(KERN_INFO PFX " %d : fid 0x%x (%d MHz), vid 0x%x\n", | 635 | printk(KERN_INFO PFX |
636 | " %d : fid 0x%x (%d MHz), vid 0x%x\n", | ||
599 | j, | 637 | j, |
600 | data->powernow_table[j].index & 0xff, | 638 | data->powernow_table[j].index & 0xff, |
601 | data->powernow_table[j].frequency/1000, | 639 | data->powernow_table[j].frequency/1000, |
@@ -604,20 +642,25 @@ static void print_basics(struct powernow_k8_data *data) | |||
604 | } | 642 | } |
605 | } | 643 | } |
606 | if (data->batps) | 644 | if (data->batps) |
607 | printk(KERN_INFO PFX "Only %d pstates on battery\n", data->batps); | 645 | printk(KERN_INFO PFX "Only %d pstates on battery\n", |
646 | data->batps); | ||
608 | } | 647 | } |
609 | 648 | ||
610 | static int fill_powernow_table(struct powernow_k8_data *data, struct pst_s *pst, u8 maxvid) | 649 | static int fill_powernow_table(struct powernow_k8_data *data, |
650 | struct pst_s *pst, u8 maxvid) | ||
611 | { | 651 | { |
612 | struct cpufreq_frequency_table *powernow_table; | 652 | struct cpufreq_frequency_table *powernow_table; |
613 | unsigned int j; | 653 | unsigned int j; |
614 | 654 | ||
615 | if (data->batps) { /* use ACPI support to get full speed on mains power */ | 655 | if (data->batps) { |
616 | printk(KERN_WARNING PFX "Only %d pstates usable (use ACPI driver for full range\n", data->batps); | 656 | /* use ACPI support to get full speed on mains power */ |
657 | printk(KERN_WARNING PFX | ||
658 | "Only %d pstates usable (use ACPI driver for full " | ||
659 | "range\n", data->batps); | ||
617 | data->numps = data->batps; | 660 | data->numps = data->batps; |
618 | } | 661 | } |
619 | 662 | ||
620 | for ( j=1; j<data->numps; j++ ) { | 663 | for (j = 1; j < data->numps; j++) { |
621 | if (pst[j-1].fid >= pst[j].fid) { | 664 | if (pst[j-1].fid >= pst[j].fid) { |
622 | printk(KERN_ERR PFX "PST out of sequence\n"); | 665 | printk(KERN_ERR PFX "PST out of sequence\n"); |
623 | return -EINVAL; | 666 | return -EINVAL; |
@@ -640,9 +683,11 @@ static int fill_powernow_table(struct powernow_k8_data *data, struct pst_s *pst, | |||
640 | } | 683 | } |
641 | 684 | ||
642 | for (j = 0; j < data->numps; j++) { | 685 | for (j = 0; j < data->numps; j++) { |
686 | int freq; | ||
643 | powernow_table[j].index = pst[j].fid; /* lower 8 bits */ | 687 | powernow_table[j].index = pst[j].fid; /* lower 8 bits */ |
644 | powernow_table[j].index |= (pst[j].vid << 8); /* upper 8 bits */ | 688 | powernow_table[j].index |= (pst[j].vid << 8); /* upper 8 bits */ |
645 | powernow_table[j].frequency = find_khz_freq_from_fid(pst[j].fid); | 689 | freq = find_khz_freq_from_fid(pst[j].fid); |
690 | powernow_table[j].frequency = freq; | ||
646 | } | 691 | } |
647 | powernow_table[data->numps].frequency = CPUFREQ_TABLE_END; | 692 | powernow_table[data->numps].frequency = CPUFREQ_TABLE_END; |
648 | powernow_table[data->numps].index = 0; | 693 | powernow_table[data->numps].index = 0; |
@@ -658,7 +703,8 @@ static int fill_powernow_table(struct powernow_k8_data *data, struct pst_s *pst, | |||
658 | print_basics(data); | 703 | print_basics(data); |
659 | 704 | ||
660 | for (j = 0; j < data->numps; j++) | 705 | for (j = 0; j < data->numps; j++) |
661 | if ((pst[j].fid==data->currfid) && (pst[j].vid==data->currvid)) | 706 | if ((pst[j].fid == data->currfid) && |
707 | (pst[j].vid == data->currvid)) | ||
662 | return 0; | 708 | return 0; |
663 | 709 | ||
664 | dprintk("currfid/vid do not match PST, ignoring\n"); | 710 | dprintk("currfid/vid do not match PST, ignoring\n"); |
@@ -698,7 +744,8 @@ static int find_psb_table(struct powernow_k8_data *data) | |||
698 | } | 744 | } |
699 | 745 | ||
700 | data->vstable = psb->vstable; | 746 | data->vstable = psb->vstable; |
701 | dprintk("voltage stabilization time: %d(*20us)\n", data->vstable); | 747 | dprintk("voltage stabilization time: %d(*20us)\n", |
748 | data->vstable); | ||
702 | 749 | ||
703 | dprintk("flags2: 0x%x\n", psb->flags2); | 750 | dprintk("flags2: 0x%x\n", psb->flags2); |
704 | data->rvo = psb->flags2 & 3; | 751 | data->rvo = psb->flags2 & 3; |
@@ -713,11 +760,12 @@ static int find_psb_table(struct powernow_k8_data *data) | |||
713 | 760 | ||
714 | dprintk("numpst: 0x%x\n", psb->num_tables); | 761 | dprintk("numpst: 0x%x\n", psb->num_tables); |
715 | cpst = psb->num_tables; | 762 | cpst = psb->num_tables; |
716 | if ((psb->cpuid == 0x00000fc0) || (psb->cpuid == 0x00000fe0) ){ | 763 | if ((psb->cpuid == 0x00000fc0) || |
764 | (psb->cpuid == 0x00000fe0)) { | ||
717 | thiscpuid = cpuid_eax(CPUID_PROCESSOR_SIGNATURE); | 765 | thiscpuid = cpuid_eax(CPUID_PROCESSOR_SIGNATURE); |
718 | if ((thiscpuid == 0x00000fc0) || (thiscpuid == 0x00000fe0) ) { | 766 | if ((thiscpuid == 0x00000fc0) || |
767 | (thiscpuid == 0x00000fe0)) | ||
719 | cpst = 1; | 768 | cpst = 1; |
720 | } | ||
721 | } | 769 | } |
722 | if (cpst != 1) { | 770 | if (cpst != 1) { |
723 | printk(KERN_ERR FW_BUG PFX "numpst must be 1\n"); | 771 | printk(KERN_ERR FW_BUG PFX "numpst must be 1\n"); |
@@ -732,7 +780,8 @@ static int find_psb_table(struct powernow_k8_data *data) | |||
732 | 780 | ||
733 | data->numps = psb->numps; | 781 | data->numps = psb->numps; |
734 | dprintk("numpstates: 0x%x\n", data->numps); | 782 | dprintk("numpstates: 0x%x\n", data->numps); |
735 | return fill_powernow_table(data, (struct pst_s *)(psb+1), maxvid); | 783 | return fill_powernow_table(data, |
784 | (struct pst_s *)(psb+1), maxvid); | ||
736 | } | 785 | } |
737 | /* | 786 | /* |
738 | * If you see this message, complain to BIOS manufacturer. If | 787 | * If you see this message, complain to BIOS manufacturer. If |
@@ -745,28 +794,31 @@ static int find_psb_table(struct powernow_k8_data *data) | |||
745 | * BIOS and Kernel Developer's Guide, which is available on | 794 | * BIOS and Kernel Developer's Guide, which is available on |
746 | * www.amd.com | 795 | * www.amd.com |
747 | */ | 796 | */ |
748 | printk(KERN_ERR PFX "BIOS error - no PSB or ACPI _PSS objects\n"); | 797 | printk(KERN_ERR FW_BUG PFX "No PSB or ACPI _PSS objects\n"); |
749 | return -ENODEV; | 798 | return -ENODEV; |
750 | } | 799 | } |
751 | 800 | ||
752 | #ifdef CONFIG_X86_POWERNOW_K8_ACPI | 801 | static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, |
753 | static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index) | 802 | unsigned int index) |
754 | { | 803 | { |
804 | acpi_integer control; | ||
805 | |||
755 | if (!data->acpi_data.state_count || (cpu_family == CPU_HW_PSTATE)) | 806 | if (!data->acpi_data.state_count || (cpu_family == CPU_HW_PSTATE)) |
756 | return; | 807 | return; |
757 | 808 | ||
758 | data->irt = (data->acpi_data.states[index].control >> IRT_SHIFT) & IRT_MASK; | 809 | control = data->acpi_data.states[index].control; data->irt = (control |
759 | data->rvo = (data->acpi_data.states[index].control >> RVO_SHIFT) & RVO_MASK; | 810 | >> IRT_SHIFT) & IRT_MASK; data->rvo = (control >> |
760 | data->exttype = (data->acpi_data.states[index].control >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK; | 811 | RVO_SHIFT) & RVO_MASK; data->exttype = (control |
761 | data->plllock = (data->acpi_data.states[index].control >> PLL_L_SHIFT) & PLL_L_MASK; | 812 | >> EXT_TYPE_SHIFT) & EXT_TYPE_MASK; |
762 | data->vidmvs = 1 << ((data->acpi_data.states[index].control >> MVS_SHIFT) & MVS_MASK); | 813 | data->plllock = (control >> PLL_L_SHIFT) & PLL_L_MASK; data->vidmvs = 1 |
763 | data->vstable = (data->acpi_data.states[index].control >> VST_SHIFT) & VST_MASK; | 814 | << ((control >> MVS_SHIFT) & MVS_MASK); data->vstable = |
764 | } | 815 | (control >> VST_SHIFT) & VST_MASK; } |
765 | 816 | ||
766 | static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) | 817 | static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) |
767 | { | 818 | { |
768 | struct cpufreq_frequency_table *powernow_table; | 819 | struct cpufreq_frequency_table *powernow_table; |
769 | int ret_val = -ENODEV; | 820 | int ret_val = -ENODEV; |
821 | acpi_integer space_id; | ||
770 | 822 | ||
771 | if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) { | 823 | if (acpi_processor_register_performance(&data->acpi_data, data->cpu)) { |
772 | dprintk("register performance failed: bad ACPI data\n"); | 824 | dprintk("register performance failed: bad ACPI data\n"); |
@@ -779,11 +831,12 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) | |||
779 | goto err_out; | 831 | goto err_out; |
780 | } | 832 | } |
781 | 833 | ||
782 | if ((data->acpi_data.control_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) || | 834 | space_id = data->acpi_data.control_register.space_id; |
783 | (data->acpi_data.status_register.space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)) { | 835 | if ((space_id != ACPI_ADR_SPACE_FIXED_HARDWARE) || |
836 | (space_id != ACPI_ADR_SPACE_FIXED_HARDWARE)) { | ||
784 | dprintk("Invalid control/status registers (%x - %x)\n", | 837 | dprintk("Invalid control/status registers (%x - %x)\n", |
785 | data->acpi_data.control_register.space_id, | 838 | data->acpi_data.control_register.space_id, |
786 | data->acpi_data.status_register.space_id); | 839 | space_id); |
787 | goto err_out; | 840 | goto err_out; |
788 | } | 841 | } |
789 | 842 | ||
@@ -802,7 +855,8 @@ static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) | |||
802 | if (ret_val) | 855 | if (ret_val) |
803 | goto err_out_mem; | 856 | goto err_out_mem; |
804 | 857 | ||
805 | powernow_table[data->acpi_data.state_count].frequency = CPUFREQ_TABLE_END; | 858 | powernow_table[data->acpi_data.state_count].frequency = |
859 | CPUFREQ_TABLE_END; | ||
806 | powernow_table[data->acpi_data.state_count].index = 0; | 860 | powernow_table[data->acpi_data.state_count].index = 0; |
807 | data->powernow_table = powernow_table; | 861 | data->powernow_table = powernow_table; |
808 | 862 | ||
@@ -830,13 +884,15 @@ err_out_mem: | |||
830 | err_out: | 884 | err_out: |
831 | acpi_processor_unregister_performance(&data->acpi_data, data->cpu); | 885 | acpi_processor_unregister_performance(&data->acpi_data, data->cpu); |
832 | 886 | ||
833 | /* data->acpi_data.state_count informs us at ->exit() whether ACPI was used */ | 887 | /* data->acpi_data.state_count informs us at ->exit() |
888 | * whether ACPI was used */ | ||
834 | data->acpi_data.state_count = 0; | 889 | data->acpi_data.state_count = 0; |
835 | 890 | ||
836 | return ret_val; | 891 | return ret_val; |
837 | } | 892 | } |
838 | 893 | ||
839 | static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table) | 894 | static int fill_powernow_table_pstate(struct powernow_k8_data *data, |
895 | struct cpufreq_frequency_table *powernow_table) | ||
840 | { | 896 | { |
841 | int i; | 897 | int i; |
842 | u32 hi = 0, lo = 0; | 898 | u32 hi = 0, lo = 0; |
@@ -848,84 +904,101 @@ static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpuf | |||
848 | 904 | ||
849 | index = data->acpi_data.states[i].control & HW_PSTATE_MASK; | 905 | index = data->acpi_data.states[i].control & HW_PSTATE_MASK; |
850 | if (index > data->max_hw_pstate) { | 906 | if (index > data->max_hw_pstate) { |
851 | printk(KERN_ERR PFX "invalid pstate %d - bad value %d.\n", i, index); | 907 | printk(KERN_ERR PFX "invalid pstate %d - " |
852 | printk(KERN_ERR PFX "Please report to BIOS manufacturer\n"); | 908 | "bad value %d.\n", i, index); |
853 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 909 | printk(KERN_ERR PFX "Please report to BIOS " |
910 | "manufacturer\n"); | ||
911 | invalidate_entry(data, i); | ||
854 | continue; | 912 | continue; |
855 | } | 913 | } |
856 | rdmsr(MSR_PSTATE_DEF_BASE + index, lo, hi); | 914 | rdmsr(MSR_PSTATE_DEF_BASE + index, lo, hi); |
857 | if (!(hi & HW_PSTATE_VALID_MASK)) { | 915 | if (!(hi & HW_PSTATE_VALID_MASK)) { |
858 | dprintk("invalid pstate %d, ignoring\n", index); | 916 | dprintk("invalid pstate %d, ignoring\n", index); |
859 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 917 | invalidate_entry(data, i); |
860 | continue; | 918 | continue; |
861 | } | 919 | } |
862 | 920 | ||
863 | powernow_table[i].index = index; | 921 | powernow_table[i].index = index; |
864 | 922 | ||
865 | powernow_table[i].frequency = data->acpi_data.states[i].core_frequency * 1000; | 923 | powernow_table[i].frequency = |
924 | data->acpi_data.states[i].core_frequency * 1000; | ||
866 | } | 925 | } |
867 | return 0; | 926 | return 0; |
868 | } | 927 | } |
869 | 928 | ||
870 | static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table) | 929 | static int fill_powernow_table_fidvid(struct powernow_k8_data *data, |
930 | struct cpufreq_frequency_table *powernow_table) | ||
871 | { | 931 | { |
872 | int i; | 932 | int i; |
873 | int cntlofreq = 0; | 933 | int cntlofreq = 0; |
934 | |||
874 | for (i = 0; i < data->acpi_data.state_count; i++) { | 935 | for (i = 0; i < data->acpi_data.state_count; i++) { |
875 | u32 fid; | 936 | u32 fid; |
876 | u32 vid; | 937 | u32 vid; |
938 | u32 freq, index; | ||
939 | acpi_integer status, control; | ||
877 | 940 | ||
878 | if (data->exttype) { | 941 | if (data->exttype) { |
879 | fid = data->acpi_data.states[i].status & EXT_FID_MASK; | 942 | status = data->acpi_data.states[i].status; |
880 | vid = (data->acpi_data.states[i].status >> VID_SHIFT) & EXT_VID_MASK; | 943 | fid = status & EXT_FID_MASK; |
944 | vid = (status >> VID_SHIFT) & EXT_VID_MASK; | ||
881 | } else { | 945 | } else { |
882 | fid = data->acpi_data.states[i].control & FID_MASK; | 946 | control = data->acpi_data.states[i].control; |
883 | vid = (data->acpi_data.states[i].control >> VID_SHIFT) & VID_MASK; | 947 | fid = control & FID_MASK; |
948 | vid = (control >> VID_SHIFT) & VID_MASK; | ||
884 | } | 949 | } |
885 | 950 | ||
886 | dprintk(" %d : fid 0x%x, vid 0x%x\n", i, fid, vid); | 951 | dprintk(" %d : fid 0x%x, vid 0x%x\n", i, fid, vid); |
887 | 952 | ||
888 | powernow_table[i].index = fid; /* lower 8 bits */ | 953 | index = fid | (vid<<8); |
889 | powernow_table[i].index |= (vid << 8); /* upper 8 bits */ | 954 | powernow_table[i].index = index; |
890 | powernow_table[i].frequency = find_khz_freq_from_fid(fid); | 955 | |
956 | freq = find_khz_freq_from_fid(fid); | ||
957 | powernow_table[i].frequency = freq; | ||
891 | 958 | ||
892 | /* verify frequency is OK */ | 959 | /* verify frequency is OK */ |
893 | if ((powernow_table[i].frequency > (MAX_FREQ * 1000)) || | 960 | if ((freq > (MAX_FREQ * 1000)) || (freq < (MIN_FREQ * 1000))) { |
894 | (powernow_table[i].frequency < (MIN_FREQ * 1000))) { | 961 | dprintk("invalid freq %u kHz, ignoring\n", freq); |
895 | dprintk("invalid freq %u kHz, ignoring\n", powernow_table[i].frequency); | 962 | invalidate_entry(data, i); |
896 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | ||
897 | continue; | 963 | continue; |
898 | } | 964 | } |
899 | 965 | ||
900 | /* verify voltage is OK - BIOSs are using "off" to indicate invalid */ | 966 | /* verify voltage is OK - |
967 | * BIOSs are using "off" to indicate invalid */ | ||
901 | if (vid == VID_OFF) { | 968 | if (vid == VID_OFF) { |
902 | dprintk("invalid vid %u, ignoring\n", vid); | 969 | dprintk("invalid vid %u, ignoring\n", vid); |
903 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 970 | invalidate_entry(data, i); |
904 | continue; | 971 | continue; |
905 | } | 972 | } |
906 | 973 | ||
907 | /* verify only 1 entry from the lo frequency table */ | 974 | /* verify only 1 entry from the lo frequency table */ |
908 | if (fid < HI_FID_TABLE_BOTTOM) { | 975 | if (fid < HI_FID_TABLE_BOTTOM) { |
909 | if (cntlofreq) { | 976 | if (cntlofreq) { |
910 | /* if both entries are the same, ignore this one ... */ | 977 | /* if both entries are the same, |
911 | if ((powernow_table[i].frequency != powernow_table[cntlofreq].frequency) || | 978 | * ignore this one ... */ |
912 | (powernow_table[i].index != powernow_table[cntlofreq].index)) { | 979 | if ((freq != powernow_table[cntlofreq].frequency) || |
913 | printk(KERN_ERR PFX "Too many lo freq table entries\n"); | 980 | (index != powernow_table[cntlofreq].index)) { |
981 | printk(KERN_ERR PFX | ||
982 | "Too many lo freq table " | ||
983 | "entries\n"); | ||
914 | return 1; | 984 | return 1; |
915 | } | 985 | } |
916 | 986 | ||
917 | dprintk("double low frequency table entry, ignoring it.\n"); | 987 | dprintk("double low frequency table entry, " |
918 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 988 | "ignoring it.\n"); |
989 | invalidate_entry(data, i); | ||
919 | continue; | 990 | continue; |
920 | } else | 991 | } else |
921 | cntlofreq = i; | 992 | cntlofreq = i; |
922 | } | 993 | } |
923 | 994 | ||
924 | if (powernow_table[i].frequency != (data->acpi_data.states[i].core_frequency * 1000)) { | 995 | if (freq != (data->acpi_data.states[i].core_frequency * 1000)) { |
925 | printk(KERN_INFO PFX "invalid freq entries %u kHz vs. %u kHz\n", | 996 | printk(KERN_INFO PFX "invalid freq entries " |
926 | powernow_table[i].frequency, | 997 | "%u kHz vs. %u kHz\n", freq, |
927 | (unsigned int) (data->acpi_data.states[i].core_frequency * 1000)); | 998 | (unsigned int) |
928 | powernow_table[i].frequency = CPUFREQ_ENTRY_INVALID; | 999 | (data->acpi_data.states[i].core_frequency |
1000 | * 1000)); | ||
1001 | invalidate_entry(data, i); | ||
929 | continue; | 1002 | continue; |
930 | } | 1003 | } |
931 | } | 1004 | } |
@@ -935,7 +1008,8 @@ static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpuf | |||
935 | static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) | 1008 | static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) |
936 | { | 1009 | { |
937 | if (data->acpi_data.state_count) | 1010 | if (data->acpi_data.state_count) |
938 | acpi_processor_unregister_performance(&data->acpi_data, data->cpu); | 1011 | acpi_processor_unregister_performance(&data->acpi_data, |
1012 | data->cpu); | ||
939 | free_cpumask_var(data->acpi_data.shared_cpu_map); | 1013 | free_cpumask_var(data->acpi_data.shared_cpu_map); |
940 | } | 1014 | } |
941 | 1015 | ||
@@ -953,15 +1027,9 @@ static int get_transition_latency(struct powernow_k8_data *data) | |||
953 | return 1000 * max_latency; | 1027 | return 1000 * max_latency; |
954 | } | 1028 | } |
955 | 1029 | ||
956 | #else | ||
957 | static int powernow_k8_cpu_init_acpi(struct powernow_k8_data *data) { return -ENODEV; } | ||
958 | static void powernow_k8_cpu_exit_acpi(struct powernow_k8_data *data) { return; } | ||
959 | static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index) { return; } | ||
960 | static int get_transition_latency(struct powernow_k8_data *data) { return 0; } | ||
961 | #endif /* CONFIG_X86_POWERNOW_K8_ACPI */ | ||
962 | |||
963 | /* Take a frequency, and issue the fid/vid transition command */ | 1030 | /* Take a frequency, and issue the fid/vid transition command */ |
964 | static int transition_frequency_fidvid(struct powernow_k8_data *data, unsigned int index) | 1031 | static int transition_frequency_fidvid(struct powernow_k8_data *data, |
1032 | unsigned int index) | ||
965 | { | 1033 | { |
966 | u32 fid = 0; | 1034 | u32 fid = 0; |
967 | u32 vid = 0; | 1035 | u32 vid = 0; |
@@ -989,7 +1057,8 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data, unsigned i | |||
989 | return 0; | 1057 | return 0; |
990 | } | 1058 | } |
991 | 1059 | ||
992 | if ((fid < HI_FID_TABLE_BOTTOM) && (data->currfid < HI_FID_TABLE_BOTTOM)) { | 1060 | if ((fid < HI_FID_TABLE_BOTTOM) && |
1061 | (data->currfid < HI_FID_TABLE_BOTTOM)) { | ||
993 | printk(KERN_ERR PFX | 1062 | printk(KERN_ERR PFX |
994 | "ignoring illegal change in lo freq table-%x to 0x%x\n", | 1063 | "ignoring illegal change in lo freq table-%x to 0x%x\n", |
995 | data->currfid, fid); | 1064 | data->currfid, fid); |
@@ -1017,7 +1086,8 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data, unsigned i | |||
1017 | } | 1086 | } |
1018 | 1087 | ||
1019 | /* Take a frequency, and issue the hardware pstate transition command */ | 1088 | /* Take a frequency, and issue the hardware pstate transition command */ |
1020 | static int transition_frequency_pstate(struct powernow_k8_data *data, unsigned int index) | 1089 | static int transition_frequency_pstate(struct powernow_k8_data *data, |
1090 | unsigned int index) | ||
1021 | { | 1091 | { |
1022 | u32 pstate = 0; | 1092 | u32 pstate = 0; |
1023 | int res, i; | 1093 | int res, i; |
@@ -1029,7 +1099,8 @@ static int transition_frequency_pstate(struct powernow_k8_data *data, unsigned i | |||
1029 | pstate = index & HW_PSTATE_MASK; | 1099 | pstate = index & HW_PSTATE_MASK; |
1030 | if (pstate > data->max_hw_pstate) | 1100 | if (pstate > data->max_hw_pstate) |
1031 | return 0; | 1101 | return 0; |
1032 | freqs.old = find_khz_freq_from_pstate(data->powernow_table, data->currpstate); | 1102 | freqs.old = find_khz_freq_from_pstate(data->powernow_table, |
1103 | data->currpstate); | ||
1033 | freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate); | 1104 | freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate); |
1034 | 1105 | ||
1035 | for_each_cpu_mask_nr(i, *(data->available_cores)) { | 1106 | for_each_cpu_mask_nr(i, *(data->available_cores)) { |
@@ -1048,7 +1119,8 @@ static int transition_frequency_pstate(struct powernow_k8_data *data, unsigned i | |||
1048 | } | 1119 | } |
1049 | 1120 | ||
1050 | /* Driver entry point to switch to the target frequency */ | 1121 | /* Driver entry point to switch to the target frequency */ |
1051 | static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsigned relation) | 1122 | static int powernowk8_target(struct cpufreq_policy *pol, |
1123 | unsigned targfreq, unsigned relation) | ||
1052 | { | 1124 | { |
1053 | cpumask_t oldmask; | 1125 | cpumask_t oldmask; |
1054 | struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); | 1126 | struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); |
@@ -1087,14 +1159,18 @@ static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsi | |||
1087 | dprintk("targ: curr fid 0x%x, vid 0x%x\n", | 1159 | dprintk("targ: curr fid 0x%x, vid 0x%x\n", |
1088 | data->currfid, data->currvid); | 1160 | data->currfid, data->currvid); |
1089 | 1161 | ||
1090 | if ((checkvid != data->currvid) || (checkfid != data->currfid)) { | 1162 | if ((checkvid != data->currvid) || |
1163 | (checkfid != data->currfid)) { | ||
1091 | printk(KERN_INFO PFX | 1164 | printk(KERN_INFO PFX |
1092 | "error - out of sync, fix 0x%x 0x%x, vid 0x%x 0x%x\n", | 1165 | "error - out of sync, fix 0x%x 0x%x, " |
1093 | checkfid, data->currfid, checkvid, data->currvid); | 1166 | "vid 0x%x 0x%x\n", |
1167 | checkfid, data->currfid, | ||
1168 | checkvid, data->currvid); | ||
1094 | } | 1169 | } |
1095 | } | 1170 | } |
1096 | 1171 | ||
1097 | if (cpufreq_frequency_table_target(pol, data->powernow_table, targfreq, relation, &newstate)) | 1172 | if (cpufreq_frequency_table_target(pol, data->powernow_table, |
1173 | targfreq, relation, &newstate)) | ||
1098 | goto err_out; | 1174 | goto err_out; |
1099 | 1175 | ||
1100 | mutex_lock(&fidvid_mutex); | 1176 | mutex_lock(&fidvid_mutex); |
@@ -1114,7 +1190,8 @@ static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsi | |||
1114 | mutex_unlock(&fidvid_mutex); | 1190 | mutex_unlock(&fidvid_mutex); |
1115 | 1191 | ||
1116 | if (cpu_family == CPU_HW_PSTATE) | 1192 | if (cpu_family == CPU_HW_PSTATE) |
1117 | pol->cur = find_khz_freq_from_pstate(data->powernow_table, newstate); | 1193 | pol->cur = find_khz_freq_from_pstate(data->powernow_table, |
1194 | newstate); | ||
1118 | else | 1195 | else |
1119 | pol->cur = find_khz_freq_from_fid(data->currfid); | 1196 | pol->cur = find_khz_freq_from_fid(data->currfid); |
1120 | ret = 0; | 1197 | ret = 0; |
@@ -1141,6 +1218,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) | |||
1141 | struct powernow_k8_data *data; | 1218 | struct powernow_k8_data *data; |
1142 | cpumask_t oldmask; | 1219 | cpumask_t oldmask; |
1143 | int rc; | 1220 | int rc; |
1221 | static int print_once; | ||
1144 | 1222 | ||
1145 | if (!cpu_online(pol->cpu)) | 1223 | if (!cpu_online(pol->cpu)) |
1146 | return -ENODEV; | 1224 | return -ENODEV; |
@@ -1163,33 +1241,31 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) | |||
1163 | * an UP version, and is deprecated by AMD. | 1241 | * an UP version, and is deprecated by AMD. |
1164 | */ | 1242 | */ |
1165 | if (num_online_cpus() != 1) { | 1243 | if (num_online_cpus() != 1) { |
1166 | #ifndef CONFIG_ACPI_PROCESSOR | 1244 | /* |
1167 | printk(KERN_ERR PFX "ACPI Processor support is required " | 1245 | * Replace this one with print_once as soon as such a |
1168 | "for SMP systems but is absent. Please load the " | 1246 | * thing gets introduced |
1169 | "ACPI Processor module before starting this " | 1247 | */ |
1170 | "driver.\n"); | 1248 | if (!print_once) { |
1171 | #else | 1249 | WARN_ONCE(1, KERN_ERR FW_BUG PFX "Your BIOS " |
1172 | printk(KERN_ERR FW_BUG PFX "Your BIOS does not provide" | 1250 | "does not provide ACPI _PSS objects " |
1173 | " ACPI _PSS objects in a way that Linux " | 1251 | "in a way that Linux understands. " |
1174 | "understands. Please report this to the Linux " | 1252 | "Please report this to the Linux ACPI" |
1175 | "ACPI maintainers and complain to your BIOS " | 1253 | " maintainers and complain to your " |
1176 | "vendor.\n"); | 1254 | "BIOS vendor.\n"); |
1177 | #endif | 1255 | print_once++; |
1178 | kfree(data); | 1256 | } |
1179 | return -ENODEV; | 1257 | goto err_out; |
1180 | } | 1258 | } |
1181 | if (pol->cpu != 0) { | 1259 | if (pol->cpu != 0) { |
1182 | printk(KERN_ERR FW_BUG PFX "No ACPI _PSS objects for " | 1260 | printk(KERN_ERR FW_BUG PFX "No ACPI _PSS objects for " |
1183 | "CPU other than CPU0. Complain to your BIOS " | 1261 | "CPU other than CPU0. Complain to your BIOS " |
1184 | "vendor.\n"); | 1262 | "vendor.\n"); |
1185 | kfree(data); | 1263 | goto err_out; |
1186 | return -ENODEV; | ||
1187 | } | 1264 | } |
1188 | rc = find_psb_table(data); | 1265 | rc = find_psb_table(data); |
1189 | if (rc) { | 1266 | if (rc) |
1190 | kfree(data); | 1267 | goto err_out; |
1191 | return -ENODEV; | 1268 | |
1192 | } | ||
1193 | /* Take a crude guess here. | 1269 | /* Take a crude guess here. |
1194 | * That guess was in microseconds, so multiply with 1000 */ | 1270 | * That guess was in microseconds, so multiply with 1000 */ |
1195 | pol->cpuinfo.transition_latency = ( | 1271 | pol->cpuinfo.transition_latency = ( |
@@ -1204,16 +1280,16 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) | |||
1204 | 1280 | ||
1205 | if (smp_processor_id() != pol->cpu) { | 1281 | if (smp_processor_id() != pol->cpu) { |
1206 | printk(KERN_ERR PFX "limiting to cpu %u failed\n", pol->cpu); | 1282 | printk(KERN_ERR PFX "limiting to cpu %u failed\n", pol->cpu); |
1207 | goto err_out; | 1283 | goto err_out_unmask; |
1208 | } | 1284 | } |
1209 | 1285 | ||
1210 | if (pending_bit_stuck()) { | 1286 | if (pending_bit_stuck()) { |
1211 | printk(KERN_ERR PFX "failing init, change pending bit set\n"); | 1287 | printk(KERN_ERR PFX "failing init, change pending bit set\n"); |
1212 | goto err_out; | 1288 | goto err_out_unmask; |
1213 | } | 1289 | } |
1214 | 1290 | ||
1215 | if (query_current_values_with_pending_wait(data)) | 1291 | if (query_current_values_with_pending_wait(data)) |
1216 | goto err_out; | 1292 | goto err_out_unmask; |
1217 | 1293 | ||
1218 | if (cpu_family == CPU_OPTERON) | 1294 | if (cpu_family == CPU_OPTERON) |
1219 | fidvid_msr_init(); | 1295 | fidvid_msr_init(); |
@@ -1228,7 +1304,8 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) | |||
1228 | data->available_cores = pol->cpus; | 1304 | data->available_cores = pol->cpus; |
1229 | 1305 | ||
1230 | if (cpu_family == CPU_HW_PSTATE) | 1306 | if (cpu_family == CPU_HW_PSTATE) |
1231 | pol->cur = find_khz_freq_from_pstate(data->powernow_table, data->currpstate); | 1307 | pol->cur = find_khz_freq_from_pstate(data->powernow_table, |
1308 | data->currpstate); | ||
1232 | else | 1309 | else |
1233 | pol->cur = find_khz_freq_from_fid(data->currfid); | 1310 | pol->cur = find_khz_freq_from_fid(data->currfid); |
1234 | dprintk("policy current frequency %d kHz\n", pol->cur); | 1311 | dprintk("policy current frequency %d kHz\n", pol->cur); |
@@ -1245,7 +1322,8 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) | |||
1245 | cpufreq_frequency_table_get_attr(data->powernow_table, pol->cpu); | 1322 | cpufreq_frequency_table_get_attr(data->powernow_table, pol->cpu); |
1246 | 1323 | ||
1247 | if (cpu_family == CPU_HW_PSTATE) | 1324 | if (cpu_family == CPU_HW_PSTATE) |
1248 | dprintk("cpu_init done, current pstate 0x%x\n", data->currpstate); | 1325 | dprintk("cpu_init done, current pstate 0x%x\n", |
1326 | data->currpstate); | ||
1249 | else | 1327 | else |
1250 | dprintk("cpu_init done, current fid 0x%x, vid 0x%x\n", | 1328 | dprintk("cpu_init done, current fid 0x%x, vid 0x%x\n", |
1251 | data->currfid, data->currvid); | 1329 | data->currfid, data->currvid); |
@@ -1254,15 +1332,16 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) | |||
1254 | 1332 | ||
1255 | return 0; | 1333 | return 0; |
1256 | 1334 | ||
1257 | err_out: | 1335 | err_out_unmask: |
1258 | set_cpus_allowed_ptr(current, &oldmask); | 1336 | set_cpus_allowed_ptr(current, &oldmask); |
1259 | powernow_k8_cpu_exit_acpi(data); | 1337 | powernow_k8_cpu_exit_acpi(data); |
1260 | 1338 | ||
1339 | err_out: | ||
1261 | kfree(data); | 1340 | kfree(data); |
1262 | return -ENODEV; | 1341 | return -ENODEV; |
1263 | } | 1342 | } |
1264 | 1343 | ||
1265 | static int __devexit powernowk8_cpu_exit (struct cpufreq_policy *pol) | 1344 | static int __devexit powernowk8_cpu_exit(struct cpufreq_policy *pol) |
1266 | { | 1345 | { |
1267 | struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); | 1346 | struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); |
1268 | 1347 | ||
@@ -1279,7 +1358,7 @@ static int __devexit powernowk8_cpu_exit (struct cpufreq_policy *pol) | |||
1279 | return 0; | 1358 | return 0; |
1280 | } | 1359 | } |
1281 | 1360 | ||
1282 | static unsigned int powernowk8_get (unsigned int cpu) | 1361 | static unsigned int powernowk8_get(unsigned int cpu) |
1283 | { | 1362 | { |
1284 | struct powernow_k8_data *data; | 1363 | struct powernow_k8_data *data; |
1285 | cpumask_t oldmask = current->cpus_allowed; | 1364 | cpumask_t oldmask = current->cpus_allowed; |
@@ -1315,7 +1394,7 @@ out: | |||
1315 | return khz; | 1394 | return khz; |
1316 | } | 1395 | } |
1317 | 1396 | ||
1318 | static struct freq_attr* powernow_k8_attr[] = { | 1397 | static struct freq_attr *powernow_k8_attr[] = { |
1319 | &cpufreq_freq_attr_scaling_available_freqs, | 1398 | &cpufreq_freq_attr_scaling_available_freqs, |
1320 | NULL, | 1399 | NULL, |
1321 | }; | 1400 | }; |
@@ -1360,7 +1439,8 @@ static void __exit powernowk8_exit(void) | |||
1360 | cpufreq_unregister_driver(&cpufreq_amd64_driver); | 1439 | cpufreq_unregister_driver(&cpufreq_amd64_driver); |
1361 | } | 1440 | } |
1362 | 1441 | ||
1363 | MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com> and Mark Langsdorf <mark.langsdorf@amd.com>"); | 1442 | MODULE_AUTHOR("Paul Devriendt <paul.devriendt@amd.com> and " |
1443 | "Mark Langsdorf <mark.langsdorf@amd.com>"); | ||
1364 | MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver."); | 1444 | MODULE_DESCRIPTION("AMD Athlon 64 and Opteron processor frequency driver."); |
1365 | MODULE_LICENSE("GPL"); | 1445 | MODULE_LICENSE("GPL"); |
1366 | 1446 | ||
diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.h b/arch/x86/kernel/cpu/cpufreq/powernow-k8.h index 8ecc75b6c7c3..6c6698feade1 100644 --- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.h +++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.h | |||
@@ -45,11 +45,10 @@ struct powernow_k8_data { | |||
45 | * frequency is in kHz */ | 45 | * frequency is in kHz */ |
46 | struct cpufreq_frequency_table *powernow_table; | 46 | struct cpufreq_frequency_table *powernow_table; |
47 | 47 | ||
48 | #ifdef CONFIG_X86_POWERNOW_K8_ACPI | ||
49 | /* the acpi table needs to be kept. it's only available if ACPI was | 48 | /* the acpi table needs to be kept. it's only available if ACPI was |
50 | * used to determine valid frequency/vid/fid states */ | 49 | * used to determine valid frequency/vid/fid states */ |
51 | struct acpi_processor_performance acpi_data; | 50 | struct acpi_processor_performance acpi_data; |
52 | #endif | 51 | |
53 | /* we need to keep track of associated cores, but let cpufreq | 52 | /* we need to keep track of associated cores, but let cpufreq |
54 | * handle hotplug events - so just point at cpufreq pol->cpus | 53 | * handle hotplug events - so just point at cpufreq pol->cpus |
55 | * structure */ | 54 | * structure */ |
@@ -222,10 +221,8 @@ static int core_frequency_transition(struct powernow_k8_data *data, u32 reqfid); | |||
222 | 221 | ||
223 | static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index); | 222 | static void powernow_k8_acpi_pst_values(struct powernow_k8_data *data, unsigned int index); |
224 | 223 | ||
225 | #ifdef CONFIG_X86_POWERNOW_K8_ACPI | ||
226 | static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table); | 224 | static int fill_powernow_table_pstate(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table); |
227 | static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table); | 225 | static int fill_powernow_table_fidvid(struct powernow_k8_data *data, struct cpufreq_frequency_table *powernow_table); |
228 | #endif | ||
229 | 226 | ||
230 | #ifdef CONFIG_SMP | 227 | #ifdef CONFIG_SMP |
231 | static inline void define_siblings(int cpu, cpumask_t cpu_sharedcore_mask[]) | 228 | static inline void define_siblings(int cpu, cpumask_t cpu_sharedcore_mask[]) |
diff --git a/arch/x86/kernel/cpu/cpufreq/sc520_freq.c b/arch/x86/kernel/cpu/cpufreq/sc520_freq.c index 42da9bd677d6..435a996a613a 100644 --- a/arch/x86/kernel/cpu/cpufreq/sc520_freq.c +++ b/arch/x86/kernel/cpu/cpufreq/sc520_freq.c | |||
@@ -19,17 +19,19 @@ | |||
19 | 19 | ||
20 | #include <linux/delay.h> | 20 | #include <linux/delay.h> |
21 | #include <linux/cpufreq.h> | 21 | #include <linux/cpufreq.h> |
22 | #include <linux/timex.h> | ||
23 | #include <linux/io.h> | ||
22 | 24 | ||
23 | #include <asm/msr.h> | 25 | #include <asm/msr.h> |
24 | #include <asm/timex.h> | ||
25 | #include <asm/io.h> | ||
26 | 26 | ||
27 | #define MMCR_BASE 0xfffef000 /* The default base address */ | 27 | #define MMCR_BASE 0xfffef000 /* The default base address */ |
28 | #define OFFS_CPUCTL 0x2 /* CPU Control Register */ | 28 | #define OFFS_CPUCTL 0x2 /* CPU Control Register */ |
29 | 29 | ||
30 | static __u8 __iomem *cpuctl; | 30 | static __u8 __iomem *cpuctl; |
31 | 31 | ||
32 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "sc520_freq", msg) | 32 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
33 | "sc520_freq", msg) | ||
34 | #define PFX "sc520_freq: " | ||
33 | 35 | ||
34 | static struct cpufreq_frequency_table sc520_freq_table[] = { | 36 | static struct cpufreq_frequency_table sc520_freq_table[] = { |
35 | {0x01, 100000}, | 37 | {0x01, 100000}, |
@@ -43,7 +45,8 @@ static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu) | |||
43 | 45 | ||
44 | switch (clockspeed_reg & 0x03) { | 46 | switch (clockspeed_reg & 0x03) { |
45 | default: | 47 | default: |
46 | printk(KERN_ERR "sc520_freq: error: cpuctl register has unexpected value %02x\n", clockspeed_reg); | 48 | printk(KERN_ERR PFX "error: cpuctl register has unexpected " |
49 | "value %02x\n", clockspeed_reg); | ||
47 | case 0x01: | 50 | case 0x01: |
48 | return 100000; | 51 | return 100000; |
49 | case 0x02: | 52 | case 0x02: |
@@ -51,7 +54,7 @@ static unsigned int sc520_freq_get_cpu_frequency(unsigned int cpu) | |||
51 | } | 54 | } |
52 | } | 55 | } |
53 | 56 | ||
54 | static void sc520_freq_set_cpu_state (unsigned int state) | 57 | static void sc520_freq_set_cpu_state(unsigned int state) |
55 | { | 58 | { |
56 | 59 | ||
57 | struct cpufreq_freqs freqs; | 60 | struct cpufreq_freqs freqs; |
@@ -76,18 +79,19 @@ static void sc520_freq_set_cpu_state (unsigned int state) | |||
76 | cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); | 79 | cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); |
77 | }; | 80 | }; |
78 | 81 | ||
79 | static int sc520_freq_verify (struct cpufreq_policy *policy) | 82 | static int sc520_freq_verify(struct cpufreq_policy *policy) |
80 | { | 83 | { |
81 | return cpufreq_frequency_table_verify(policy, &sc520_freq_table[0]); | 84 | return cpufreq_frequency_table_verify(policy, &sc520_freq_table[0]); |
82 | } | 85 | } |
83 | 86 | ||
84 | static int sc520_freq_target (struct cpufreq_policy *policy, | 87 | static int sc520_freq_target(struct cpufreq_policy *policy, |
85 | unsigned int target_freq, | 88 | unsigned int target_freq, |
86 | unsigned int relation) | 89 | unsigned int relation) |
87 | { | 90 | { |
88 | unsigned int newstate = 0; | 91 | unsigned int newstate = 0; |
89 | 92 | ||
90 | if (cpufreq_frequency_table_target(policy, sc520_freq_table, target_freq, relation, &newstate)) | 93 | if (cpufreq_frequency_table_target(policy, sc520_freq_table, |
94 | target_freq, relation, &newstate)) | ||
91 | return -EINVAL; | 95 | return -EINVAL; |
92 | 96 | ||
93 | sc520_freq_set_cpu_state(newstate); | 97 | sc520_freq_set_cpu_state(newstate); |
@@ -116,7 +120,7 @@ static int sc520_freq_cpu_init(struct cpufreq_policy *policy) | |||
116 | 120 | ||
117 | result = cpufreq_frequency_table_cpuinfo(policy, sc520_freq_table); | 121 | result = cpufreq_frequency_table_cpuinfo(policy, sc520_freq_table); |
118 | if (result) | 122 | if (result) |
119 | return (result); | 123 | return result; |
120 | 124 | ||
121 | cpufreq_frequency_table_get_attr(sc520_freq_table, policy->cpu); | 125 | cpufreq_frequency_table_get_attr(sc520_freq_table, policy->cpu); |
122 | 126 | ||
@@ -131,7 +135,7 @@ static int sc520_freq_cpu_exit(struct cpufreq_policy *policy) | |||
131 | } | 135 | } |
132 | 136 | ||
133 | 137 | ||
134 | static struct freq_attr* sc520_freq_attr[] = { | 138 | static struct freq_attr *sc520_freq_attr[] = { |
135 | &cpufreq_freq_attr_scaling_available_freqs, | 139 | &cpufreq_freq_attr_scaling_available_freqs, |
136 | NULL, | 140 | NULL, |
137 | }; | 141 | }; |
@@ -155,13 +159,13 @@ static int __init sc520_freq_init(void) | |||
155 | int err; | 159 | int err; |
156 | 160 | ||
157 | /* Test if we have the right hardware */ | 161 | /* Test if we have the right hardware */ |
158 | if(c->x86_vendor != X86_VENDOR_AMD || | 162 | if (c->x86_vendor != X86_VENDOR_AMD || |
159 | c->x86 != 4 || c->x86_model != 9) { | 163 | c->x86 != 4 || c->x86_model != 9) { |
160 | dprintk("no Elan SC520 processor found!\n"); | 164 | dprintk("no Elan SC520 processor found!\n"); |
161 | return -ENODEV; | 165 | return -ENODEV; |
162 | } | 166 | } |
163 | cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1); | 167 | cpuctl = ioremap((unsigned long)(MMCR_BASE + OFFS_CPUCTL), 1); |
164 | if(!cpuctl) { | 168 | if (!cpuctl) { |
165 | printk(KERN_ERR "sc520_freq: error: failed to remap memory\n"); | 169 | printk(KERN_ERR "sc520_freq: error: failed to remap memory\n"); |
166 | return -ENOMEM; | 170 | return -ENOMEM; |
167 | } | 171 | } |
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c index dedc1e98f168..8bbb11adb315 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c | |||
@@ -39,7 +39,7 @@ static struct pci_dev *speedstep_chipset_dev; | |||
39 | 39 | ||
40 | /* speedstep_processor | 40 | /* speedstep_processor |
41 | */ | 41 | */ |
42 | static unsigned int speedstep_processor = 0; | 42 | static unsigned int speedstep_processor; |
43 | 43 | ||
44 | static u32 pmbase; | 44 | static u32 pmbase; |
45 | 45 | ||
@@ -54,7 +54,8 @@ static struct cpufreq_frequency_table speedstep_freqs[] = { | |||
54 | }; | 54 | }; |
55 | 55 | ||
56 | 56 | ||
57 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "speedstep-ich", msg) | 57 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
58 | "speedstep-ich", msg) | ||
58 | 59 | ||
59 | 60 | ||
60 | /** | 61 | /** |
@@ -62,7 +63,7 @@ static struct cpufreq_frequency_table speedstep_freqs[] = { | |||
62 | * | 63 | * |
63 | * Returns: -ENODEV if no register could be found | 64 | * Returns: -ENODEV if no register could be found |
64 | */ | 65 | */ |
65 | static int speedstep_find_register (void) | 66 | static int speedstep_find_register(void) |
66 | { | 67 | { |
67 | if (!speedstep_chipset_dev) | 68 | if (!speedstep_chipset_dev) |
68 | return -ENODEV; | 69 | return -ENODEV; |
@@ -90,7 +91,7 @@ static int speedstep_find_register (void) | |||
90 | * | 91 | * |
91 | * Tries to change the SpeedStep state. | 92 | * Tries to change the SpeedStep state. |
92 | */ | 93 | */ |
93 | static void speedstep_set_state (unsigned int state) | 94 | static void speedstep_set_state(unsigned int state) |
94 | { | 95 | { |
95 | u8 pm2_blk; | 96 | u8 pm2_blk; |
96 | u8 value; | 97 | u8 value; |
@@ -133,11 +134,11 @@ static void speedstep_set_state (unsigned int state) | |||
133 | 134 | ||
134 | dprintk("read at pmbase 0x%x + 0x50 returned 0x%x\n", pmbase, value); | 135 | dprintk("read at pmbase 0x%x + 0x50 returned 0x%x\n", pmbase, value); |
135 | 136 | ||
136 | if (state == (value & 0x1)) { | 137 | if (state == (value & 0x1)) |
137 | dprintk("change to %u MHz succeeded\n", (speedstep_get_processor_frequency(speedstep_processor) / 1000)); | 138 | dprintk("change to %u MHz succeeded\n", |
138 | } else { | 139 | speedstep_get_frequency(speedstep_processor) / 1000); |
139 | printk (KERN_ERR "cpufreq: change failed - I/O error\n"); | 140 | else |
140 | } | 141 | printk(KERN_ERR "cpufreq: change failed - I/O error\n"); |
141 | 142 | ||
142 | return; | 143 | return; |
143 | } | 144 | } |
@@ -149,7 +150,7 @@ static void speedstep_set_state (unsigned int state) | |||
149 | * Tries to activate the SpeedStep status and control registers. | 150 | * Tries to activate the SpeedStep status and control registers. |
150 | * Returns -EINVAL on an unsupported chipset, and zero on success. | 151 | * Returns -EINVAL on an unsupported chipset, and zero on success. |
151 | */ | 152 | */ |
152 | static int speedstep_activate (void) | 153 | static int speedstep_activate(void) |
153 | { | 154 | { |
154 | u16 value = 0; | 155 | u16 value = 0; |
155 | 156 | ||
@@ -175,20 +176,18 @@ static int speedstep_activate (void) | |||
175 | * functions. Returns the SPEEDSTEP_CHIPSET_-number for the detected | 176 | * functions. Returns the SPEEDSTEP_CHIPSET_-number for the detected |
176 | * chipset, or zero on failure. | 177 | * chipset, or zero on failure. |
177 | */ | 178 | */ |
178 | static unsigned int speedstep_detect_chipset (void) | 179 | static unsigned int speedstep_detect_chipset(void) |
179 | { | 180 | { |
180 | speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, | 181 | speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, |
181 | PCI_DEVICE_ID_INTEL_82801DB_12, | 182 | PCI_DEVICE_ID_INTEL_82801DB_12, |
182 | PCI_ANY_ID, | 183 | PCI_ANY_ID, PCI_ANY_ID, |
183 | PCI_ANY_ID, | ||
184 | NULL); | 184 | NULL); |
185 | if (speedstep_chipset_dev) | 185 | if (speedstep_chipset_dev) |
186 | return 4; /* 4-M */ | 186 | return 4; /* 4-M */ |
187 | 187 | ||
188 | speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, | 188 | speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, |
189 | PCI_DEVICE_ID_INTEL_82801CA_12, | 189 | PCI_DEVICE_ID_INTEL_82801CA_12, |
190 | PCI_ANY_ID, | 190 | PCI_ANY_ID, PCI_ANY_ID, |
191 | PCI_ANY_ID, | ||
192 | NULL); | 191 | NULL); |
193 | if (speedstep_chipset_dev) | 192 | if (speedstep_chipset_dev) |
194 | return 3; /* 3-M */ | 193 | return 3; /* 3-M */ |
@@ -196,8 +195,7 @@ static unsigned int speedstep_detect_chipset (void) | |||
196 | 195 | ||
197 | speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, | 196 | speedstep_chipset_dev = pci_get_subsys(PCI_VENDOR_ID_INTEL, |
198 | PCI_DEVICE_ID_INTEL_82801BA_10, | 197 | PCI_DEVICE_ID_INTEL_82801BA_10, |
199 | PCI_ANY_ID, | 198 | PCI_ANY_ID, PCI_ANY_ID, |
200 | PCI_ANY_ID, | ||
201 | NULL); | 199 | NULL); |
202 | if (speedstep_chipset_dev) { | 200 | if (speedstep_chipset_dev) { |
203 | /* speedstep.c causes lockups on Dell Inspirons 8000 and | 201 | /* speedstep.c causes lockups on Dell Inspirons 8000 and |
@@ -208,8 +206,7 @@ static unsigned int speedstep_detect_chipset (void) | |||
208 | 206 | ||
209 | hostbridge = pci_get_subsys(PCI_VENDOR_ID_INTEL, | 207 | hostbridge = pci_get_subsys(PCI_VENDOR_ID_INTEL, |
210 | PCI_DEVICE_ID_INTEL_82815_MC, | 208 | PCI_DEVICE_ID_INTEL_82815_MC, |
211 | PCI_ANY_ID, | 209 | PCI_ANY_ID, PCI_ANY_ID, |
212 | PCI_ANY_ID, | ||
213 | NULL); | 210 | NULL); |
214 | 211 | ||
215 | if (!hostbridge) | 212 | if (!hostbridge) |
@@ -236,7 +233,7 @@ static unsigned int _speedstep_get(const struct cpumask *cpus) | |||
236 | 233 | ||
237 | cpus_allowed = current->cpus_allowed; | 234 | cpus_allowed = current->cpus_allowed; |
238 | set_cpus_allowed_ptr(current, cpus); | 235 | set_cpus_allowed_ptr(current, cpus); |
239 | speed = speedstep_get_processor_frequency(speedstep_processor); | 236 | speed = speedstep_get_frequency(speedstep_processor); |
240 | set_cpus_allowed_ptr(current, &cpus_allowed); | 237 | set_cpus_allowed_ptr(current, &cpus_allowed); |
241 | dprintk("detected %u kHz as current frequency\n", speed); | 238 | dprintk("detected %u kHz as current frequency\n", speed); |
242 | return speed; | 239 | return speed; |
@@ -251,11 +248,12 @@ static unsigned int speedstep_get(unsigned int cpu) | |||
251 | * speedstep_target - set a new CPUFreq policy | 248 | * speedstep_target - set a new CPUFreq policy |
252 | * @policy: new policy | 249 | * @policy: new policy |
253 | * @target_freq: the target frequency | 250 | * @target_freq: the target frequency |
254 | * @relation: how that frequency relates to achieved frequency (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) | 251 | * @relation: how that frequency relates to achieved frequency |
252 | * (CPUFREQ_RELATION_L or CPUFREQ_RELATION_H) | ||
255 | * | 253 | * |
256 | * Sets a new CPUFreq policy. | 254 | * Sets a new CPUFreq policy. |
257 | */ | 255 | */ |
258 | static int speedstep_target (struct cpufreq_policy *policy, | 256 | static int speedstep_target(struct cpufreq_policy *policy, |
259 | unsigned int target_freq, | 257 | unsigned int target_freq, |
260 | unsigned int relation) | 258 | unsigned int relation) |
261 | { | 259 | { |
@@ -264,7 +262,8 @@ static int speedstep_target (struct cpufreq_policy *policy, | |||
264 | cpumask_t cpus_allowed; | 262 | cpumask_t cpus_allowed; |
265 | int i; | 263 | int i; |
266 | 264 | ||
267 | if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], target_freq, relation, &newstate)) | 265 | if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], |
266 | target_freq, relation, &newstate)) | ||
268 | return -EINVAL; | 267 | return -EINVAL; |
269 | 268 | ||
270 | freqs.old = _speedstep_get(policy->cpus); | 269 | freqs.old = _speedstep_get(policy->cpus); |
@@ -308,7 +307,7 @@ static int speedstep_target (struct cpufreq_policy *policy, | |||
308 | * Limit must be within speedstep_low_freq and speedstep_high_freq, with | 307 | * Limit must be within speedstep_low_freq and speedstep_high_freq, with |
309 | * at least one border included. | 308 | * at least one border included. |
310 | */ | 309 | */ |
311 | static int speedstep_verify (struct cpufreq_policy *policy) | 310 | static int speedstep_verify(struct cpufreq_policy *policy) |
312 | { | 311 | { |
313 | return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]); | 312 | return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]); |
314 | } | 313 | } |
@@ -344,7 +343,8 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) | |||
344 | return -EIO; | 343 | return -EIO; |
345 | 344 | ||
346 | dprintk("currently at %s speed setting - %i MHz\n", | 345 | dprintk("currently at %s speed setting - %i MHz\n", |
347 | (speed == speedstep_freqs[SPEEDSTEP_LOW].frequency) ? "low" : "high", | 346 | (speed == speedstep_freqs[SPEEDSTEP_LOW].frequency) |
347 | ? "low" : "high", | ||
348 | (speed / 1000)); | 348 | (speed / 1000)); |
349 | 349 | ||
350 | /* cpuinfo and default policy values */ | 350 | /* cpuinfo and default policy values */ |
@@ -352,9 +352,9 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) | |||
352 | 352 | ||
353 | result = cpufreq_frequency_table_cpuinfo(policy, speedstep_freqs); | 353 | result = cpufreq_frequency_table_cpuinfo(policy, speedstep_freqs); |
354 | if (result) | 354 | if (result) |
355 | return (result); | 355 | return result; |
356 | 356 | ||
357 | cpufreq_frequency_table_get_attr(speedstep_freqs, policy->cpu); | 357 | cpufreq_frequency_table_get_attr(speedstep_freqs, policy->cpu); |
358 | 358 | ||
359 | return 0; | 359 | return 0; |
360 | } | 360 | } |
@@ -366,7 +366,7 @@ static int speedstep_cpu_exit(struct cpufreq_policy *policy) | |||
366 | return 0; | 366 | return 0; |
367 | } | 367 | } |
368 | 368 | ||
369 | static struct freq_attr* speedstep_attr[] = { | 369 | static struct freq_attr *speedstep_attr[] = { |
370 | &cpufreq_freq_attr_scaling_available_freqs, | 370 | &cpufreq_freq_attr_scaling_available_freqs, |
371 | NULL, | 371 | NULL, |
372 | }; | 372 | }; |
@@ -396,13 +396,15 @@ static int __init speedstep_init(void) | |||
396 | /* detect processor */ | 396 | /* detect processor */ |
397 | speedstep_processor = speedstep_detect_processor(); | 397 | speedstep_processor = speedstep_detect_processor(); |
398 | if (!speedstep_processor) { | 398 | if (!speedstep_processor) { |
399 | dprintk("Intel(R) SpeedStep(TM) capable processor not found\n"); | 399 | dprintk("Intel(R) SpeedStep(TM) capable processor " |
400 | "not found\n"); | ||
400 | return -ENODEV; | 401 | return -ENODEV; |
401 | } | 402 | } |
402 | 403 | ||
403 | /* detect chipset */ | 404 | /* detect chipset */ |
404 | if (!speedstep_detect_chipset()) { | 405 | if (!speedstep_detect_chipset()) { |
405 | dprintk("Intel(R) SpeedStep(TM) for this chipset not (yet) available.\n"); | 406 | dprintk("Intel(R) SpeedStep(TM) for this chipset not " |
407 | "(yet) available.\n"); | ||
406 | return -ENODEV; | 408 | return -ENODEV; |
407 | } | 409 | } |
408 | 410 | ||
@@ -431,9 +433,11 @@ static void __exit speedstep_exit(void) | |||
431 | } | 433 | } |
432 | 434 | ||
433 | 435 | ||
434 | MODULE_AUTHOR ("Dave Jones <davej@redhat.com>, Dominik Brodowski <linux@brodo.de>"); | 436 | MODULE_AUTHOR("Dave Jones <davej@redhat.com>, " |
435 | MODULE_DESCRIPTION ("Speedstep driver for Intel mobile processors on chipsets with ICH-M southbridges."); | 437 | "Dominik Brodowski <linux@brodo.de>"); |
436 | MODULE_LICENSE ("GPL"); | 438 | MODULE_DESCRIPTION("Speedstep driver for Intel mobile processors on chipsets " |
439 | "with ICH-M southbridges."); | ||
440 | MODULE_LICENSE("GPL"); | ||
437 | 441 | ||
438 | module_init(speedstep_init); | 442 | module_init(speedstep_init); |
439 | module_exit(speedstep_exit); | 443 | module_exit(speedstep_exit); |
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c index cdac7d62369b..2e3c6862657b 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.c | |||
@@ -16,12 +16,16 @@ | |||
16 | #include <linux/slab.h> | 16 | #include <linux/slab.h> |
17 | 17 | ||
18 | #include <asm/msr.h> | 18 | #include <asm/msr.h> |
19 | #include <asm/tsc.h> | ||
19 | #include "speedstep-lib.h" | 20 | #include "speedstep-lib.h" |
20 | 21 | ||
21 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "speedstep-lib", msg) | 22 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
23 | "speedstep-lib", msg) | ||
24 | |||
25 | #define PFX "speedstep-lib: " | ||
22 | 26 | ||
23 | #ifdef CONFIG_X86_SPEEDSTEP_RELAXED_CAP_CHECK | 27 | #ifdef CONFIG_X86_SPEEDSTEP_RELAXED_CAP_CHECK |
24 | static int relaxed_check = 0; | 28 | static int relaxed_check; |
25 | #else | 29 | #else |
26 | #define relaxed_check 0 | 30 | #define relaxed_check 0 |
27 | #endif | 31 | #endif |
@@ -30,14 +34,14 @@ static int relaxed_check = 0; | |||
30 | * GET PROCESSOR CORE SPEED IN KHZ * | 34 | * GET PROCESSOR CORE SPEED IN KHZ * |
31 | *********************************************************************/ | 35 | *********************************************************************/ |
32 | 36 | ||
33 | static unsigned int pentium3_get_frequency (unsigned int processor) | 37 | static unsigned int pentium3_get_frequency(unsigned int processor) |
34 | { | 38 | { |
35 | /* See table 14 of p3_ds.pdf and table 22 of 29834003.pdf */ | 39 | /* See table 14 of p3_ds.pdf and table 22 of 29834003.pdf */ |
36 | struct { | 40 | struct { |
37 | unsigned int ratio; /* Frequency Multiplier (x10) */ | 41 | unsigned int ratio; /* Frequency Multiplier (x10) */ |
38 | u8 bitmap; /* power on configuration bits | 42 | u8 bitmap; /* power on configuration bits |
39 | [27, 25:22] (in MSR 0x2a) */ | 43 | [27, 25:22] (in MSR 0x2a) */ |
40 | } msr_decode_mult [] = { | 44 | } msr_decode_mult[] = { |
41 | { 30, 0x01 }, | 45 | { 30, 0x01 }, |
42 | { 35, 0x05 }, | 46 | { 35, 0x05 }, |
43 | { 40, 0x02 }, | 47 | { 40, 0x02 }, |
@@ -52,7 +56,7 @@ static unsigned int pentium3_get_frequency (unsigned int processor) | |||
52 | { 85, 0x26 }, | 56 | { 85, 0x26 }, |
53 | { 90, 0x20 }, | 57 | { 90, 0x20 }, |
54 | { 100, 0x2b }, | 58 | { 100, 0x2b }, |
55 | { 0, 0xff } /* error or unknown value */ | 59 | { 0, 0xff } /* error or unknown value */ |
56 | }; | 60 | }; |
57 | 61 | ||
58 | /* PIII(-M) FSB settings: see table b1-b of 24547206.pdf */ | 62 | /* PIII(-M) FSB settings: see table b1-b of 24547206.pdf */ |
@@ -60,7 +64,7 @@ static unsigned int pentium3_get_frequency (unsigned int processor) | |||
60 | unsigned int value; /* Front Side Bus speed in MHz */ | 64 | unsigned int value; /* Front Side Bus speed in MHz */ |
61 | u8 bitmap; /* power on configuration bits [18: 19] | 65 | u8 bitmap; /* power on configuration bits [18: 19] |
62 | (in MSR 0x2a) */ | 66 | (in MSR 0x2a) */ |
63 | } msr_decode_fsb [] = { | 67 | } msr_decode_fsb[] = { |
64 | { 66, 0x0 }, | 68 | { 66, 0x0 }, |
65 | { 100, 0x2 }, | 69 | { 100, 0x2 }, |
66 | { 133, 0x1 }, | 70 | { 133, 0x1 }, |
@@ -85,7 +89,7 @@ static unsigned int pentium3_get_frequency (unsigned int processor) | |||
85 | } | 89 | } |
86 | 90 | ||
87 | /* decode the multiplier */ | 91 | /* decode the multiplier */ |
88 | if (processor == SPEEDSTEP_PROCESSOR_PIII_C_EARLY) { | 92 | if (processor == SPEEDSTEP_CPU_PIII_C_EARLY) { |
89 | dprintk("workaround for early PIIIs\n"); | 93 | dprintk("workaround for early PIIIs\n"); |
90 | msr_lo &= 0x03c00000; | 94 | msr_lo &= 0x03c00000; |
91 | } else | 95 | } else |
@@ -97,9 +101,10 @@ static unsigned int pentium3_get_frequency (unsigned int processor) | |||
97 | j++; | 101 | j++; |
98 | } | 102 | } |
99 | 103 | ||
100 | dprintk("speed is %u\n", (msr_decode_mult[j].ratio * msr_decode_fsb[i].value * 100)); | 104 | dprintk("speed is %u\n", |
105 | (msr_decode_mult[j].ratio * msr_decode_fsb[i].value * 100)); | ||
101 | 106 | ||
102 | return (msr_decode_mult[j].ratio * msr_decode_fsb[i].value * 100); | 107 | return msr_decode_mult[j].ratio * msr_decode_fsb[i].value * 100; |
103 | } | 108 | } |
104 | 109 | ||
105 | 110 | ||
@@ -112,20 +117,23 @@ static unsigned int pentiumM_get_frequency(void) | |||
112 | 117 | ||
113 | /* see table B-2 of 24547212.pdf */ | 118 | /* see table B-2 of 24547212.pdf */ |
114 | if (msr_lo & 0x00040000) { | 119 | if (msr_lo & 0x00040000) { |
115 | printk(KERN_DEBUG "speedstep-lib: PM - invalid FSB: 0x%x 0x%x\n", msr_lo, msr_tmp); | 120 | printk(KERN_DEBUG PFX "PM - invalid FSB: 0x%x 0x%x\n", |
121 | msr_lo, msr_tmp); | ||
116 | return 0; | 122 | return 0; |
117 | } | 123 | } |
118 | 124 | ||
119 | msr_tmp = (msr_lo >> 22) & 0x1f; | 125 | msr_tmp = (msr_lo >> 22) & 0x1f; |
120 | dprintk("bits 22-26 are 0x%x, speed is %u\n", msr_tmp, (msr_tmp * 100 * 1000)); | 126 | dprintk("bits 22-26 are 0x%x, speed is %u\n", |
127 | msr_tmp, (msr_tmp * 100 * 1000)); | ||
121 | 128 | ||
122 | return (msr_tmp * 100 * 1000); | 129 | return msr_tmp * 100 * 1000; |
123 | } | 130 | } |
124 | 131 | ||
125 | static unsigned int pentium_core_get_frequency(void) | 132 | static unsigned int pentium_core_get_frequency(void) |
126 | { | 133 | { |
127 | u32 fsb = 0; | 134 | u32 fsb = 0; |
128 | u32 msr_lo, msr_tmp; | 135 | u32 msr_lo, msr_tmp; |
136 | int ret; | ||
129 | 137 | ||
130 | rdmsr(MSR_FSB_FREQ, msr_lo, msr_tmp); | 138 | rdmsr(MSR_FSB_FREQ, msr_lo, msr_tmp); |
131 | /* see table B-2 of 25366920.pdf */ | 139 | /* see table B-2 of 25366920.pdf */ |
@@ -153,12 +161,15 @@ static unsigned int pentium_core_get_frequency(void) | |||
153 | } | 161 | } |
154 | 162 | ||
155 | rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp); | 163 | rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_tmp); |
156 | dprintk("PCORE - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", msr_lo, msr_tmp); | 164 | dprintk("PCORE - MSR_IA32_EBL_CR_POWERON: 0x%x 0x%x\n", |
165 | msr_lo, msr_tmp); | ||
157 | 166 | ||
158 | msr_tmp = (msr_lo >> 22) & 0x1f; | 167 | msr_tmp = (msr_lo >> 22) & 0x1f; |
159 | dprintk("bits 22-26 are 0x%x, speed is %u\n", msr_tmp, (msr_tmp * fsb)); | 168 | dprintk("bits 22-26 are 0x%x, speed is %u\n", |
169 | msr_tmp, (msr_tmp * fsb)); | ||
160 | 170 | ||
161 | return (msr_tmp * fsb); | 171 | ret = (msr_tmp * fsb); |
172 | return ret; | ||
162 | } | 173 | } |
163 | 174 | ||
164 | 175 | ||
@@ -167,6 +178,16 @@ static unsigned int pentium4_get_frequency(void) | |||
167 | struct cpuinfo_x86 *c = &boot_cpu_data; | 178 | struct cpuinfo_x86 *c = &boot_cpu_data; |
168 | u32 msr_lo, msr_hi, mult; | 179 | u32 msr_lo, msr_hi, mult; |
169 | unsigned int fsb = 0; | 180 | unsigned int fsb = 0; |
181 | unsigned int ret; | ||
182 | u8 fsb_code; | ||
183 | |||
184 | /* Pentium 4 Model 0 and 1 do not have the Core Clock Frequency | ||
185 | * to System Bus Frequency Ratio Field in the Processor Frequency | ||
186 | * Configuration Register of the MSR. Therefore the current | ||
187 | * frequency cannot be calculated and has to be measured. | ||
188 | */ | ||
189 | if (c->x86_model < 2) | ||
190 | return cpu_khz; | ||
170 | 191 | ||
171 | rdmsr(0x2c, msr_lo, msr_hi); | 192 | rdmsr(0x2c, msr_lo, msr_hi); |
172 | 193 | ||
@@ -177,62 +198,61 @@ static unsigned int pentium4_get_frequency(void) | |||
177 | * revision #12 in Table B-1: MSRs in the Pentium 4 and | 198 | * revision #12 in Table B-1: MSRs in the Pentium 4 and |
178 | * Intel Xeon Processors, on page B-4 and B-5. | 199 | * Intel Xeon Processors, on page B-4 and B-5. |
179 | */ | 200 | */ |
180 | if (c->x86_model < 2) | 201 | fsb_code = (msr_lo >> 16) & 0x7; |
202 | switch (fsb_code) { | ||
203 | case 0: | ||
181 | fsb = 100 * 1000; | 204 | fsb = 100 * 1000; |
182 | else { | 205 | break; |
183 | u8 fsb_code = (msr_lo >> 16) & 0x7; | 206 | case 1: |
184 | switch (fsb_code) { | 207 | fsb = 13333 * 10; |
185 | case 0: | 208 | break; |
186 | fsb = 100 * 1000; | 209 | case 2: |
187 | break; | 210 | fsb = 200 * 1000; |
188 | case 1: | 211 | break; |
189 | fsb = 13333 * 10; | ||
190 | break; | ||
191 | case 2: | ||
192 | fsb = 200 * 1000; | ||
193 | break; | ||
194 | } | ||
195 | } | 212 | } |
196 | 213 | ||
197 | if (!fsb) | 214 | if (!fsb) |
198 | printk(KERN_DEBUG "speedstep-lib: couldn't detect FSB speed. Please send an e-mail to <linux@brodo.de>\n"); | 215 | printk(KERN_DEBUG PFX "couldn't detect FSB speed. " |
216 | "Please send an e-mail to <linux@brodo.de>\n"); | ||
199 | 217 | ||
200 | /* Multiplier. */ | 218 | /* Multiplier. */ |
201 | mult = msr_lo >> 24; | 219 | mult = msr_lo >> 24; |
202 | 220 | ||
203 | dprintk("P4 - FSB %u kHz; Multiplier %u; Speed %u kHz\n", fsb, mult, (fsb * mult)); | 221 | dprintk("P4 - FSB %u kHz; Multiplier %u; Speed %u kHz\n", |
222 | fsb, mult, (fsb * mult)); | ||
204 | 223 | ||
205 | return (fsb * mult); | 224 | ret = (fsb * mult); |
225 | return ret; | ||
206 | } | 226 | } |
207 | 227 | ||
208 | 228 | ||
209 | unsigned int speedstep_get_processor_frequency(unsigned int processor) | 229 | unsigned int speedstep_get_frequency(unsigned int processor) |
210 | { | 230 | { |
211 | switch (processor) { | 231 | switch (processor) { |
212 | case SPEEDSTEP_PROCESSOR_PCORE: | 232 | case SPEEDSTEP_CPU_PCORE: |
213 | return pentium_core_get_frequency(); | 233 | return pentium_core_get_frequency(); |
214 | case SPEEDSTEP_PROCESSOR_PM: | 234 | case SPEEDSTEP_CPU_PM: |
215 | return pentiumM_get_frequency(); | 235 | return pentiumM_get_frequency(); |
216 | case SPEEDSTEP_PROCESSOR_P4D: | 236 | case SPEEDSTEP_CPU_P4D: |
217 | case SPEEDSTEP_PROCESSOR_P4M: | 237 | case SPEEDSTEP_CPU_P4M: |
218 | return pentium4_get_frequency(); | 238 | return pentium4_get_frequency(); |
219 | case SPEEDSTEP_PROCESSOR_PIII_T: | 239 | case SPEEDSTEP_CPU_PIII_T: |
220 | case SPEEDSTEP_PROCESSOR_PIII_C: | 240 | case SPEEDSTEP_CPU_PIII_C: |
221 | case SPEEDSTEP_PROCESSOR_PIII_C_EARLY: | 241 | case SPEEDSTEP_CPU_PIII_C_EARLY: |
222 | return pentium3_get_frequency(processor); | 242 | return pentium3_get_frequency(processor); |
223 | default: | 243 | default: |
224 | return 0; | 244 | return 0; |
225 | }; | 245 | }; |
226 | return 0; | 246 | return 0; |
227 | } | 247 | } |
228 | EXPORT_SYMBOL_GPL(speedstep_get_processor_frequency); | 248 | EXPORT_SYMBOL_GPL(speedstep_get_frequency); |
229 | 249 | ||
230 | 250 | ||
231 | /********************************************************************* | 251 | /********************************************************************* |
232 | * DETECT SPEEDSTEP-CAPABLE PROCESSOR * | 252 | * DETECT SPEEDSTEP-CAPABLE PROCESSOR * |
233 | *********************************************************************/ | 253 | *********************************************************************/ |
234 | 254 | ||
235 | unsigned int speedstep_detect_processor (void) | 255 | unsigned int speedstep_detect_processor(void) |
236 | { | 256 | { |
237 | struct cpuinfo_x86 *c = &cpu_data(0); | 257 | struct cpuinfo_x86 *c = &cpu_data(0); |
238 | u32 ebx, msr_lo, msr_hi; | 258 | u32 ebx, msr_lo, msr_hi; |
@@ -261,7 +281,7 @@ unsigned int speedstep_detect_processor (void) | |||
261 | * sample has ebx = 0x0f, production has 0x0e. | 281 | * sample has ebx = 0x0f, production has 0x0e. |
262 | */ | 282 | */ |
263 | if ((ebx == 0x0e) || (ebx == 0x0f)) | 283 | if ((ebx == 0x0e) || (ebx == 0x0f)) |
264 | return SPEEDSTEP_PROCESSOR_P4M; | 284 | return SPEEDSTEP_CPU_P4M; |
265 | break; | 285 | break; |
266 | case 7: | 286 | case 7: |
267 | /* | 287 | /* |
@@ -272,7 +292,7 @@ unsigned int speedstep_detect_processor (void) | |||
272 | * samples are only of B-stepping... | 292 | * samples are only of B-stepping... |
273 | */ | 293 | */ |
274 | if (ebx == 0x0e) | 294 | if (ebx == 0x0e) |
275 | return SPEEDSTEP_PROCESSOR_P4M; | 295 | return SPEEDSTEP_CPU_P4M; |
276 | break; | 296 | break; |
277 | case 9: | 297 | case 9: |
278 | /* | 298 | /* |
@@ -288,10 +308,13 @@ unsigned int speedstep_detect_processor (void) | |||
288 | * M-P4-Ms may have either ebx=0xe or 0xf [see above] | 308 | * M-P4-Ms may have either ebx=0xe or 0xf [see above] |
289 | * M-P4/533 have either ebx=0xe or 0xf. [25317607.pdf] | 309 | * M-P4/533 have either ebx=0xe or 0xf. [25317607.pdf] |
290 | * also, M-P4M HTs have ebx=0x8, too | 310 | * also, M-P4M HTs have ebx=0x8, too |
291 | * For now, they are distinguished by the model_id string | 311 | * For now, they are distinguished by the model_id |
312 | * string | ||
292 | */ | 313 | */ |
293 | if ((ebx == 0x0e) || (strstr(c->x86_model_id,"Mobile Intel(R) Pentium(R) 4") != NULL)) | 314 | if ((ebx == 0x0e) || |
294 | return SPEEDSTEP_PROCESSOR_P4M; | 315 | (strstr(c->x86_model_id, |
316 | "Mobile Intel(R) Pentium(R) 4") != NULL)) | ||
317 | return SPEEDSTEP_CPU_P4M; | ||
295 | break; | 318 | break; |
296 | default: | 319 | default: |
297 | break; | 320 | break; |
@@ -301,7 +324,8 @@ unsigned int speedstep_detect_processor (void) | |||
301 | 324 | ||
302 | switch (c->x86_model) { | 325 | switch (c->x86_model) { |
303 | case 0x0B: /* Intel PIII [Tualatin] */ | 326 | case 0x0B: /* Intel PIII [Tualatin] */ |
304 | /* cpuid_ebx(1) is 0x04 for desktop PIII, 0x06 for mobile PIII-M */ | 327 | /* cpuid_ebx(1) is 0x04 for desktop PIII, |
328 | * 0x06 for mobile PIII-M */ | ||
305 | ebx = cpuid_ebx(0x00000001); | 329 | ebx = cpuid_ebx(0x00000001); |
306 | dprintk("ebx is %x\n", ebx); | 330 | dprintk("ebx is %x\n", ebx); |
307 | 331 | ||
@@ -313,14 +337,15 @@ unsigned int speedstep_detect_processor (void) | |||
313 | /* So far all PIII-M processors support SpeedStep. See | 337 | /* So far all PIII-M processors support SpeedStep. See |
314 | * Intel's 24540640.pdf of June 2003 | 338 | * Intel's 24540640.pdf of June 2003 |
315 | */ | 339 | */ |
316 | return SPEEDSTEP_PROCESSOR_PIII_T; | 340 | return SPEEDSTEP_CPU_PIII_T; |
317 | 341 | ||
318 | case 0x08: /* Intel PIII [Coppermine] */ | 342 | case 0x08: /* Intel PIII [Coppermine] */ |
319 | 343 | ||
320 | /* all mobile PIII Coppermines have FSB 100 MHz | 344 | /* all mobile PIII Coppermines have FSB 100 MHz |
321 | * ==> sort out a few desktop PIIIs. */ | 345 | * ==> sort out a few desktop PIIIs. */ |
322 | rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_hi); | 346 | rdmsr(MSR_IA32_EBL_CR_POWERON, msr_lo, msr_hi); |
323 | dprintk("Coppermine: MSR_IA32_EBL_CR_POWERON is 0x%x, 0x%x\n", msr_lo, msr_hi); | 347 | dprintk("Coppermine: MSR_IA32_EBL_CR_POWERON is 0x%x, 0x%x\n", |
348 | msr_lo, msr_hi); | ||
324 | msr_lo &= 0x00c0000; | 349 | msr_lo &= 0x00c0000; |
325 | if (msr_lo != 0x0080000) | 350 | if (msr_lo != 0x0080000) |
326 | return 0; | 351 | return 0; |
@@ -332,13 +357,15 @@ unsigned int speedstep_detect_processor (void) | |||
332 | * bit 56 or 57 is set | 357 | * bit 56 or 57 is set |
333 | */ | 358 | */ |
334 | rdmsr(MSR_IA32_PLATFORM_ID, msr_lo, msr_hi); | 359 | rdmsr(MSR_IA32_PLATFORM_ID, msr_lo, msr_hi); |
335 | dprintk("Coppermine: MSR_IA32_PLATFORM ID is 0x%x, 0x%x\n", msr_lo, msr_hi); | 360 | dprintk("Coppermine: MSR_IA32_PLATFORM ID is 0x%x, 0x%x\n", |
336 | if ((msr_hi & (1<<18)) && (relaxed_check ? 1 : (msr_hi & (3<<24)))) { | 361 | msr_lo, msr_hi); |
362 | if ((msr_hi & (1<<18)) && | ||
363 | (relaxed_check ? 1 : (msr_hi & (3<<24)))) { | ||
337 | if (c->x86_mask == 0x01) { | 364 | if (c->x86_mask == 0x01) { |
338 | dprintk("early PIII version\n"); | 365 | dprintk("early PIII version\n"); |
339 | return SPEEDSTEP_PROCESSOR_PIII_C_EARLY; | 366 | return SPEEDSTEP_CPU_PIII_C_EARLY; |
340 | } else | 367 | } else |
341 | return SPEEDSTEP_PROCESSOR_PIII_C; | 368 | return SPEEDSTEP_CPU_PIII_C; |
342 | } | 369 | } |
343 | 370 | ||
344 | default: | 371 | default: |
@@ -369,7 +396,7 @@ unsigned int speedstep_get_freqs(unsigned int processor, | |||
369 | dprintk("trying to determine both speeds\n"); | 396 | dprintk("trying to determine both speeds\n"); |
370 | 397 | ||
371 | /* get current speed */ | 398 | /* get current speed */ |
372 | prev_speed = speedstep_get_processor_frequency(processor); | 399 | prev_speed = speedstep_get_frequency(processor); |
373 | if (!prev_speed) | 400 | if (!prev_speed) |
374 | return -EIO; | 401 | return -EIO; |
375 | 402 | ||
@@ -379,7 +406,7 @@ unsigned int speedstep_get_freqs(unsigned int processor, | |||
379 | 406 | ||
380 | /* switch to low state */ | 407 | /* switch to low state */ |
381 | set_state(SPEEDSTEP_LOW); | 408 | set_state(SPEEDSTEP_LOW); |
382 | *low_speed = speedstep_get_processor_frequency(processor); | 409 | *low_speed = speedstep_get_frequency(processor); |
383 | if (!*low_speed) { | 410 | if (!*low_speed) { |
384 | ret = -EIO; | 411 | ret = -EIO; |
385 | goto out; | 412 | goto out; |
@@ -398,7 +425,7 @@ unsigned int speedstep_get_freqs(unsigned int processor, | |||
398 | if (transition_latency) | 425 | if (transition_latency) |
399 | do_gettimeofday(&tv2); | 426 | do_gettimeofday(&tv2); |
400 | 427 | ||
401 | *high_speed = speedstep_get_processor_frequency(processor); | 428 | *high_speed = speedstep_get_frequency(processor); |
402 | if (!*high_speed) { | 429 | if (!*high_speed) { |
403 | ret = -EIO; | 430 | ret = -EIO; |
404 | goto out; | 431 | goto out; |
@@ -426,9 +453,12 @@ unsigned int speedstep_get_freqs(unsigned int processor, | |||
426 | /* check if the latency measurement is too high or too low | 453 | /* check if the latency measurement is too high or too low |
427 | * and set it to a safe value (500uSec) in that case | 454 | * and set it to a safe value (500uSec) in that case |
428 | */ | 455 | */ |
429 | if (*transition_latency > 10000000 || *transition_latency < 50000) { | 456 | if (*transition_latency > 10000000 || |
430 | printk (KERN_WARNING "speedstep: frequency transition measured seems out of " | 457 | *transition_latency < 50000) { |
431 | "range (%u nSec), falling back to a safe one of %u nSec.\n", | 458 | printk(KERN_WARNING PFX "frequency transition " |
459 | "measured seems out of range (%u " | ||
460 | "nSec), falling back to a safe one of" | ||
461 | "%u nSec.\n", | ||
432 | *transition_latency, 500000); | 462 | *transition_latency, 500000); |
433 | *transition_latency = 500000; | 463 | *transition_latency = 500000; |
434 | } | 464 | } |
@@ -436,15 +466,16 @@ unsigned int speedstep_get_freqs(unsigned int processor, | |||
436 | 466 | ||
437 | out: | 467 | out: |
438 | local_irq_restore(flags); | 468 | local_irq_restore(flags); |
439 | return (ret); | 469 | return ret; |
440 | } | 470 | } |
441 | EXPORT_SYMBOL_GPL(speedstep_get_freqs); | 471 | EXPORT_SYMBOL_GPL(speedstep_get_freqs); |
442 | 472 | ||
443 | #ifdef CONFIG_X86_SPEEDSTEP_RELAXED_CAP_CHECK | 473 | #ifdef CONFIG_X86_SPEEDSTEP_RELAXED_CAP_CHECK |
444 | module_param(relaxed_check, int, 0444); | 474 | module_param(relaxed_check, int, 0444); |
445 | MODULE_PARM_DESC(relaxed_check, "Don't do all checks for speedstep capability."); | 475 | MODULE_PARM_DESC(relaxed_check, |
476 | "Don't do all checks for speedstep capability."); | ||
446 | #endif | 477 | #endif |
447 | 478 | ||
448 | MODULE_AUTHOR ("Dominik Brodowski <linux@brodo.de>"); | 479 | MODULE_AUTHOR("Dominik Brodowski <linux@brodo.de>"); |
449 | MODULE_DESCRIPTION ("Library for Intel SpeedStep 1 or 2 cpufreq drivers."); | 480 | MODULE_DESCRIPTION("Library for Intel SpeedStep 1 or 2 cpufreq drivers."); |
450 | MODULE_LICENSE ("GPL"); | 481 | MODULE_LICENSE("GPL"); |
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.h b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.h index b11bcc608cac..2b6c04e5a304 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-lib.h +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-lib.h | |||
@@ -12,17 +12,17 @@ | |||
12 | 12 | ||
13 | /* processors */ | 13 | /* processors */ |
14 | 14 | ||
15 | #define SPEEDSTEP_PROCESSOR_PIII_C_EARLY 0x00000001 /* Coppermine core */ | 15 | #define SPEEDSTEP_CPU_PIII_C_EARLY 0x00000001 /* Coppermine core */ |
16 | #define SPEEDSTEP_PROCESSOR_PIII_C 0x00000002 /* Coppermine core */ | 16 | #define SPEEDSTEP_CPU_PIII_C 0x00000002 /* Coppermine core */ |
17 | #define SPEEDSTEP_PROCESSOR_PIII_T 0x00000003 /* Tualatin core */ | 17 | #define SPEEDSTEP_CPU_PIII_T 0x00000003 /* Tualatin core */ |
18 | #define SPEEDSTEP_PROCESSOR_P4M 0x00000004 /* P4-M */ | 18 | #define SPEEDSTEP_CPU_P4M 0x00000004 /* P4-M */ |
19 | 19 | ||
20 | /* the following processors are not speedstep-capable and are not auto-detected | 20 | /* the following processors are not speedstep-capable and are not auto-detected |
21 | * in speedstep_detect_processor(). However, their speed can be detected using | 21 | * in speedstep_detect_processor(). However, their speed can be detected using |
22 | * the speedstep_get_processor_frequency() call. */ | 22 | * the speedstep_get_frequency() call. */ |
23 | #define SPEEDSTEP_PROCESSOR_PM 0xFFFFFF03 /* Pentium M */ | 23 | #define SPEEDSTEP_CPU_PM 0xFFFFFF03 /* Pentium M */ |
24 | #define SPEEDSTEP_PROCESSOR_P4D 0xFFFFFF04 /* desktop P4 */ | 24 | #define SPEEDSTEP_CPU_P4D 0xFFFFFF04 /* desktop P4 */ |
25 | #define SPEEDSTEP_PROCESSOR_PCORE 0xFFFFFF05 /* Core */ | 25 | #define SPEEDSTEP_CPU_PCORE 0xFFFFFF05 /* Core */ |
26 | 26 | ||
27 | /* speedstep states -- only two of them */ | 27 | /* speedstep states -- only two of them */ |
28 | 28 | ||
@@ -34,7 +34,7 @@ | |||
34 | extern unsigned int speedstep_detect_processor (void); | 34 | extern unsigned int speedstep_detect_processor (void); |
35 | 35 | ||
36 | /* detect the current speed (in khz) of the processor */ | 36 | /* detect the current speed (in khz) of the processor */ |
37 | extern unsigned int speedstep_get_processor_frequency(unsigned int processor); | 37 | extern unsigned int speedstep_get_frequency(unsigned int processor); |
38 | 38 | ||
39 | 39 | ||
40 | /* detect the low and high speeds of the processor. The callback | 40 | /* detect the low and high speeds of the processor. The callback |
diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-smi.c b/arch/x86/kernel/cpu/cpufreq/speedstep-smi.c index 8a85c93bd62a..befea088e4f5 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-smi.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-smi.c | |||
@@ -19,8 +19,8 @@ | |||
19 | #include <linux/cpufreq.h> | 19 | #include <linux/cpufreq.h> |
20 | #include <linux/slab.h> | 20 | #include <linux/slab.h> |
21 | #include <linux/delay.h> | 21 | #include <linux/delay.h> |
22 | #include <linux/io.h> | ||
22 | #include <asm/ist.h> | 23 | #include <asm/ist.h> |
23 | #include <asm/io.h> | ||
24 | 24 | ||
25 | #include "speedstep-lib.h" | 25 | #include "speedstep-lib.h" |
26 | 26 | ||
@@ -30,12 +30,12 @@ | |||
30 | * If user gives it, these are used. | 30 | * If user gives it, these are used. |
31 | * | 31 | * |
32 | */ | 32 | */ |
33 | static int smi_port = 0; | 33 | static int smi_port; |
34 | static int smi_cmd = 0; | 34 | static int smi_cmd; |
35 | static unsigned int smi_sig = 0; | 35 | static unsigned int smi_sig; |
36 | 36 | ||
37 | /* info about the processor */ | 37 | /* info about the processor */ |
38 | static unsigned int speedstep_processor = 0; | 38 | static unsigned int speedstep_processor; |
39 | 39 | ||
40 | /* | 40 | /* |
41 | * There are only two frequency states for each processor. Values | 41 | * There are only two frequency states for each processor. Values |
@@ -56,12 +56,13 @@ static struct cpufreq_frequency_table speedstep_freqs[] = { | |||
56 | * of DMA activity going on? */ | 56 | * of DMA activity going on? */ |
57 | #define SMI_TRIES 5 | 57 | #define SMI_TRIES 5 |
58 | 58 | ||
59 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, "speedstep-smi", msg) | 59 | #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_DRIVER, \ |
60 | "speedstep-smi", msg) | ||
60 | 61 | ||
61 | /** | 62 | /** |
62 | * speedstep_smi_ownership | 63 | * speedstep_smi_ownership |
63 | */ | 64 | */ |
64 | static int speedstep_smi_ownership (void) | 65 | static int speedstep_smi_ownership(void) |
65 | { | 66 | { |
66 | u32 command, result, magic, dummy; | 67 | u32 command, result, magic, dummy; |
67 | u32 function = GET_SPEEDSTEP_OWNER; | 68 | u32 function = GET_SPEEDSTEP_OWNER; |
@@ -70,16 +71,18 @@ static int speedstep_smi_ownership (void) | |||
70 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); | 71 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); |
71 | magic = virt_to_phys(magic_data); | 72 | magic = virt_to_phys(magic_data); |
72 | 73 | ||
73 | dprintk("trying to obtain ownership with command %x at port %x\n", command, smi_port); | 74 | dprintk("trying to obtain ownership with command %x at port %x\n", |
75 | command, smi_port); | ||
74 | 76 | ||
75 | __asm__ __volatile__( | 77 | __asm__ __volatile__( |
76 | "push %%ebp\n" | 78 | "push %%ebp\n" |
77 | "out %%al, (%%dx)\n" | 79 | "out %%al, (%%dx)\n" |
78 | "pop %%ebp\n" | 80 | "pop %%ebp\n" |
79 | : "=D" (result), "=a" (dummy), "=b" (dummy), "=c" (dummy), "=d" (dummy), | 81 | : "=D" (result), |
80 | "=S" (dummy) | 82 | "=a" (dummy), "=b" (dummy), "=c" (dummy), "=d" (dummy), |
83 | "=S" (dummy) | ||
81 | : "a" (command), "b" (function), "c" (0), "d" (smi_port), | 84 | : "a" (command), "b" (function), "c" (0), "d" (smi_port), |
82 | "D" (0), "S" (magic) | 85 | "D" (0), "S" (magic) |
83 | : "memory" | 86 | : "memory" |
84 | ); | 87 | ); |
85 | 88 | ||
@@ -97,10 +100,10 @@ static int speedstep_smi_ownership (void) | |||
97 | * even hangs [cf. bugme.osdl.org # 1422] on earlier systems. Empirical testing | 100 | * even hangs [cf. bugme.osdl.org # 1422] on earlier systems. Empirical testing |
98 | * shows that the latter occurs if !(ist_info.event & 0xFFFF). | 101 | * shows that the latter occurs if !(ist_info.event & 0xFFFF). |
99 | */ | 102 | */ |
100 | static int speedstep_smi_get_freqs (unsigned int *low, unsigned int *high) | 103 | static int speedstep_smi_get_freqs(unsigned int *low, unsigned int *high) |
101 | { | 104 | { |
102 | u32 command, result = 0, edi, high_mhz, low_mhz, dummy; | 105 | u32 command, result = 0, edi, high_mhz, low_mhz, dummy; |
103 | u32 state=0; | 106 | u32 state = 0; |
104 | u32 function = GET_SPEEDSTEP_FREQS; | 107 | u32 function = GET_SPEEDSTEP_FREQS; |
105 | 108 | ||
106 | if (!(ist_info.event & 0xFFFF)) { | 109 | if (!(ist_info.event & 0xFFFF)) { |
@@ -110,17 +113,25 @@ static int speedstep_smi_get_freqs (unsigned int *low, unsigned int *high) | |||
110 | 113 | ||
111 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); | 114 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); |
112 | 115 | ||
113 | dprintk("trying to determine frequencies with command %x at port %x\n", command, smi_port); | 116 | dprintk("trying to determine frequencies with command %x at port %x\n", |
117 | command, smi_port); | ||
114 | 118 | ||
115 | __asm__ __volatile__( | 119 | __asm__ __volatile__( |
116 | "push %%ebp\n" | 120 | "push %%ebp\n" |
117 | "out %%al, (%%dx)\n" | 121 | "out %%al, (%%dx)\n" |
118 | "pop %%ebp" | 122 | "pop %%ebp" |
119 | : "=a" (result), "=b" (high_mhz), "=c" (low_mhz), "=d" (state), "=D" (edi), "=S" (dummy) | 123 | : "=a" (result), |
120 | : "a" (command), "b" (function), "c" (state), "d" (smi_port), "S" (0), "D" (0) | 124 | "=b" (high_mhz), |
125 | "=c" (low_mhz), | ||
126 | "=d" (state), "=D" (edi), "=S" (dummy) | ||
127 | : "a" (command), | ||
128 | "b" (function), | ||
129 | "c" (state), | ||
130 | "d" (smi_port), "S" (0), "D" (0) | ||
121 | ); | 131 | ); |
122 | 132 | ||
123 | dprintk("result %x, low_freq %u, high_freq %u\n", result, low_mhz, high_mhz); | 133 | dprintk("result %x, low_freq %u, high_freq %u\n", |
134 | result, low_mhz, high_mhz); | ||
124 | 135 | ||
125 | /* abort if results are obviously incorrect... */ | 136 | /* abort if results are obviously incorrect... */ |
126 | if ((high_mhz + low_mhz) < 600) | 137 | if ((high_mhz + low_mhz) < 600) |
@@ -137,26 +148,30 @@ static int speedstep_smi_get_freqs (unsigned int *low, unsigned int *high) | |||
137 | * @state: processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH) | 148 | * @state: processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH) |
138 | * | 149 | * |
139 | */ | 150 | */ |
140 | static int speedstep_get_state (void) | 151 | static int speedstep_get_state(void) |
141 | { | 152 | { |
142 | u32 function=GET_SPEEDSTEP_STATE; | 153 | u32 function = GET_SPEEDSTEP_STATE; |
143 | u32 result, state, edi, command, dummy; | 154 | u32 result, state, edi, command, dummy; |
144 | 155 | ||
145 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); | 156 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); |
146 | 157 | ||
147 | dprintk("trying to determine current setting with command %x at port %x\n", command, smi_port); | 158 | dprintk("trying to determine current setting with command %x " |
159 | "at port %x\n", command, smi_port); | ||
148 | 160 | ||
149 | __asm__ __volatile__( | 161 | __asm__ __volatile__( |
150 | "push %%ebp\n" | 162 | "push %%ebp\n" |
151 | "out %%al, (%%dx)\n" | 163 | "out %%al, (%%dx)\n" |
152 | "pop %%ebp\n" | 164 | "pop %%ebp\n" |
153 | : "=a" (result), "=b" (state), "=D" (edi), "=c" (dummy), "=d" (dummy), "=S" (dummy) | 165 | : "=a" (result), |
154 | : "a" (command), "b" (function), "c" (0), "d" (smi_port), "S" (0), "D" (0) | 166 | "=b" (state), "=D" (edi), |
167 | "=c" (dummy), "=d" (dummy), "=S" (dummy) | ||
168 | : "a" (command), "b" (function), "c" (0), | ||
169 | "d" (smi_port), "S" (0), "D" (0) | ||
155 | ); | 170 | ); |
156 | 171 | ||
157 | dprintk("state is %x, result is %x\n", state, result); | 172 | dprintk("state is %x, result is %x\n", state, result); |
158 | 173 | ||
159 | return (state & 1); | 174 | return state & 1; |
160 | } | 175 | } |
161 | 176 | ||
162 | 177 | ||
@@ -165,11 +180,11 @@ static int speedstep_get_state (void) | |||
165 | * @state: new processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH) | 180 | * @state: new processor frequency state (SPEEDSTEP_LOW or SPEEDSTEP_HIGH) |
166 | * | 181 | * |
167 | */ | 182 | */ |
168 | static void speedstep_set_state (unsigned int state) | 183 | static void speedstep_set_state(unsigned int state) |
169 | { | 184 | { |
170 | unsigned int result = 0, command, new_state, dummy; | 185 | unsigned int result = 0, command, new_state, dummy; |
171 | unsigned long flags; | 186 | unsigned long flags; |
172 | unsigned int function=SET_SPEEDSTEP_STATE; | 187 | unsigned int function = SET_SPEEDSTEP_STATE; |
173 | unsigned int retry = 0; | 188 | unsigned int retry = 0; |
174 | 189 | ||
175 | if (state > 0x1) | 190 | if (state > 0x1) |
@@ -180,11 +195,14 @@ static void speedstep_set_state (unsigned int state) | |||
180 | 195 | ||
181 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); | 196 | command = (smi_sig & 0xffffff00) | (smi_cmd & 0xff); |
182 | 197 | ||
183 | dprintk("trying to set frequency to state %u with command %x at port %x\n", state, command, smi_port); | 198 | dprintk("trying to set frequency to state %u " |
199 | "with command %x at port %x\n", | ||
200 | state, command, smi_port); | ||
184 | 201 | ||
185 | do { | 202 | do { |
186 | if (retry) { | 203 | if (retry) { |
187 | dprintk("retry %u, previous result %u, waiting...\n", retry, result); | 204 | dprintk("retry %u, previous result %u, waiting...\n", |
205 | retry, result); | ||
188 | mdelay(retry * 50); | 206 | mdelay(retry * 50); |
189 | } | 207 | } |
190 | retry++; | 208 | retry++; |
@@ -192,20 +210,26 @@ static void speedstep_set_state (unsigned int state) | |||
192 | "push %%ebp\n" | 210 | "push %%ebp\n" |
193 | "out %%al, (%%dx)\n" | 211 | "out %%al, (%%dx)\n" |
194 | "pop %%ebp" | 212 | "pop %%ebp" |
195 | : "=b" (new_state), "=D" (result), "=c" (dummy), "=a" (dummy), | 213 | : "=b" (new_state), "=D" (result), |
196 | "=d" (dummy), "=S" (dummy) | 214 | "=c" (dummy), "=a" (dummy), |
197 | : "a" (command), "b" (function), "c" (state), "d" (smi_port), "S" (0), "D" (0) | 215 | "=d" (dummy), "=S" (dummy) |
216 | : "a" (command), "b" (function), "c" (state), | ||
217 | "d" (smi_port), "S" (0), "D" (0) | ||
198 | ); | 218 | ); |
199 | } while ((new_state != state) && (retry <= SMI_TRIES)); | 219 | } while ((new_state != state) && (retry <= SMI_TRIES)); |
200 | 220 | ||
201 | /* enable IRQs */ | 221 | /* enable IRQs */ |
202 | local_irq_restore(flags); | 222 | local_irq_restore(flags); |
203 | 223 | ||
204 | if (new_state == state) { | 224 | if (new_state == state) |
205 | dprintk("change to %u MHz succeeded after %u tries with result %u\n", (speedstep_freqs[new_state].frequency / 1000), retry, result); | 225 | dprintk("change to %u MHz succeeded after %u tries " |
206 | } else { | 226 | "with result %u\n", |
207 | printk(KERN_ERR "cpufreq: change to state %u failed with new_state %u and result %u\n", state, new_state, result); | 227 | (speedstep_freqs[new_state].frequency / 1000), |
208 | } | 228 | retry, result); |
229 | else | ||
230 | printk(KERN_ERR "cpufreq: change to state %u " | ||
231 | "failed with new_state %u and result %u\n", | ||
232 | state, new_state, result); | ||
209 | 233 | ||
210 | return; | 234 | return; |
211 | } | 235 | } |
@@ -219,13 +243,14 @@ static void speedstep_set_state (unsigned int state) | |||
219 | * | 243 | * |
220 | * Sets a new CPUFreq policy/freq. | 244 | * Sets a new CPUFreq policy/freq. |
221 | */ | 245 | */ |
222 | static int speedstep_target (struct cpufreq_policy *policy, | 246 | static int speedstep_target(struct cpufreq_policy *policy, |
223 | unsigned int target_freq, unsigned int relation) | 247 | unsigned int target_freq, unsigned int relation) |
224 | { | 248 | { |
225 | unsigned int newstate = 0; | 249 | unsigned int newstate = 0; |
226 | struct cpufreq_freqs freqs; | 250 | struct cpufreq_freqs freqs; |
227 | 251 | ||
228 | if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], target_freq, relation, &newstate)) | 252 | if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], |
253 | target_freq, relation, &newstate)) | ||
229 | return -EINVAL; | 254 | return -EINVAL; |
230 | 255 | ||
231 | freqs.old = speedstep_freqs[speedstep_get_state()].frequency; | 256 | freqs.old = speedstep_freqs[speedstep_get_state()].frequency; |
@@ -250,7 +275,7 @@ static int speedstep_target (struct cpufreq_policy *policy, | |||
250 | * Limit must be within speedstep_low_freq and speedstep_high_freq, with | 275 | * Limit must be within speedstep_low_freq and speedstep_high_freq, with |
251 | * at least one border included. | 276 | * at least one border included. |
252 | */ | 277 | */ |
253 | static int speedstep_verify (struct cpufreq_policy *policy) | 278 | static int speedstep_verify(struct cpufreq_policy *policy) |
254 | { | 279 | { |
255 | return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]); | 280 | return cpufreq_frequency_table_verify(policy, &speedstep_freqs[0]); |
256 | } | 281 | } |
@@ -259,7 +284,8 @@ static int speedstep_verify (struct cpufreq_policy *policy) | |||
259 | static int speedstep_cpu_init(struct cpufreq_policy *policy) | 284 | static int speedstep_cpu_init(struct cpufreq_policy *policy) |
260 | { | 285 | { |
261 | int result; | 286 | int result; |
262 | unsigned int speed,state; | 287 | unsigned int speed, state; |
288 | unsigned int *low, *high; | ||
263 | 289 | ||
264 | /* capability check */ | 290 | /* capability check */ |
265 | if (policy->cpu != 0) | 291 | if (policy->cpu != 0) |
@@ -272,19 +298,23 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) | |||
272 | } | 298 | } |
273 | 299 | ||
274 | /* detect low and high frequency */ | 300 | /* detect low and high frequency */ |
275 | result = speedstep_smi_get_freqs(&speedstep_freqs[SPEEDSTEP_LOW].frequency, | 301 | low = &speedstep_freqs[SPEEDSTEP_LOW].frequency; |
276 | &speedstep_freqs[SPEEDSTEP_HIGH].frequency); | 302 | high = &speedstep_freqs[SPEEDSTEP_HIGH].frequency; |
303 | |||
304 | result = speedstep_smi_get_freqs(low, high); | ||
277 | if (result) { | 305 | if (result) { |
278 | /* fall back to speedstep_lib.c dection mechanism: try both states out */ | 306 | /* fall back to speedstep_lib.c dection mechanism: |
279 | dprintk("could not detect low and high frequencies by SMI call.\n"); | 307 | * try both states out */ |
308 | dprintk("could not detect low and high frequencies " | ||
309 | "by SMI call.\n"); | ||
280 | result = speedstep_get_freqs(speedstep_processor, | 310 | result = speedstep_get_freqs(speedstep_processor, |
281 | &speedstep_freqs[SPEEDSTEP_LOW].frequency, | 311 | low, high, |
282 | &speedstep_freqs[SPEEDSTEP_HIGH].frequency, | ||
283 | NULL, | 312 | NULL, |
284 | &speedstep_set_state); | 313 | &speedstep_set_state); |
285 | 314 | ||
286 | if (result) { | 315 | if (result) { |
287 | dprintk("could not detect two different speeds -- aborting.\n"); | 316 | dprintk("could not detect two different speeds" |
317 | " -- aborting.\n"); | ||
288 | return result; | 318 | return result; |
289 | } else | 319 | } else |
290 | dprintk("workaround worked.\n"); | 320 | dprintk("workaround worked.\n"); |
@@ -295,7 +325,8 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) | |||
295 | speed = speedstep_freqs[state].frequency; | 325 | speed = speedstep_freqs[state].frequency; |
296 | 326 | ||
297 | dprintk("currently at %s speed setting - %i MHz\n", | 327 | dprintk("currently at %s speed setting - %i MHz\n", |
298 | (speed == speedstep_freqs[SPEEDSTEP_LOW].frequency) ? "low" : "high", | 328 | (speed == speedstep_freqs[SPEEDSTEP_LOW].frequency) |
329 | ? "low" : "high", | ||
299 | (speed / 1000)); | 330 | (speed / 1000)); |
300 | 331 | ||
301 | /* cpuinfo and default policy values */ | 332 | /* cpuinfo and default policy values */ |
@@ -304,7 +335,7 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) | |||
304 | 335 | ||
305 | result = cpufreq_frequency_table_cpuinfo(policy, speedstep_freqs); | 336 | result = cpufreq_frequency_table_cpuinfo(policy, speedstep_freqs); |
306 | if (result) | 337 | if (result) |
307 | return (result); | 338 | return result; |
308 | 339 | ||
309 | cpufreq_frequency_table_get_attr(speedstep_freqs, policy->cpu); | 340 | cpufreq_frequency_table_get_attr(speedstep_freqs, policy->cpu); |
310 | 341 | ||
@@ -321,7 +352,7 @@ static unsigned int speedstep_get(unsigned int cpu) | |||
321 | { | 352 | { |
322 | if (cpu) | 353 | if (cpu) |
323 | return -ENODEV; | 354 | return -ENODEV; |
324 | return speedstep_get_processor_frequency(speedstep_processor); | 355 | return speedstep_get_frequency(speedstep_processor); |
325 | } | 356 | } |
326 | 357 | ||
327 | 358 | ||
@@ -335,7 +366,7 @@ static int speedstep_resume(struct cpufreq_policy *policy) | |||
335 | return result; | 366 | return result; |
336 | } | 367 | } |
337 | 368 | ||
338 | static struct freq_attr* speedstep_attr[] = { | 369 | static struct freq_attr *speedstep_attr[] = { |
339 | &cpufreq_freq_attr_scaling_available_freqs, | 370 | &cpufreq_freq_attr_scaling_available_freqs, |
340 | NULL, | 371 | NULL, |
341 | }; | 372 | }; |
@@ -364,21 +395,23 @@ static int __init speedstep_init(void) | |||
364 | speedstep_processor = speedstep_detect_processor(); | 395 | speedstep_processor = speedstep_detect_processor(); |
365 | 396 | ||
366 | switch (speedstep_processor) { | 397 | switch (speedstep_processor) { |
367 | case SPEEDSTEP_PROCESSOR_PIII_T: | 398 | case SPEEDSTEP_CPU_PIII_T: |
368 | case SPEEDSTEP_PROCESSOR_PIII_C: | 399 | case SPEEDSTEP_CPU_PIII_C: |
369 | case SPEEDSTEP_PROCESSOR_PIII_C_EARLY: | 400 | case SPEEDSTEP_CPU_PIII_C_EARLY: |
370 | break; | 401 | break; |
371 | default: | 402 | default: |
372 | speedstep_processor = 0; | 403 | speedstep_processor = 0; |
373 | } | 404 | } |
374 | 405 | ||
375 | if (!speedstep_processor) { | 406 | if (!speedstep_processor) { |
376 | dprintk ("No supported Intel CPU detected.\n"); | 407 | dprintk("No supported Intel CPU detected.\n"); |
377 | return -ENODEV; | 408 | return -ENODEV; |
378 | } | 409 | } |
379 | 410 | ||
380 | dprintk("signature:0x%.8lx, command:0x%.8lx, event:0x%.8lx, perf_level:0x%.8lx.\n", | 411 | dprintk("signature:0x%.8lx, command:0x%.8lx, " |
381 | ist_info.signature, ist_info.command, ist_info.event, ist_info.perf_level); | 412 | "event:0x%.8lx, perf_level:0x%.8lx.\n", |
413 | ist_info.signature, ist_info.command, | ||
414 | ist_info.event, ist_info.perf_level); | ||
382 | 415 | ||
383 | /* Error if no IST-SMI BIOS or no PARM | 416 | /* Error if no IST-SMI BIOS or no PARM |
384 | sig= 'ISGE' aka 'Intel Speedstep Gate E' */ | 417 | sig= 'ISGE' aka 'Intel Speedstep Gate E' */ |
@@ -416,17 +449,20 @@ static void __exit speedstep_exit(void) | |||
416 | cpufreq_unregister_driver(&speedstep_driver); | 449 | cpufreq_unregister_driver(&speedstep_driver); |
417 | } | 450 | } |
418 | 451 | ||
419 | module_param(smi_port, int, 0444); | 452 | module_param(smi_port, int, 0444); |
420 | module_param(smi_cmd, int, 0444); | 453 | module_param(smi_cmd, int, 0444); |
421 | module_param(smi_sig, uint, 0444); | 454 | module_param(smi_sig, uint, 0444); |
422 | 455 | ||
423 | MODULE_PARM_DESC(smi_port, "Override the BIOS-given IST port with this value -- Intel's default setting is 0xb2"); | 456 | MODULE_PARM_DESC(smi_port, "Override the BIOS-given IST port with this value " |
424 | MODULE_PARM_DESC(smi_cmd, "Override the BIOS-given IST command with this value -- Intel's default setting is 0x82"); | 457 | "-- Intel's default setting is 0xb2"); |
425 | MODULE_PARM_DESC(smi_sig, "Set to 1 to fake the IST signature when using the SMI interface."); | 458 | MODULE_PARM_DESC(smi_cmd, "Override the BIOS-given IST command with this value " |
459 | "-- Intel's default setting is 0x82"); | ||
460 | MODULE_PARM_DESC(smi_sig, "Set to 1 to fake the IST signature when using the " | ||
461 | "SMI interface."); | ||
426 | 462 | ||
427 | MODULE_AUTHOR ("Hiroshi Miura"); | 463 | MODULE_AUTHOR("Hiroshi Miura"); |
428 | MODULE_DESCRIPTION ("Speedstep driver for IST applet SMI interface."); | 464 | MODULE_DESCRIPTION("Speedstep driver for IST applet SMI interface."); |
429 | MODULE_LICENSE ("GPL"); | 465 | MODULE_LICENSE("GPL"); |
430 | 466 | ||
431 | module_init(speedstep_init); | 467 | module_init(speedstep_init); |
432 | module_exit(speedstep_exit); | 468 | module_exit(speedstep_exit); |
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index b09d4eb52bb9..7437fa133c02 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c | |||
@@ -4,6 +4,7 @@ | |||
4 | #include <linux/string.h> | 4 | #include <linux/string.h> |
5 | #include <linux/bitops.h> | 5 | #include <linux/bitops.h> |
6 | #include <linux/smp.h> | 6 | #include <linux/smp.h> |
7 | #include <linux/sched.h> | ||
7 | #include <linux/thread_info.h> | 8 | #include <linux/thread_info.h> |
8 | #include <linux/module.h> | 9 | #include <linux/module.h> |
9 | 10 | ||
@@ -61,11 +62,16 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c) | |||
61 | 62 | ||
62 | /* | 63 | /* |
63 | * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate | 64 | * c->x86_power is 8000_0007 edx. Bit 8 is TSC runs at constant rate |
64 | * with P/T states and does not stop in deep C-states | 65 | * with P/T states and does not stop in deep C-states. |
66 | * | ||
67 | * It is also reliable across cores and sockets. (but not across | ||
68 | * cabinets - we turn it off in that case explicitly.) | ||
65 | */ | 69 | */ |
66 | if (c->x86_power & (1 << 8)) { | 70 | if (c->x86_power & (1 << 8)) { |
67 | set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); | 71 | set_cpu_cap(c, X86_FEATURE_CONSTANT_TSC); |
68 | set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC); | 72 | set_cpu_cap(c, X86_FEATURE_NONSTOP_TSC); |
73 | set_cpu_cap(c, X86_FEATURE_TSC_RELIABLE); | ||
74 | sched_clock_stable = 1; | ||
69 | } | 75 | } |
70 | 76 | ||
71 | /* | 77 | /* |
diff --git a/arch/x86/kernel/cpu/mtrr/if.c b/arch/x86/kernel/cpu/mtrr/if.c index 4c4214690dd1..fb73a52913a4 100644 --- a/arch/x86/kernel/cpu/mtrr/if.c +++ b/arch/x86/kernel/cpu/mtrr/if.c | |||
@@ -377,10 +377,6 @@ static const struct file_operations mtrr_fops = { | |||
377 | .release = mtrr_close, | 377 | .release = mtrr_close, |
378 | }; | 378 | }; |
379 | 379 | ||
380 | |||
381 | static struct proc_dir_entry *proc_root_mtrr; | ||
382 | |||
383 | |||
384 | static int mtrr_seq_show(struct seq_file *seq, void *offset) | 380 | static int mtrr_seq_show(struct seq_file *seq, void *offset) |
385 | { | 381 | { |
386 | char factor; | 382 | char factor; |
@@ -423,11 +419,7 @@ static int __init mtrr_if_init(void) | |||
423 | (!cpu_has(c, X86_FEATURE_CENTAUR_MCR))) | 419 | (!cpu_has(c, X86_FEATURE_CENTAUR_MCR))) |
424 | return -ENODEV; | 420 | return -ENODEV; |
425 | 421 | ||
426 | proc_root_mtrr = | 422 | proc_create("mtrr", S_IWUSR | S_IRUGO, NULL, &mtrr_fops); |
427 | proc_create("mtrr", S_IWUSR | S_IRUGO, NULL, &mtrr_fops); | ||
428 | |||
429 | if (proc_root_mtrr) | ||
430 | proc_root_mtrr->owner = THIS_MODULE; | ||
431 | return 0; | 423 | return 0; |
432 | } | 424 | } |
433 | 425 | ||
diff --git a/arch/x86/kernel/dumpstack.c b/arch/x86/kernel/dumpstack.c index 87d103ded1c3..dd2130b0fb3e 100644 --- a/arch/x86/kernel/dumpstack.c +++ b/arch/x86/kernel/dumpstack.c | |||
@@ -10,6 +10,7 @@ | |||
10 | #include <linux/kdebug.h> | 10 | #include <linux/kdebug.h> |
11 | #include <linux/module.h> | 11 | #include <linux/module.h> |
12 | #include <linux/ptrace.h> | 12 | #include <linux/ptrace.h> |
13 | #include <linux/ftrace.h> | ||
13 | #include <linux/kexec.h> | 14 | #include <linux/kexec.h> |
14 | #include <linux/bug.h> | 15 | #include <linux/bug.h> |
15 | #include <linux/nmi.h> | 16 | #include <linux/nmi.h> |
diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index fb638d9ce6d2..ef2c3563357d 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c | |||
@@ -233,7 +233,7 @@ void __init e820_print_map(char *who) | |||
233 | */ | 233 | */ |
234 | 234 | ||
235 | int __init sanitize_e820_map(struct e820entry *biosmap, int max_nr_map, | 235 | int __init sanitize_e820_map(struct e820entry *biosmap, int max_nr_map, |
236 | int *pnr_map) | 236 | u32 *pnr_map) |
237 | { | 237 | { |
238 | struct change_member { | 238 | struct change_member { |
239 | struct e820entry *pbios; /* pointer to original bios entry */ | 239 | struct e820entry *pbios; /* pointer to original bios entry */ |
@@ -552,7 +552,7 @@ u64 __init e820_remove_range(u64 start, u64 size, unsigned old_type, | |||
552 | 552 | ||
553 | void __init update_e820(void) | 553 | void __init update_e820(void) |
554 | { | 554 | { |
555 | int nr_map; | 555 | u32 nr_map; |
556 | 556 | ||
557 | nr_map = e820.nr_map; | 557 | nr_map = e820.nr_map; |
558 | if (sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &nr_map)) | 558 | if (sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &nr_map)) |
@@ -563,7 +563,7 @@ void __init update_e820(void) | |||
563 | } | 563 | } |
564 | static void __init update_e820_saved(void) | 564 | static void __init update_e820_saved(void) |
565 | { | 565 | { |
566 | int nr_map; | 566 | u32 nr_map; |
567 | 567 | ||
568 | nr_map = e820_saved.nr_map; | 568 | nr_map = e820_saved.nr_map; |
569 | if (sanitize_e820_map(e820_saved.map, ARRAY_SIZE(e820_saved.map), &nr_map)) | 569 | if (sanitize_e820_map(e820_saved.map, ARRAY_SIZE(e820_saved.map), &nr_map)) |
@@ -1303,7 +1303,7 @@ early_param("memmap", parse_memmap_opt); | |||
1303 | void __init finish_e820_parsing(void) | 1303 | void __init finish_e820_parsing(void) |
1304 | { | 1304 | { |
1305 | if (userdef) { | 1305 | if (userdef) { |
1306 | int nr = e820.nr_map; | 1306 | u32 nr = e820.nr_map; |
1307 | 1307 | ||
1308 | if (sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &nr) < 0) | 1308 | if (sanitize_e820_map(e820.map, ARRAY_SIZE(e820.map), &nr) < 0) |
1309 | early_panic("Invalid user supplied memory map"); | 1309 | early_panic("Invalid user supplied memory map"); |
@@ -1386,7 +1386,7 @@ void __init e820_reserve_resources_late(void) | |||
1386 | char *__init default_machine_specific_memory_setup(void) | 1386 | char *__init default_machine_specific_memory_setup(void) |
1387 | { | 1387 | { |
1388 | char *who = "BIOS-e820"; | 1388 | char *who = "BIOS-e820"; |
1389 | int new_nr; | 1389 | u32 new_nr; |
1390 | /* | 1390 | /* |
1391 | * Try to copy the BIOS-supplied E820-map. | 1391 | * Try to copy the BIOS-supplied E820-map. |
1392 | * | 1392 | * |
diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c index 231bdd3c5b1c..76f7141e0f91 100644 --- a/arch/x86/kernel/ftrace.c +++ b/arch/x86/kernel/ftrace.c | |||
@@ -389,79 +389,6 @@ void ftrace_nmi_exit(void) | |||
389 | 389 | ||
390 | #endif /* !CONFIG_DYNAMIC_FTRACE */ | 390 | #endif /* !CONFIG_DYNAMIC_FTRACE */ |
391 | 391 | ||
392 | /* Add a function return address to the trace stack on thread info.*/ | ||
393 | static int push_return_trace(unsigned long ret, unsigned long long time, | ||
394 | unsigned long func, int *depth) | ||
395 | { | ||
396 | int index; | ||
397 | |||
398 | if (!current->ret_stack) | ||
399 | return -EBUSY; | ||
400 | |||
401 | /* The return trace stack is full */ | ||
402 | if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) { | ||
403 | atomic_inc(¤t->trace_overrun); | ||
404 | return -EBUSY; | ||
405 | } | ||
406 | |||
407 | index = ++current->curr_ret_stack; | ||
408 | barrier(); | ||
409 | current->ret_stack[index].ret = ret; | ||
410 | current->ret_stack[index].func = func; | ||
411 | current->ret_stack[index].calltime = time; | ||
412 | *depth = index; | ||
413 | |||
414 | return 0; | ||
415 | } | ||
416 | |||
417 | /* Retrieve a function return address to the trace stack on thread info.*/ | ||
418 | static void pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret) | ||
419 | { | ||
420 | int index; | ||
421 | |||
422 | index = current->curr_ret_stack; | ||
423 | |||
424 | if (unlikely(index < 0)) { | ||
425 | ftrace_graph_stop(); | ||
426 | WARN_ON(1); | ||
427 | /* Might as well panic, otherwise we have no where to go */ | ||
428 | *ret = (unsigned long)panic; | ||
429 | return; | ||
430 | } | ||
431 | |||
432 | *ret = current->ret_stack[index].ret; | ||
433 | trace->func = current->ret_stack[index].func; | ||
434 | trace->calltime = current->ret_stack[index].calltime; | ||
435 | trace->overrun = atomic_read(¤t->trace_overrun); | ||
436 | trace->depth = index; | ||
437 | barrier(); | ||
438 | current->curr_ret_stack--; | ||
439 | |||
440 | } | ||
441 | |||
442 | /* | ||
443 | * Send the trace to the ring-buffer. | ||
444 | * @return the original return address. | ||
445 | */ | ||
446 | unsigned long ftrace_return_to_handler(void) | ||
447 | { | ||
448 | struct ftrace_graph_ret trace; | ||
449 | unsigned long ret; | ||
450 | |||
451 | pop_return_trace(&trace, &ret); | ||
452 | trace.rettime = cpu_clock(raw_smp_processor_id()); | ||
453 | ftrace_graph_return(&trace); | ||
454 | |||
455 | if (unlikely(!ret)) { | ||
456 | ftrace_graph_stop(); | ||
457 | WARN_ON(1); | ||
458 | /* Might as well panic. What else to do? */ | ||
459 | ret = (unsigned long)panic; | ||
460 | } | ||
461 | |||
462 | return ret; | ||
463 | } | ||
464 | |||
465 | /* | 392 | /* |
466 | * Hook the return address and push it in the stack of return addrs | 393 | * Hook the return address and push it in the stack of return addrs |
467 | * in current thread info. | 394 | * in current thread info. |
@@ -521,7 +448,7 @@ void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr) | |||
521 | 448 | ||
522 | calltime = cpu_clock(raw_smp_processor_id()); | 449 | calltime = cpu_clock(raw_smp_processor_id()); |
523 | 450 | ||
524 | if (push_return_trace(old, calltime, | 451 | if (ftrace_push_return_trace(old, calltime, |
525 | self_addr, &trace.depth) == -EBUSY) { | 452 | self_addr, &trace.depth) == -EBUSY) { |
526 | *parent = old; | 453 | *parent = old; |
527 | return; | 454 | return; |
diff --git a/arch/x86/kernel/hpet.c b/arch/x86/kernel/hpet.c index a00545fe5cdd..648b3a2a3a44 100644 --- a/arch/x86/kernel/hpet.c +++ b/arch/x86/kernel/hpet.c | |||
@@ -80,6 +80,7 @@ static inline void hpet_clear_mapping(void) | |||
80 | */ | 80 | */ |
81 | static int boot_hpet_disable; | 81 | static int boot_hpet_disable; |
82 | int hpet_force_user; | 82 | int hpet_force_user; |
83 | static int hpet_verbose; | ||
83 | 84 | ||
84 | static int __init hpet_setup(char *str) | 85 | static int __init hpet_setup(char *str) |
85 | { | 86 | { |
@@ -88,6 +89,8 @@ static int __init hpet_setup(char *str) | |||
88 | boot_hpet_disable = 1; | 89 | boot_hpet_disable = 1; |
89 | if (!strncmp("force", str, 5)) | 90 | if (!strncmp("force", str, 5)) |
90 | hpet_force_user = 1; | 91 | hpet_force_user = 1; |
92 | if (!strncmp("verbose", str, 7)) | ||
93 | hpet_verbose = 1; | ||
91 | } | 94 | } |
92 | return 1; | 95 | return 1; |
93 | } | 96 | } |
@@ -119,6 +122,43 @@ int is_hpet_enabled(void) | |||
119 | } | 122 | } |
120 | EXPORT_SYMBOL_GPL(is_hpet_enabled); | 123 | EXPORT_SYMBOL_GPL(is_hpet_enabled); |
121 | 124 | ||
125 | static void _hpet_print_config(const char *function, int line) | ||
126 | { | ||
127 | u32 i, timers, l, h; | ||
128 | printk(KERN_INFO "hpet: %s(%d):\n", function, line); | ||
129 | l = hpet_readl(HPET_ID); | ||
130 | h = hpet_readl(HPET_PERIOD); | ||
131 | timers = ((l & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT) + 1; | ||
132 | printk(KERN_INFO "hpet: ID: 0x%x, PERIOD: 0x%x\n", l, h); | ||
133 | l = hpet_readl(HPET_CFG); | ||
134 | h = hpet_readl(HPET_STATUS); | ||
135 | printk(KERN_INFO "hpet: CFG: 0x%x, STATUS: 0x%x\n", l, h); | ||
136 | l = hpet_readl(HPET_COUNTER); | ||
137 | h = hpet_readl(HPET_COUNTER+4); | ||
138 | printk(KERN_INFO "hpet: COUNTER_l: 0x%x, COUNTER_h: 0x%x\n", l, h); | ||
139 | |||
140 | for (i = 0; i < timers; i++) { | ||
141 | l = hpet_readl(HPET_Tn_CFG(i)); | ||
142 | h = hpet_readl(HPET_Tn_CFG(i)+4); | ||
143 | printk(KERN_INFO "hpet: T%d: CFG_l: 0x%x, CFG_h: 0x%x\n", | ||
144 | i, l, h); | ||
145 | l = hpet_readl(HPET_Tn_CMP(i)); | ||
146 | h = hpet_readl(HPET_Tn_CMP(i)+4); | ||
147 | printk(KERN_INFO "hpet: T%d: CMP_l: 0x%x, CMP_h: 0x%x\n", | ||
148 | i, l, h); | ||
149 | l = hpet_readl(HPET_Tn_ROUTE(i)); | ||
150 | h = hpet_readl(HPET_Tn_ROUTE(i)+4); | ||
151 | printk(KERN_INFO "hpet: T%d ROUTE_l: 0x%x, ROUTE_h: 0x%x\n", | ||
152 | i, l, h); | ||
153 | } | ||
154 | } | ||
155 | |||
156 | #define hpet_print_config() \ | ||
157 | do { \ | ||
158 | if (hpet_verbose) \ | ||
159 | _hpet_print_config(__FUNCTION__, __LINE__); \ | ||
160 | } while (0) | ||
161 | |||
122 | /* | 162 | /* |
123 | * When the hpet driver (/dev/hpet) is enabled, we need to reserve | 163 | * When the hpet driver (/dev/hpet) is enabled, we need to reserve |
124 | * timer 0 and timer 1 in case of RTC emulation. | 164 | * timer 0 and timer 1 in case of RTC emulation. |
@@ -191,27 +231,37 @@ static struct clock_event_device hpet_clockevent = { | |||
191 | .rating = 50, | 231 | .rating = 50, |
192 | }; | 232 | }; |
193 | 233 | ||
194 | static void hpet_start_counter(void) | 234 | static void hpet_stop_counter(void) |
195 | { | 235 | { |
196 | unsigned long cfg = hpet_readl(HPET_CFG); | 236 | unsigned long cfg = hpet_readl(HPET_CFG); |
197 | |||
198 | cfg &= ~HPET_CFG_ENABLE; | 237 | cfg &= ~HPET_CFG_ENABLE; |
199 | hpet_writel(cfg, HPET_CFG); | 238 | hpet_writel(cfg, HPET_CFG); |
200 | hpet_writel(0, HPET_COUNTER); | 239 | hpet_writel(0, HPET_COUNTER); |
201 | hpet_writel(0, HPET_COUNTER + 4); | 240 | hpet_writel(0, HPET_COUNTER + 4); |
241 | } | ||
242 | |||
243 | static void hpet_start_counter(void) | ||
244 | { | ||
245 | unsigned long cfg = hpet_readl(HPET_CFG); | ||
202 | cfg |= HPET_CFG_ENABLE; | 246 | cfg |= HPET_CFG_ENABLE; |
203 | hpet_writel(cfg, HPET_CFG); | 247 | hpet_writel(cfg, HPET_CFG); |
204 | } | 248 | } |
205 | 249 | ||
250 | static void hpet_restart_counter(void) | ||
251 | { | ||
252 | hpet_stop_counter(); | ||
253 | hpet_start_counter(); | ||
254 | } | ||
255 | |||
206 | static void hpet_resume_device(void) | 256 | static void hpet_resume_device(void) |
207 | { | 257 | { |
208 | force_hpet_resume(); | 258 | force_hpet_resume(); |
209 | } | 259 | } |
210 | 260 | ||
211 | static void hpet_restart_counter(void) | 261 | static void hpet_resume_counter(void) |
212 | { | 262 | { |
213 | hpet_resume_device(); | 263 | hpet_resume_device(); |
214 | hpet_start_counter(); | 264 | hpet_restart_counter(); |
215 | } | 265 | } |
216 | 266 | ||
217 | static void hpet_enable_legacy_int(void) | 267 | static void hpet_enable_legacy_int(void) |
@@ -259,29 +309,23 @@ static int hpet_setup_msi_irq(unsigned int irq); | |||
259 | static void hpet_set_mode(enum clock_event_mode mode, | 309 | static void hpet_set_mode(enum clock_event_mode mode, |
260 | struct clock_event_device *evt, int timer) | 310 | struct clock_event_device *evt, int timer) |
261 | { | 311 | { |
262 | unsigned long cfg, cmp, now; | 312 | unsigned long cfg; |
263 | uint64_t delta; | 313 | uint64_t delta; |
264 | 314 | ||
265 | switch (mode) { | 315 | switch (mode) { |
266 | case CLOCK_EVT_MODE_PERIODIC: | 316 | case CLOCK_EVT_MODE_PERIODIC: |
317 | hpet_stop_counter(); | ||
267 | delta = ((uint64_t)(NSEC_PER_SEC/HZ)) * evt->mult; | 318 | delta = ((uint64_t)(NSEC_PER_SEC/HZ)) * evt->mult; |
268 | delta >>= evt->shift; | 319 | delta >>= evt->shift; |
269 | now = hpet_readl(HPET_COUNTER); | ||
270 | cmp = now + (unsigned long) delta; | ||
271 | cfg = hpet_readl(HPET_Tn_CFG(timer)); | 320 | cfg = hpet_readl(HPET_Tn_CFG(timer)); |
272 | /* Make sure we use edge triggered interrupts */ | 321 | /* Make sure we use edge triggered interrupts */ |
273 | cfg &= ~HPET_TN_LEVEL; | 322 | cfg &= ~HPET_TN_LEVEL; |
274 | cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | | 323 | cfg |= HPET_TN_ENABLE | HPET_TN_PERIODIC | |
275 | HPET_TN_SETVAL | HPET_TN_32BIT; | 324 | HPET_TN_SETVAL | HPET_TN_32BIT; |
276 | hpet_writel(cfg, HPET_Tn_CFG(timer)); | 325 | hpet_writel(cfg, HPET_Tn_CFG(timer)); |
277 | /* | ||
278 | * The first write after writing TN_SETVAL to the | ||
279 | * config register sets the counter value, the second | ||
280 | * write sets the period. | ||
281 | */ | ||
282 | hpet_writel(cmp, HPET_Tn_CMP(timer)); | ||
283 | udelay(1); | ||
284 | hpet_writel((unsigned long) delta, HPET_Tn_CMP(timer)); | 326 | hpet_writel((unsigned long) delta, HPET_Tn_CMP(timer)); |
327 | hpet_start_counter(); | ||
328 | hpet_print_config(); | ||
285 | break; | 329 | break; |
286 | 330 | ||
287 | case CLOCK_EVT_MODE_ONESHOT: | 331 | case CLOCK_EVT_MODE_ONESHOT: |
@@ -308,6 +352,7 @@ static void hpet_set_mode(enum clock_event_mode mode, | |||
308 | irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu)); | 352 | irq_set_affinity(hdev->irq, cpumask_of(hdev->cpu)); |
309 | enable_irq(hdev->irq); | 353 | enable_irq(hdev->irq); |
310 | } | 354 | } |
355 | hpet_print_config(); | ||
311 | break; | 356 | break; |
312 | } | 357 | } |
313 | } | 358 | } |
@@ -526,6 +571,7 @@ static void hpet_msi_capability_lookup(unsigned int start_timer) | |||
526 | 571 | ||
527 | num_timers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT); | 572 | num_timers = ((id & HPET_ID_NUMBER) >> HPET_ID_NUMBER_SHIFT); |
528 | num_timers++; /* Value read out starts from 0 */ | 573 | num_timers++; /* Value read out starts from 0 */ |
574 | hpet_print_config(); | ||
529 | 575 | ||
530 | hpet_devs = kzalloc(sizeof(struct hpet_dev) * num_timers, GFP_KERNEL); | 576 | hpet_devs = kzalloc(sizeof(struct hpet_dev) * num_timers, GFP_KERNEL); |
531 | if (!hpet_devs) | 577 | if (!hpet_devs) |
@@ -695,7 +741,7 @@ static struct clocksource clocksource_hpet = { | |||
695 | .mask = HPET_MASK, | 741 | .mask = HPET_MASK, |
696 | .shift = HPET_SHIFT, | 742 | .shift = HPET_SHIFT, |
697 | .flags = CLOCK_SOURCE_IS_CONTINUOUS, | 743 | .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
698 | .resume = hpet_restart_counter, | 744 | .resume = hpet_resume_counter, |
699 | #ifdef CONFIG_X86_64 | 745 | #ifdef CONFIG_X86_64 |
700 | .vread = vread_hpet, | 746 | .vread = vread_hpet, |
701 | #endif | 747 | #endif |
@@ -707,7 +753,7 @@ static int hpet_clocksource_register(void) | |||
707 | cycle_t t1; | 753 | cycle_t t1; |
708 | 754 | ||
709 | /* Start the counter */ | 755 | /* Start the counter */ |
710 | hpet_start_counter(); | 756 | hpet_restart_counter(); |
711 | 757 | ||
712 | /* Verify whether hpet counter works */ | 758 | /* Verify whether hpet counter works */ |
713 | t1 = read_hpet(); | 759 | t1 = read_hpet(); |
@@ -793,6 +839,7 @@ int __init hpet_enable(void) | |||
793 | * information and the number of channels | 839 | * information and the number of channels |
794 | */ | 840 | */ |
795 | id = hpet_readl(HPET_ID); | 841 | id = hpet_readl(HPET_ID); |
842 | hpet_print_config(); | ||
796 | 843 | ||
797 | #ifdef CONFIG_HPET_EMULATE_RTC | 844 | #ifdef CONFIG_HPET_EMULATE_RTC |
798 | /* | 845 | /* |
@@ -845,6 +892,7 @@ static __init int hpet_late_init(void) | |||
845 | return -ENODEV; | 892 | return -ENODEV; |
846 | 893 | ||
847 | hpet_reserve_platform_timers(hpet_readl(HPET_ID)); | 894 | hpet_reserve_platform_timers(hpet_readl(HPET_ID)); |
895 | hpet_print_config(); | ||
848 | 896 | ||
849 | for_each_online_cpu(cpu) { | 897 | for_each_online_cpu(cpu) { |
850 | hpet_cpuhp_notify(NULL, CPU_ONLINE, (void *)(long)cpu); | 898 | hpet_cpuhp_notify(NULL, CPU_ONLINE, (void *)(long)cpu); |
diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c index 10f92fb532f3..3475440baa54 100644 --- a/arch/x86/kernel/i8253.c +++ b/arch/x86/kernel/i8253.c | |||
@@ -3,17 +3,17 @@ | |||
3 | * | 3 | * |
4 | */ | 4 | */ |
5 | #include <linux/clockchips.h> | 5 | #include <linux/clockchips.h> |
6 | #include <linux/init.h> | ||
7 | #include <linux/interrupt.h> | 6 | #include <linux/interrupt.h> |
7 | #include <linux/spinlock.h> | ||
8 | #include <linux/jiffies.h> | 8 | #include <linux/jiffies.h> |
9 | #include <linux/module.h> | 9 | #include <linux/module.h> |
10 | #include <linux/spinlock.h> | 10 | #include <linux/delay.h> |
11 | #include <linux/init.h> | ||
12 | #include <linux/io.h> | ||
11 | 13 | ||
12 | #include <asm/smp.h> | ||
13 | #include <asm/delay.h> | ||
14 | #include <asm/i8253.h> | 14 | #include <asm/i8253.h> |
15 | #include <asm/io.h> | ||
16 | #include <asm/hpet.h> | 15 | #include <asm/hpet.h> |
16 | #include <asm/smp.h> | ||
17 | 17 | ||
18 | DEFINE_SPINLOCK(i8253_lock); | 18 | DEFINE_SPINLOCK(i8253_lock); |
19 | EXPORT_SYMBOL(i8253_lock); | 19 | EXPORT_SYMBOL(i8253_lock); |
@@ -40,7 +40,7 @@ static void init_pit_timer(enum clock_event_mode mode, | |||
40 | { | 40 | { |
41 | spin_lock(&i8253_lock); | 41 | spin_lock(&i8253_lock); |
42 | 42 | ||
43 | switch(mode) { | 43 | switch (mode) { |
44 | case CLOCK_EVT_MODE_PERIODIC: | 44 | case CLOCK_EVT_MODE_PERIODIC: |
45 | /* binary, mode 2, LSB/MSB, ch 0 */ | 45 | /* binary, mode 2, LSB/MSB, ch 0 */ |
46 | outb_pit(0x34, PIT_MODE); | 46 | outb_pit(0x34, PIT_MODE); |
@@ -95,7 +95,7 @@ static int pit_next_event(unsigned long delta, struct clock_event_device *evt) | |||
95 | * registered. This mechanism replaces the previous #ifdef LOCAL_APIC - | 95 | * registered. This mechanism replaces the previous #ifdef LOCAL_APIC - |
96 | * !using_apic_timer decisions in do_timer_interrupt_hook() | 96 | * !using_apic_timer decisions in do_timer_interrupt_hook() |
97 | */ | 97 | */ |
98 | static struct clock_event_device pit_clockevent = { | 98 | static struct clock_event_device pit_ce = { |
99 | .name = "pit", | 99 | .name = "pit", |
100 | .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, | 100 | .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, |
101 | .set_mode = init_pit_timer, | 101 | .set_mode = init_pit_timer, |
@@ -114,15 +114,13 @@ void __init setup_pit_timer(void) | |||
114 | * Start pit with the boot cpu mask and make it global after the | 114 | * Start pit with the boot cpu mask and make it global after the |
115 | * IO_APIC has been initialized. | 115 | * IO_APIC has been initialized. |
116 | */ | 116 | */ |
117 | pit_clockevent.cpumask = cpumask_of(smp_processor_id()); | 117 | pit_ce.cpumask = cpumask_of(smp_processor_id()); |
118 | pit_clockevent.mult = div_sc(CLOCK_TICK_RATE, NSEC_PER_SEC, | 118 | pit_ce.mult = div_sc(CLOCK_TICK_RATE, NSEC_PER_SEC, pit_ce.shift); |
119 | pit_clockevent.shift); | 119 | pit_ce.max_delta_ns = clockevent_delta2ns(0x7FFF, &pit_ce); |
120 | pit_clockevent.max_delta_ns = | 120 | pit_ce.min_delta_ns = clockevent_delta2ns(0xF, &pit_ce); |
121 | clockevent_delta2ns(0x7FFF, &pit_clockevent); | 121 | |
122 | pit_clockevent.min_delta_ns = | 122 | clockevents_register_device(&pit_ce); |
123 | clockevent_delta2ns(0xF, &pit_clockevent); | 123 | global_clock_event = &pit_ce; |
124 | clockevents_register_device(&pit_clockevent); | ||
125 | global_clock_event = &pit_clockevent; | ||
126 | } | 124 | } |
127 | 125 | ||
128 | #ifndef CONFIG_X86_64 | 126 | #ifndef CONFIG_X86_64 |
@@ -133,11 +131,11 @@ void __init setup_pit_timer(void) | |||
133 | */ | 131 | */ |
134 | static cycle_t pit_read(void) | 132 | static cycle_t pit_read(void) |
135 | { | 133 | { |
134 | static int old_count; | ||
135 | static u32 old_jifs; | ||
136 | unsigned long flags; | 136 | unsigned long flags; |
137 | int count; | 137 | int count; |
138 | u32 jifs; | 138 | u32 jifs; |
139 | static int old_count; | ||
140 | static u32 old_jifs; | ||
141 | 139 | ||
142 | spin_lock_irqsave(&i8253_lock, flags); | 140 | spin_lock_irqsave(&i8253_lock, flags); |
143 | /* | 141 | /* |
@@ -179,9 +177,9 @@ static cycle_t pit_read(void) | |||
179 | * Previous attempts to handle these cases intelligently were | 177 | * Previous attempts to handle these cases intelligently were |
180 | * buggy, so we just do the simple thing now. | 178 | * buggy, so we just do the simple thing now. |
181 | */ | 179 | */ |
182 | if (count > old_count && jifs == old_jifs) { | 180 | if (count > old_count && jifs == old_jifs) |
183 | count = old_count; | 181 | count = old_count; |
184 | } | 182 | |
185 | old_count = count; | 183 | old_count = count; |
186 | old_jifs = jifs; | 184 | old_jifs = jifs; |
187 | 185 | ||
@@ -192,13 +190,13 @@ static cycle_t pit_read(void) | |||
192 | return (cycle_t)(jifs * LATCH) + count; | 190 | return (cycle_t)(jifs * LATCH) + count; |
193 | } | 191 | } |
194 | 192 | ||
195 | static struct clocksource clocksource_pit = { | 193 | static struct clocksource pit_cs = { |
196 | .name = "pit", | 194 | .name = "pit", |
197 | .rating = 110, | 195 | .rating = 110, |
198 | .read = pit_read, | 196 | .read = pit_read, |
199 | .mask = CLOCKSOURCE_MASK(32), | 197 | .mask = CLOCKSOURCE_MASK(32), |
200 | .mult = 0, | 198 | .mult = 0, |
201 | .shift = 20, | 199 | .shift = 20, |
202 | }; | 200 | }; |
203 | 201 | ||
204 | static void pit_disable_clocksource(void) | 202 | static void pit_disable_clocksource(void) |
@@ -206,9 +204,9 @@ static void pit_disable_clocksource(void) | |||
206 | /* | 204 | /* |
207 | * Use mult to check whether it is registered or not | 205 | * Use mult to check whether it is registered or not |
208 | */ | 206 | */ |
209 | if (clocksource_pit.mult) { | 207 | if (pit_cs.mult) { |
210 | clocksource_unregister(&clocksource_pit); | 208 | clocksource_unregister(&pit_cs); |
211 | clocksource_pit.mult = 0; | 209 | pit_cs.mult = 0; |
212 | } | 210 | } |
213 | } | 211 | } |
214 | 212 | ||
@@ -222,13 +220,13 @@ static int __init init_pit_clocksource(void) | |||
222 | * - when local APIC timer is active (PIT is switched off) | 220 | * - when local APIC timer is active (PIT is switched off) |
223 | */ | 221 | */ |
224 | if (num_possible_cpus() > 1 || is_hpet_enabled() || | 222 | if (num_possible_cpus() > 1 || is_hpet_enabled() || |
225 | pit_clockevent.mode != CLOCK_EVT_MODE_PERIODIC) | 223 | pit_ce.mode != CLOCK_EVT_MODE_PERIODIC) |
226 | return 0; | 224 | return 0; |
227 | 225 | ||
228 | clocksource_pit.mult = clocksource_hz2mult(CLOCK_TICK_RATE, | 226 | pit_cs.mult = clocksource_hz2mult(CLOCK_TICK_RATE, pit_cs.shift); |
229 | clocksource_pit.shift); | 227 | |
230 | return clocksource_register(&clocksource_pit); | 228 | return clocksource_register(&pit_cs); |
231 | } | 229 | } |
232 | arch_initcall(init_pit_clocksource); | 230 | arch_initcall(init_pit_clocksource); |
233 | 231 | ||
234 | #endif | 232 | #endif /* !CONFIG_X86_64 */ |
diff --git a/arch/x86/kernel/io_delay.c b/arch/x86/kernel/io_delay.c index 720d2607aacb..a979b5bd2fc0 100644 --- a/arch/x86/kernel/io_delay.c +++ b/arch/x86/kernel/io_delay.c | |||
@@ -7,10 +7,10 @@ | |||
7 | */ | 7 | */ |
8 | #include <linux/kernel.h> | 8 | #include <linux/kernel.h> |
9 | #include <linux/module.h> | 9 | #include <linux/module.h> |
10 | #include <linux/init.h> | ||
11 | #include <linux/delay.h> | 10 | #include <linux/delay.h> |
11 | #include <linux/init.h> | ||
12 | #include <linux/dmi.h> | 12 | #include <linux/dmi.h> |
13 | #include <asm/io.h> | 13 | #include <linux/io.h> |
14 | 14 | ||
15 | int io_delay_type __read_mostly = CONFIG_DEFAULT_IO_DELAY_TYPE; | 15 | int io_delay_type __read_mostly = CONFIG_DEFAULT_IO_DELAY_TYPE; |
16 | 16 | ||
@@ -47,8 +47,7 @@ EXPORT_SYMBOL(native_io_delay); | |||
47 | static int __init dmi_io_delay_0xed_port(const struct dmi_system_id *id) | 47 | static int __init dmi_io_delay_0xed_port(const struct dmi_system_id *id) |
48 | { | 48 | { |
49 | if (io_delay_type == CONFIG_IO_DELAY_TYPE_0X80) { | 49 | if (io_delay_type == CONFIG_IO_DELAY_TYPE_0X80) { |
50 | printk(KERN_NOTICE "%s: using 0xed I/O delay port\n", | 50 | pr_notice("%s: using 0xed I/O delay port\n", id->ident); |
51 | id->ident); | ||
52 | io_delay_type = CONFIG_IO_DELAY_TYPE_0XED; | 51 | io_delay_type = CONFIG_IO_DELAY_TYPE_0XED; |
53 | } | 52 | } |
54 | 53 | ||
@@ -64,40 +63,40 @@ static struct dmi_system_id __initdata io_delay_0xed_port_dmi_table[] = { | |||
64 | .callback = dmi_io_delay_0xed_port, | 63 | .callback = dmi_io_delay_0xed_port, |
65 | .ident = "Compaq Presario V6000", | 64 | .ident = "Compaq Presario V6000", |
66 | .matches = { | 65 | .matches = { |
67 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), | 66 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), |
68 | DMI_MATCH(DMI_BOARD_NAME, "30B7") | 67 | DMI_MATCH(DMI_BOARD_NAME, "30B7") |
69 | } | 68 | } |
70 | }, | 69 | }, |
71 | { | 70 | { |
72 | .callback = dmi_io_delay_0xed_port, | 71 | .callback = dmi_io_delay_0xed_port, |
73 | .ident = "HP Pavilion dv9000z", | 72 | .ident = "HP Pavilion dv9000z", |
74 | .matches = { | 73 | .matches = { |
75 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), | 74 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), |
76 | DMI_MATCH(DMI_BOARD_NAME, "30B9") | 75 | DMI_MATCH(DMI_BOARD_NAME, "30B9") |
77 | } | 76 | } |
78 | }, | 77 | }, |
79 | { | 78 | { |
80 | .callback = dmi_io_delay_0xed_port, | 79 | .callback = dmi_io_delay_0xed_port, |
81 | .ident = "HP Pavilion dv6000", | 80 | .ident = "HP Pavilion dv6000", |
82 | .matches = { | 81 | .matches = { |
83 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), | 82 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), |
84 | DMI_MATCH(DMI_BOARD_NAME, "30B8") | 83 | DMI_MATCH(DMI_BOARD_NAME, "30B8") |
85 | } | 84 | } |
86 | }, | 85 | }, |
87 | { | 86 | { |
88 | .callback = dmi_io_delay_0xed_port, | 87 | .callback = dmi_io_delay_0xed_port, |
89 | .ident = "HP Pavilion tx1000", | 88 | .ident = "HP Pavilion tx1000", |
90 | .matches = { | 89 | .matches = { |
91 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), | 90 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), |
92 | DMI_MATCH(DMI_BOARD_NAME, "30BF") | 91 | DMI_MATCH(DMI_BOARD_NAME, "30BF") |
93 | } | 92 | } |
94 | }, | 93 | }, |
95 | { | 94 | { |
96 | .callback = dmi_io_delay_0xed_port, | 95 | .callback = dmi_io_delay_0xed_port, |
97 | .ident = "Presario F700", | 96 | .ident = "Presario F700", |
98 | .matches = { | 97 | .matches = { |
99 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), | 98 | DMI_MATCH(DMI_BOARD_VENDOR, "Quanta"), |
100 | DMI_MATCH(DMI_BOARD_NAME, "30D3") | 99 | DMI_MATCH(DMI_BOARD_NAME, "30D3") |
101 | } | 100 | } |
102 | }, | 101 | }, |
103 | { } | 102 | { } |
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index b8ac3b6cf776..3aaf7b9e3a8b 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c | |||
@@ -58,6 +58,11 @@ static int show_other_interrupts(struct seq_file *p, int prec) | |||
58 | for_each_online_cpu(j) | 58 | for_each_online_cpu(j) |
59 | seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs); | 59 | seq_printf(p, "%10u ", irq_stats(j)->apic_timer_irqs); |
60 | seq_printf(p, " Local timer interrupts\n"); | 60 | seq_printf(p, " Local timer interrupts\n"); |
61 | |||
62 | seq_printf(p, "%*s: ", prec, "SPU"); | ||
63 | for_each_online_cpu(j) | ||
64 | seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count); | ||
65 | seq_printf(p, " Spurious interrupts\n"); | ||
61 | #endif | 66 | #endif |
62 | if (generic_interrupt_extension) { | 67 | if (generic_interrupt_extension) { |
63 | seq_printf(p, "PLT: "); | 68 | seq_printf(p, "PLT: "); |
@@ -91,12 +96,6 @@ static int show_other_interrupts(struct seq_file *p, int prec) | |||
91 | seq_printf(p, " Threshold APIC interrupts\n"); | 96 | seq_printf(p, " Threshold APIC interrupts\n"); |
92 | # endif | 97 | # endif |
93 | #endif | 98 | #endif |
94 | #ifdef CONFIG_X86_LOCAL_APIC | ||
95 | seq_printf(p, "%*s: ", prec, "SPU"); | ||
96 | for_each_online_cpu(j) | ||
97 | seq_printf(p, "%10u ", irq_stats(j)->irq_spurious_count); | ||
98 | seq_printf(p, " Spurious interrupts\n"); | ||
99 | #endif | ||
100 | seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); | 99 | seq_printf(p, "%*s: %10u\n", prec, "ERR", atomic_read(&irq_err_count)); |
101 | #if defined(CONFIG_X86_IO_APIC) | 100 | #if defined(CONFIG_X86_IO_APIC) |
102 | seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count)); | 101 | seq_printf(p, "%*s: %10u\n", prec, "MIS", atomic_read(&irq_mis_count)); |
@@ -133,23 +132,15 @@ int show_interrupts(struct seq_file *p, void *v) | |||
133 | return 0; | 132 | return 0; |
134 | 133 | ||
135 | spin_lock_irqsave(&desc->lock, flags); | 134 | spin_lock_irqsave(&desc->lock, flags); |
136 | #ifndef CONFIG_SMP | ||
137 | any_count = kstat_irqs(i); | ||
138 | #else | ||
139 | for_each_online_cpu(j) | 135 | for_each_online_cpu(j) |
140 | any_count |= kstat_irqs_cpu(i, j); | 136 | any_count |= kstat_irqs_cpu(i, j); |
141 | #endif | ||
142 | action = desc->action; | 137 | action = desc->action; |
143 | if (!action && !any_count) | 138 | if (!action && !any_count) |
144 | goto out; | 139 | goto out; |
145 | 140 | ||
146 | seq_printf(p, "%*d: ", prec, i); | 141 | seq_printf(p, "%*d: ", prec, i); |
147 | #ifndef CONFIG_SMP | ||
148 | seq_printf(p, "%10u ", kstat_irqs(i)); | ||
149 | #else | ||
150 | for_each_online_cpu(j) | 142 | for_each_online_cpu(j) |
151 | seq_printf(p, "%10u ", kstat_irqs_cpu(i, j)); | 143 | seq_printf(p, "%10u ", kstat_irqs_cpu(i, j)); |
152 | #endif | ||
153 | seq_printf(p, " %8s", desc->chip->name); | 144 | seq_printf(p, " %8s", desc->chip->name); |
154 | seq_printf(p, "-%-8s", desc->name); | 145 | seq_printf(p, "-%-8s", desc->name); |
155 | 146 | ||
@@ -174,6 +165,7 @@ u64 arch_irq_stat_cpu(unsigned int cpu) | |||
174 | 165 | ||
175 | #ifdef CONFIG_X86_LOCAL_APIC | 166 | #ifdef CONFIG_X86_LOCAL_APIC |
176 | sum += irq_stats(cpu)->apic_timer_irqs; | 167 | sum += irq_stats(cpu)->apic_timer_irqs; |
168 | sum += irq_stats(cpu)->irq_spurious_count; | ||
177 | #endif | 169 | #endif |
178 | if (generic_interrupt_extension) | 170 | if (generic_interrupt_extension) |
179 | sum += irq_stats(cpu)->generic_irqs; | 171 | sum += irq_stats(cpu)->generic_irqs; |
@@ -188,9 +180,6 @@ u64 arch_irq_stat_cpu(unsigned int cpu) | |||
188 | sum += irq_stats(cpu)->irq_threshold_count; | 180 | sum += irq_stats(cpu)->irq_threshold_count; |
189 | #endif | 181 | #endif |
190 | #endif | 182 | #endif |
191 | #ifdef CONFIG_X86_LOCAL_APIC | ||
192 | sum += irq_stats(cpu)->irq_spurious_count; | ||
193 | #endif | ||
194 | return sum; | 183 | return sum; |
195 | } | 184 | } |
196 | 185 | ||
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c index bc1326105448..368b0a8836f9 100644 --- a/arch/x86/kernel/irqinit_32.c +++ b/arch/x86/kernel/irqinit_32.c | |||
@@ -50,7 +50,6 @@ static irqreturn_t math_error_irq(int cpl, void *dev_id) | |||
50 | */ | 50 | */ |
51 | static struct irqaction fpu_irq = { | 51 | static struct irqaction fpu_irq = { |
52 | .handler = math_error_irq, | 52 | .handler = math_error_irq, |
53 | .mask = CPU_MASK_NONE, | ||
54 | .name = "fpu", | 53 | .name = "fpu", |
55 | }; | 54 | }; |
56 | 55 | ||
@@ -83,7 +82,6 @@ void __init init_ISA_irqs(void) | |||
83 | */ | 82 | */ |
84 | static struct irqaction irq2 = { | 83 | static struct irqaction irq2 = { |
85 | .handler = no_action, | 84 | .handler = no_action, |
86 | .mask = CPU_MASK_NONE, | ||
87 | .name = "cascade", | 85 | .name = "cascade", |
88 | }; | 86 | }; |
89 | 87 | ||
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c index c7a49e0ffbfb..8cd10537fd46 100644 --- a/arch/x86/kernel/irqinit_64.c +++ b/arch/x86/kernel/irqinit_64.c | |||
@@ -45,7 +45,6 @@ | |||
45 | 45 | ||
46 | static struct irqaction irq2 = { | 46 | static struct irqaction irq2 = { |
47 | .handler = no_action, | 47 | .handler = no_action, |
48 | .mask = CPU_MASK_NONE, | ||
49 | .name = "cascade", | 48 | .name = "cascade", |
50 | }; | 49 | }; |
51 | DEFINE_PER_CPU(vector_irq_t, vector_irq) = { | 50 | DEFINE_PER_CPU(vector_irq_t, vector_irq) = { |
diff --git a/arch/x86/kernel/kdebugfs.c b/arch/x86/kernel/kdebugfs.c index ff7d3b0124f1..e444357375ce 100644 --- a/arch/x86/kernel/kdebugfs.c +++ b/arch/x86/kernel/kdebugfs.c | |||
@@ -8,11 +8,11 @@ | |||
8 | */ | 8 | */ |
9 | #include <linux/debugfs.h> | 9 | #include <linux/debugfs.h> |
10 | #include <linux/uaccess.h> | 10 | #include <linux/uaccess.h> |
11 | #include <linux/stat.h> | 11 | #include <linux/module.h> |
12 | #include <linux/init.h> | 12 | #include <linux/init.h> |
13 | #include <linux/stat.h> | ||
13 | #include <linux/io.h> | 14 | #include <linux/io.h> |
14 | #include <linux/mm.h> | 15 | #include <linux/mm.h> |
15 | #include <linux/module.h> | ||
16 | 16 | ||
17 | #include <asm/setup.h> | 17 | #include <asm/setup.h> |
18 | 18 | ||
@@ -26,9 +26,8 @@ struct setup_data_node { | |||
26 | u32 len; | 26 | u32 len; |
27 | }; | 27 | }; |
28 | 28 | ||
29 | static ssize_t | 29 | static ssize_t setup_data_read(struct file *file, char __user *user_buf, |
30 | setup_data_read(struct file *file, char __user *user_buf, size_t count, | 30 | size_t count, loff_t *ppos) |
31 | loff_t *ppos) | ||
32 | { | 31 | { |
33 | struct setup_data_node *node = file->private_data; | 32 | struct setup_data_node *node = file->private_data; |
34 | unsigned long remain; | 33 | unsigned long remain; |
@@ -39,20 +38,21 @@ setup_data_read(struct file *file, char __user *user_buf, size_t count, | |||
39 | 38 | ||
40 | if (pos < 0) | 39 | if (pos < 0) |
41 | return -EINVAL; | 40 | return -EINVAL; |
41 | |||
42 | if (pos >= node->len) | 42 | if (pos >= node->len) |
43 | return 0; | 43 | return 0; |
44 | 44 | ||
45 | if (count > node->len - pos) | 45 | if (count > node->len - pos) |
46 | count = node->len - pos; | 46 | count = node->len - pos; |
47 | |||
47 | pa = node->paddr + sizeof(struct setup_data) + pos; | 48 | pa = node->paddr + sizeof(struct setup_data) + pos; |
48 | pg = pfn_to_page((pa + count - 1) >> PAGE_SHIFT); | 49 | pg = pfn_to_page((pa + count - 1) >> PAGE_SHIFT); |
49 | if (PageHighMem(pg)) { | 50 | if (PageHighMem(pg)) { |
50 | p = ioremap_cache(pa, count); | 51 | p = ioremap_cache(pa, count); |
51 | if (!p) | 52 | if (!p) |
52 | return -ENXIO; | 53 | return -ENXIO; |
53 | } else { | 54 | } else |
54 | p = __va(pa); | 55 | p = __va(pa); |
55 | } | ||
56 | 56 | ||
57 | remain = copy_to_user(user_buf, p, count); | 57 | remain = copy_to_user(user_buf, p, count); |
58 | 58 | ||
@@ -70,12 +70,13 @@ setup_data_read(struct file *file, char __user *user_buf, size_t count, | |||
70 | static int setup_data_open(struct inode *inode, struct file *file) | 70 | static int setup_data_open(struct inode *inode, struct file *file) |
71 | { | 71 | { |
72 | file->private_data = inode->i_private; | 72 | file->private_data = inode->i_private; |
73 | |||
73 | return 0; | 74 | return 0; |
74 | } | 75 | } |
75 | 76 | ||
76 | static const struct file_operations fops_setup_data = { | 77 | static const struct file_operations fops_setup_data = { |
77 | .read = setup_data_read, | 78 | .read = setup_data_read, |
78 | .open = setup_data_open, | 79 | .open = setup_data_open, |
79 | }; | 80 | }; |
80 | 81 | ||
81 | static int __init | 82 | static int __init |
@@ -84,57 +85,50 @@ create_setup_data_node(struct dentry *parent, int no, | |||
84 | { | 85 | { |
85 | struct dentry *d, *type, *data; | 86 | struct dentry *d, *type, *data; |
86 | char buf[16]; | 87 | char buf[16]; |
87 | int error; | ||
88 | 88 | ||
89 | sprintf(buf, "%d", no); | 89 | sprintf(buf, "%d", no); |
90 | d = debugfs_create_dir(buf, parent); | 90 | d = debugfs_create_dir(buf, parent); |
91 | if (!d) { | 91 | if (!d) |
92 | error = -ENOMEM; | 92 | return -ENOMEM; |
93 | goto err_return; | 93 | |
94 | } | ||
95 | type = debugfs_create_x32("type", S_IRUGO, d, &node->type); | 94 | type = debugfs_create_x32("type", S_IRUGO, d, &node->type); |
96 | if (!type) { | 95 | if (!type) |
97 | error = -ENOMEM; | ||
98 | goto err_dir; | 96 | goto err_dir; |
99 | } | 97 | |
100 | data = debugfs_create_file("data", S_IRUGO, d, node, &fops_setup_data); | 98 | data = debugfs_create_file("data", S_IRUGO, d, node, &fops_setup_data); |
101 | if (!data) { | 99 | if (!data) |
102 | error = -ENOMEM; | ||
103 | goto err_type; | 100 | goto err_type; |
104 | } | 101 | |
105 | return 0; | 102 | return 0; |
106 | 103 | ||
107 | err_type: | 104 | err_type: |
108 | debugfs_remove(type); | 105 | debugfs_remove(type); |
109 | err_dir: | 106 | err_dir: |
110 | debugfs_remove(d); | 107 | debugfs_remove(d); |
111 | err_return: | 108 | return -ENOMEM; |
112 | return error; | ||
113 | } | 109 | } |
114 | 110 | ||
115 | static int __init create_setup_data_nodes(struct dentry *parent) | 111 | static int __init create_setup_data_nodes(struct dentry *parent) |
116 | { | 112 | { |
117 | struct setup_data_node *node; | 113 | struct setup_data_node *node; |
118 | struct setup_data *data; | 114 | struct setup_data *data; |
119 | int error, no = 0; | 115 | int error = -ENOMEM; |
120 | struct dentry *d; | 116 | struct dentry *d; |
121 | struct page *pg; | 117 | struct page *pg; |
122 | u64 pa_data; | 118 | u64 pa_data; |
119 | int no = 0; | ||
123 | 120 | ||
124 | d = debugfs_create_dir("setup_data", parent); | 121 | d = debugfs_create_dir("setup_data", parent); |
125 | if (!d) { | 122 | if (!d) |
126 | error = -ENOMEM; | 123 | return -ENOMEM; |
127 | goto err_return; | ||
128 | } | ||
129 | 124 | ||
130 | pa_data = boot_params.hdr.setup_data; | 125 | pa_data = boot_params.hdr.setup_data; |
131 | 126 | ||
132 | while (pa_data) { | 127 | while (pa_data) { |
133 | node = kmalloc(sizeof(*node), GFP_KERNEL); | 128 | node = kmalloc(sizeof(*node), GFP_KERNEL); |
134 | if (!node) { | 129 | if (!node) |
135 | error = -ENOMEM; | ||
136 | goto err_dir; | 130 | goto err_dir; |
137 | } | 131 | |
138 | pg = pfn_to_page((pa_data+sizeof(*data)-1) >> PAGE_SHIFT); | 132 | pg = pfn_to_page((pa_data+sizeof(*data)-1) >> PAGE_SHIFT); |
139 | if (PageHighMem(pg)) { | 133 | if (PageHighMem(pg)) { |
140 | data = ioremap_cache(pa_data, sizeof(*data)); | 134 | data = ioremap_cache(pa_data, sizeof(*data)); |
@@ -143,9 +137,8 @@ static int __init create_setup_data_nodes(struct dentry *parent) | |||
143 | error = -ENXIO; | 137 | error = -ENXIO; |
144 | goto err_dir; | 138 | goto err_dir; |
145 | } | 139 | } |
146 | } else { | 140 | } else |
147 | data = __va(pa_data); | 141 | data = __va(pa_data); |
148 | } | ||
149 | 142 | ||
150 | node->paddr = pa_data; | 143 | node->paddr = pa_data; |
151 | node->type = data->type; | 144 | node->type = data->type; |
@@ -159,11 +152,11 @@ static int __init create_setup_data_nodes(struct dentry *parent) | |||
159 | goto err_dir; | 152 | goto err_dir; |
160 | no++; | 153 | no++; |
161 | } | 154 | } |
155 | |||
162 | return 0; | 156 | return 0; |
163 | 157 | ||
164 | err_dir: | 158 | err_dir: |
165 | debugfs_remove(d); | 159 | debugfs_remove(d); |
166 | err_return: | ||
167 | return error; | 160 | return error; |
168 | } | 161 | } |
169 | 162 | ||
@@ -175,28 +168,26 @@ static struct debugfs_blob_wrapper boot_params_blob = { | |||
175 | static int __init boot_params_kdebugfs_init(void) | 168 | static int __init boot_params_kdebugfs_init(void) |
176 | { | 169 | { |
177 | struct dentry *dbp, *version, *data; | 170 | struct dentry *dbp, *version, *data; |
178 | int error; | 171 | int error = -ENOMEM; |
179 | 172 | ||
180 | dbp = debugfs_create_dir("boot_params", NULL); | 173 | dbp = debugfs_create_dir("boot_params", NULL); |
181 | if (!dbp) { | 174 | if (!dbp) |
182 | error = -ENOMEM; | 175 | return -ENOMEM; |
183 | goto err_return; | 176 | |
184 | } | ||
185 | version = debugfs_create_x16("version", S_IRUGO, dbp, | 177 | version = debugfs_create_x16("version", S_IRUGO, dbp, |
186 | &boot_params.hdr.version); | 178 | &boot_params.hdr.version); |
187 | if (!version) { | 179 | if (!version) |
188 | error = -ENOMEM; | ||
189 | goto err_dir; | 180 | goto err_dir; |
190 | } | 181 | |
191 | data = debugfs_create_blob("data", S_IRUGO, dbp, | 182 | data = debugfs_create_blob("data", S_IRUGO, dbp, |
192 | &boot_params_blob); | 183 | &boot_params_blob); |
193 | if (!data) { | 184 | if (!data) |
194 | error = -ENOMEM; | ||
195 | goto err_version; | 185 | goto err_version; |
196 | } | 186 | |
197 | error = create_setup_data_nodes(dbp); | 187 | error = create_setup_data_nodes(dbp); |
198 | if (error) | 188 | if (error) |
199 | goto err_data; | 189 | goto err_data; |
190 | |||
200 | return 0; | 191 | return 0; |
201 | 192 | ||
202 | err_data: | 193 | err_data: |
@@ -205,10 +196,9 @@ err_version: | |||
205 | debugfs_remove(version); | 196 | debugfs_remove(version); |
206 | err_dir: | 197 | err_dir: |
207 | debugfs_remove(dbp); | 198 | debugfs_remove(dbp); |
208 | err_return: | ||
209 | return error; | 199 | return error; |
210 | } | 200 | } |
211 | #endif | 201 | #endif /* CONFIG_DEBUG_BOOT_PARAMS */ |
212 | 202 | ||
213 | static int __init arch_kdebugfs_init(void) | 203 | static int __init arch_kdebugfs_init(void) |
214 | { | 204 | { |
diff --git a/arch/x86/kernel/mfgpt_32.c b/arch/x86/kernel/mfgpt_32.c index 8815f3c7fec7..846510b78a09 100644 --- a/arch/x86/kernel/mfgpt_32.c +++ b/arch/x86/kernel/mfgpt_32.c | |||
@@ -348,7 +348,6 @@ static irqreturn_t mfgpt_tick(int irq, void *dev_id) | |||
348 | static struct irqaction mfgptirq = { | 348 | static struct irqaction mfgptirq = { |
349 | .handler = mfgpt_tick, | 349 | .handler = mfgpt_tick, |
350 | .flags = IRQF_DISABLED | IRQF_NOBALANCING, | 350 | .flags = IRQF_DISABLED | IRQF_NOBALANCING, |
351 | .mask = CPU_MASK_NONE, | ||
352 | .name = "mfgpt-timer" | 351 | .name = "mfgpt-timer" |
353 | }; | 352 | }; |
354 | 353 | ||
diff --git a/arch/x86/kernel/mpparse.c b/arch/x86/kernel/mpparse.c index 290cb57f4697..dce99dca6cf8 100644 --- a/arch/x86/kernel/mpparse.c +++ b/arch/x86/kernel/mpparse.c | |||
@@ -282,6 +282,14 @@ static void skip_entry(unsigned char **ptr, int *count, int size) | |||
282 | *count += size; | 282 | *count += size; |
283 | } | 283 | } |
284 | 284 | ||
285 | static void __init smp_dump_mptable(struct mpc_table *mpc, unsigned char *mpt) | ||
286 | { | ||
287 | printk(KERN_ERR "Your mptable is wrong, contact your HW vendor!\n" | ||
288 | "type %x\n", *mpt); | ||
289 | print_hex_dump(KERN_ERR, " ", DUMP_PREFIX_ADDRESS, 16, | ||
290 | 1, mpc, mpc->length, 1); | ||
291 | } | ||
292 | |||
285 | static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early) | 293 | static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early) |
286 | { | 294 | { |
287 | char str[16]; | 295 | char str[16]; |
@@ -340,10 +348,7 @@ static int __init smp_read_mpc(struct mpc_table *mpc, unsigned early) | |||
340 | break; | 348 | break; |
341 | default: | 349 | default: |
342 | /* wrong mptable */ | 350 | /* wrong mptable */ |
343 | printk(KERN_ERR "Your mptable is wrong, contact your HW vendor!\n"); | 351 | smp_dump_mptable(mpc, mpt); |
344 | printk(KERN_ERR "type %x\n", *mpt); | ||
345 | print_hex_dump(KERN_ERR, " ", DUMP_PREFIX_ADDRESS, 16, | ||
346 | 1, mpc, mpc->length, 1); | ||
347 | count = mpc->length; | 352 | count = mpc->length; |
348 | break; | 353 | break; |
349 | } | 354 | } |
@@ -550,6 +555,55 @@ static unsigned long __init get_mpc_size(unsigned long physptr) | |||
550 | return size; | 555 | return size; |
551 | } | 556 | } |
552 | 557 | ||
558 | static int __init check_physptr(struct mpf_intel *mpf, unsigned int early) | ||
559 | { | ||
560 | struct mpc_table *mpc; | ||
561 | unsigned long size; | ||
562 | |||
563 | size = get_mpc_size(mpf->physptr); | ||
564 | mpc = early_ioremap(mpf->physptr, size); | ||
565 | /* | ||
566 | * Read the physical hardware table. Anything here will | ||
567 | * override the defaults. | ||
568 | */ | ||
569 | if (!smp_read_mpc(mpc, early)) { | ||
570 | #ifdef CONFIG_X86_LOCAL_APIC | ||
571 | smp_found_config = 0; | ||
572 | #endif | ||
573 | printk(KERN_ERR "BIOS bug, MP table errors detected!...\n" | ||
574 | "... disabling SMP support. (tell your hw vendor)\n"); | ||
575 | early_iounmap(mpc, size); | ||
576 | return -1; | ||
577 | } | ||
578 | early_iounmap(mpc, size); | ||
579 | |||
580 | if (early) | ||
581 | return -1; | ||
582 | |||
583 | #ifdef CONFIG_X86_IO_APIC | ||
584 | /* | ||
585 | * If there are no explicit MP IRQ entries, then we are | ||
586 | * broken. We set up most of the low 16 IO-APIC pins to | ||
587 | * ISA defaults and hope it will work. | ||
588 | */ | ||
589 | if (!mp_irq_entries) { | ||
590 | struct mpc_bus bus; | ||
591 | |||
592 | printk(KERN_ERR "BIOS bug, no explicit IRQ entries, " | ||
593 | "using default mptable. (tell your hw vendor)\n"); | ||
594 | |||
595 | bus.type = MP_BUS; | ||
596 | bus.busid = 0; | ||
597 | memcpy(bus.bustype, "ISA ", 6); | ||
598 | MP_bus_info(&bus); | ||
599 | |||
600 | construct_default_ioirq_mptable(0); | ||
601 | } | ||
602 | #endif | ||
603 | |||
604 | return 0; | ||
605 | } | ||
606 | |||
553 | /* | 607 | /* |
554 | * Scan the memory blocks for an SMP configuration block. | 608 | * Scan the memory blocks for an SMP configuration block. |
555 | */ | 609 | */ |
@@ -603,51 +657,8 @@ static void __init __get_smp_config(unsigned int early) | |||
603 | construct_default_ISA_mptable(mpf->feature1); | 657 | construct_default_ISA_mptable(mpf->feature1); |
604 | 658 | ||
605 | } else if (mpf->physptr) { | 659 | } else if (mpf->physptr) { |
606 | struct mpc_table *mpc; | 660 | if (check_physptr(mpf, early)) |
607 | unsigned long size; | ||
608 | |||
609 | size = get_mpc_size(mpf->physptr); | ||
610 | mpc = early_ioremap(mpf->physptr, size); | ||
611 | /* | ||
612 | * Read the physical hardware table. Anything here will | ||
613 | * override the defaults. | ||
614 | */ | ||
615 | if (!smp_read_mpc(mpc, early)) { | ||
616 | #ifdef CONFIG_X86_LOCAL_APIC | ||
617 | smp_found_config = 0; | ||
618 | #endif | ||
619 | printk(KERN_ERR | ||
620 | "BIOS bug, MP table errors detected!...\n"); | ||
621 | printk(KERN_ERR "... disabling SMP support. " | ||
622 | "(tell your hw vendor)\n"); | ||
623 | early_iounmap(mpc, size); | ||
624 | return; | 661 | return; |
625 | } | ||
626 | early_iounmap(mpc, size); | ||
627 | |||
628 | if (early) | ||
629 | return; | ||
630 | #ifdef CONFIG_X86_IO_APIC | ||
631 | /* | ||
632 | * If there are no explicit MP IRQ entries, then we are | ||
633 | * broken. We set up most of the low 16 IO-APIC pins to | ||
634 | * ISA defaults and hope it will work. | ||
635 | */ | ||
636 | if (!mp_irq_entries) { | ||
637 | struct mpc_bus bus; | ||
638 | |||
639 | printk(KERN_ERR "BIOS bug, no explicit IRQ entries, " | ||
640 | "using default mptable. " | ||
641 | "(tell your hw vendor)\n"); | ||
642 | |||
643 | bus.type = MP_BUS; | ||
644 | bus.busid = 0; | ||
645 | memcpy(bus.bustype, "ISA ", 6); | ||
646 | MP_bus_info(&bus); | ||
647 | |||
648 | construct_default_ioirq_mptable(0); | ||
649 | } | ||
650 | #endif | ||
651 | } else | 662 | } else |
652 | BUG(); | 663 | BUG(); |
653 | 664 | ||
@@ -910,10 +921,7 @@ static int __init replace_intsrc_all(struct mpc_table *mpc, | |||
910 | break; | 921 | break; |
911 | default: | 922 | default: |
912 | /* wrong mptable */ | 923 | /* wrong mptable */ |
913 | printk(KERN_ERR "Your mptable is wrong, contact your HW vendor!\n"); | 924 | smp_dump_mptable(mpc, mpt); |
914 | printk(KERN_ERR "type %x\n", *mpt); | ||
915 | print_hex_dump(KERN_ERR, " ", DUMP_PREFIX_ADDRESS, 16, | ||
916 | 1, mpc, mpc->length, 1); | ||
917 | goto out; | 925 | goto out; |
918 | } | 926 | } |
919 | } | 927 | } |
diff --git a/arch/x86/kernel/pci-calgary_64.c b/arch/x86/kernel/pci-calgary_64.c index d28bbdc35e4e..755c21e906f3 100644 --- a/arch/x86/kernel/pci-calgary_64.c +++ b/arch/x86/kernel/pci-calgary_64.c | |||
@@ -380,8 +380,9 @@ static inline struct iommu_table *find_iommu_table(struct device *dev) | |||
380 | return tbl; | 380 | return tbl; |
381 | } | 381 | } |
382 | 382 | ||
383 | static void calgary_unmap_sg(struct device *dev, | 383 | static void calgary_unmap_sg(struct device *dev, struct scatterlist *sglist, |
384 | struct scatterlist *sglist, int nelems, int direction) | 384 | int nelems,enum dma_data_direction dir, |
385 | struct dma_attrs *attrs) | ||
385 | { | 386 | { |
386 | struct iommu_table *tbl = find_iommu_table(dev); | 387 | struct iommu_table *tbl = find_iommu_table(dev); |
387 | struct scatterlist *s; | 388 | struct scatterlist *s; |
@@ -404,7 +405,8 @@ static void calgary_unmap_sg(struct device *dev, | |||
404 | } | 405 | } |
405 | 406 | ||
406 | static int calgary_map_sg(struct device *dev, struct scatterlist *sg, | 407 | static int calgary_map_sg(struct device *dev, struct scatterlist *sg, |
407 | int nelems, int direction) | 408 | int nelems, enum dma_data_direction dir, |
409 | struct dma_attrs *attrs) | ||
408 | { | 410 | { |
409 | struct iommu_table *tbl = find_iommu_table(dev); | 411 | struct iommu_table *tbl = find_iommu_table(dev); |
410 | struct scatterlist *s; | 412 | struct scatterlist *s; |
@@ -429,15 +431,14 @@ static int calgary_map_sg(struct device *dev, struct scatterlist *sg, | |||
429 | s->dma_address = (entry << PAGE_SHIFT) | s->offset; | 431 | s->dma_address = (entry << PAGE_SHIFT) | s->offset; |
430 | 432 | ||
431 | /* insert into HW table */ | 433 | /* insert into HW table */ |
432 | tce_build(tbl, entry, npages, vaddr & PAGE_MASK, | 434 | tce_build(tbl, entry, npages, vaddr & PAGE_MASK, dir); |
433 | direction); | ||
434 | 435 | ||
435 | s->dma_length = s->length; | 436 | s->dma_length = s->length; |
436 | } | 437 | } |
437 | 438 | ||
438 | return nelems; | 439 | return nelems; |
439 | error: | 440 | error: |
440 | calgary_unmap_sg(dev, sg, nelems, direction); | 441 | calgary_unmap_sg(dev, sg, nelems, dir, NULL); |
441 | for_each_sg(sg, s, nelems, i) { | 442 | for_each_sg(sg, s, nelems, i) { |
442 | sg->dma_address = bad_dma_address; | 443 | sg->dma_address = bad_dma_address; |
443 | sg->dma_length = 0; | 444 | sg->dma_length = 0; |
@@ -445,10 +446,12 @@ error: | |||
445 | return 0; | 446 | return 0; |
446 | } | 447 | } |
447 | 448 | ||
448 | static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr, | 449 | static dma_addr_t calgary_map_page(struct device *dev, struct page *page, |
449 | size_t size, int direction) | 450 | unsigned long offset, size_t size, |
451 | enum dma_data_direction dir, | ||
452 | struct dma_attrs *attrs) | ||
450 | { | 453 | { |
451 | void *vaddr = phys_to_virt(paddr); | 454 | void *vaddr = page_address(page) + offset; |
452 | unsigned long uaddr; | 455 | unsigned long uaddr; |
453 | unsigned int npages; | 456 | unsigned int npages; |
454 | struct iommu_table *tbl = find_iommu_table(dev); | 457 | struct iommu_table *tbl = find_iommu_table(dev); |
@@ -456,17 +459,18 @@ static dma_addr_t calgary_map_single(struct device *dev, phys_addr_t paddr, | |||
456 | uaddr = (unsigned long)vaddr; | 459 | uaddr = (unsigned long)vaddr; |
457 | npages = iommu_num_pages(uaddr, size, PAGE_SIZE); | 460 | npages = iommu_num_pages(uaddr, size, PAGE_SIZE); |
458 | 461 | ||
459 | return iommu_alloc(dev, tbl, vaddr, npages, direction); | 462 | return iommu_alloc(dev, tbl, vaddr, npages, dir); |
460 | } | 463 | } |
461 | 464 | ||
462 | static void calgary_unmap_single(struct device *dev, dma_addr_t dma_handle, | 465 | static void calgary_unmap_page(struct device *dev, dma_addr_t dma_addr, |
463 | size_t size, int direction) | 466 | size_t size, enum dma_data_direction dir, |
467 | struct dma_attrs *attrs) | ||
464 | { | 468 | { |
465 | struct iommu_table *tbl = find_iommu_table(dev); | 469 | struct iommu_table *tbl = find_iommu_table(dev); |
466 | unsigned int npages; | 470 | unsigned int npages; |
467 | 471 | ||
468 | npages = iommu_num_pages(dma_handle, size, PAGE_SIZE); | 472 | npages = iommu_num_pages(dma_addr, size, PAGE_SIZE); |
469 | iommu_free(tbl, dma_handle, npages); | 473 | iommu_free(tbl, dma_addr, npages); |
470 | } | 474 | } |
471 | 475 | ||
472 | static void* calgary_alloc_coherent(struct device *dev, size_t size, | 476 | static void* calgary_alloc_coherent(struct device *dev, size_t size, |
@@ -515,13 +519,13 @@ static void calgary_free_coherent(struct device *dev, size_t size, | |||
515 | free_pages((unsigned long)vaddr, get_order(size)); | 519 | free_pages((unsigned long)vaddr, get_order(size)); |
516 | } | 520 | } |
517 | 521 | ||
518 | static struct dma_mapping_ops calgary_dma_ops = { | 522 | static struct dma_map_ops calgary_dma_ops = { |
519 | .alloc_coherent = calgary_alloc_coherent, | 523 | .alloc_coherent = calgary_alloc_coherent, |
520 | .free_coherent = calgary_free_coherent, | 524 | .free_coherent = calgary_free_coherent, |
521 | .map_single = calgary_map_single, | ||
522 | .unmap_single = calgary_unmap_single, | ||
523 | .map_sg = calgary_map_sg, | 525 | .map_sg = calgary_map_sg, |
524 | .unmap_sg = calgary_unmap_sg, | 526 | .unmap_sg = calgary_unmap_sg, |
527 | .map_page = calgary_map_page, | ||
528 | .unmap_page = calgary_unmap_page, | ||
525 | }; | 529 | }; |
526 | 530 | ||
527 | static inline void __iomem * busno_to_bbar(unsigned char num) | 531 | static inline void __iomem * busno_to_bbar(unsigned char num) |
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c index b25428533141..90f5b9ef5def 100644 --- a/arch/x86/kernel/pci-dma.c +++ b/arch/x86/kernel/pci-dma.c | |||
@@ -1,4 +1,5 @@ | |||
1 | #include <linux/dma-mapping.h> | 1 | #include <linux/dma-mapping.h> |
2 | #include <linux/dma-debug.h> | ||
2 | #include <linux/dmar.h> | 3 | #include <linux/dmar.h> |
3 | #include <linux/bootmem.h> | 4 | #include <linux/bootmem.h> |
4 | #include <linux/pci.h> | 5 | #include <linux/pci.h> |
@@ -12,7 +13,7 @@ | |||
12 | 13 | ||
13 | static int forbid_dac __read_mostly; | 14 | static int forbid_dac __read_mostly; |
14 | 15 | ||
15 | struct dma_mapping_ops *dma_ops; | 16 | struct dma_map_ops *dma_ops; |
16 | EXPORT_SYMBOL(dma_ops); | 17 | EXPORT_SYMBOL(dma_ops); |
17 | 18 | ||
18 | static int iommu_sac_force __read_mostly; | 19 | static int iommu_sac_force __read_mostly; |
@@ -44,6 +45,9 @@ struct device x86_dma_fallback_dev = { | |||
44 | }; | 45 | }; |
45 | EXPORT_SYMBOL(x86_dma_fallback_dev); | 46 | EXPORT_SYMBOL(x86_dma_fallback_dev); |
46 | 47 | ||
48 | /* Number of entries preallocated for DMA-API debugging */ | ||
49 | #define PREALLOC_DMA_DEBUG_ENTRIES 32768 | ||
50 | |||
47 | int dma_set_mask(struct device *dev, u64 mask) | 51 | int dma_set_mask(struct device *dev, u64 mask) |
48 | { | 52 | { |
49 | if (!dev->dma_mask || !dma_supported(dev, mask)) | 53 | if (!dev->dma_mask || !dma_supported(dev, mask)) |
@@ -224,7 +228,7 @@ early_param("iommu", iommu_setup); | |||
224 | 228 | ||
225 | int dma_supported(struct device *dev, u64 mask) | 229 | int dma_supported(struct device *dev, u64 mask) |
226 | { | 230 | { |
227 | struct dma_mapping_ops *ops = get_dma_ops(dev); | 231 | struct dma_map_ops *ops = get_dma_ops(dev); |
228 | 232 | ||
229 | #ifdef CONFIG_PCI | 233 | #ifdef CONFIG_PCI |
230 | if (mask > 0xffffffff && forbid_dac > 0) { | 234 | if (mask > 0xffffffff && forbid_dac > 0) { |
@@ -265,6 +269,12 @@ EXPORT_SYMBOL(dma_supported); | |||
265 | 269 | ||
266 | static int __init pci_iommu_init(void) | 270 | static int __init pci_iommu_init(void) |
267 | { | 271 | { |
272 | dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); | ||
273 | |||
274 | #ifdef CONFIG_PCI | ||
275 | dma_debug_add_bus(&pci_bus_type); | ||
276 | #endif | ||
277 | |||
268 | calgary_iommu_init(); | 278 | calgary_iommu_init(); |
269 | 279 | ||
270 | intel_iommu_init(); | 280 | intel_iommu_init(); |
@@ -290,8 +300,7 @@ fs_initcall(pci_iommu_init); | |||
290 | static __devinit void via_no_dac(struct pci_dev *dev) | 300 | static __devinit void via_no_dac(struct pci_dev *dev) |
291 | { | 301 | { |
292 | if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { | 302 | if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { |
293 | printk(KERN_INFO | 303 | dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); |
294 | "PCI: VIA PCI bridge detected. Disabling DAC.\n"); | ||
295 | forbid_dac = 1; | 304 | forbid_dac = 1; |
296 | } | 305 | } |
297 | } | 306 | } |
diff --git a/arch/x86/kernel/pci-gart_64.c b/arch/x86/kernel/pci-gart_64.c index d5768b1af080..b284b58c035c 100644 --- a/arch/x86/kernel/pci-gart_64.c +++ b/arch/x86/kernel/pci-gart_64.c | |||
@@ -255,10 +255,13 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, | |||
255 | } | 255 | } |
256 | 256 | ||
257 | /* Map a single area into the IOMMU */ | 257 | /* Map a single area into the IOMMU */ |
258 | static dma_addr_t | 258 | static dma_addr_t gart_map_page(struct device *dev, struct page *page, |
259 | gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir) | 259 | unsigned long offset, size_t size, |
260 | enum dma_data_direction dir, | ||
261 | struct dma_attrs *attrs) | ||
260 | { | 262 | { |
261 | unsigned long bus; | 263 | unsigned long bus; |
264 | phys_addr_t paddr = page_to_phys(page) + offset; | ||
262 | 265 | ||
263 | if (!dev) | 266 | if (!dev) |
264 | dev = &x86_dma_fallback_dev; | 267 | dev = &x86_dma_fallback_dev; |
@@ -275,8 +278,9 @@ gart_map_single(struct device *dev, phys_addr_t paddr, size_t size, int dir) | |||
275 | /* | 278 | /* |
276 | * Free a DMA mapping. | 279 | * Free a DMA mapping. |
277 | */ | 280 | */ |
278 | static void gart_unmap_single(struct device *dev, dma_addr_t dma_addr, | 281 | static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, |
279 | size_t size, int direction) | 282 | size_t size, enum dma_data_direction dir, |
283 | struct dma_attrs *attrs) | ||
280 | { | 284 | { |
281 | unsigned long iommu_page; | 285 | unsigned long iommu_page; |
282 | int npages; | 286 | int npages; |
@@ -298,8 +302,8 @@ static void gart_unmap_single(struct device *dev, dma_addr_t dma_addr, | |||
298 | /* | 302 | /* |
299 | * Wrapper for pci_unmap_single working with scatterlists. | 303 | * Wrapper for pci_unmap_single working with scatterlists. |
300 | */ | 304 | */ |
301 | static void | 305 | static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, |
302 | gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir) | 306 | enum dma_data_direction dir, struct dma_attrs *attrs) |
303 | { | 307 | { |
304 | struct scatterlist *s; | 308 | struct scatterlist *s; |
305 | int i; | 309 | int i; |
@@ -307,7 +311,7 @@ gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir) | |||
307 | for_each_sg(sg, s, nents, i) { | 311 | for_each_sg(sg, s, nents, i) { |
308 | if (!s->dma_length || !s->length) | 312 | if (!s->dma_length || !s->length) |
309 | break; | 313 | break; |
310 | gart_unmap_single(dev, s->dma_address, s->dma_length, dir); | 314 | gart_unmap_page(dev, s->dma_address, s->dma_length, dir, NULL); |
311 | } | 315 | } |
312 | } | 316 | } |
313 | 317 | ||
@@ -329,7 +333,7 @@ static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg, | |||
329 | addr = dma_map_area(dev, addr, s->length, dir, 0); | 333 | addr = dma_map_area(dev, addr, s->length, dir, 0); |
330 | if (addr == bad_dma_address) { | 334 | if (addr == bad_dma_address) { |
331 | if (i > 0) | 335 | if (i > 0) |
332 | gart_unmap_sg(dev, sg, i, dir); | 336 | gart_unmap_sg(dev, sg, i, dir, NULL); |
333 | nents = 0; | 337 | nents = 0; |
334 | sg[0].dma_length = 0; | 338 | sg[0].dma_length = 0; |
335 | break; | 339 | break; |
@@ -400,8 +404,8 @@ dma_map_cont(struct device *dev, struct scatterlist *start, int nelems, | |||
400 | * DMA map all entries in a scatterlist. | 404 | * DMA map all entries in a scatterlist. |
401 | * Merge chunks that have page aligned sizes into a continuous mapping. | 405 | * Merge chunks that have page aligned sizes into a continuous mapping. |
402 | */ | 406 | */ |
403 | static int | 407 | static int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, |
404 | gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir) | 408 | enum dma_data_direction dir, struct dma_attrs *attrs) |
405 | { | 409 | { |
406 | struct scatterlist *s, *ps, *start_sg, *sgmap; | 410 | struct scatterlist *s, *ps, *start_sg, *sgmap; |
407 | int need = 0, nextneed, i, out, start; | 411 | int need = 0, nextneed, i, out, start; |
@@ -468,7 +472,7 @@ gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir) | |||
468 | 472 | ||
469 | error: | 473 | error: |
470 | flush_gart(); | 474 | flush_gart(); |
471 | gart_unmap_sg(dev, sg, out, dir); | 475 | gart_unmap_sg(dev, sg, out, dir, NULL); |
472 | 476 | ||
473 | /* When it was forced or merged try again in a dumb way */ | 477 | /* When it was forced or merged try again in a dumb way */ |
474 | if (force_iommu || iommu_merge) { | 478 | if (force_iommu || iommu_merge) { |
@@ -521,7 +525,7 @@ static void | |||
521 | gart_free_coherent(struct device *dev, size_t size, void *vaddr, | 525 | gart_free_coherent(struct device *dev, size_t size, void *vaddr, |
522 | dma_addr_t dma_addr) | 526 | dma_addr_t dma_addr) |
523 | { | 527 | { |
524 | gart_unmap_single(dev, dma_addr, size, DMA_BIDIRECTIONAL); | 528 | gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, NULL); |
525 | free_pages((unsigned long)vaddr, get_order(size)); | 529 | free_pages((unsigned long)vaddr, get_order(size)); |
526 | } | 530 | } |
527 | 531 | ||
@@ -707,11 +711,11 @@ static __init int init_k8_gatt(struct agp_kern_info *info) | |||
707 | return -1; | 711 | return -1; |
708 | } | 712 | } |
709 | 713 | ||
710 | static struct dma_mapping_ops gart_dma_ops = { | 714 | static struct dma_map_ops gart_dma_ops = { |
711 | .map_single = gart_map_single, | ||
712 | .unmap_single = gart_unmap_single, | ||
713 | .map_sg = gart_map_sg, | 715 | .map_sg = gart_map_sg, |
714 | .unmap_sg = gart_unmap_sg, | 716 | .unmap_sg = gart_unmap_sg, |
717 | .map_page = gart_map_page, | ||
718 | .unmap_page = gart_unmap_page, | ||
715 | .alloc_coherent = gart_alloc_coherent, | 719 | .alloc_coherent = gart_alloc_coherent, |
716 | .free_coherent = gart_free_coherent, | 720 | .free_coherent = gart_free_coherent, |
717 | }; | 721 | }; |
diff --git a/arch/x86/kernel/pci-nommu.c b/arch/x86/kernel/pci-nommu.c index c70ab5a5d4c8..c6d703b39326 100644 --- a/arch/x86/kernel/pci-nommu.c +++ b/arch/x86/kernel/pci-nommu.c | |||
@@ -1,14 +1,14 @@ | |||
1 | /* Fallback functions when the main IOMMU code is not compiled in. This | 1 | /* Fallback functions when the main IOMMU code is not compiled in. This |
2 | code is roughly equivalent to i386. */ | 2 | code is roughly equivalent to i386. */ |
3 | #include <linux/mm.h> | ||
4 | #include <linux/init.h> | ||
5 | #include <linux/pci.h> | ||
6 | #include <linux/string.h> | ||
7 | #include <linux/dma-mapping.h> | 3 | #include <linux/dma-mapping.h> |
8 | #include <linux/scatterlist.h> | 4 | #include <linux/scatterlist.h> |
5 | #include <linux/string.h> | ||
6 | #include <linux/init.h> | ||
7 | #include <linux/pci.h> | ||
8 | #include <linux/mm.h> | ||
9 | 9 | ||
10 | #include <asm/iommu.h> | ||
11 | #include <asm/processor.h> | 10 | #include <asm/processor.h> |
11 | #include <asm/iommu.h> | ||
12 | #include <asm/dma.h> | 12 | #include <asm/dma.h> |
13 | 13 | ||
14 | static int | 14 | static int |
@@ -25,19 +25,19 @@ check_addr(char *name, struct device *hwdev, dma_addr_t bus, size_t size) | |||
25 | return 1; | 25 | return 1; |
26 | } | 26 | } |
27 | 27 | ||
28 | static dma_addr_t | 28 | static dma_addr_t nommu_map_page(struct device *dev, struct page *page, |
29 | nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size, | 29 | unsigned long offset, size_t size, |
30 | int direction) | 30 | enum dma_data_direction dir, |
31 | struct dma_attrs *attrs) | ||
31 | { | 32 | { |
32 | dma_addr_t bus = paddr; | 33 | dma_addr_t bus = page_to_phys(page) + offset; |
33 | WARN_ON(size == 0); | 34 | WARN_ON(size == 0); |
34 | if (!check_addr("map_single", hwdev, bus, size)) | 35 | if (!check_addr("map_single", dev, bus, size)) |
35 | return bad_dma_address; | 36 | return bad_dma_address; |
36 | flush_write_buffers(); | 37 | flush_write_buffers(); |
37 | return bus; | 38 | return bus; |
38 | } | 39 | } |
39 | 40 | ||
40 | |||
41 | /* Map a set of buffers described by scatterlist in streaming | 41 | /* Map a set of buffers described by scatterlist in streaming |
42 | * mode for DMA. This is the scatter-gather version of the | 42 | * mode for DMA. This is the scatter-gather version of the |
43 | * above pci_map_single interface. Here the scatter gather list | 43 | * above pci_map_single interface. Here the scatter gather list |
@@ -54,7 +54,8 @@ nommu_map_single(struct device *hwdev, phys_addr_t paddr, size_t size, | |||
54 | * the same here. | 54 | * the same here. |
55 | */ | 55 | */ |
56 | static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg, | 56 | static int nommu_map_sg(struct device *hwdev, struct scatterlist *sg, |
57 | int nents, int direction) | 57 | int nents, enum dma_data_direction dir, |
58 | struct dma_attrs *attrs) | ||
58 | { | 59 | { |
59 | struct scatterlist *s; | 60 | struct scatterlist *s; |
60 | int i; | 61 | int i; |
@@ -78,12 +79,12 @@ static void nommu_free_coherent(struct device *dev, size_t size, void *vaddr, | |||
78 | free_pages((unsigned long)vaddr, get_order(size)); | 79 | free_pages((unsigned long)vaddr, get_order(size)); |
79 | } | 80 | } |
80 | 81 | ||
81 | struct dma_mapping_ops nommu_dma_ops = { | 82 | struct dma_map_ops nommu_dma_ops = { |
82 | .alloc_coherent = dma_generic_alloc_coherent, | 83 | .alloc_coherent = dma_generic_alloc_coherent, |
83 | .free_coherent = nommu_free_coherent, | 84 | .free_coherent = nommu_free_coherent, |
84 | .map_single = nommu_map_single, | 85 | .map_sg = nommu_map_sg, |
85 | .map_sg = nommu_map_sg, | 86 | .map_page = nommu_map_page, |
86 | .is_phys = 1, | 87 | .is_phys = 1, |
87 | }; | 88 | }; |
88 | 89 | ||
89 | void __init no_iommu_init(void) | 90 | void __init no_iommu_init(void) |
diff --git a/arch/x86/kernel/pci-swiotlb_64.c b/arch/x86/kernel/pci-swiotlb.c index d59c91747665..34f12e9996ed 100644 --- a/arch/x86/kernel/pci-swiotlb_64.c +++ b/arch/x86/kernel/pci-swiotlb.c | |||
@@ -33,18 +33,11 @@ phys_addr_t swiotlb_bus_to_phys(dma_addr_t baddr) | |||
33 | return baddr; | 33 | return baddr; |
34 | } | 34 | } |
35 | 35 | ||
36 | int __weak swiotlb_arch_range_needs_mapping(void *ptr, size_t size) | 36 | int __weak swiotlb_arch_range_needs_mapping(phys_addr_t paddr, size_t size) |
37 | { | 37 | { |
38 | return 0; | 38 | return 0; |
39 | } | 39 | } |
40 | 40 | ||
41 | static dma_addr_t | ||
42 | swiotlb_map_single_phys(struct device *hwdev, phys_addr_t paddr, size_t size, | ||
43 | int direction) | ||
44 | { | ||
45 | return swiotlb_map_single(hwdev, phys_to_virt(paddr), size, direction); | ||
46 | } | ||
47 | |||
48 | static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size, | 41 | static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size, |
49 | dma_addr_t *dma_handle, gfp_t flags) | 42 | dma_addr_t *dma_handle, gfp_t flags) |
50 | { | 43 | { |
@@ -57,20 +50,20 @@ static void *x86_swiotlb_alloc_coherent(struct device *hwdev, size_t size, | |||
57 | return swiotlb_alloc_coherent(hwdev, size, dma_handle, flags); | 50 | return swiotlb_alloc_coherent(hwdev, size, dma_handle, flags); |
58 | } | 51 | } |
59 | 52 | ||
60 | struct dma_mapping_ops swiotlb_dma_ops = { | 53 | struct dma_map_ops swiotlb_dma_ops = { |
61 | .mapping_error = swiotlb_dma_mapping_error, | 54 | .mapping_error = swiotlb_dma_mapping_error, |
62 | .alloc_coherent = x86_swiotlb_alloc_coherent, | 55 | .alloc_coherent = x86_swiotlb_alloc_coherent, |
63 | .free_coherent = swiotlb_free_coherent, | 56 | .free_coherent = swiotlb_free_coherent, |
64 | .map_single = swiotlb_map_single_phys, | ||
65 | .unmap_single = swiotlb_unmap_single, | ||
66 | .sync_single_for_cpu = swiotlb_sync_single_for_cpu, | 57 | .sync_single_for_cpu = swiotlb_sync_single_for_cpu, |
67 | .sync_single_for_device = swiotlb_sync_single_for_device, | 58 | .sync_single_for_device = swiotlb_sync_single_for_device, |
68 | .sync_single_range_for_cpu = swiotlb_sync_single_range_for_cpu, | 59 | .sync_single_range_for_cpu = swiotlb_sync_single_range_for_cpu, |
69 | .sync_single_range_for_device = swiotlb_sync_single_range_for_device, | 60 | .sync_single_range_for_device = swiotlb_sync_single_range_for_device, |
70 | .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, | 61 | .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu, |
71 | .sync_sg_for_device = swiotlb_sync_sg_for_device, | 62 | .sync_sg_for_device = swiotlb_sync_sg_for_device, |
72 | .map_sg = swiotlb_map_sg, | 63 | .map_sg = swiotlb_map_sg_attrs, |
73 | .unmap_sg = swiotlb_unmap_sg, | 64 | .unmap_sg = swiotlb_unmap_sg_attrs, |
65 | .map_page = swiotlb_map_page, | ||
66 | .unmap_page = swiotlb_unmap_page, | ||
74 | .dma_supported = NULL, | 67 | .dma_supported = NULL, |
75 | }; | 68 | }; |
76 | 69 | ||
diff --git a/arch/x86/kernel/quirks.c b/arch/x86/kernel/quirks.c index 6a5a2970f4c5..e95022e4f5d5 100644 --- a/arch/x86/kernel/quirks.c +++ b/arch/x86/kernel/quirks.c | |||
@@ -171,7 +171,8 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH8_4, | |||
171 | ich_force_enable_hpet); | 171 | ich_force_enable_hpet); |
172 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH9_7, | 172 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_ICH9_7, |
173 | ich_force_enable_hpet); | 173 | ich_force_enable_hpet); |
174 | 174 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x3a16, /* ICH10 */ | |
175 | ich_force_enable_hpet); | ||
175 | 176 | ||
176 | static struct pci_dev *cached_dev; | 177 | static struct pci_dev *cached_dev; |
177 | 178 | ||
diff --git a/arch/x86/kernel/rtc.c b/arch/x86/kernel/rtc.c index dd6f2b71561b..5d465b207e72 100644 --- a/arch/x86/kernel/rtc.c +++ b/arch/x86/kernel/rtc.c | |||
@@ -1,14 +1,14 @@ | |||
1 | /* | 1 | /* |
2 | * RTC related functions | 2 | * RTC related functions |
3 | */ | 3 | */ |
4 | #include <linux/platform_device.h> | ||
5 | #include <linux/mc146818rtc.h> | ||
4 | #include <linux/acpi.h> | 6 | #include <linux/acpi.h> |
5 | #include <linux/bcd.h> | 7 | #include <linux/bcd.h> |
6 | #include <linux/mc146818rtc.h> | ||
7 | #include <linux/platform_device.h> | ||
8 | #include <linux/pnp.h> | 8 | #include <linux/pnp.h> |
9 | 9 | ||
10 | #include <asm/time.h> | ||
11 | #include <asm/vsyscall.h> | 10 | #include <asm/vsyscall.h> |
11 | #include <asm/time.h> | ||
12 | 12 | ||
13 | #ifdef CONFIG_X86_32 | 13 | #ifdef CONFIG_X86_32 |
14 | /* | 14 | /* |
@@ -16,9 +16,9 @@ | |||
16 | * register we are working with. It is required for NMI access to the | 16 | * register we are working with. It is required for NMI access to the |
17 | * CMOS/RTC registers. See include/asm-i386/mc146818rtc.h for details. | 17 | * CMOS/RTC registers. See include/asm-i386/mc146818rtc.h for details. |
18 | */ | 18 | */ |
19 | volatile unsigned long cmos_lock = 0; | 19 | volatile unsigned long cmos_lock; |
20 | EXPORT_SYMBOL(cmos_lock); | 20 | EXPORT_SYMBOL(cmos_lock); |
21 | #endif | 21 | #endif /* CONFIG_X86_32 */ |
22 | 22 | ||
23 | /* For two digit years assume time is always after that */ | 23 | /* For two digit years assume time is always after that */ |
24 | #define CMOS_YEARS_OFFS 2000 | 24 | #define CMOS_YEARS_OFFS 2000 |
@@ -38,9 +38,9 @@ EXPORT_SYMBOL(rtc_lock); | |||
38 | */ | 38 | */ |
39 | int mach_set_rtc_mmss(unsigned long nowtime) | 39 | int mach_set_rtc_mmss(unsigned long nowtime) |
40 | { | 40 | { |
41 | int retval = 0; | ||
42 | int real_seconds, real_minutes, cmos_minutes; | 41 | int real_seconds, real_minutes, cmos_minutes; |
43 | unsigned char save_control, save_freq_select; | 42 | unsigned char save_control, save_freq_select; |
43 | int retval = 0; | ||
44 | 44 | ||
45 | /* tell the clock it's being set */ | 45 | /* tell the clock it's being set */ |
46 | save_control = CMOS_READ(RTC_CONTROL); | 46 | save_control = CMOS_READ(RTC_CONTROL); |
@@ -72,8 +72,8 @@ int mach_set_rtc_mmss(unsigned long nowtime) | |||
72 | real_seconds = bin2bcd(real_seconds); | 72 | real_seconds = bin2bcd(real_seconds); |
73 | real_minutes = bin2bcd(real_minutes); | 73 | real_minutes = bin2bcd(real_minutes); |
74 | } | 74 | } |
75 | CMOS_WRITE(real_seconds,RTC_SECONDS); | 75 | CMOS_WRITE(real_seconds, RTC_SECONDS); |
76 | CMOS_WRITE(real_minutes,RTC_MINUTES); | 76 | CMOS_WRITE(real_minutes, RTC_MINUTES); |
77 | } else { | 77 | } else { |
78 | printk(KERN_WARNING | 78 | printk(KERN_WARNING |
79 | "set_rtc_mmss: can't update from %d to %d\n", | 79 | "set_rtc_mmss: can't update from %d to %d\n", |
@@ -151,6 +151,7 @@ unsigned char rtc_cmos_read(unsigned char addr) | |||
151 | outb(addr, RTC_PORT(0)); | 151 | outb(addr, RTC_PORT(0)); |
152 | val = inb(RTC_PORT(1)); | 152 | val = inb(RTC_PORT(1)); |
153 | lock_cmos_suffix(addr); | 153 | lock_cmos_suffix(addr); |
154 | |||
154 | return val; | 155 | return val; |
155 | } | 156 | } |
156 | EXPORT_SYMBOL(rtc_cmos_read); | 157 | EXPORT_SYMBOL(rtc_cmos_read); |
@@ -166,8 +167,8 @@ EXPORT_SYMBOL(rtc_cmos_write); | |||
166 | 167 | ||
167 | static int set_rtc_mmss(unsigned long nowtime) | 168 | static int set_rtc_mmss(unsigned long nowtime) |
168 | { | 169 | { |
169 | int retval; | ||
170 | unsigned long flags; | 170 | unsigned long flags; |
171 | int retval; | ||
171 | 172 | ||
172 | spin_lock_irqsave(&rtc_lock, flags); | 173 | spin_lock_irqsave(&rtc_lock, flags); |
173 | retval = set_wallclock(nowtime); | 174 | retval = set_wallclock(nowtime); |
@@ -242,6 +243,7 @@ static __init int add_rtc_cmos(void) | |||
242 | platform_device_register(&rtc_device); | 243 | platform_device_register(&rtc_device); |
243 | dev_info(&rtc_device.dev, | 244 | dev_info(&rtc_device.dev, |
244 | "registered platform RTC device (no PNP device found)\n"); | 245 | "registered platform RTC device (no PNP device found)\n"); |
246 | |||
245 | return 0; | 247 | return 0; |
246 | } | 248 | } |
247 | device_initcall(add_rtc_cmos); | 249 | device_initcall(add_rtc_cmos); |
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index a0d26237d7cf..b4158439bf63 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c | |||
@@ -1049,7 +1049,6 @@ void __init x86_quirk_trap_init(void) | |||
1049 | static struct irqaction irq0 = { | 1049 | static struct irqaction irq0 = { |
1050 | .handler = timer_interrupt, | 1050 | .handler = timer_interrupt, |
1051 | .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_IRQPOLL | IRQF_TIMER, | 1051 | .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_IRQPOLL | IRQF_TIMER, |
1052 | .mask = CPU_MASK_NONE, | ||
1053 | .name = "timer" | 1052 | .name = "timer" |
1054 | }; | 1053 | }; |
1055 | 1054 | ||
diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index d2cc6428c587..dfcc74ab0ab6 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c | |||
@@ -211,31 +211,27 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size, | |||
211 | { | 211 | { |
212 | /* Default to using normal stack */ | 212 | /* Default to using normal stack */ |
213 | unsigned long sp = regs->sp; | 213 | unsigned long sp = regs->sp; |
214 | int onsigstack = on_sig_stack(sp); | ||
214 | 215 | ||
215 | #ifdef CONFIG_X86_64 | 216 | #ifdef CONFIG_X86_64 |
216 | /* redzone */ | 217 | /* redzone */ |
217 | sp -= 128; | 218 | sp -= 128; |
218 | #endif /* CONFIG_X86_64 */ | 219 | #endif /* CONFIG_X86_64 */ |
219 | 220 | ||
220 | /* | 221 | if (!onsigstack) { |
221 | * If we are on the alternate signal stack and would overflow it, don't. | 222 | /* This is the X/Open sanctioned signal stack switching. */ |
222 | * Return an always-bogus address instead so we will die with SIGSEGV. | 223 | if (ka->sa.sa_flags & SA_ONSTACK) { |
223 | */ | 224 | if (sas_ss_flags(sp) == 0) |
224 | if (on_sig_stack(sp) && !likely(on_sig_stack(sp - frame_size))) | 225 | sp = current->sas_ss_sp + current->sas_ss_size; |
225 | return (void __user *) -1L; | 226 | } else { |
226 | |||
227 | /* This is the X/Open sanctioned signal stack switching. */ | ||
228 | if (ka->sa.sa_flags & SA_ONSTACK) { | ||
229 | if (sas_ss_flags(sp) == 0) | ||
230 | sp = current->sas_ss_sp + current->sas_ss_size; | ||
231 | } else { | ||
232 | #ifdef CONFIG_X86_32 | 227 | #ifdef CONFIG_X86_32 |
233 | /* This is the legacy signal stack switching. */ | 228 | /* This is the legacy signal stack switching. */ |
234 | if ((regs->ss & 0xffff) != __USER_DS && | 229 | if ((regs->ss & 0xffff) != __USER_DS && |
235 | !(ka->sa.sa_flags & SA_RESTORER) && | 230 | !(ka->sa.sa_flags & SA_RESTORER) && |
236 | ka->sa.sa_restorer) | 231 | ka->sa.sa_restorer) |
237 | sp = (unsigned long) ka->sa.sa_restorer; | 232 | sp = (unsigned long) ka->sa.sa_restorer; |
238 | #endif /* CONFIG_X86_32 */ | 233 | #endif /* CONFIG_X86_32 */ |
234 | } | ||
239 | } | 235 | } |
240 | 236 | ||
241 | if (used_math()) { | 237 | if (used_math()) { |
@@ -244,12 +240,22 @@ get_sigframe(struct k_sigaction *ka, struct pt_regs *regs, size_t frame_size, | |||
244 | sp = round_down(sp, 64); | 240 | sp = round_down(sp, 64); |
245 | #endif /* CONFIG_X86_64 */ | 241 | #endif /* CONFIG_X86_64 */ |
246 | *fpstate = (void __user *)sp; | 242 | *fpstate = (void __user *)sp; |
247 | |||
248 | if (save_i387_xstate(*fpstate) < 0) | ||
249 | return (void __user *)-1L; | ||
250 | } | 243 | } |
251 | 244 | ||
252 | return (void __user *)align_sigframe(sp - frame_size); | 245 | sp = align_sigframe(sp - frame_size); |
246 | |||
247 | /* | ||
248 | * If we are on the alternate signal stack and would overflow it, don't. | ||
249 | * Return an always-bogus address instead so we will die with SIGSEGV. | ||
250 | */ | ||
251 | if (onsigstack && !likely(on_sig_stack(sp))) | ||
252 | return (void __user *)-1L; | ||
253 | |||
254 | /* save i387 state */ | ||
255 | if (used_math() && save_i387_xstate(*fpstate) < 0) | ||
256 | return (void __user *)-1L; | ||
257 | |||
258 | return (void __user *)sp; | ||
253 | } | 259 | } |
254 | 260 | ||
255 | #ifdef CONFIG_X86_32 | 261 | #ifdef CONFIG_X86_32 |
diff --git a/arch/x86/kernel/time_64.c b/arch/x86/kernel/time_64.c index 241ec3923f61..5ba343e61844 100644 --- a/arch/x86/kernel/time_64.c +++ b/arch/x86/kernel/time_64.c | |||
@@ -116,7 +116,6 @@ unsigned long __init calibrate_cpu(void) | |||
116 | static struct irqaction irq0 = { | 116 | static struct irqaction irq0 = { |
117 | .handler = timer_interrupt, | 117 | .handler = timer_interrupt, |
118 | .flags = IRQF_DISABLED | IRQF_IRQPOLL | IRQF_NOBALANCING | IRQF_TIMER, | 118 | .flags = IRQF_DISABLED | IRQF_IRQPOLL | IRQF_NOBALANCING | IRQF_TIMER, |
119 | .mask = CPU_MASK_NONE, | ||
120 | .name = "timer" | 119 | .name = "timer" |
121 | }; | 120 | }; |
122 | 121 | ||
@@ -125,7 +124,6 @@ void __init hpet_time_init(void) | |||
125 | if (!hpet_enable()) | 124 | if (!hpet_enable()) |
126 | setup_pit_timer(); | 125 | setup_pit_timer(); |
127 | 126 | ||
128 | irq0.mask = cpumask_of_cpu(0); | ||
129 | setup_irq(0, &irq0); | 127 | setup_irq(0, &irq0); |
130 | } | 128 | } |
131 | 129 | ||
diff --git a/arch/x86/kernel/topology.c b/arch/x86/kernel/topology.c index 0fcc95a354f7..7e4515957a1c 100644 --- a/arch/x86/kernel/topology.c +++ b/arch/x86/kernel/topology.c | |||
@@ -25,10 +25,10 @@ | |||
25 | * | 25 | * |
26 | * Send feedback to <colpatch@us.ibm.com> | 26 | * Send feedback to <colpatch@us.ibm.com> |
27 | */ | 27 | */ |
28 | #include <linux/init.h> | ||
29 | #include <linux/smp.h> | ||
30 | #include <linux/nodemask.h> | 28 | #include <linux/nodemask.h> |
31 | #include <linux/mmzone.h> | 29 | #include <linux/mmzone.h> |
30 | #include <linux/init.h> | ||
31 | #include <linux/smp.h> | ||
32 | #include <asm/cpu.h> | 32 | #include <asm/cpu.h> |
33 | 33 | ||
34 | static DEFINE_PER_CPU(struct x86_cpu, cpu_devices); | 34 | static DEFINE_PER_CPU(struct x86_cpu, cpu_devices); |
@@ -47,6 +47,7 @@ int __ref arch_register_cpu(int num) | |||
47 | */ | 47 | */ |
48 | if (num) | 48 | if (num) |
49 | per_cpu(cpu_devices, num).cpu.hotpluggable = 1; | 49 | per_cpu(cpu_devices, num).cpu.hotpluggable = 1; |
50 | |||
50 | return register_cpu(&per_cpu(cpu_devices, num).cpu, num); | 51 | return register_cpu(&per_cpu(cpu_devices, num).cpu, num); |
51 | } | 52 | } |
52 | EXPORT_SYMBOL(arch_register_cpu); | 53 | EXPORT_SYMBOL(arch_register_cpu); |
@@ -56,12 +57,13 @@ void arch_unregister_cpu(int num) | |||
56 | unregister_cpu(&per_cpu(cpu_devices, num).cpu); | 57 | unregister_cpu(&per_cpu(cpu_devices, num).cpu); |
57 | } | 58 | } |
58 | EXPORT_SYMBOL(arch_unregister_cpu); | 59 | EXPORT_SYMBOL(arch_unregister_cpu); |
59 | #else | 60 | #else /* CONFIG_HOTPLUG_CPU */ |
61 | |||
60 | static int __init arch_register_cpu(int num) | 62 | static int __init arch_register_cpu(int num) |
61 | { | 63 | { |
62 | return register_cpu(&per_cpu(cpu_devices, num).cpu, num); | 64 | return register_cpu(&per_cpu(cpu_devices, num).cpu, num); |
63 | } | 65 | } |
64 | #endif /*CONFIG_HOTPLUG_CPU*/ | 66 | #endif /* CONFIG_HOTPLUG_CPU */ |
65 | 67 | ||
66 | static int __init topology_init(void) | 68 | static int __init topology_init(void) |
67 | { | 69 | { |
@@ -70,11 +72,11 @@ static int __init topology_init(void) | |||
70 | #ifdef CONFIG_NUMA | 72 | #ifdef CONFIG_NUMA |
71 | for_each_online_node(i) | 73 | for_each_online_node(i) |
72 | register_one_node(i); | 74 | register_one_node(i); |
73 | #endif /* CONFIG_NUMA */ | 75 | #endif |
74 | 76 | ||
75 | for_each_present_cpu(i) | 77 | for_each_present_cpu(i) |
76 | arch_register_cpu(i); | 78 | arch_register_cpu(i); |
79 | |||
77 | return 0; | 80 | return 0; |
78 | } | 81 | } |
79 | |||
80 | subsys_initcall(topology_init); | 82 | subsys_initcall(topology_init); |
diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c index 462b9ba67e92..7a567ebe6361 100644 --- a/arch/x86/kernel/tsc.c +++ b/arch/x86/kernel/tsc.c | |||
@@ -17,20 +17,21 @@ | |||
17 | #include <asm/delay.h> | 17 | #include <asm/delay.h> |
18 | #include <asm/hypervisor.h> | 18 | #include <asm/hypervisor.h> |
19 | 19 | ||
20 | unsigned int cpu_khz; /* TSC clocks / usec, not used here */ | 20 | unsigned int __read_mostly cpu_khz; /* TSC clocks / usec, not used here */ |
21 | EXPORT_SYMBOL(cpu_khz); | 21 | EXPORT_SYMBOL(cpu_khz); |
22 | unsigned int tsc_khz; | 22 | |
23 | unsigned int __read_mostly tsc_khz; | ||
23 | EXPORT_SYMBOL(tsc_khz); | 24 | EXPORT_SYMBOL(tsc_khz); |
24 | 25 | ||
25 | /* | 26 | /* |
26 | * TSC can be unstable due to cpufreq or due to unsynced TSCs | 27 | * TSC can be unstable due to cpufreq or due to unsynced TSCs |
27 | */ | 28 | */ |
28 | static int tsc_unstable; | 29 | static int __read_mostly tsc_unstable; |
29 | 30 | ||
30 | /* native_sched_clock() is called before tsc_init(), so | 31 | /* native_sched_clock() is called before tsc_init(), so |
31 | we must start with the TSC soft disabled to prevent | 32 | we must start with the TSC soft disabled to prevent |
32 | erroneous rdtsc usage on !cpu_has_tsc processors */ | 33 | erroneous rdtsc usage on !cpu_has_tsc processors */ |
33 | static int tsc_disabled = -1; | 34 | static int __read_mostly tsc_disabled = -1; |
34 | 35 | ||
35 | static int tsc_clocksource_reliable; | 36 | static int tsc_clocksource_reliable; |
36 | /* | 37 | /* |
@@ -543,8 +544,6 @@ unsigned long native_calibrate_tsc(void) | |||
543 | return tsc_pit_min; | 544 | return tsc_pit_min; |
544 | } | 545 | } |
545 | 546 | ||
546 | #ifdef CONFIG_X86_32 | ||
547 | /* Only called from the Powernow K7 cpu freq driver */ | ||
548 | int recalibrate_cpu_khz(void) | 547 | int recalibrate_cpu_khz(void) |
549 | { | 548 | { |
550 | #ifndef CONFIG_SMP | 549 | #ifndef CONFIG_SMP |
@@ -566,7 +565,6 @@ int recalibrate_cpu_khz(void) | |||
566 | 565 | ||
567 | EXPORT_SYMBOL(recalibrate_cpu_khz); | 566 | EXPORT_SYMBOL(recalibrate_cpu_khz); |
568 | 567 | ||
569 | #endif /* CONFIG_X86_32 */ | ||
570 | 568 | ||
571 | /* Accelerators for sched_clock() | 569 | /* Accelerators for sched_clock() |
572 | * convert from cycles(64bits) => nanoseconds (64bits) | 570 | * convert from cycles(64bits) => nanoseconds (64bits) |
diff --git a/arch/x86/kernel/vmiclock_32.c b/arch/x86/kernel/vmiclock_32.c index 33a788d5879c..d303369a7bad 100644 --- a/arch/x86/kernel/vmiclock_32.c +++ b/arch/x86/kernel/vmiclock_32.c | |||
@@ -202,7 +202,6 @@ static struct irqaction vmi_clock_action = { | |||
202 | .name = "vmi-timer", | 202 | .name = "vmi-timer", |
203 | .handler = vmi_timer_interrupt, | 203 | .handler = vmi_timer_interrupt, |
204 | .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_TIMER, | 204 | .flags = IRQF_DISABLED | IRQF_NOBALANCING | IRQF_TIMER, |
205 | .mask = CPU_MASK_ALL, | ||
206 | }; | 205 | }; |
207 | 206 | ||
208 | static void __devinit vmi_time_init_clockevent(void) | 207 | static void __devinit vmi_time_init_clockevent(void) |
diff --git a/arch/x86/kernel/vsmp_64.c b/arch/x86/kernel/vsmp_64.c index 74de562812cc..a1d804bcd483 100644 --- a/arch/x86/kernel/vsmp_64.c +++ b/arch/x86/kernel/vsmp_64.c | |||
@@ -22,7 +22,7 @@ | |||
22 | #include <asm/paravirt.h> | 22 | #include <asm/paravirt.h> |
23 | #include <asm/setup.h> | 23 | #include <asm/setup.h> |
24 | 24 | ||
25 | #ifdef CONFIG_PARAVIRT | 25 | #if defined CONFIG_PCI && defined CONFIG_PARAVIRT |
26 | /* | 26 | /* |
27 | * Interrupt control on vSMPowered systems: | 27 | * Interrupt control on vSMPowered systems: |
28 | * ~AC is a shadow of IF. If IF is 'on' AC should be 'off' | 28 | * ~AC is a shadow of IF. If IF is 'on' AC should be 'off' |
@@ -114,6 +114,7 @@ static void __init set_vsmp_pv_ops(void) | |||
114 | } | 114 | } |
115 | #endif | 115 | #endif |
116 | 116 | ||
117 | #ifdef CONFIG_PCI | ||
117 | static int is_vsmp = -1; | 118 | static int is_vsmp = -1; |
118 | 119 | ||
119 | static void __init detect_vsmp_box(void) | 120 | static void __init detect_vsmp_box(void) |
@@ -139,6 +140,15 @@ int is_vsmp_box(void) | |||
139 | } | 140 | } |
140 | } | 141 | } |
141 | 142 | ||
143 | #else | ||
144 | static void __init detect_vsmp_box(void) | ||
145 | { | ||
146 | } | ||
147 | int is_vsmp_box(void) | ||
148 | { | ||
149 | return 0; | ||
150 | } | ||
151 | #endif | ||
142 | void __init vsmp_init(void) | 152 | void __init vsmp_init(void) |
143 | { | 153 | { |
144 | detect_vsmp_box(); | 154 | detect_vsmp_box(); |
diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index b81125f0bdee..0a303c3ed11f 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig | |||
@@ -4,6 +4,10 @@ | |||
4 | config HAVE_KVM | 4 | config HAVE_KVM |
5 | bool | 5 | bool |
6 | 6 | ||
7 | config HAVE_KVM_IRQCHIP | ||
8 | bool | ||
9 | default y | ||
10 | |||
7 | menuconfig VIRTUALIZATION | 11 | menuconfig VIRTUALIZATION |
8 | bool "Virtualization" | 12 | bool "Virtualization" |
9 | depends on HAVE_KVM || X86 | 13 | depends on HAVE_KVM || X86 |
diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index 72bd275a9b5c..c13bb92d3157 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c | |||
@@ -201,6 +201,9 @@ static int __pit_timer_fn(struct kvm_kpit_state *ps) | |||
201 | if (!atomic_inc_and_test(&pt->pending)) | 201 | if (!atomic_inc_and_test(&pt->pending)) |
202 | set_bit(KVM_REQ_PENDING_TIMER, &vcpu0->requests); | 202 | set_bit(KVM_REQ_PENDING_TIMER, &vcpu0->requests); |
203 | 203 | ||
204 | if (!pt->reinject) | ||
205 | atomic_set(&pt->pending, 1); | ||
206 | |||
204 | if (vcpu0 && waitqueue_active(&vcpu0->wq)) | 207 | if (vcpu0 && waitqueue_active(&vcpu0->wq)) |
205 | wake_up_interruptible(&vcpu0->wq); | 208 | wake_up_interruptible(&vcpu0->wq); |
206 | 209 | ||
@@ -536,6 +539,16 @@ void kvm_pit_reset(struct kvm_pit *pit) | |||
536 | pit->pit_state.irq_ack = 1; | 539 | pit->pit_state.irq_ack = 1; |
537 | } | 540 | } |
538 | 541 | ||
542 | static void pit_mask_notifer(struct kvm_irq_mask_notifier *kimn, bool mask) | ||
543 | { | ||
544 | struct kvm_pit *pit = container_of(kimn, struct kvm_pit, mask_notifier); | ||
545 | |||
546 | if (!mask) { | ||
547 | atomic_set(&pit->pit_state.pit_timer.pending, 0); | ||
548 | pit->pit_state.irq_ack = 1; | ||
549 | } | ||
550 | } | ||
551 | |||
539 | struct kvm_pit *kvm_create_pit(struct kvm *kvm) | 552 | struct kvm_pit *kvm_create_pit(struct kvm *kvm) |
540 | { | 553 | { |
541 | struct kvm_pit *pit; | 554 | struct kvm_pit *pit; |
@@ -545,9 +558,7 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm) | |||
545 | if (!pit) | 558 | if (!pit) |
546 | return NULL; | 559 | return NULL; |
547 | 560 | ||
548 | mutex_lock(&kvm->lock); | ||
549 | pit->irq_source_id = kvm_request_irq_source_id(kvm); | 561 | pit->irq_source_id = kvm_request_irq_source_id(kvm); |
550 | mutex_unlock(&kvm->lock); | ||
551 | if (pit->irq_source_id < 0) { | 562 | if (pit->irq_source_id < 0) { |
552 | kfree(pit); | 563 | kfree(pit); |
553 | return NULL; | 564 | return NULL; |
@@ -580,10 +591,14 @@ struct kvm_pit *kvm_create_pit(struct kvm *kvm) | |||
580 | pit_state->irq_ack_notifier.gsi = 0; | 591 | pit_state->irq_ack_notifier.gsi = 0; |
581 | pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq; | 592 | pit_state->irq_ack_notifier.irq_acked = kvm_pit_ack_irq; |
582 | kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier); | 593 | kvm_register_irq_ack_notifier(kvm, &pit_state->irq_ack_notifier); |
594 | pit_state->pit_timer.reinject = true; | ||
583 | mutex_unlock(&pit->pit_state.lock); | 595 | mutex_unlock(&pit->pit_state.lock); |
584 | 596 | ||
585 | kvm_pit_reset(pit); | 597 | kvm_pit_reset(pit); |
586 | 598 | ||
599 | pit->mask_notifier.func = pit_mask_notifer; | ||
600 | kvm_register_irq_mask_notifier(kvm, 0, &pit->mask_notifier); | ||
601 | |||
587 | return pit; | 602 | return pit; |
588 | } | 603 | } |
589 | 604 | ||
@@ -592,6 +607,8 @@ void kvm_free_pit(struct kvm *kvm) | |||
592 | struct hrtimer *timer; | 607 | struct hrtimer *timer; |
593 | 608 | ||
594 | if (kvm->arch.vpit) { | 609 | if (kvm->arch.vpit) { |
610 | kvm_unregister_irq_mask_notifier(kvm, 0, | ||
611 | &kvm->arch.vpit->mask_notifier); | ||
595 | mutex_lock(&kvm->arch.vpit->pit_state.lock); | 612 | mutex_lock(&kvm->arch.vpit->pit_state.lock); |
596 | timer = &kvm->arch.vpit->pit_state.pit_timer.timer; | 613 | timer = &kvm->arch.vpit->pit_state.pit_timer.timer; |
597 | hrtimer_cancel(timer); | 614 | hrtimer_cancel(timer); |
diff --git a/arch/x86/kvm/i8254.h b/arch/x86/kvm/i8254.h index 4178022b97aa..6acbe4b505d5 100644 --- a/arch/x86/kvm/i8254.h +++ b/arch/x86/kvm/i8254.h | |||
@@ -9,6 +9,7 @@ struct kvm_kpit_timer { | |||
9 | s64 period; /* unit: ns */ | 9 | s64 period; /* unit: ns */ |
10 | s64 scheduled; | 10 | s64 scheduled; |
11 | atomic_t pending; | 11 | atomic_t pending; |
12 | bool reinject; | ||
12 | }; | 13 | }; |
13 | 14 | ||
14 | struct kvm_kpit_channel_state { | 15 | struct kvm_kpit_channel_state { |
@@ -45,6 +46,7 @@ struct kvm_pit { | |||
45 | struct kvm *kvm; | 46 | struct kvm *kvm; |
46 | struct kvm_kpit_state pit_state; | 47 | struct kvm_kpit_state pit_state; |
47 | int irq_source_id; | 48 | int irq_source_id; |
49 | struct kvm_irq_mask_notifier mask_notifier; | ||
48 | }; | 50 | }; |
49 | 51 | ||
50 | #define KVM_PIT_BASE_ADDRESS 0x40 | 52 | #define KVM_PIT_BASE_ADDRESS 0x40 |
diff --git a/arch/x86/kvm/i8259.c b/arch/x86/kvm/i8259.c index 179dcb0103fd..1ccb50c74f18 100644 --- a/arch/x86/kvm/i8259.c +++ b/arch/x86/kvm/i8259.c | |||
@@ -32,11 +32,13 @@ | |||
32 | #include <linux/kvm_host.h> | 32 | #include <linux/kvm_host.h> |
33 | 33 | ||
34 | static void pic_lock(struct kvm_pic *s) | 34 | static void pic_lock(struct kvm_pic *s) |
35 | __acquires(&s->lock) | ||
35 | { | 36 | { |
36 | spin_lock(&s->lock); | 37 | spin_lock(&s->lock); |
37 | } | 38 | } |
38 | 39 | ||
39 | static void pic_unlock(struct kvm_pic *s) | 40 | static void pic_unlock(struct kvm_pic *s) |
41 | __releases(&s->lock) | ||
40 | { | 42 | { |
41 | struct kvm *kvm = s->kvm; | 43 | struct kvm *kvm = s->kvm; |
42 | unsigned acks = s->pending_acks; | 44 | unsigned acks = s->pending_acks; |
@@ -49,7 +51,8 @@ static void pic_unlock(struct kvm_pic *s) | |||
49 | spin_unlock(&s->lock); | 51 | spin_unlock(&s->lock); |
50 | 52 | ||
51 | while (acks) { | 53 | while (acks) { |
52 | kvm_notify_acked_irq(kvm, __ffs(acks)); | 54 | kvm_notify_acked_irq(kvm, SELECT_PIC(__ffs(acks)), |
55 | __ffs(acks)); | ||
53 | acks &= acks - 1; | 56 | acks &= acks - 1; |
54 | } | 57 | } |
55 | 58 | ||
@@ -76,12 +79,13 @@ void kvm_pic_clear_isr_ack(struct kvm *kvm) | |||
76 | /* | 79 | /* |
77 | * set irq level. If an edge is detected, then the IRR is set to 1 | 80 | * set irq level. If an edge is detected, then the IRR is set to 1 |
78 | */ | 81 | */ |
79 | static inline void pic_set_irq1(struct kvm_kpic_state *s, int irq, int level) | 82 | static inline int pic_set_irq1(struct kvm_kpic_state *s, int irq, int level) |
80 | { | 83 | { |
81 | int mask; | 84 | int mask, ret = 1; |
82 | mask = 1 << irq; | 85 | mask = 1 << irq; |
83 | if (s->elcr & mask) /* level triggered */ | 86 | if (s->elcr & mask) /* level triggered */ |
84 | if (level) { | 87 | if (level) { |
88 | ret = !(s->irr & mask); | ||
85 | s->irr |= mask; | 89 | s->irr |= mask; |
86 | s->last_irr |= mask; | 90 | s->last_irr |= mask; |
87 | } else { | 91 | } else { |
@@ -90,11 +94,15 @@ static inline void pic_set_irq1(struct kvm_kpic_state *s, int irq, int level) | |||
90 | } | 94 | } |
91 | else /* edge triggered */ | 95 | else /* edge triggered */ |
92 | if (level) { | 96 | if (level) { |
93 | if ((s->last_irr & mask) == 0) | 97 | if ((s->last_irr & mask) == 0) { |
98 | ret = !(s->irr & mask); | ||
94 | s->irr |= mask; | 99 | s->irr |= mask; |
100 | } | ||
95 | s->last_irr |= mask; | 101 | s->last_irr |= mask; |
96 | } else | 102 | } else |
97 | s->last_irr &= ~mask; | 103 | s->last_irr &= ~mask; |
104 | |||
105 | return (s->imr & mask) ? -1 : ret; | ||
98 | } | 106 | } |
99 | 107 | ||
100 | /* | 108 | /* |
@@ -171,16 +179,19 @@ void kvm_pic_update_irq(struct kvm_pic *s) | |||
171 | pic_unlock(s); | 179 | pic_unlock(s); |
172 | } | 180 | } |
173 | 181 | ||
174 | void kvm_pic_set_irq(void *opaque, int irq, int level) | 182 | int kvm_pic_set_irq(void *opaque, int irq, int level) |
175 | { | 183 | { |
176 | struct kvm_pic *s = opaque; | 184 | struct kvm_pic *s = opaque; |
185 | int ret = -1; | ||
177 | 186 | ||
178 | pic_lock(s); | 187 | pic_lock(s); |
179 | if (irq >= 0 && irq < PIC_NUM_PINS) { | 188 | if (irq >= 0 && irq < PIC_NUM_PINS) { |
180 | pic_set_irq1(&s->pics[irq >> 3], irq & 7, level); | 189 | ret = pic_set_irq1(&s->pics[irq >> 3], irq & 7, level); |
181 | pic_update_irq(s); | 190 | pic_update_irq(s); |
182 | } | 191 | } |
183 | pic_unlock(s); | 192 | pic_unlock(s); |
193 | |||
194 | return ret; | ||
184 | } | 195 | } |
185 | 196 | ||
186 | /* | 197 | /* |
@@ -232,7 +243,7 @@ int kvm_pic_read_irq(struct kvm *kvm) | |||
232 | } | 243 | } |
233 | pic_update_irq(s); | 244 | pic_update_irq(s); |
234 | pic_unlock(s); | 245 | pic_unlock(s); |
235 | kvm_notify_acked_irq(kvm, irq); | 246 | kvm_notify_acked_irq(kvm, SELECT_PIC(irq), irq); |
236 | 247 | ||
237 | return intno; | 248 | return intno; |
238 | } | 249 | } |
diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 82579ee538d0..9f593188129e 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h | |||
@@ -32,6 +32,8 @@ | |||
32 | #include "lapic.h" | 32 | #include "lapic.h" |
33 | 33 | ||
34 | #define PIC_NUM_PINS 16 | 34 | #define PIC_NUM_PINS 16 |
35 | #define SELECT_PIC(irq) \ | ||
36 | ((irq) < 8 ? KVM_IRQCHIP_PIC_MASTER : KVM_IRQCHIP_PIC_SLAVE) | ||
35 | 37 | ||
36 | struct kvm; | 38 | struct kvm; |
37 | struct kvm_vcpu; | 39 | struct kvm_vcpu; |
diff --git a/arch/x86/kvm/kvm_svm.h b/arch/x86/kvm/kvm_svm.h index 8e5ee99551f6..ed66e4c078dc 100644 --- a/arch/x86/kvm/kvm_svm.h +++ b/arch/x86/kvm/kvm_svm.h | |||
@@ -18,7 +18,6 @@ static const u32 host_save_user_msrs[] = { | |||
18 | }; | 18 | }; |
19 | 19 | ||
20 | #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) | 20 | #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs) |
21 | #define NUM_DB_REGS 4 | ||
22 | 21 | ||
23 | struct kvm_vcpu; | 22 | struct kvm_vcpu; |
24 | 23 | ||
@@ -29,18 +28,23 @@ struct vcpu_svm { | |||
29 | struct svm_cpu_data *svm_data; | 28 | struct svm_cpu_data *svm_data; |
30 | uint64_t asid_generation; | 29 | uint64_t asid_generation; |
31 | 30 | ||
32 | unsigned long db_regs[NUM_DB_REGS]; | ||
33 | |||
34 | u64 next_rip; | 31 | u64 next_rip; |
35 | 32 | ||
36 | u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; | 33 | u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS]; |
37 | u64 host_gs_base; | 34 | u64 host_gs_base; |
38 | unsigned long host_cr2; | 35 | unsigned long host_cr2; |
39 | unsigned long host_db_regs[NUM_DB_REGS]; | ||
40 | unsigned long host_dr6; | ||
41 | unsigned long host_dr7; | ||
42 | 36 | ||
43 | u32 *msrpm; | 37 | u32 *msrpm; |
38 | struct vmcb *hsave; | ||
39 | u64 hsave_msr; | ||
40 | |||
41 | u64 nested_vmcb; | ||
42 | |||
43 | /* These are the merged vectors */ | ||
44 | u32 *nested_msrpm; | ||
45 | |||
46 | /* gpa pointers to the real vectors */ | ||
47 | u64 nested_vmcb_msrpm; | ||
44 | }; | 48 | }; |
45 | 49 | ||
46 | #endif | 50 | #endif |
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c index 2d4477c71473..2a36f7f7c4c7 100644 --- a/arch/x86/kvm/mmu.c +++ b/arch/x86/kvm/mmu.c | |||
@@ -145,11 +145,20 @@ struct kvm_rmap_desc { | |||
145 | struct kvm_rmap_desc *more; | 145 | struct kvm_rmap_desc *more; |
146 | }; | 146 | }; |
147 | 147 | ||
148 | struct kvm_shadow_walk { | 148 | struct kvm_shadow_walk_iterator { |
149 | int (*entry)(struct kvm_shadow_walk *walk, struct kvm_vcpu *vcpu, | 149 | u64 addr; |
150 | u64 addr, u64 *spte, int level); | 150 | hpa_t shadow_addr; |
151 | int level; | ||
152 | u64 *sptep; | ||
153 | unsigned index; | ||
151 | }; | 154 | }; |
152 | 155 | ||
156 | #define for_each_shadow_entry(_vcpu, _addr, _walker) \ | ||
157 | for (shadow_walk_init(&(_walker), _vcpu, _addr); \ | ||
158 | shadow_walk_okay(&(_walker)); \ | ||
159 | shadow_walk_next(&(_walker))) | ||
160 | |||
161 | |||
153 | struct kvm_unsync_walk { | 162 | struct kvm_unsync_walk { |
154 | int (*entry) (struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk); | 163 | int (*entry) (struct kvm_mmu_page *sp, struct kvm_unsync_walk *walk); |
155 | }; | 164 | }; |
@@ -343,7 +352,6 @@ static void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc, | |||
343 | 352 | ||
344 | BUG_ON(!mc->nobjs); | 353 | BUG_ON(!mc->nobjs); |
345 | p = mc->objects[--mc->nobjs]; | 354 | p = mc->objects[--mc->nobjs]; |
346 | memset(p, 0, size); | ||
347 | return p; | 355 | return p; |
348 | } | 356 | } |
349 | 357 | ||
@@ -794,10 +802,8 @@ static struct kvm_mmu_page *kvm_mmu_alloc_page(struct kvm_vcpu *vcpu, | |||
794 | set_page_private(virt_to_page(sp->spt), (unsigned long)sp); | 802 | set_page_private(virt_to_page(sp->spt), (unsigned long)sp); |
795 | list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); | 803 | list_add(&sp->link, &vcpu->kvm->arch.active_mmu_pages); |
796 | INIT_LIST_HEAD(&sp->oos_link); | 804 | INIT_LIST_HEAD(&sp->oos_link); |
797 | ASSERT(is_empty_shadow_page(sp->spt)); | ||
798 | bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS); | 805 | bitmap_zero(sp->slot_bitmap, KVM_MEMORY_SLOTS + KVM_PRIVATE_MEM_SLOTS); |
799 | sp->multimapped = 0; | 806 | sp->multimapped = 0; |
800 | sp->global = 1; | ||
801 | sp->parent_pte = parent_pte; | 807 | sp->parent_pte = parent_pte; |
802 | --vcpu->kvm->arch.n_free_mmu_pages; | 808 | --vcpu->kvm->arch.n_free_mmu_pages; |
803 | return sp; | 809 | return sp; |
@@ -983,8 +989,8 @@ struct kvm_mmu_pages { | |||
983 | idx < 512; \ | 989 | idx < 512; \ |
984 | idx = find_next_bit(bitmap, 512, idx+1)) | 990 | idx = find_next_bit(bitmap, 512, idx+1)) |
985 | 991 | ||
986 | int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp, | 992 | static int mmu_pages_add(struct kvm_mmu_pages *pvec, struct kvm_mmu_page *sp, |
987 | int idx) | 993 | int idx) |
988 | { | 994 | { |
989 | int i; | 995 | int i; |
990 | 996 | ||
@@ -1059,7 +1065,7 @@ static struct kvm_mmu_page *kvm_mmu_lookup_page(struct kvm *kvm, gfn_t gfn) | |||
1059 | index = kvm_page_table_hashfn(gfn); | 1065 | index = kvm_page_table_hashfn(gfn); |
1060 | bucket = &kvm->arch.mmu_page_hash[index]; | 1066 | bucket = &kvm->arch.mmu_page_hash[index]; |
1061 | hlist_for_each_entry(sp, node, bucket, hash_link) | 1067 | hlist_for_each_entry(sp, node, bucket, hash_link) |
1062 | if (sp->gfn == gfn && !sp->role.metaphysical | 1068 | if (sp->gfn == gfn && !sp->role.direct |
1063 | && !sp->role.invalid) { | 1069 | && !sp->role.invalid) { |
1064 | pgprintk("%s: found role %x\n", | 1070 | pgprintk("%s: found role %x\n", |
1065 | __func__, sp->role.word); | 1071 | __func__, sp->role.word); |
@@ -1115,8 +1121,9 @@ struct mmu_page_path { | |||
1115 | i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \ | 1121 | i < pvec.nr && ({ sp = pvec.page[i].sp; 1;}); \ |
1116 | i = mmu_pages_next(&pvec, &parents, i)) | 1122 | i = mmu_pages_next(&pvec, &parents, i)) |
1117 | 1123 | ||
1118 | int mmu_pages_next(struct kvm_mmu_pages *pvec, struct mmu_page_path *parents, | 1124 | static int mmu_pages_next(struct kvm_mmu_pages *pvec, |
1119 | int i) | 1125 | struct mmu_page_path *parents, |
1126 | int i) | ||
1120 | { | 1127 | { |
1121 | int n; | 1128 | int n; |
1122 | 1129 | ||
@@ -1135,7 +1142,7 @@ int mmu_pages_next(struct kvm_mmu_pages *pvec, struct mmu_page_path *parents, | |||
1135 | return n; | 1142 | return n; |
1136 | } | 1143 | } |
1137 | 1144 | ||
1138 | void mmu_pages_clear_parents(struct mmu_page_path *parents) | 1145 | static void mmu_pages_clear_parents(struct mmu_page_path *parents) |
1139 | { | 1146 | { |
1140 | struct kvm_mmu_page *sp; | 1147 | struct kvm_mmu_page *sp; |
1141 | unsigned int level = 0; | 1148 | unsigned int level = 0; |
@@ -1193,7 +1200,7 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, | |||
1193 | gfn_t gfn, | 1200 | gfn_t gfn, |
1194 | gva_t gaddr, | 1201 | gva_t gaddr, |
1195 | unsigned level, | 1202 | unsigned level, |
1196 | int metaphysical, | 1203 | int direct, |
1197 | unsigned access, | 1204 | unsigned access, |
1198 | u64 *parent_pte) | 1205 | u64 *parent_pte) |
1199 | { | 1206 | { |
@@ -1204,10 +1211,9 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, | |||
1204 | struct kvm_mmu_page *sp; | 1211 | struct kvm_mmu_page *sp; |
1205 | struct hlist_node *node, *tmp; | 1212 | struct hlist_node *node, *tmp; |
1206 | 1213 | ||
1207 | role.word = 0; | 1214 | role = vcpu->arch.mmu.base_role; |
1208 | role.glevels = vcpu->arch.mmu.root_level; | ||
1209 | role.level = level; | 1215 | role.level = level; |
1210 | role.metaphysical = metaphysical; | 1216 | role.direct = direct; |
1211 | role.access = access; | 1217 | role.access = access; |
1212 | if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) { | 1218 | if (vcpu->arch.mmu.root_level <= PT32_ROOT_LEVEL) { |
1213 | quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); | 1219 | quadrant = gaddr >> (PAGE_SHIFT + (PT64_PT_BITS * level)); |
@@ -1242,8 +1248,9 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, | |||
1242 | pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word); | 1248 | pgprintk("%s: adding gfn %lx role %x\n", __func__, gfn, role.word); |
1243 | sp->gfn = gfn; | 1249 | sp->gfn = gfn; |
1244 | sp->role = role; | 1250 | sp->role = role; |
1251 | sp->global = role.cr4_pge; | ||
1245 | hlist_add_head(&sp->hash_link, bucket); | 1252 | hlist_add_head(&sp->hash_link, bucket); |
1246 | if (!metaphysical) { | 1253 | if (!direct) { |
1247 | if (rmap_write_protect(vcpu->kvm, gfn)) | 1254 | if (rmap_write_protect(vcpu->kvm, gfn)) |
1248 | kvm_flush_remote_tlbs(vcpu->kvm); | 1255 | kvm_flush_remote_tlbs(vcpu->kvm); |
1249 | account_shadowed(vcpu->kvm, gfn); | 1256 | account_shadowed(vcpu->kvm, gfn); |
@@ -1255,35 +1262,35 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, | |||
1255 | return sp; | 1262 | return sp; |
1256 | } | 1263 | } |
1257 | 1264 | ||
1258 | static int walk_shadow(struct kvm_shadow_walk *walker, | 1265 | static void shadow_walk_init(struct kvm_shadow_walk_iterator *iterator, |
1259 | struct kvm_vcpu *vcpu, u64 addr) | 1266 | struct kvm_vcpu *vcpu, u64 addr) |
1260 | { | 1267 | { |
1261 | hpa_t shadow_addr; | 1268 | iterator->addr = addr; |
1262 | int level; | 1269 | iterator->shadow_addr = vcpu->arch.mmu.root_hpa; |
1263 | int r; | 1270 | iterator->level = vcpu->arch.mmu.shadow_root_level; |
1264 | u64 *sptep; | 1271 | if (iterator->level == PT32E_ROOT_LEVEL) { |
1265 | unsigned index; | 1272 | iterator->shadow_addr |
1266 | 1273 | = vcpu->arch.mmu.pae_root[(addr >> 30) & 3]; | |
1267 | shadow_addr = vcpu->arch.mmu.root_hpa; | 1274 | iterator->shadow_addr &= PT64_BASE_ADDR_MASK; |
1268 | level = vcpu->arch.mmu.shadow_root_level; | 1275 | --iterator->level; |
1269 | if (level == PT32E_ROOT_LEVEL) { | 1276 | if (!iterator->shadow_addr) |
1270 | shadow_addr = vcpu->arch.mmu.pae_root[(addr >> 30) & 3]; | 1277 | iterator->level = 0; |
1271 | shadow_addr &= PT64_BASE_ADDR_MASK; | ||
1272 | if (!shadow_addr) | ||
1273 | return 1; | ||
1274 | --level; | ||
1275 | } | 1278 | } |
1279 | } | ||
1276 | 1280 | ||
1277 | while (level >= PT_PAGE_TABLE_LEVEL) { | 1281 | static bool shadow_walk_okay(struct kvm_shadow_walk_iterator *iterator) |
1278 | index = SHADOW_PT_INDEX(addr, level); | 1282 | { |
1279 | sptep = ((u64 *)__va(shadow_addr)) + index; | 1283 | if (iterator->level < PT_PAGE_TABLE_LEVEL) |
1280 | r = walker->entry(walker, vcpu, addr, sptep, level); | 1284 | return false; |
1281 | if (r) | 1285 | iterator->index = SHADOW_PT_INDEX(iterator->addr, iterator->level); |
1282 | return r; | 1286 | iterator->sptep = ((u64 *)__va(iterator->shadow_addr)) + iterator->index; |
1283 | shadow_addr = *sptep & PT64_BASE_ADDR_MASK; | 1287 | return true; |
1284 | --level; | 1288 | } |
1285 | } | 1289 | |
1286 | return 0; | 1290 | static void shadow_walk_next(struct kvm_shadow_walk_iterator *iterator) |
1291 | { | ||
1292 | iterator->shadow_addr = *iterator->sptep & PT64_BASE_ADDR_MASK; | ||
1293 | --iterator->level; | ||
1287 | } | 1294 | } |
1288 | 1295 | ||
1289 | static void kvm_mmu_page_unlink_children(struct kvm *kvm, | 1296 | static void kvm_mmu_page_unlink_children(struct kvm *kvm, |
@@ -1388,7 +1395,7 @@ static int kvm_mmu_zap_page(struct kvm *kvm, struct kvm_mmu_page *sp) | |||
1388 | kvm_mmu_page_unlink_children(kvm, sp); | 1395 | kvm_mmu_page_unlink_children(kvm, sp); |
1389 | kvm_mmu_unlink_parents(kvm, sp); | 1396 | kvm_mmu_unlink_parents(kvm, sp); |
1390 | kvm_flush_remote_tlbs(kvm); | 1397 | kvm_flush_remote_tlbs(kvm); |
1391 | if (!sp->role.invalid && !sp->role.metaphysical) | 1398 | if (!sp->role.invalid && !sp->role.direct) |
1392 | unaccount_shadowed(kvm, sp->gfn); | 1399 | unaccount_shadowed(kvm, sp->gfn); |
1393 | if (sp->unsync) | 1400 | if (sp->unsync) |
1394 | kvm_unlink_unsync_page(kvm, sp); | 1401 | kvm_unlink_unsync_page(kvm, sp); |
@@ -1451,7 +1458,7 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) | |||
1451 | index = kvm_page_table_hashfn(gfn); | 1458 | index = kvm_page_table_hashfn(gfn); |
1452 | bucket = &kvm->arch.mmu_page_hash[index]; | 1459 | bucket = &kvm->arch.mmu_page_hash[index]; |
1453 | hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) | 1460 | hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) |
1454 | if (sp->gfn == gfn && !sp->role.metaphysical) { | 1461 | if (sp->gfn == gfn && !sp->role.direct) { |
1455 | pgprintk("%s: gfn %lx role %x\n", __func__, gfn, | 1462 | pgprintk("%s: gfn %lx role %x\n", __func__, gfn, |
1456 | sp->role.word); | 1463 | sp->role.word); |
1457 | r = 1; | 1464 | r = 1; |
@@ -1463,11 +1470,20 @@ static int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn) | |||
1463 | 1470 | ||
1464 | static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) | 1471 | static void mmu_unshadow(struct kvm *kvm, gfn_t gfn) |
1465 | { | 1472 | { |
1473 | unsigned index; | ||
1474 | struct hlist_head *bucket; | ||
1466 | struct kvm_mmu_page *sp; | 1475 | struct kvm_mmu_page *sp; |
1476 | struct hlist_node *node, *nn; | ||
1467 | 1477 | ||
1468 | while ((sp = kvm_mmu_lookup_page(kvm, gfn)) != NULL) { | 1478 | index = kvm_page_table_hashfn(gfn); |
1469 | pgprintk("%s: zap %lx %x\n", __func__, gfn, sp->role.word); | 1479 | bucket = &kvm->arch.mmu_page_hash[index]; |
1470 | kvm_mmu_zap_page(kvm, sp); | 1480 | hlist_for_each_entry_safe(sp, node, nn, bucket, hash_link) { |
1481 | if (sp->gfn == gfn && !sp->role.direct | ||
1482 | && !sp->role.invalid) { | ||
1483 | pgprintk("%s: zap %lx %x\n", | ||
1484 | __func__, gfn, sp->role.word); | ||
1485 | kvm_mmu_zap_page(kvm, sp); | ||
1486 | } | ||
1471 | } | 1487 | } |
1472 | } | 1488 | } |
1473 | 1489 | ||
@@ -1622,7 +1638,7 @@ static int kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) | |||
1622 | bucket = &vcpu->kvm->arch.mmu_page_hash[index]; | 1638 | bucket = &vcpu->kvm->arch.mmu_page_hash[index]; |
1623 | /* don't unsync if pagetable is shadowed with multiple roles */ | 1639 | /* don't unsync if pagetable is shadowed with multiple roles */ |
1624 | hlist_for_each_entry_safe(s, node, n, bucket, hash_link) { | 1640 | hlist_for_each_entry_safe(s, node, n, bucket, hash_link) { |
1625 | if (s->gfn != sp->gfn || s->role.metaphysical) | 1641 | if (s->gfn != sp->gfn || s->role.direct) |
1626 | continue; | 1642 | continue; |
1627 | if (s->role.word != sp->role.word) | 1643 | if (s->role.word != sp->role.word) |
1628 | return 1; | 1644 | return 1; |
@@ -1669,8 +1685,6 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, | |||
1669 | u64 mt_mask = shadow_mt_mask; | 1685 | u64 mt_mask = shadow_mt_mask; |
1670 | struct kvm_mmu_page *sp = page_header(__pa(shadow_pte)); | 1686 | struct kvm_mmu_page *sp = page_header(__pa(shadow_pte)); |
1671 | 1687 | ||
1672 | if (!(vcpu->arch.cr4 & X86_CR4_PGE)) | ||
1673 | global = 0; | ||
1674 | if (!global && sp->global) { | 1688 | if (!global && sp->global) { |
1675 | sp->global = 0; | 1689 | sp->global = 0; |
1676 | if (sp->unsync) { | 1690 | if (sp->unsync) { |
@@ -1777,12 +1791,8 @@ static void mmu_set_spte(struct kvm_vcpu *vcpu, u64 *shadow_pte, | |||
1777 | pgprintk("hfn old %lx new %lx\n", | 1791 | pgprintk("hfn old %lx new %lx\n", |
1778 | spte_to_pfn(*shadow_pte), pfn); | 1792 | spte_to_pfn(*shadow_pte), pfn); |
1779 | rmap_remove(vcpu->kvm, shadow_pte); | 1793 | rmap_remove(vcpu->kvm, shadow_pte); |
1780 | } else { | 1794 | } else |
1781 | if (largepage) | 1795 | was_rmapped = 1; |
1782 | was_rmapped = is_large_pte(*shadow_pte); | ||
1783 | else | ||
1784 | was_rmapped = 1; | ||
1785 | } | ||
1786 | } | 1796 | } |
1787 | if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault, | 1797 | if (set_spte(vcpu, shadow_pte, pte_access, user_fault, write_fault, |
1788 | dirty, largepage, global, gfn, pfn, speculative, true)) { | 1798 | dirty, largepage, global, gfn, pfn, speculative, true)) { |
@@ -1820,67 +1830,42 @@ static void nonpaging_new_cr3(struct kvm_vcpu *vcpu) | |||
1820 | { | 1830 | { |
1821 | } | 1831 | } |
1822 | 1832 | ||
1823 | struct direct_shadow_walk { | 1833 | static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, |
1824 | struct kvm_shadow_walk walker; | 1834 | int largepage, gfn_t gfn, pfn_t pfn) |
1825 | pfn_t pfn; | ||
1826 | int write; | ||
1827 | int largepage; | ||
1828 | int pt_write; | ||
1829 | }; | ||
1830 | |||
1831 | static int direct_map_entry(struct kvm_shadow_walk *_walk, | ||
1832 | struct kvm_vcpu *vcpu, | ||
1833 | u64 addr, u64 *sptep, int level) | ||
1834 | { | 1835 | { |
1835 | struct direct_shadow_walk *walk = | 1836 | struct kvm_shadow_walk_iterator iterator; |
1836 | container_of(_walk, struct direct_shadow_walk, walker); | ||
1837 | struct kvm_mmu_page *sp; | 1837 | struct kvm_mmu_page *sp; |
1838 | int pt_write = 0; | ||
1838 | gfn_t pseudo_gfn; | 1839 | gfn_t pseudo_gfn; |
1839 | gfn_t gfn = addr >> PAGE_SHIFT; | ||
1840 | |||
1841 | if (level == PT_PAGE_TABLE_LEVEL | ||
1842 | || (walk->largepage && level == PT_DIRECTORY_LEVEL)) { | ||
1843 | mmu_set_spte(vcpu, sptep, ACC_ALL, ACC_ALL, | ||
1844 | 0, walk->write, 1, &walk->pt_write, | ||
1845 | walk->largepage, 0, gfn, walk->pfn, false); | ||
1846 | ++vcpu->stat.pf_fixed; | ||
1847 | return 1; | ||
1848 | } | ||
1849 | 1840 | ||
1850 | if (*sptep == shadow_trap_nonpresent_pte) { | 1841 | for_each_shadow_entry(vcpu, (u64)gfn << PAGE_SHIFT, iterator) { |
1851 | pseudo_gfn = (addr & PT64_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; | 1842 | if (iterator.level == PT_PAGE_TABLE_LEVEL |
1852 | sp = kvm_mmu_get_page(vcpu, pseudo_gfn, (gva_t)addr, level - 1, | 1843 | || (largepage && iterator.level == PT_DIRECTORY_LEVEL)) { |
1853 | 1, ACC_ALL, sptep); | 1844 | mmu_set_spte(vcpu, iterator.sptep, ACC_ALL, ACC_ALL, |
1854 | if (!sp) { | 1845 | 0, write, 1, &pt_write, |
1855 | pgprintk("nonpaging_map: ENOMEM\n"); | 1846 | largepage, 0, gfn, pfn, false); |
1856 | kvm_release_pfn_clean(walk->pfn); | 1847 | ++vcpu->stat.pf_fixed; |
1857 | return -ENOMEM; | 1848 | break; |
1858 | } | 1849 | } |
1859 | 1850 | ||
1860 | set_shadow_pte(sptep, | 1851 | if (*iterator.sptep == shadow_trap_nonpresent_pte) { |
1861 | __pa(sp->spt) | 1852 | pseudo_gfn = (iterator.addr & PT64_DIR_BASE_ADDR_MASK) >> PAGE_SHIFT; |
1862 | | PT_PRESENT_MASK | PT_WRITABLE_MASK | 1853 | sp = kvm_mmu_get_page(vcpu, pseudo_gfn, iterator.addr, |
1863 | | shadow_user_mask | shadow_x_mask); | 1854 | iterator.level - 1, |
1864 | } | 1855 | 1, ACC_ALL, iterator.sptep); |
1865 | return 0; | 1856 | if (!sp) { |
1866 | } | 1857 | pgprintk("nonpaging_map: ENOMEM\n"); |
1858 | kvm_release_pfn_clean(pfn); | ||
1859 | return -ENOMEM; | ||
1860 | } | ||
1867 | 1861 | ||
1868 | static int __direct_map(struct kvm_vcpu *vcpu, gpa_t v, int write, | 1862 | set_shadow_pte(iterator.sptep, |
1869 | int largepage, gfn_t gfn, pfn_t pfn) | 1863 | __pa(sp->spt) |
1870 | { | 1864 | | PT_PRESENT_MASK | PT_WRITABLE_MASK |
1871 | int r; | 1865 | | shadow_user_mask | shadow_x_mask); |
1872 | struct direct_shadow_walk walker = { | 1866 | } |
1873 | .walker = { .entry = direct_map_entry, }, | 1867 | } |
1874 | .pfn = pfn, | 1868 | return pt_write; |
1875 | .largepage = largepage, | ||
1876 | .write = write, | ||
1877 | .pt_write = 0, | ||
1878 | }; | ||
1879 | |||
1880 | r = walk_shadow(&walker.walker, vcpu, gfn << PAGE_SHIFT); | ||
1881 | if (r < 0) | ||
1882 | return r; | ||
1883 | return walker.pt_write; | ||
1884 | } | 1869 | } |
1885 | 1870 | ||
1886 | static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) | 1871 | static int nonpaging_map(struct kvm_vcpu *vcpu, gva_t v, int write, gfn_t gfn) |
@@ -1962,7 +1947,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) | |||
1962 | int i; | 1947 | int i; |
1963 | gfn_t root_gfn; | 1948 | gfn_t root_gfn; |
1964 | struct kvm_mmu_page *sp; | 1949 | struct kvm_mmu_page *sp; |
1965 | int metaphysical = 0; | 1950 | int direct = 0; |
1966 | 1951 | ||
1967 | root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT; | 1952 | root_gfn = vcpu->arch.cr3 >> PAGE_SHIFT; |
1968 | 1953 | ||
@@ -1971,18 +1956,18 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) | |||
1971 | 1956 | ||
1972 | ASSERT(!VALID_PAGE(root)); | 1957 | ASSERT(!VALID_PAGE(root)); |
1973 | if (tdp_enabled) | 1958 | if (tdp_enabled) |
1974 | metaphysical = 1; | 1959 | direct = 1; |
1975 | sp = kvm_mmu_get_page(vcpu, root_gfn, 0, | 1960 | sp = kvm_mmu_get_page(vcpu, root_gfn, 0, |
1976 | PT64_ROOT_LEVEL, metaphysical, | 1961 | PT64_ROOT_LEVEL, direct, |
1977 | ACC_ALL, NULL); | 1962 | ACC_ALL, NULL); |
1978 | root = __pa(sp->spt); | 1963 | root = __pa(sp->spt); |
1979 | ++sp->root_count; | 1964 | ++sp->root_count; |
1980 | vcpu->arch.mmu.root_hpa = root; | 1965 | vcpu->arch.mmu.root_hpa = root; |
1981 | return; | 1966 | return; |
1982 | } | 1967 | } |
1983 | metaphysical = !is_paging(vcpu); | 1968 | direct = !is_paging(vcpu); |
1984 | if (tdp_enabled) | 1969 | if (tdp_enabled) |
1985 | metaphysical = 1; | 1970 | direct = 1; |
1986 | for (i = 0; i < 4; ++i) { | 1971 | for (i = 0; i < 4; ++i) { |
1987 | hpa_t root = vcpu->arch.mmu.pae_root[i]; | 1972 | hpa_t root = vcpu->arch.mmu.pae_root[i]; |
1988 | 1973 | ||
@@ -1996,7 +1981,7 @@ static void mmu_alloc_roots(struct kvm_vcpu *vcpu) | |||
1996 | } else if (vcpu->arch.mmu.root_level == 0) | 1981 | } else if (vcpu->arch.mmu.root_level == 0) |
1997 | root_gfn = 0; | 1982 | root_gfn = 0; |
1998 | sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, | 1983 | sp = kvm_mmu_get_page(vcpu, root_gfn, i << 30, |
1999 | PT32_ROOT_LEVEL, metaphysical, | 1984 | PT32_ROOT_LEVEL, direct, |
2000 | ACC_ALL, NULL); | 1985 | ACC_ALL, NULL); |
2001 | root = __pa(sp->spt); | 1986 | root = __pa(sp->spt); |
2002 | ++sp->root_count; | 1987 | ++sp->root_count; |
@@ -2251,17 +2236,23 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) | |||
2251 | 2236 | ||
2252 | static int init_kvm_softmmu(struct kvm_vcpu *vcpu) | 2237 | static int init_kvm_softmmu(struct kvm_vcpu *vcpu) |
2253 | { | 2238 | { |
2239 | int r; | ||
2240 | |||
2254 | ASSERT(vcpu); | 2241 | ASSERT(vcpu); |
2255 | ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); | 2242 | ASSERT(!VALID_PAGE(vcpu->arch.mmu.root_hpa)); |
2256 | 2243 | ||
2257 | if (!is_paging(vcpu)) | 2244 | if (!is_paging(vcpu)) |
2258 | return nonpaging_init_context(vcpu); | 2245 | r = nonpaging_init_context(vcpu); |
2259 | else if (is_long_mode(vcpu)) | 2246 | else if (is_long_mode(vcpu)) |
2260 | return paging64_init_context(vcpu); | 2247 | r = paging64_init_context(vcpu); |
2261 | else if (is_pae(vcpu)) | 2248 | else if (is_pae(vcpu)) |
2262 | return paging32E_init_context(vcpu); | 2249 | r = paging32E_init_context(vcpu); |
2263 | else | 2250 | else |
2264 | return paging32_init_context(vcpu); | 2251 | r = paging32_init_context(vcpu); |
2252 | |||
2253 | vcpu->arch.mmu.base_role.glevels = vcpu->arch.mmu.root_level; | ||
2254 | |||
2255 | return r; | ||
2265 | } | 2256 | } |
2266 | 2257 | ||
2267 | static int init_kvm_mmu(struct kvm_vcpu *vcpu) | 2258 | static int init_kvm_mmu(struct kvm_vcpu *vcpu) |
@@ -2492,7 +2483,7 @@ void kvm_mmu_pte_write(struct kvm_vcpu *vcpu, gpa_t gpa, | |||
2492 | index = kvm_page_table_hashfn(gfn); | 2483 | index = kvm_page_table_hashfn(gfn); |
2493 | bucket = &vcpu->kvm->arch.mmu_page_hash[index]; | 2484 | bucket = &vcpu->kvm->arch.mmu_page_hash[index]; |
2494 | hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { | 2485 | hlist_for_each_entry_safe(sp, node, n, bucket, hash_link) { |
2495 | if (sp->gfn != gfn || sp->role.metaphysical || sp->role.invalid) | 2486 | if (sp->gfn != gfn || sp->role.direct || sp->role.invalid) |
2496 | continue; | 2487 | continue; |
2497 | pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; | 2488 | pte_size = sp->role.glevels == PT32_ROOT_LEVEL ? 4 : 8; |
2498 | misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); | 2489 | misaligned = (offset ^ (offset + bytes - 1)) & ~(pte_size - 1); |
@@ -3130,7 +3121,7 @@ static void audit_write_protection(struct kvm_vcpu *vcpu) | |||
3130 | gfn_t gfn; | 3121 | gfn_t gfn; |
3131 | 3122 | ||
3132 | list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { | 3123 | list_for_each_entry(sp, &vcpu->kvm->arch.active_mmu_pages, link) { |
3133 | if (sp->role.metaphysical) | 3124 | if (sp->role.direct) |
3134 | continue; | 3125 | continue; |
3135 | 3126 | ||
3136 | gfn = unalias_gfn(vcpu->kvm, sp->gfn); | 3127 | gfn = unalias_gfn(vcpu->kvm, sp->gfn); |
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 258e5d56298e..eaab2145f62b 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h | |||
@@ -54,7 +54,7 @@ static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu) | |||
54 | static inline int is_long_mode(struct kvm_vcpu *vcpu) | 54 | static inline int is_long_mode(struct kvm_vcpu *vcpu) |
55 | { | 55 | { |
56 | #ifdef CONFIG_X86_64 | 56 | #ifdef CONFIG_X86_64 |
57 | return vcpu->arch.shadow_efer & EFER_LME; | 57 | return vcpu->arch.shadow_efer & EFER_LMA; |
58 | #else | 58 | #else |
59 | return 0; | 59 | return 0; |
60 | #endif | 60 | #endif |
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h index 9fd78b6e17ad..6bd70206c561 100644 --- a/arch/x86/kvm/paging_tmpl.h +++ b/arch/x86/kvm/paging_tmpl.h | |||
@@ -25,7 +25,6 @@ | |||
25 | #if PTTYPE == 64 | 25 | #if PTTYPE == 64 |
26 | #define pt_element_t u64 | 26 | #define pt_element_t u64 |
27 | #define guest_walker guest_walker64 | 27 | #define guest_walker guest_walker64 |
28 | #define shadow_walker shadow_walker64 | ||
29 | #define FNAME(name) paging##64_##name | 28 | #define FNAME(name) paging##64_##name |
30 | #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK | 29 | #define PT_BASE_ADDR_MASK PT64_BASE_ADDR_MASK |
31 | #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK | 30 | #define PT_DIR_BASE_ADDR_MASK PT64_DIR_BASE_ADDR_MASK |
@@ -42,7 +41,6 @@ | |||
42 | #elif PTTYPE == 32 | 41 | #elif PTTYPE == 32 |
43 | #define pt_element_t u32 | 42 | #define pt_element_t u32 |
44 | #define guest_walker guest_walker32 | 43 | #define guest_walker guest_walker32 |
45 | #define shadow_walker shadow_walker32 | ||
46 | #define FNAME(name) paging##32_##name | 44 | #define FNAME(name) paging##32_##name |
47 | #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK | 45 | #define PT_BASE_ADDR_MASK PT32_BASE_ADDR_MASK |
48 | #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK | 46 | #define PT_DIR_BASE_ADDR_MASK PT32_DIR_BASE_ADDR_MASK |
@@ -73,18 +71,6 @@ struct guest_walker { | |||
73 | u32 error_code; | 71 | u32 error_code; |
74 | }; | 72 | }; |
75 | 73 | ||
76 | struct shadow_walker { | ||
77 | struct kvm_shadow_walk walker; | ||
78 | struct guest_walker *guest_walker; | ||
79 | int user_fault; | ||
80 | int write_fault; | ||
81 | int largepage; | ||
82 | int *ptwrite; | ||
83 | pfn_t pfn; | ||
84 | u64 *sptep; | ||
85 | gpa_t pte_gpa; | ||
86 | }; | ||
87 | |||
88 | static gfn_t gpte_to_gfn(pt_element_t gpte) | 74 | static gfn_t gpte_to_gfn(pt_element_t gpte) |
89 | { | 75 | { |
90 | return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; | 76 | return (gpte & PT_BASE_ADDR_MASK) >> PAGE_SHIFT; |
@@ -283,91 +269,79 @@ static void FNAME(update_pte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *page, | |||
283 | /* | 269 | /* |
284 | * Fetch a shadow pte for a specific level in the paging hierarchy. | 270 | * Fetch a shadow pte for a specific level in the paging hierarchy. |
285 | */ | 271 | */ |
286 | static int FNAME(shadow_walk_entry)(struct kvm_shadow_walk *_sw, | 272 | static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, |
287 | struct kvm_vcpu *vcpu, u64 addr, | 273 | struct guest_walker *gw, |
288 | u64 *sptep, int level) | 274 | int user_fault, int write_fault, int largepage, |
275 | int *ptwrite, pfn_t pfn) | ||
289 | { | 276 | { |
290 | struct shadow_walker *sw = | ||
291 | container_of(_sw, struct shadow_walker, walker); | ||
292 | struct guest_walker *gw = sw->guest_walker; | ||
293 | unsigned access = gw->pt_access; | 277 | unsigned access = gw->pt_access; |
294 | struct kvm_mmu_page *shadow_page; | 278 | struct kvm_mmu_page *shadow_page; |
295 | u64 spte; | 279 | u64 spte, *sptep; |
296 | int metaphysical; | 280 | int direct; |
297 | gfn_t table_gfn; | 281 | gfn_t table_gfn; |
298 | int r; | 282 | int r; |
283 | int level; | ||
299 | pt_element_t curr_pte; | 284 | pt_element_t curr_pte; |
285 | struct kvm_shadow_walk_iterator iterator; | ||
300 | 286 | ||
301 | if (level == PT_PAGE_TABLE_LEVEL | 287 | if (!is_present_pte(gw->ptes[gw->level - 1])) |
302 | || (sw->largepage && level == PT_DIRECTORY_LEVEL)) { | 288 | return NULL; |
303 | mmu_set_spte(vcpu, sptep, access, gw->pte_access & access, | ||
304 | sw->user_fault, sw->write_fault, | ||
305 | gw->ptes[gw->level-1] & PT_DIRTY_MASK, | ||
306 | sw->ptwrite, sw->largepage, | ||
307 | gw->ptes[gw->level-1] & PT_GLOBAL_MASK, | ||
308 | gw->gfn, sw->pfn, false); | ||
309 | sw->sptep = sptep; | ||
310 | return 1; | ||
311 | } | ||
312 | 289 | ||
313 | if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) | 290 | for_each_shadow_entry(vcpu, addr, iterator) { |
314 | return 0; | 291 | level = iterator.level; |
292 | sptep = iterator.sptep; | ||
293 | if (level == PT_PAGE_TABLE_LEVEL | ||
294 | || (largepage && level == PT_DIRECTORY_LEVEL)) { | ||
295 | mmu_set_spte(vcpu, sptep, access, | ||
296 | gw->pte_access & access, | ||
297 | user_fault, write_fault, | ||
298 | gw->ptes[gw->level-1] & PT_DIRTY_MASK, | ||
299 | ptwrite, largepage, | ||
300 | gw->ptes[gw->level-1] & PT_GLOBAL_MASK, | ||
301 | gw->gfn, pfn, false); | ||
302 | break; | ||
303 | } | ||
315 | 304 | ||
316 | if (is_large_pte(*sptep)) { | 305 | if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) |
317 | set_shadow_pte(sptep, shadow_trap_nonpresent_pte); | 306 | continue; |
318 | kvm_flush_remote_tlbs(vcpu->kvm); | ||
319 | rmap_remove(vcpu->kvm, sptep); | ||
320 | } | ||
321 | 307 | ||
322 | if (level == PT_DIRECTORY_LEVEL && gw->level == PT_DIRECTORY_LEVEL) { | 308 | if (is_large_pte(*sptep)) { |
323 | metaphysical = 1; | 309 | rmap_remove(vcpu->kvm, sptep); |
324 | if (!is_dirty_pte(gw->ptes[level - 1])) | 310 | set_shadow_pte(sptep, shadow_trap_nonpresent_pte); |
325 | access &= ~ACC_WRITE_MASK; | 311 | kvm_flush_remote_tlbs(vcpu->kvm); |
326 | table_gfn = gpte_to_gfn(gw->ptes[level - 1]); | ||
327 | } else { | ||
328 | metaphysical = 0; | ||
329 | table_gfn = gw->table_gfn[level - 2]; | ||
330 | } | ||
331 | shadow_page = kvm_mmu_get_page(vcpu, table_gfn, (gva_t)addr, level-1, | ||
332 | metaphysical, access, sptep); | ||
333 | if (!metaphysical) { | ||
334 | r = kvm_read_guest_atomic(vcpu->kvm, gw->pte_gpa[level - 2], | ||
335 | &curr_pte, sizeof(curr_pte)); | ||
336 | if (r || curr_pte != gw->ptes[level - 2]) { | ||
337 | kvm_mmu_put_page(shadow_page, sptep); | ||
338 | kvm_release_pfn_clean(sw->pfn); | ||
339 | sw->sptep = NULL; | ||
340 | return 1; | ||
341 | } | 312 | } |
342 | } | ||
343 | 313 | ||
344 | spte = __pa(shadow_page->spt) | PT_PRESENT_MASK | PT_ACCESSED_MASK | 314 | if (level == PT_DIRECTORY_LEVEL |
345 | | PT_WRITABLE_MASK | PT_USER_MASK; | 315 | && gw->level == PT_DIRECTORY_LEVEL) { |
346 | *sptep = spte; | 316 | direct = 1; |
347 | return 0; | 317 | if (!is_dirty_pte(gw->ptes[level - 1])) |
348 | } | 318 | access &= ~ACC_WRITE_MASK; |
349 | 319 | table_gfn = gpte_to_gfn(gw->ptes[level - 1]); | |
350 | static u64 *FNAME(fetch)(struct kvm_vcpu *vcpu, gva_t addr, | 320 | } else { |
351 | struct guest_walker *guest_walker, | 321 | direct = 0; |
352 | int user_fault, int write_fault, int largepage, | 322 | table_gfn = gw->table_gfn[level - 2]; |
353 | int *ptwrite, pfn_t pfn) | 323 | } |
354 | { | 324 | shadow_page = kvm_mmu_get_page(vcpu, table_gfn, addr, level-1, |
355 | struct shadow_walker walker = { | 325 | direct, access, sptep); |
356 | .walker = { .entry = FNAME(shadow_walk_entry), }, | 326 | if (!direct) { |
357 | .guest_walker = guest_walker, | 327 | r = kvm_read_guest_atomic(vcpu->kvm, |
358 | .user_fault = user_fault, | 328 | gw->pte_gpa[level - 2], |
359 | .write_fault = write_fault, | 329 | &curr_pte, sizeof(curr_pte)); |
360 | .largepage = largepage, | 330 | if (r || curr_pte != gw->ptes[level - 2]) { |
361 | .ptwrite = ptwrite, | 331 | kvm_mmu_put_page(shadow_page, sptep); |
362 | .pfn = pfn, | 332 | kvm_release_pfn_clean(pfn); |
363 | }; | 333 | sptep = NULL; |
364 | 334 | break; | |
365 | if (!is_present_pte(guest_walker->ptes[guest_walker->level - 1])) | 335 | } |
366 | return NULL; | 336 | } |
367 | 337 | ||
368 | walk_shadow(&walker.walker, vcpu, addr); | 338 | spte = __pa(shadow_page->spt) |
339 | | PT_PRESENT_MASK | PT_ACCESSED_MASK | ||
340 | | PT_WRITABLE_MASK | PT_USER_MASK; | ||
341 | *sptep = spte; | ||
342 | } | ||
369 | 343 | ||
370 | return walker.sptep; | 344 | return sptep; |
371 | } | 345 | } |
372 | 346 | ||
373 | /* | 347 | /* |
@@ -465,54 +439,56 @@ out_unlock: | |||
465 | return 0; | 439 | return 0; |
466 | } | 440 | } |
467 | 441 | ||
468 | static int FNAME(shadow_invlpg_entry)(struct kvm_shadow_walk *_sw, | 442 | static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva) |
469 | struct kvm_vcpu *vcpu, u64 addr, | ||
470 | u64 *sptep, int level) | ||
471 | { | 443 | { |
472 | struct shadow_walker *sw = | 444 | struct kvm_shadow_walk_iterator iterator; |
473 | container_of(_sw, struct shadow_walker, walker); | 445 | pt_element_t gpte; |
474 | 446 | gpa_t pte_gpa = -1; | |
475 | /* FIXME: properly handle invlpg on large guest pages */ | 447 | int level; |
476 | if (level == PT_PAGE_TABLE_LEVEL || | 448 | u64 *sptep; |
477 | ((level == PT_DIRECTORY_LEVEL) && is_large_pte(*sptep))) { | 449 | int need_flush = 0; |
478 | struct kvm_mmu_page *sp = page_header(__pa(sptep)); | ||
479 | 450 | ||
480 | sw->pte_gpa = (sp->gfn << PAGE_SHIFT); | 451 | spin_lock(&vcpu->kvm->mmu_lock); |
481 | sw->pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t); | ||
482 | 452 | ||
483 | if (is_shadow_present_pte(*sptep)) { | 453 | for_each_shadow_entry(vcpu, gva, iterator) { |
484 | rmap_remove(vcpu->kvm, sptep); | 454 | level = iterator.level; |
485 | if (is_large_pte(*sptep)) | 455 | sptep = iterator.sptep; |
486 | --vcpu->kvm->stat.lpages; | 456 | |
457 | /* FIXME: properly handle invlpg on large guest pages */ | ||
458 | if (level == PT_PAGE_TABLE_LEVEL || | ||
459 | ((level == PT_DIRECTORY_LEVEL) && is_large_pte(*sptep))) { | ||
460 | struct kvm_mmu_page *sp = page_header(__pa(sptep)); | ||
461 | |||
462 | pte_gpa = (sp->gfn << PAGE_SHIFT); | ||
463 | pte_gpa += (sptep - sp->spt) * sizeof(pt_element_t); | ||
464 | |||
465 | if (is_shadow_present_pte(*sptep)) { | ||
466 | rmap_remove(vcpu->kvm, sptep); | ||
467 | if (is_large_pte(*sptep)) | ||
468 | --vcpu->kvm->stat.lpages; | ||
469 | need_flush = 1; | ||
470 | } | ||
471 | set_shadow_pte(sptep, shadow_trap_nonpresent_pte); | ||
472 | break; | ||
487 | } | 473 | } |
488 | set_shadow_pte(sptep, shadow_trap_nonpresent_pte); | ||
489 | return 1; | ||
490 | } | ||
491 | if (!is_shadow_present_pte(*sptep)) | ||
492 | return 1; | ||
493 | return 0; | ||
494 | } | ||
495 | 474 | ||
496 | static void FNAME(invlpg)(struct kvm_vcpu *vcpu, gva_t gva) | 475 | if (!is_shadow_present_pte(*sptep)) |
497 | { | 476 | break; |
498 | pt_element_t gpte; | 477 | } |
499 | struct shadow_walker walker = { | ||
500 | .walker = { .entry = FNAME(shadow_invlpg_entry), }, | ||
501 | .pte_gpa = -1, | ||
502 | }; | ||
503 | 478 | ||
504 | spin_lock(&vcpu->kvm->mmu_lock); | 479 | if (need_flush) |
505 | walk_shadow(&walker.walker, vcpu, gva); | 480 | kvm_flush_remote_tlbs(vcpu->kvm); |
506 | spin_unlock(&vcpu->kvm->mmu_lock); | 481 | spin_unlock(&vcpu->kvm->mmu_lock); |
507 | if (walker.pte_gpa == -1) | 482 | |
483 | if (pte_gpa == -1) | ||
508 | return; | 484 | return; |
509 | if (kvm_read_guest_atomic(vcpu->kvm, walker.pte_gpa, &gpte, | 485 | if (kvm_read_guest_atomic(vcpu->kvm, pte_gpa, &gpte, |
510 | sizeof(pt_element_t))) | 486 | sizeof(pt_element_t))) |
511 | return; | 487 | return; |
512 | if (is_present_pte(gpte) && (gpte & PT_ACCESSED_MASK)) { | 488 | if (is_present_pte(gpte) && (gpte & PT_ACCESSED_MASK)) { |
513 | if (mmu_topup_memory_caches(vcpu)) | 489 | if (mmu_topup_memory_caches(vcpu)) |
514 | return; | 490 | return; |
515 | kvm_mmu_pte_write(vcpu, walker.pte_gpa, (const u8 *)&gpte, | 491 | kvm_mmu_pte_write(vcpu, pte_gpa, (const u8 *)&gpte, |
516 | sizeof(pt_element_t), 0); | 492 | sizeof(pt_element_t), 0); |
517 | } | 493 | } |
518 | } | 494 | } |
@@ -540,7 +516,7 @@ static void FNAME(prefetch_page)(struct kvm_vcpu *vcpu, | |||
540 | pt_element_t pt[256 / sizeof(pt_element_t)]; | 516 | pt_element_t pt[256 / sizeof(pt_element_t)]; |
541 | gpa_t pte_gpa; | 517 | gpa_t pte_gpa; |
542 | 518 | ||
543 | if (sp->role.metaphysical | 519 | if (sp->role.direct |
544 | || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { | 520 | || (PTTYPE == 32 && sp->role.level > PT_PAGE_TABLE_LEVEL)) { |
545 | nonpaging_prefetch_page(vcpu, sp); | 521 | nonpaging_prefetch_page(vcpu, sp); |
546 | return; | 522 | return; |
@@ -619,7 +595,6 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) | |||
619 | 595 | ||
620 | #undef pt_element_t | 596 | #undef pt_element_t |
621 | #undef guest_walker | 597 | #undef guest_walker |
622 | #undef shadow_walker | ||
623 | #undef FNAME | 598 | #undef FNAME |
624 | #undef PT_BASE_ADDR_MASK | 599 | #undef PT_BASE_ADDR_MASK |
625 | #undef PT_INDEX | 600 | #undef PT_INDEX |
diff --git a/arch/x86/kvm/svm.c b/arch/x86/kvm/svm.c index a9e769e4e251..1821c2078199 100644 --- a/arch/x86/kvm/svm.c +++ b/arch/x86/kvm/svm.c | |||
@@ -38,9 +38,6 @@ MODULE_LICENSE("GPL"); | |||
38 | #define IOPM_ALLOC_ORDER 2 | 38 | #define IOPM_ALLOC_ORDER 2 |
39 | #define MSRPM_ALLOC_ORDER 1 | 39 | #define MSRPM_ALLOC_ORDER 1 |
40 | 40 | ||
41 | #define DR7_GD_MASK (1 << 13) | ||
42 | #define DR6_BD_MASK (1 << 13) | ||
43 | |||
44 | #define SEG_TYPE_LDT 2 | 41 | #define SEG_TYPE_LDT 2 |
45 | #define SEG_TYPE_BUSY_TSS16 3 | 42 | #define SEG_TYPE_BUSY_TSS16 3 |
46 | 43 | ||
@@ -50,6 +47,15 @@ MODULE_LICENSE("GPL"); | |||
50 | 47 | ||
51 | #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) | 48 | #define DEBUGCTL_RESERVED_BITS (~(0x3fULL)) |
52 | 49 | ||
50 | /* Turn on to get debugging output*/ | ||
51 | /* #define NESTED_DEBUG */ | ||
52 | |||
53 | #ifdef NESTED_DEBUG | ||
54 | #define nsvm_printk(fmt, args...) printk(KERN_INFO fmt, ## args) | ||
55 | #else | ||
56 | #define nsvm_printk(fmt, args...) do {} while(0) | ||
57 | #endif | ||
58 | |||
53 | /* enable NPT for AMD64 and X86 with PAE */ | 59 | /* enable NPT for AMD64 and X86 with PAE */ |
54 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) | 60 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
55 | static bool npt_enabled = true; | 61 | static bool npt_enabled = true; |
@@ -60,14 +66,29 @@ static int npt = 1; | |||
60 | 66 | ||
61 | module_param(npt, int, S_IRUGO); | 67 | module_param(npt, int, S_IRUGO); |
62 | 68 | ||
69 | static int nested = 0; | ||
70 | module_param(nested, int, S_IRUGO); | ||
71 | |||
63 | static void kvm_reput_irq(struct vcpu_svm *svm); | 72 | static void kvm_reput_irq(struct vcpu_svm *svm); |
64 | static void svm_flush_tlb(struct kvm_vcpu *vcpu); | 73 | static void svm_flush_tlb(struct kvm_vcpu *vcpu); |
65 | 74 | ||
75 | static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override); | ||
76 | static int nested_svm_vmexit(struct vcpu_svm *svm); | ||
77 | static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb, | ||
78 | void *arg2, void *opaque); | ||
79 | static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, | ||
80 | bool has_error_code, u32 error_code); | ||
81 | |||
66 | static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) | 82 | static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu) |
67 | { | 83 | { |
68 | return container_of(vcpu, struct vcpu_svm, vcpu); | 84 | return container_of(vcpu, struct vcpu_svm, vcpu); |
69 | } | 85 | } |
70 | 86 | ||
87 | static inline bool is_nested(struct vcpu_svm *svm) | ||
88 | { | ||
89 | return svm->nested_vmcb; | ||
90 | } | ||
91 | |||
71 | static unsigned long iopm_base; | 92 | static unsigned long iopm_base; |
72 | 93 | ||
73 | struct kvm_ldttss_desc { | 94 | struct kvm_ldttss_desc { |
@@ -157,32 +178,6 @@ static inline void kvm_write_cr2(unsigned long val) | |||
157 | asm volatile ("mov %0, %%cr2" :: "r" (val)); | 178 | asm volatile ("mov %0, %%cr2" :: "r" (val)); |
158 | } | 179 | } |
159 | 180 | ||
160 | static inline unsigned long read_dr6(void) | ||
161 | { | ||
162 | unsigned long dr6; | ||
163 | |||
164 | asm volatile ("mov %%dr6, %0" : "=r" (dr6)); | ||
165 | return dr6; | ||
166 | } | ||
167 | |||
168 | static inline void write_dr6(unsigned long val) | ||
169 | { | ||
170 | asm volatile ("mov %0, %%dr6" :: "r" (val)); | ||
171 | } | ||
172 | |||
173 | static inline unsigned long read_dr7(void) | ||
174 | { | ||
175 | unsigned long dr7; | ||
176 | |||
177 | asm volatile ("mov %%dr7, %0" : "=r" (dr7)); | ||
178 | return dr7; | ||
179 | } | ||
180 | |||
181 | static inline void write_dr7(unsigned long val) | ||
182 | { | ||
183 | asm volatile ("mov %0, %%dr7" :: "r" (val)); | ||
184 | } | ||
185 | |||
186 | static inline void force_new_asid(struct kvm_vcpu *vcpu) | 181 | static inline void force_new_asid(struct kvm_vcpu *vcpu) |
187 | { | 182 | { |
188 | to_svm(vcpu)->asid_generation--; | 183 | to_svm(vcpu)->asid_generation--; |
@@ -198,7 +193,7 @@ static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer) | |||
198 | if (!npt_enabled && !(efer & EFER_LMA)) | 193 | if (!npt_enabled && !(efer & EFER_LMA)) |
199 | efer &= ~EFER_LME; | 194 | efer &= ~EFER_LME; |
200 | 195 | ||
201 | to_svm(vcpu)->vmcb->save.efer = efer | MSR_EFER_SVME_MASK; | 196 | to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME; |
202 | vcpu->arch.shadow_efer = efer; | 197 | vcpu->arch.shadow_efer = efer; |
203 | } | 198 | } |
204 | 199 | ||
@@ -207,6 +202,11 @@ static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, | |||
207 | { | 202 | { |
208 | struct vcpu_svm *svm = to_svm(vcpu); | 203 | struct vcpu_svm *svm = to_svm(vcpu); |
209 | 204 | ||
205 | /* If we are within a nested VM we'd better #VMEXIT and let the | ||
206 | guest handle the exception */ | ||
207 | if (nested_svm_check_exception(svm, nr, has_error_code, error_code)) | ||
208 | return; | ||
209 | |||
210 | svm->vmcb->control.event_inj = nr | 210 | svm->vmcb->control.event_inj = nr |
211 | | SVM_EVTINJ_VALID | 211 | | SVM_EVTINJ_VALID |
212 | | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) | 212 | | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0) |
@@ -242,7 +242,7 @@ static void skip_emulated_instruction(struct kvm_vcpu *vcpu) | |||
242 | kvm_rip_write(vcpu, svm->next_rip); | 242 | kvm_rip_write(vcpu, svm->next_rip); |
243 | svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; | 243 | svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; |
244 | 244 | ||
245 | vcpu->arch.interrupt_window_open = 1; | 245 | vcpu->arch.interrupt_window_open = (svm->vcpu.arch.hflags & HF_GIF_MASK); |
246 | } | 246 | } |
247 | 247 | ||
248 | static int has_svm(void) | 248 | static int has_svm(void) |
@@ -250,7 +250,7 @@ static int has_svm(void) | |||
250 | const char *msg; | 250 | const char *msg; |
251 | 251 | ||
252 | if (!cpu_has_svm(&msg)) { | 252 | if (!cpu_has_svm(&msg)) { |
253 | printk(KERN_INFO "has_svn: %s\n", msg); | 253 | printk(KERN_INFO "has_svm: %s\n", msg); |
254 | return 0; | 254 | return 0; |
255 | } | 255 | } |
256 | 256 | ||
@@ -292,7 +292,7 @@ static void svm_hardware_enable(void *garbage) | |||
292 | svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); | 292 | svm_data->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS); |
293 | 293 | ||
294 | rdmsrl(MSR_EFER, efer); | 294 | rdmsrl(MSR_EFER, efer); |
295 | wrmsrl(MSR_EFER, efer | MSR_EFER_SVME_MASK); | 295 | wrmsrl(MSR_EFER, efer | EFER_SVME); |
296 | 296 | ||
297 | wrmsrl(MSR_VM_HSAVE_PA, | 297 | wrmsrl(MSR_VM_HSAVE_PA, |
298 | page_to_pfn(svm_data->save_area) << PAGE_SHIFT); | 298 | page_to_pfn(svm_data->save_area) << PAGE_SHIFT); |
@@ -417,6 +417,14 @@ static __init int svm_hardware_setup(void) | |||
417 | if (boot_cpu_has(X86_FEATURE_NX)) | 417 | if (boot_cpu_has(X86_FEATURE_NX)) |
418 | kvm_enable_efer_bits(EFER_NX); | 418 | kvm_enable_efer_bits(EFER_NX); |
419 | 419 | ||
420 | if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) | ||
421 | kvm_enable_efer_bits(EFER_FFXSR); | ||
422 | |||
423 | if (nested) { | ||
424 | printk(KERN_INFO "kvm: Nested Virtualization enabled\n"); | ||
425 | kvm_enable_efer_bits(EFER_SVME); | ||
426 | } | ||
427 | |||
420 | for_each_online_cpu(cpu) { | 428 | for_each_online_cpu(cpu) { |
421 | r = svm_cpu_init(cpu); | 429 | r = svm_cpu_init(cpu); |
422 | if (r) | 430 | if (r) |
@@ -559,7 +567,7 @@ static void init_vmcb(struct vcpu_svm *svm) | |||
559 | init_sys_seg(&save->ldtr, SEG_TYPE_LDT); | 567 | init_sys_seg(&save->ldtr, SEG_TYPE_LDT); |
560 | init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); | 568 | init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); |
561 | 569 | ||
562 | save->efer = MSR_EFER_SVME_MASK; | 570 | save->efer = EFER_SVME; |
563 | save->dr6 = 0xffff0ff0; | 571 | save->dr6 = 0xffff0ff0; |
564 | save->dr7 = 0x400; | 572 | save->dr7 = 0x400; |
565 | save->rflags = 2; | 573 | save->rflags = 2; |
@@ -591,6 +599,9 @@ static void init_vmcb(struct vcpu_svm *svm) | |||
591 | save->cr4 = 0; | 599 | save->cr4 = 0; |
592 | } | 600 | } |
593 | force_new_asid(&svm->vcpu); | 601 | force_new_asid(&svm->vcpu); |
602 | |||
603 | svm->nested_vmcb = 0; | ||
604 | svm->vcpu.arch.hflags = HF_GIF_MASK; | ||
594 | } | 605 | } |
595 | 606 | ||
596 | static int svm_vcpu_reset(struct kvm_vcpu *vcpu) | 607 | static int svm_vcpu_reset(struct kvm_vcpu *vcpu) |
@@ -615,6 +626,8 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) | |||
615 | struct vcpu_svm *svm; | 626 | struct vcpu_svm *svm; |
616 | struct page *page; | 627 | struct page *page; |
617 | struct page *msrpm_pages; | 628 | struct page *msrpm_pages; |
629 | struct page *hsave_page; | ||
630 | struct page *nested_msrpm_pages; | ||
618 | int err; | 631 | int err; |
619 | 632 | ||
620 | svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); | 633 | svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); |
@@ -637,14 +650,25 @@ static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id) | |||
637 | msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); | 650 | msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); |
638 | if (!msrpm_pages) | 651 | if (!msrpm_pages) |
639 | goto uninit; | 652 | goto uninit; |
653 | |||
654 | nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER); | ||
655 | if (!nested_msrpm_pages) | ||
656 | goto uninit; | ||
657 | |||
640 | svm->msrpm = page_address(msrpm_pages); | 658 | svm->msrpm = page_address(msrpm_pages); |
641 | svm_vcpu_init_msrpm(svm->msrpm); | 659 | svm_vcpu_init_msrpm(svm->msrpm); |
642 | 660 | ||
661 | hsave_page = alloc_page(GFP_KERNEL); | ||
662 | if (!hsave_page) | ||
663 | goto uninit; | ||
664 | svm->hsave = page_address(hsave_page); | ||
665 | |||
666 | svm->nested_msrpm = page_address(nested_msrpm_pages); | ||
667 | |||
643 | svm->vmcb = page_address(page); | 668 | svm->vmcb = page_address(page); |
644 | clear_page(svm->vmcb); | 669 | clear_page(svm->vmcb); |
645 | svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT; | 670 | svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT; |
646 | svm->asid_generation = 0; | 671 | svm->asid_generation = 0; |
647 | memset(svm->db_regs, 0, sizeof(svm->db_regs)); | ||
648 | init_vmcb(svm); | 672 | init_vmcb(svm); |
649 | 673 | ||
650 | fx_init(&svm->vcpu); | 674 | fx_init(&svm->vcpu); |
@@ -669,6 +693,8 @@ static void svm_free_vcpu(struct kvm_vcpu *vcpu) | |||
669 | 693 | ||
670 | __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT)); | 694 | __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT)); |
671 | __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER); | 695 | __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER); |
696 | __free_page(virt_to_page(svm->hsave)); | ||
697 | __free_pages(virt_to_page(svm->nested_msrpm), MSRPM_ALLOC_ORDER); | ||
672 | kvm_vcpu_uninit(vcpu); | 698 | kvm_vcpu_uninit(vcpu); |
673 | kmem_cache_free(kvm_vcpu_cache, svm); | 699 | kmem_cache_free(kvm_vcpu_cache, svm); |
674 | } | 700 | } |
@@ -718,6 +744,16 @@ static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags) | |||
718 | to_svm(vcpu)->vmcb->save.rflags = rflags; | 744 | to_svm(vcpu)->vmcb->save.rflags = rflags; |
719 | } | 745 | } |
720 | 746 | ||
747 | static void svm_set_vintr(struct vcpu_svm *svm) | ||
748 | { | ||
749 | svm->vmcb->control.intercept |= 1ULL << INTERCEPT_VINTR; | ||
750 | } | ||
751 | |||
752 | static void svm_clear_vintr(struct vcpu_svm *svm) | ||
753 | { | ||
754 | svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); | ||
755 | } | ||
756 | |||
721 | static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) | 757 | static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg) |
722 | { | 758 | { |
723 | struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; | 759 | struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; |
@@ -760,20 +796,37 @@ static void svm_get_segment(struct kvm_vcpu *vcpu, | |||
760 | var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; | 796 | var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; |
761 | var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1; | 797 | var->g = (s->attrib >> SVM_SELECTOR_G_SHIFT) & 1; |
762 | 798 | ||
763 | /* | 799 | switch (seg) { |
764 | * SVM always stores 0 for the 'G' bit in the CS selector in | 800 | case VCPU_SREG_CS: |
765 | * the VMCB on a VMEXIT. This hurts cross-vendor migration: | 801 | /* |
766 | * Intel's VMENTRY has a check on the 'G' bit. | 802 | * SVM always stores 0 for the 'G' bit in the CS selector in |
767 | */ | 803 | * the VMCB on a VMEXIT. This hurts cross-vendor migration: |
768 | if (seg == VCPU_SREG_CS) | 804 | * Intel's VMENTRY has a check on the 'G' bit. |
805 | */ | ||
769 | var->g = s->limit > 0xfffff; | 806 | var->g = s->limit > 0xfffff; |
770 | 807 | break; | |
771 | /* | 808 | case VCPU_SREG_TR: |
772 | * Work around a bug where the busy flag in the tr selector | 809 | /* |
773 | * isn't exposed | 810 | * Work around a bug where the busy flag in the tr selector |
774 | */ | 811 | * isn't exposed |
775 | if (seg == VCPU_SREG_TR) | 812 | */ |
776 | var->type |= 0x2; | 813 | var->type |= 0x2; |
814 | break; | ||
815 | case VCPU_SREG_DS: | ||
816 | case VCPU_SREG_ES: | ||
817 | case VCPU_SREG_FS: | ||
818 | case VCPU_SREG_GS: | ||
819 | /* | ||
820 | * The accessed bit must always be set in the segment | ||
821 | * descriptor cache, although it can be cleared in the | ||
822 | * descriptor, the cached bit always remains at 1. Since | ||
823 | * Intel has a check on this, set it here to support | ||
824 | * cross-vendor migration. | ||
825 | */ | ||
826 | if (!var->unusable) | ||
827 | var->type |= 0x1; | ||
828 | break; | ||
829 | } | ||
777 | 830 | ||
778 | var->unusable = !var->present; | 831 | var->unusable = !var->present; |
779 | } | 832 | } |
@@ -905,9 +958,37 @@ static void svm_set_segment(struct kvm_vcpu *vcpu, | |||
905 | 958 | ||
906 | } | 959 | } |
907 | 960 | ||
908 | static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) | 961 | static int svm_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) |
909 | { | 962 | { |
910 | return -EOPNOTSUPP; | 963 | int old_debug = vcpu->guest_debug; |
964 | struct vcpu_svm *svm = to_svm(vcpu); | ||
965 | |||
966 | vcpu->guest_debug = dbg->control; | ||
967 | |||
968 | svm->vmcb->control.intercept_exceptions &= | ||
969 | ~((1 << DB_VECTOR) | (1 << BP_VECTOR)); | ||
970 | if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { | ||
971 | if (vcpu->guest_debug & | ||
972 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) | ||
973 | svm->vmcb->control.intercept_exceptions |= | ||
974 | 1 << DB_VECTOR; | ||
975 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) | ||
976 | svm->vmcb->control.intercept_exceptions |= | ||
977 | 1 << BP_VECTOR; | ||
978 | } else | ||
979 | vcpu->guest_debug = 0; | ||
980 | |||
981 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | ||
982 | svm->vmcb->save.dr7 = dbg->arch.debugreg[7]; | ||
983 | else | ||
984 | svm->vmcb->save.dr7 = vcpu->arch.dr7; | ||
985 | |||
986 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
987 | svm->vmcb->save.rflags |= X86_EFLAGS_TF | X86_EFLAGS_RF; | ||
988 | else if (old_debug & KVM_GUESTDBG_SINGLESTEP) | ||
989 | svm->vmcb->save.rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | ||
990 | |||
991 | return 0; | ||
911 | } | 992 | } |
912 | 993 | ||
913 | static int svm_get_irq(struct kvm_vcpu *vcpu) | 994 | static int svm_get_irq(struct kvm_vcpu *vcpu) |
@@ -949,7 +1030,29 @@ static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *svm_data) | |||
949 | 1030 | ||
950 | static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr) | 1031 | static unsigned long svm_get_dr(struct kvm_vcpu *vcpu, int dr) |
951 | { | 1032 | { |
952 | unsigned long val = to_svm(vcpu)->db_regs[dr]; | 1033 | struct vcpu_svm *svm = to_svm(vcpu); |
1034 | unsigned long val; | ||
1035 | |||
1036 | switch (dr) { | ||
1037 | case 0 ... 3: | ||
1038 | val = vcpu->arch.db[dr]; | ||
1039 | break; | ||
1040 | case 6: | ||
1041 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | ||
1042 | val = vcpu->arch.dr6; | ||
1043 | else | ||
1044 | val = svm->vmcb->save.dr6; | ||
1045 | break; | ||
1046 | case 7: | ||
1047 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) | ||
1048 | val = vcpu->arch.dr7; | ||
1049 | else | ||
1050 | val = svm->vmcb->save.dr7; | ||
1051 | break; | ||
1052 | default: | ||
1053 | val = 0; | ||
1054 | } | ||
1055 | |||
953 | KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler); | 1056 | KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler); |
954 | return val; | 1057 | return val; |
955 | } | 1058 | } |
@@ -959,33 +1062,40 @@ static void svm_set_dr(struct kvm_vcpu *vcpu, int dr, unsigned long value, | |||
959 | { | 1062 | { |
960 | struct vcpu_svm *svm = to_svm(vcpu); | 1063 | struct vcpu_svm *svm = to_svm(vcpu); |
961 | 1064 | ||
962 | *exception = 0; | 1065 | KVMTRACE_2D(DR_WRITE, vcpu, (u32)dr, (u32)value, handler); |
963 | 1066 | ||
964 | if (svm->vmcb->save.dr7 & DR7_GD_MASK) { | 1067 | *exception = 0; |
965 | svm->vmcb->save.dr7 &= ~DR7_GD_MASK; | ||
966 | svm->vmcb->save.dr6 |= DR6_BD_MASK; | ||
967 | *exception = DB_VECTOR; | ||
968 | return; | ||
969 | } | ||
970 | 1068 | ||
971 | switch (dr) { | 1069 | switch (dr) { |
972 | case 0 ... 3: | 1070 | case 0 ... 3: |
973 | svm->db_regs[dr] = value; | 1071 | vcpu->arch.db[dr] = value; |
1072 | if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) | ||
1073 | vcpu->arch.eff_db[dr] = value; | ||
974 | return; | 1074 | return; |
975 | case 4 ... 5: | 1075 | case 4 ... 5: |
976 | if (vcpu->arch.cr4 & X86_CR4_DE) { | 1076 | if (vcpu->arch.cr4 & X86_CR4_DE) |
977 | *exception = UD_VECTOR; | 1077 | *exception = UD_VECTOR; |
1078 | return; | ||
1079 | case 6: | ||
1080 | if (value & 0xffffffff00000000ULL) { | ||
1081 | *exception = GP_VECTOR; | ||
978 | return; | 1082 | return; |
979 | } | 1083 | } |
980 | case 7: { | 1084 | vcpu->arch.dr6 = (value & DR6_VOLATILE) | DR6_FIXED_1; |
981 | if (value & ~((1ULL << 32) - 1)) { | 1085 | return; |
1086 | case 7: | ||
1087 | if (value & 0xffffffff00000000ULL) { | ||
982 | *exception = GP_VECTOR; | 1088 | *exception = GP_VECTOR; |
983 | return; | 1089 | return; |
984 | } | 1090 | } |
985 | svm->vmcb->save.dr7 = value; | 1091 | vcpu->arch.dr7 = (value & DR7_VOLATILE) | DR7_FIXED_1; |
1092 | if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { | ||
1093 | svm->vmcb->save.dr7 = vcpu->arch.dr7; | ||
1094 | vcpu->arch.switch_db_regs = (value & DR7_BP_EN_MASK); | ||
1095 | } | ||
986 | return; | 1096 | return; |
987 | } | ||
988 | default: | 1097 | default: |
1098 | /* FIXME: Possible case? */ | ||
989 | printk(KERN_DEBUG "%s: unexpected dr %u\n", | 1099 | printk(KERN_DEBUG "%s: unexpected dr %u\n", |
990 | __func__, dr); | 1100 | __func__, dr); |
991 | *exception = UD_VECTOR; | 1101 | *exception = UD_VECTOR; |
@@ -1031,6 +1141,27 @@ static int pf_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1031 | return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); | 1141 | return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code); |
1032 | } | 1142 | } |
1033 | 1143 | ||
1144 | static int db_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1145 | { | ||
1146 | if (!(svm->vcpu.guest_debug & | ||
1147 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { | ||
1148 | kvm_queue_exception(&svm->vcpu, DB_VECTOR); | ||
1149 | return 1; | ||
1150 | } | ||
1151 | kvm_run->exit_reason = KVM_EXIT_DEBUG; | ||
1152 | kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; | ||
1153 | kvm_run->debug.arch.exception = DB_VECTOR; | ||
1154 | return 0; | ||
1155 | } | ||
1156 | |||
1157 | static int bp_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1158 | { | ||
1159 | kvm_run->exit_reason = KVM_EXIT_DEBUG; | ||
1160 | kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; | ||
1161 | kvm_run->debug.arch.exception = BP_VECTOR; | ||
1162 | return 0; | ||
1163 | } | ||
1164 | |||
1034 | static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1165 | static int ud_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) |
1035 | { | 1166 | { |
1036 | int er; | 1167 | int er; |
@@ -1080,7 +1211,7 @@ static int shutdown_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1080 | static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | 1211 | static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) |
1081 | { | 1212 | { |
1082 | u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ | 1213 | u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ |
1083 | int size, down, in, string, rep; | 1214 | int size, in, string; |
1084 | unsigned port; | 1215 | unsigned port; |
1085 | 1216 | ||
1086 | ++svm->vcpu.stat.io_exits; | 1217 | ++svm->vcpu.stat.io_exits; |
@@ -1099,8 +1230,6 @@ static int io_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1099 | in = (io_info & SVM_IOIO_TYPE_MASK) != 0; | 1230 | in = (io_info & SVM_IOIO_TYPE_MASK) != 0; |
1100 | port = io_info >> 16; | 1231 | port = io_info >> 16; |
1101 | size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; | 1232 | size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT; |
1102 | rep = (io_info & SVM_IOIO_REP_MASK) != 0; | ||
1103 | down = (svm->vmcb->save.rflags & X86_EFLAGS_DF) != 0; | ||
1104 | 1233 | ||
1105 | skip_emulated_instruction(&svm->vcpu); | 1234 | skip_emulated_instruction(&svm->vcpu); |
1106 | return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port); | 1235 | return kvm_emulate_pio(&svm->vcpu, kvm_run, in, size, port); |
@@ -1139,6 +1268,567 @@ static int vmmcall_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | |||
1139 | return 1; | 1268 | return 1; |
1140 | } | 1269 | } |
1141 | 1270 | ||
1271 | static int nested_svm_check_permissions(struct vcpu_svm *svm) | ||
1272 | { | ||
1273 | if (!(svm->vcpu.arch.shadow_efer & EFER_SVME) | ||
1274 | || !is_paging(&svm->vcpu)) { | ||
1275 | kvm_queue_exception(&svm->vcpu, UD_VECTOR); | ||
1276 | return 1; | ||
1277 | } | ||
1278 | |||
1279 | if (svm->vmcb->save.cpl) { | ||
1280 | kvm_inject_gp(&svm->vcpu, 0); | ||
1281 | return 1; | ||
1282 | } | ||
1283 | |||
1284 | return 0; | ||
1285 | } | ||
1286 | |||
1287 | static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr, | ||
1288 | bool has_error_code, u32 error_code) | ||
1289 | { | ||
1290 | if (is_nested(svm)) { | ||
1291 | svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr; | ||
1292 | svm->vmcb->control.exit_code_hi = 0; | ||
1293 | svm->vmcb->control.exit_info_1 = error_code; | ||
1294 | svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2; | ||
1295 | if (nested_svm_exit_handled(svm, false)) { | ||
1296 | nsvm_printk("VMexit -> EXCP 0x%x\n", nr); | ||
1297 | |||
1298 | nested_svm_vmexit(svm); | ||
1299 | return 1; | ||
1300 | } | ||
1301 | } | ||
1302 | |||
1303 | return 0; | ||
1304 | } | ||
1305 | |||
1306 | static inline int nested_svm_intr(struct vcpu_svm *svm) | ||
1307 | { | ||
1308 | if (is_nested(svm)) { | ||
1309 | if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) | ||
1310 | return 0; | ||
1311 | |||
1312 | if (!(svm->vcpu.arch.hflags & HF_HIF_MASK)) | ||
1313 | return 0; | ||
1314 | |||
1315 | svm->vmcb->control.exit_code = SVM_EXIT_INTR; | ||
1316 | |||
1317 | if (nested_svm_exit_handled(svm, false)) { | ||
1318 | nsvm_printk("VMexit -> INTR\n"); | ||
1319 | nested_svm_vmexit(svm); | ||
1320 | return 1; | ||
1321 | } | ||
1322 | } | ||
1323 | |||
1324 | return 0; | ||
1325 | } | ||
1326 | |||
1327 | static struct page *nested_svm_get_page(struct vcpu_svm *svm, u64 gpa) | ||
1328 | { | ||
1329 | struct page *page; | ||
1330 | |||
1331 | down_read(¤t->mm->mmap_sem); | ||
1332 | page = gfn_to_page(svm->vcpu.kvm, gpa >> PAGE_SHIFT); | ||
1333 | up_read(¤t->mm->mmap_sem); | ||
1334 | |||
1335 | if (is_error_page(page)) { | ||
1336 | printk(KERN_INFO "%s: could not find page at 0x%llx\n", | ||
1337 | __func__, gpa); | ||
1338 | kvm_release_page_clean(page); | ||
1339 | kvm_inject_gp(&svm->vcpu, 0); | ||
1340 | return NULL; | ||
1341 | } | ||
1342 | return page; | ||
1343 | } | ||
1344 | |||
1345 | static int nested_svm_do(struct vcpu_svm *svm, | ||
1346 | u64 arg1_gpa, u64 arg2_gpa, void *opaque, | ||
1347 | int (*handler)(struct vcpu_svm *svm, | ||
1348 | void *arg1, | ||
1349 | void *arg2, | ||
1350 | void *opaque)) | ||
1351 | { | ||
1352 | struct page *arg1_page; | ||
1353 | struct page *arg2_page = NULL; | ||
1354 | void *arg1; | ||
1355 | void *arg2 = NULL; | ||
1356 | int retval; | ||
1357 | |||
1358 | arg1_page = nested_svm_get_page(svm, arg1_gpa); | ||
1359 | if(arg1_page == NULL) | ||
1360 | return 1; | ||
1361 | |||
1362 | if (arg2_gpa) { | ||
1363 | arg2_page = nested_svm_get_page(svm, arg2_gpa); | ||
1364 | if(arg2_page == NULL) { | ||
1365 | kvm_release_page_clean(arg1_page); | ||
1366 | return 1; | ||
1367 | } | ||
1368 | } | ||
1369 | |||
1370 | arg1 = kmap_atomic(arg1_page, KM_USER0); | ||
1371 | if (arg2_gpa) | ||
1372 | arg2 = kmap_atomic(arg2_page, KM_USER1); | ||
1373 | |||
1374 | retval = handler(svm, arg1, arg2, opaque); | ||
1375 | |||
1376 | kunmap_atomic(arg1, KM_USER0); | ||
1377 | if (arg2_gpa) | ||
1378 | kunmap_atomic(arg2, KM_USER1); | ||
1379 | |||
1380 | kvm_release_page_dirty(arg1_page); | ||
1381 | if (arg2_gpa) | ||
1382 | kvm_release_page_dirty(arg2_page); | ||
1383 | |||
1384 | return retval; | ||
1385 | } | ||
1386 | |||
1387 | static int nested_svm_exit_handled_real(struct vcpu_svm *svm, | ||
1388 | void *arg1, | ||
1389 | void *arg2, | ||
1390 | void *opaque) | ||
1391 | { | ||
1392 | struct vmcb *nested_vmcb = (struct vmcb *)arg1; | ||
1393 | bool kvm_overrides = *(bool *)opaque; | ||
1394 | u32 exit_code = svm->vmcb->control.exit_code; | ||
1395 | |||
1396 | if (kvm_overrides) { | ||
1397 | switch (exit_code) { | ||
1398 | case SVM_EXIT_INTR: | ||
1399 | case SVM_EXIT_NMI: | ||
1400 | return 0; | ||
1401 | /* For now we are always handling NPFs when using them */ | ||
1402 | case SVM_EXIT_NPF: | ||
1403 | if (npt_enabled) | ||
1404 | return 0; | ||
1405 | break; | ||
1406 | /* When we're shadowing, trap PFs */ | ||
1407 | case SVM_EXIT_EXCP_BASE + PF_VECTOR: | ||
1408 | if (!npt_enabled) | ||
1409 | return 0; | ||
1410 | break; | ||
1411 | default: | ||
1412 | break; | ||
1413 | } | ||
1414 | } | ||
1415 | |||
1416 | switch (exit_code) { | ||
1417 | case SVM_EXIT_READ_CR0 ... SVM_EXIT_READ_CR8: { | ||
1418 | u32 cr_bits = 1 << (exit_code - SVM_EXIT_READ_CR0); | ||
1419 | if (nested_vmcb->control.intercept_cr_read & cr_bits) | ||
1420 | return 1; | ||
1421 | break; | ||
1422 | } | ||
1423 | case SVM_EXIT_WRITE_CR0 ... SVM_EXIT_WRITE_CR8: { | ||
1424 | u32 cr_bits = 1 << (exit_code - SVM_EXIT_WRITE_CR0); | ||
1425 | if (nested_vmcb->control.intercept_cr_write & cr_bits) | ||
1426 | return 1; | ||
1427 | break; | ||
1428 | } | ||
1429 | case SVM_EXIT_READ_DR0 ... SVM_EXIT_READ_DR7: { | ||
1430 | u32 dr_bits = 1 << (exit_code - SVM_EXIT_READ_DR0); | ||
1431 | if (nested_vmcb->control.intercept_dr_read & dr_bits) | ||
1432 | return 1; | ||
1433 | break; | ||
1434 | } | ||
1435 | case SVM_EXIT_WRITE_DR0 ... SVM_EXIT_WRITE_DR7: { | ||
1436 | u32 dr_bits = 1 << (exit_code - SVM_EXIT_WRITE_DR0); | ||
1437 | if (nested_vmcb->control.intercept_dr_write & dr_bits) | ||
1438 | return 1; | ||
1439 | break; | ||
1440 | } | ||
1441 | case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: { | ||
1442 | u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE); | ||
1443 | if (nested_vmcb->control.intercept_exceptions & excp_bits) | ||
1444 | return 1; | ||
1445 | break; | ||
1446 | } | ||
1447 | default: { | ||
1448 | u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR); | ||
1449 | nsvm_printk("exit code: 0x%x\n", exit_code); | ||
1450 | if (nested_vmcb->control.intercept & exit_bits) | ||
1451 | return 1; | ||
1452 | } | ||
1453 | } | ||
1454 | |||
1455 | return 0; | ||
1456 | } | ||
1457 | |||
1458 | static int nested_svm_exit_handled_msr(struct vcpu_svm *svm, | ||
1459 | void *arg1, void *arg2, | ||
1460 | void *opaque) | ||
1461 | { | ||
1462 | struct vmcb *nested_vmcb = (struct vmcb *)arg1; | ||
1463 | u8 *msrpm = (u8 *)arg2; | ||
1464 | u32 t0, t1; | ||
1465 | u32 msr = svm->vcpu.arch.regs[VCPU_REGS_RCX]; | ||
1466 | u32 param = svm->vmcb->control.exit_info_1 & 1; | ||
1467 | |||
1468 | if (!(nested_vmcb->control.intercept & (1ULL << INTERCEPT_MSR_PROT))) | ||
1469 | return 0; | ||
1470 | |||
1471 | switch(msr) { | ||
1472 | case 0 ... 0x1fff: | ||
1473 | t0 = (msr * 2) % 8; | ||
1474 | t1 = msr / 8; | ||
1475 | break; | ||
1476 | case 0xc0000000 ... 0xc0001fff: | ||
1477 | t0 = (8192 + msr - 0xc0000000) * 2; | ||
1478 | t1 = (t0 / 8); | ||
1479 | t0 %= 8; | ||
1480 | break; | ||
1481 | case 0xc0010000 ... 0xc0011fff: | ||
1482 | t0 = (16384 + msr - 0xc0010000) * 2; | ||
1483 | t1 = (t0 / 8); | ||
1484 | t0 %= 8; | ||
1485 | break; | ||
1486 | default: | ||
1487 | return 1; | ||
1488 | break; | ||
1489 | } | ||
1490 | if (msrpm[t1] & ((1 << param) << t0)) | ||
1491 | return 1; | ||
1492 | |||
1493 | return 0; | ||
1494 | } | ||
1495 | |||
1496 | static int nested_svm_exit_handled(struct vcpu_svm *svm, bool kvm_override) | ||
1497 | { | ||
1498 | bool k = kvm_override; | ||
1499 | |||
1500 | switch (svm->vmcb->control.exit_code) { | ||
1501 | case SVM_EXIT_MSR: | ||
1502 | return nested_svm_do(svm, svm->nested_vmcb, | ||
1503 | svm->nested_vmcb_msrpm, NULL, | ||
1504 | nested_svm_exit_handled_msr); | ||
1505 | default: break; | ||
1506 | } | ||
1507 | |||
1508 | return nested_svm_do(svm, svm->nested_vmcb, 0, &k, | ||
1509 | nested_svm_exit_handled_real); | ||
1510 | } | ||
1511 | |||
1512 | static int nested_svm_vmexit_real(struct vcpu_svm *svm, void *arg1, | ||
1513 | void *arg2, void *opaque) | ||
1514 | { | ||
1515 | struct vmcb *nested_vmcb = (struct vmcb *)arg1; | ||
1516 | struct vmcb *hsave = svm->hsave; | ||
1517 | u64 nested_save[] = { nested_vmcb->save.cr0, | ||
1518 | nested_vmcb->save.cr3, | ||
1519 | nested_vmcb->save.cr4, | ||
1520 | nested_vmcb->save.efer, | ||
1521 | nested_vmcb->control.intercept_cr_read, | ||
1522 | nested_vmcb->control.intercept_cr_write, | ||
1523 | nested_vmcb->control.intercept_dr_read, | ||
1524 | nested_vmcb->control.intercept_dr_write, | ||
1525 | nested_vmcb->control.intercept_exceptions, | ||
1526 | nested_vmcb->control.intercept, | ||
1527 | nested_vmcb->control.msrpm_base_pa, | ||
1528 | nested_vmcb->control.iopm_base_pa, | ||
1529 | nested_vmcb->control.tsc_offset }; | ||
1530 | |||
1531 | /* Give the current vmcb to the guest */ | ||
1532 | memcpy(nested_vmcb, svm->vmcb, sizeof(struct vmcb)); | ||
1533 | nested_vmcb->save.cr0 = nested_save[0]; | ||
1534 | if (!npt_enabled) | ||
1535 | nested_vmcb->save.cr3 = nested_save[1]; | ||
1536 | nested_vmcb->save.cr4 = nested_save[2]; | ||
1537 | nested_vmcb->save.efer = nested_save[3]; | ||
1538 | nested_vmcb->control.intercept_cr_read = nested_save[4]; | ||
1539 | nested_vmcb->control.intercept_cr_write = nested_save[5]; | ||
1540 | nested_vmcb->control.intercept_dr_read = nested_save[6]; | ||
1541 | nested_vmcb->control.intercept_dr_write = nested_save[7]; | ||
1542 | nested_vmcb->control.intercept_exceptions = nested_save[8]; | ||
1543 | nested_vmcb->control.intercept = nested_save[9]; | ||
1544 | nested_vmcb->control.msrpm_base_pa = nested_save[10]; | ||
1545 | nested_vmcb->control.iopm_base_pa = nested_save[11]; | ||
1546 | nested_vmcb->control.tsc_offset = nested_save[12]; | ||
1547 | |||
1548 | /* We always set V_INTR_MASKING and remember the old value in hflags */ | ||
1549 | if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK)) | ||
1550 | nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK; | ||
1551 | |||
1552 | if ((nested_vmcb->control.int_ctl & V_IRQ_MASK) && | ||
1553 | (nested_vmcb->control.int_vector)) { | ||
1554 | nsvm_printk("WARNING: IRQ 0x%x still enabled on #VMEXIT\n", | ||
1555 | nested_vmcb->control.int_vector); | ||
1556 | } | ||
1557 | |||
1558 | /* Restore the original control entries */ | ||
1559 | svm->vmcb->control = hsave->control; | ||
1560 | |||
1561 | /* Kill any pending exceptions */ | ||
1562 | if (svm->vcpu.arch.exception.pending == true) | ||
1563 | nsvm_printk("WARNING: Pending Exception\n"); | ||
1564 | svm->vcpu.arch.exception.pending = false; | ||
1565 | |||
1566 | /* Restore selected save entries */ | ||
1567 | svm->vmcb->save.es = hsave->save.es; | ||
1568 | svm->vmcb->save.cs = hsave->save.cs; | ||
1569 | svm->vmcb->save.ss = hsave->save.ss; | ||
1570 | svm->vmcb->save.ds = hsave->save.ds; | ||
1571 | svm->vmcb->save.gdtr = hsave->save.gdtr; | ||
1572 | svm->vmcb->save.idtr = hsave->save.idtr; | ||
1573 | svm->vmcb->save.rflags = hsave->save.rflags; | ||
1574 | svm_set_efer(&svm->vcpu, hsave->save.efer); | ||
1575 | svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE); | ||
1576 | svm_set_cr4(&svm->vcpu, hsave->save.cr4); | ||
1577 | if (npt_enabled) { | ||
1578 | svm->vmcb->save.cr3 = hsave->save.cr3; | ||
1579 | svm->vcpu.arch.cr3 = hsave->save.cr3; | ||
1580 | } else { | ||
1581 | kvm_set_cr3(&svm->vcpu, hsave->save.cr3); | ||
1582 | } | ||
1583 | kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax); | ||
1584 | kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp); | ||
1585 | kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip); | ||
1586 | svm->vmcb->save.dr7 = 0; | ||
1587 | svm->vmcb->save.cpl = 0; | ||
1588 | svm->vmcb->control.exit_int_info = 0; | ||
1589 | |||
1590 | svm->vcpu.arch.hflags &= ~HF_GIF_MASK; | ||
1591 | /* Exit nested SVM mode */ | ||
1592 | svm->nested_vmcb = 0; | ||
1593 | |||
1594 | return 0; | ||
1595 | } | ||
1596 | |||
1597 | static int nested_svm_vmexit(struct vcpu_svm *svm) | ||
1598 | { | ||
1599 | nsvm_printk("VMexit\n"); | ||
1600 | if (nested_svm_do(svm, svm->nested_vmcb, 0, | ||
1601 | NULL, nested_svm_vmexit_real)) | ||
1602 | return 1; | ||
1603 | |||
1604 | kvm_mmu_reset_context(&svm->vcpu); | ||
1605 | kvm_mmu_load(&svm->vcpu); | ||
1606 | |||
1607 | return 0; | ||
1608 | } | ||
1609 | |||
1610 | static int nested_svm_vmrun_msrpm(struct vcpu_svm *svm, void *arg1, | ||
1611 | void *arg2, void *opaque) | ||
1612 | { | ||
1613 | int i; | ||
1614 | u32 *nested_msrpm = (u32*)arg1; | ||
1615 | for (i=0; i< PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER) / 4; i++) | ||
1616 | svm->nested_msrpm[i] = svm->msrpm[i] | nested_msrpm[i]; | ||
1617 | svm->vmcb->control.msrpm_base_pa = __pa(svm->nested_msrpm); | ||
1618 | |||
1619 | return 0; | ||
1620 | } | ||
1621 | |||
1622 | static int nested_svm_vmrun(struct vcpu_svm *svm, void *arg1, | ||
1623 | void *arg2, void *opaque) | ||
1624 | { | ||
1625 | struct vmcb *nested_vmcb = (struct vmcb *)arg1; | ||
1626 | struct vmcb *hsave = svm->hsave; | ||
1627 | |||
1628 | /* nested_vmcb is our indicator if nested SVM is activated */ | ||
1629 | svm->nested_vmcb = svm->vmcb->save.rax; | ||
1630 | |||
1631 | /* Clear internal status */ | ||
1632 | svm->vcpu.arch.exception.pending = false; | ||
1633 | |||
1634 | /* Save the old vmcb, so we don't need to pick what we save, but | ||
1635 | can restore everything when a VMEXIT occurs */ | ||
1636 | memcpy(hsave, svm->vmcb, sizeof(struct vmcb)); | ||
1637 | /* We need to remember the original CR3 in the SPT case */ | ||
1638 | if (!npt_enabled) | ||
1639 | hsave->save.cr3 = svm->vcpu.arch.cr3; | ||
1640 | hsave->save.cr4 = svm->vcpu.arch.cr4; | ||
1641 | hsave->save.rip = svm->next_rip; | ||
1642 | |||
1643 | if (svm->vmcb->save.rflags & X86_EFLAGS_IF) | ||
1644 | svm->vcpu.arch.hflags |= HF_HIF_MASK; | ||
1645 | else | ||
1646 | svm->vcpu.arch.hflags &= ~HF_HIF_MASK; | ||
1647 | |||
1648 | /* Load the nested guest state */ | ||
1649 | svm->vmcb->save.es = nested_vmcb->save.es; | ||
1650 | svm->vmcb->save.cs = nested_vmcb->save.cs; | ||
1651 | svm->vmcb->save.ss = nested_vmcb->save.ss; | ||
1652 | svm->vmcb->save.ds = nested_vmcb->save.ds; | ||
1653 | svm->vmcb->save.gdtr = nested_vmcb->save.gdtr; | ||
1654 | svm->vmcb->save.idtr = nested_vmcb->save.idtr; | ||
1655 | svm->vmcb->save.rflags = nested_vmcb->save.rflags; | ||
1656 | svm_set_efer(&svm->vcpu, nested_vmcb->save.efer); | ||
1657 | svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0); | ||
1658 | svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4); | ||
1659 | if (npt_enabled) { | ||
1660 | svm->vmcb->save.cr3 = nested_vmcb->save.cr3; | ||
1661 | svm->vcpu.arch.cr3 = nested_vmcb->save.cr3; | ||
1662 | } else { | ||
1663 | kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3); | ||
1664 | kvm_mmu_reset_context(&svm->vcpu); | ||
1665 | } | ||
1666 | svm->vmcb->save.cr2 = nested_vmcb->save.cr2; | ||
1667 | kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax); | ||
1668 | kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp); | ||
1669 | kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip); | ||
1670 | /* In case we don't even reach vcpu_run, the fields are not updated */ | ||
1671 | svm->vmcb->save.rax = nested_vmcb->save.rax; | ||
1672 | svm->vmcb->save.rsp = nested_vmcb->save.rsp; | ||
1673 | svm->vmcb->save.rip = nested_vmcb->save.rip; | ||
1674 | svm->vmcb->save.dr7 = nested_vmcb->save.dr7; | ||
1675 | svm->vmcb->save.dr6 = nested_vmcb->save.dr6; | ||
1676 | svm->vmcb->save.cpl = nested_vmcb->save.cpl; | ||
1677 | |||
1678 | /* We don't want a nested guest to be more powerful than the guest, | ||
1679 | so all intercepts are ORed */ | ||
1680 | svm->vmcb->control.intercept_cr_read |= | ||
1681 | nested_vmcb->control.intercept_cr_read; | ||
1682 | svm->vmcb->control.intercept_cr_write |= | ||
1683 | nested_vmcb->control.intercept_cr_write; | ||
1684 | svm->vmcb->control.intercept_dr_read |= | ||
1685 | nested_vmcb->control.intercept_dr_read; | ||
1686 | svm->vmcb->control.intercept_dr_write |= | ||
1687 | nested_vmcb->control.intercept_dr_write; | ||
1688 | svm->vmcb->control.intercept_exceptions |= | ||
1689 | nested_vmcb->control.intercept_exceptions; | ||
1690 | |||
1691 | svm->vmcb->control.intercept |= nested_vmcb->control.intercept; | ||
1692 | |||
1693 | svm->nested_vmcb_msrpm = nested_vmcb->control.msrpm_base_pa; | ||
1694 | |||
1695 | force_new_asid(&svm->vcpu); | ||
1696 | svm->vmcb->control.exit_int_info = nested_vmcb->control.exit_int_info; | ||
1697 | svm->vmcb->control.exit_int_info_err = nested_vmcb->control.exit_int_info_err; | ||
1698 | svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK; | ||
1699 | if (nested_vmcb->control.int_ctl & V_IRQ_MASK) { | ||
1700 | nsvm_printk("nSVM Injecting Interrupt: 0x%x\n", | ||
1701 | nested_vmcb->control.int_ctl); | ||
1702 | } | ||
1703 | if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK) | ||
1704 | svm->vcpu.arch.hflags |= HF_VINTR_MASK; | ||
1705 | else | ||
1706 | svm->vcpu.arch.hflags &= ~HF_VINTR_MASK; | ||
1707 | |||
1708 | nsvm_printk("nSVM exit_int_info: 0x%x | int_state: 0x%x\n", | ||
1709 | nested_vmcb->control.exit_int_info, | ||
1710 | nested_vmcb->control.int_state); | ||
1711 | |||
1712 | svm->vmcb->control.int_vector = nested_vmcb->control.int_vector; | ||
1713 | svm->vmcb->control.int_state = nested_vmcb->control.int_state; | ||
1714 | svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset; | ||
1715 | if (nested_vmcb->control.event_inj & SVM_EVTINJ_VALID) | ||
1716 | nsvm_printk("Injecting Event: 0x%x\n", | ||
1717 | nested_vmcb->control.event_inj); | ||
1718 | svm->vmcb->control.event_inj = nested_vmcb->control.event_inj; | ||
1719 | svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err; | ||
1720 | |||
1721 | svm->vcpu.arch.hflags |= HF_GIF_MASK; | ||
1722 | |||
1723 | return 0; | ||
1724 | } | ||
1725 | |||
1726 | static int nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb) | ||
1727 | { | ||
1728 | to_vmcb->save.fs = from_vmcb->save.fs; | ||
1729 | to_vmcb->save.gs = from_vmcb->save.gs; | ||
1730 | to_vmcb->save.tr = from_vmcb->save.tr; | ||
1731 | to_vmcb->save.ldtr = from_vmcb->save.ldtr; | ||
1732 | to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base; | ||
1733 | to_vmcb->save.star = from_vmcb->save.star; | ||
1734 | to_vmcb->save.lstar = from_vmcb->save.lstar; | ||
1735 | to_vmcb->save.cstar = from_vmcb->save.cstar; | ||
1736 | to_vmcb->save.sfmask = from_vmcb->save.sfmask; | ||
1737 | to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs; | ||
1738 | to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp; | ||
1739 | to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip; | ||
1740 | |||
1741 | return 1; | ||
1742 | } | ||
1743 | |||
1744 | static int nested_svm_vmload(struct vcpu_svm *svm, void *nested_vmcb, | ||
1745 | void *arg2, void *opaque) | ||
1746 | { | ||
1747 | return nested_svm_vmloadsave((struct vmcb *)nested_vmcb, svm->vmcb); | ||
1748 | } | ||
1749 | |||
1750 | static int nested_svm_vmsave(struct vcpu_svm *svm, void *nested_vmcb, | ||
1751 | void *arg2, void *opaque) | ||
1752 | { | ||
1753 | return nested_svm_vmloadsave(svm->vmcb, (struct vmcb *)nested_vmcb); | ||
1754 | } | ||
1755 | |||
1756 | static int vmload_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1757 | { | ||
1758 | if (nested_svm_check_permissions(svm)) | ||
1759 | return 1; | ||
1760 | |||
1761 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; | ||
1762 | skip_emulated_instruction(&svm->vcpu); | ||
1763 | |||
1764 | nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmload); | ||
1765 | |||
1766 | return 1; | ||
1767 | } | ||
1768 | |||
1769 | static int vmsave_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1770 | { | ||
1771 | if (nested_svm_check_permissions(svm)) | ||
1772 | return 1; | ||
1773 | |||
1774 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; | ||
1775 | skip_emulated_instruction(&svm->vcpu); | ||
1776 | |||
1777 | nested_svm_do(svm, svm->vmcb->save.rax, 0, NULL, nested_svm_vmsave); | ||
1778 | |||
1779 | return 1; | ||
1780 | } | ||
1781 | |||
1782 | static int vmrun_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1783 | { | ||
1784 | nsvm_printk("VMrun\n"); | ||
1785 | if (nested_svm_check_permissions(svm)) | ||
1786 | return 1; | ||
1787 | |||
1788 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; | ||
1789 | skip_emulated_instruction(&svm->vcpu); | ||
1790 | |||
1791 | if (nested_svm_do(svm, svm->vmcb->save.rax, 0, | ||
1792 | NULL, nested_svm_vmrun)) | ||
1793 | return 1; | ||
1794 | |||
1795 | if (nested_svm_do(svm, svm->nested_vmcb_msrpm, 0, | ||
1796 | NULL, nested_svm_vmrun_msrpm)) | ||
1797 | return 1; | ||
1798 | |||
1799 | return 1; | ||
1800 | } | ||
1801 | |||
1802 | static int stgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1803 | { | ||
1804 | if (nested_svm_check_permissions(svm)) | ||
1805 | return 1; | ||
1806 | |||
1807 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; | ||
1808 | skip_emulated_instruction(&svm->vcpu); | ||
1809 | |||
1810 | svm->vcpu.arch.hflags |= HF_GIF_MASK; | ||
1811 | |||
1812 | return 1; | ||
1813 | } | ||
1814 | |||
1815 | static int clgi_interception(struct vcpu_svm *svm, struct kvm_run *kvm_run) | ||
1816 | { | ||
1817 | if (nested_svm_check_permissions(svm)) | ||
1818 | return 1; | ||
1819 | |||
1820 | svm->next_rip = kvm_rip_read(&svm->vcpu) + 3; | ||
1821 | skip_emulated_instruction(&svm->vcpu); | ||
1822 | |||
1823 | svm->vcpu.arch.hflags &= ~HF_GIF_MASK; | ||
1824 | |||
1825 | /* After a CLGI no interrupts should come */ | ||
1826 | svm_clear_vintr(svm); | ||
1827 | svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; | ||
1828 | |||
1829 | return 1; | ||
1830 | } | ||
1831 | |||
1142 | static int invalid_op_interception(struct vcpu_svm *svm, | 1832 | static int invalid_op_interception(struct vcpu_svm *svm, |
1143 | struct kvm_run *kvm_run) | 1833 | struct kvm_run *kvm_run) |
1144 | { | 1834 | { |
@@ -1250,6 +1940,15 @@ static int svm_get_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 *data) | |||
1250 | case MSR_IA32_LASTINTTOIP: | 1940 | case MSR_IA32_LASTINTTOIP: |
1251 | *data = svm->vmcb->save.last_excp_to; | 1941 | *data = svm->vmcb->save.last_excp_to; |
1252 | break; | 1942 | break; |
1943 | case MSR_VM_HSAVE_PA: | ||
1944 | *data = svm->hsave_msr; | ||
1945 | break; | ||
1946 | case MSR_VM_CR: | ||
1947 | *data = 0; | ||
1948 | break; | ||
1949 | case MSR_IA32_UCODE_REV: | ||
1950 | *data = 0x01000065; | ||
1951 | break; | ||
1253 | default: | 1952 | default: |
1254 | return kvm_get_msr_common(vcpu, ecx, data); | 1953 | return kvm_get_msr_common(vcpu, ecx, data); |
1255 | } | 1954 | } |
@@ -1344,6 +2043,9 @@ static int svm_set_msr(struct kvm_vcpu *vcpu, unsigned ecx, u64 data) | |||
1344 | pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", ecx, data); | 2043 | pr_unimpl(vcpu, "unimplemented perfctr wrmsr: 0x%x data 0x%llx\n", ecx, data); |
1345 | 2044 | ||
1346 | break; | 2045 | break; |
2046 | case MSR_VM_HSAVE_PA: | ||
2047 | svm->hsave_msr = data; | ||
2048 | break; | ||
1347 | default: | 2049 | default: |
1348 | return kvm_set_msr_common(vcpu, ecx, data); | 2050 | return kvm_set_msr_common(vcpu, ecx, data); |
1349 | } | 2051 | } |
@@ -1380,7 +2082,7 @@ static int interrupt_window_interception(struct vcpu_svm *svm, | |||
1380 | { | 2082 | { |
1381 | KVMTRACE_0D(PEND_INTR, &svm->vcpu, handler); | 2083 | KVMTRACE_0D(PEND_INTR, &svm->vcpu, handler); |
1382 | 2084 | ||
1383 | svm->vmcb->control.intercept &= ~(1ULL << INTERCEPT_VINTR); | 2085 | svm_clear_vintr(svm); |
1384 | svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; | 2086 | svm->vmcb->control.int_ctl &= ~V_IRQ_MASK; |
1385 | /* | 2087 | /* |
1386 | * If the user space waits to inject interrupts, exit as soon as | 2088 | * If the user space waits to inject interrupts, exit as soon as |
@@ -1417,6 +2119,8 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, | |||
1417 | [SVM_EXIT_WRITE_DR3] = emulate_on_interception, | 2119 | [SVM_EXIT_WRITE_DR3] = emulate_on_interception, |
1418 | [SVM_EXIT_WRITE_DR5] = emulate_on_interception, | 2120 | [SVM_EXIT_WRITE_DR5] = emulate_on_interception, |
1419 | [SVM_EXIT_WRITE_DR7] = emulate_on_interception, | 2121 | [SVM_EXIT_WRITE_DR7] = emulate_on_interception, |
2122 | [SVM_EXIT_EXCP_BASE + DB_VECTOR] = db_interception, | ||
2123 | [SVM_EXIT_EXCP_BASE + BP_VECTOR] = bp_interception, | ||
1420 | [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, | 2124 | [SVM_EXIT_EXCP_BASE + UD_VECTOR] = ud_interception, |
1421 | [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, | 2125 | [SVM_EXIT_EXCP_BASE + PF_VECTOR] = pf_interception, |
1422 | [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception, | 2126 | [SVM_EXIT_EXCP_BASE + NM_VECTOR] = nm_interception, |
@@ -1436,12 +2140,12 @@ static int (*svm_exit_handlers[])(struct vcpu_svm *svm, | |||
1436 | [SVM_EXIT_MSR] = msr_interception, | 2140 | [SVM_EXIT_MSR] = msr_interception, |
1437 | [SVM_EXIT_TASK_SWITCH] = task_switch_interception, | 2141 | [SVM_EXIT_TASK_SWITCH] = task_switch_interception, |
1438 | [SVM_EXIT_SHUTDOWN] = shutdown_interception, | 2142 | [SVM_EXIT_SHUTDOWN] = shutdown_interception, |
1439 | [SVM_EXIT_VMRUN] = invalid_op_interception, | 2143 | [SVM_EXIT_VMRUN] = vmrun_interception, |
1440 | [SVM_EXIT_VMMCALL] = vmmcall_interception, | 2144 | [SVM_EXIT_VMMCALL] = vmmcall_interception, |
1441 | [SVM_EXIT_VMLOAD] = invalid_op_interception, | 2145 | [SVM_EXIT_VMLOAD] = vmload_interception, |
1442 | [SVM_EXIT_VMSAVE] = invalid_op_interception, | 2146 | [SVM_EXIT_VMSAVE] = vmsave_interception, |
1443 | [SVM_EXIT_STGI] = invalid_op_interception, | 2147 | [SVM_EXIT_STGI] = stgi_interception, |
1444 | [SVM_EXIT_CLGI] = invalid_op_interception, | 2148 | [SVM_EXIT_CLGI] = clgi_interception, |
1445 | [SVM_EXIT_SKINIT] = invalid_op_interception, | 2149 | [SVM_EXIT_SKINIT] = invalid_op_interception, |
1446 | [SVM_EXIT_WBINVD] = emulate_on_interception, | 2150 | [SVM_EXIT_WBINVD] = emulate_on_interception, |
1447 | [SVM_EXIT_MONITOR] = invalid_op_interception, | 2151 | [SVM_EXIT_MONITOR] = invalid_op_interception, |
@@ -1457,6 +2161,17 @@ static int handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | |||
1457 | KVMTRACE_3D(VMEXIT, vcpu, exit_code, (u32)svm->vmcb->save.rip, | 2161 | KVMTRACE_3D(VMEXIT, vcpu, exit_code, (u32)svm->vmcb->save.rip, |
1458 | (u32)((u64)svm->vmcb->save.rip >> 32), entryexit); | 2162 | (u32)((u64)svm->vmcb->save.rip >> 32), entryexit); |
1459 | 2163 | ||
2164 | if (is_nested(svm)) { | ||
2165 | nsvm_printk("nested handle_exit: 0x%x | 0x%lx | 0x%lx | 0x%lx\n", | ||
2166 | exit_code, svm->vmcb->control.exit_info_1, | ||
2167 | svm->vmcb->control.exit_info_2, svm->vmcb->save.rip); | ||
2168 | if (nested_svm_exit_handled(svm, true)) { | ||
2169 | nested_svm_vmexit(svm); | ||
2170 | nsvm_printk("-> #VMEXIT\n"); | ||
2171 | return 1; | ||
2172 | } | ||
2173 | } | ||
2174 | |||
1460 | if (npt_enabled) { | 2175 | if (npt_enabled) { |
1461 | int mmu_reload = 0; | 2176 | int mmu_reload = 0; |
1462 | if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) { | 2177 | if ((vcpu->arch.cr0 ^ svm->vmcb->save.cr0) & X86_CR0_PG) { |
@@ -1544,6 +2259,8 @@ static void svm_set_irq(struct kvm_vcpu *vcpu, int irq) | |||
1544 | { | 2259 | { |
1545 | struct vcpu_svm *svm = to_svm(vcpu); | 2260 | struct vcpu_svm *svm = to_svm(vcpu); |
1546 | 2261 | ||
2262 | nested_svm_intr(svm); | ||
2263 | |||
1547 | svm_inject_irq(svm, irq); | 2264 | svm_inject_irq(svm, irq); |
1548 | } | 2265 | } |
1549 | 2266 | ||
@@ -1589,11 +2306,17 @@ static void svm_intr_assist(struct kvm_vcpu *vcpu) | |||
1589 | if (!kvm_cpu_has_interrupt(vcpu)) | 2306 | if (!kvm_cpu_has_interrupt(vcpu)) |
1590 | goto out; | 2307 | goto out; |
1591 | 2308 | ||
2309 | if (nested_svm_intr(svm)) | ||
2310 | goto out; | ||
2311 | |||
2312 | if (!(svm->vcpu.arch.hflags & HF_GIF_MASK)) | ||
2313 | goto out; | ||
2314 | |||
1592 | if (!(vmcb->save.rflags & X86_EFLAGS_IF) || | 2315 | if (!(vmcb->save.rflags & X86_EFLAGS_IF) || |
1593 | (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || | 2316 | (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) || |
1594 | (vmcb->control.event_inj & SVM_EVTINJ_VALID)) { | 2317 | (vmcb->control.event_inj & SVM_EVTINJ_VALID)) { |
1595 | /* unable to deliver irq, set pending irq */ | 2318 | /* unable to deliver irq, set pending irq */ |
1596 | vmcb->control.intercept |= (1ULL << INTERCEPT_VINTR); | 2319 | svm_set_vintr(svm); |
1597 | svm_inject_irq(svm, 0x0); | 2320 | svm_inject_irq(svm, 0x0); |
1598 | goto out; | 2321 | goto out; |
1599 | } | 2322 | } |
@@ -1615,7 +2338,8 @@ static void kvm_reput_irq(struct vcpu_svm *svm) | |||
1615 | } | 2338 | } |
1616 | 2339 | ||
1617 | svm->vcpu.arch.interrupt_window_open = | 2340 | svm->vcpu.arch.interrupt_window_open = |
1618 | !(control->int_state & SVM_INTERRUPT_SHADOW_MASK); | 2341 | !(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && |
2342 | (svm->vcpu.arch.hflags & HF_GIF_MASK); | ||
1619 | } | 2343 | } |
1620 | 2344 | ||
1621 | static void svm_do_inject_vector(struct vcpu_svm *svm) | 2345 | static void svm_do_inject_vector(struct vcpu_svm *svm) |
@@ -1637,9 +2361,13 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, | |||
1637 | struct vcpu_svm *svm = to_svm(vcpu); | 2361 | struct vcpu_svm *svm = to_svm(vcpu); |
1638 | struct vmcb_control_area *control = &svm->vmcb->control; | 2362 | struct vmcb_control_area *control = &svm->vmcb->control; |
1639 | 2363 | ||
2364 | if (nested_svm_intr(svm)) | ||
2365 | return; | ||
2366 | |||
1640 | svm->vcpu.arch.interrupt_window_open = | 2367 | svm->vcpu.arch.interrupt_window_open = |
1641 | (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && | 2368 | (!(control->int_state & SVM_INTERRUPT_SHADOW_MASK) && |
1642 | (svm->vmcb->save.rflags & X86_EFLAGS_IF)); | 2369 | (svm->vmcb->save.rflags & X86_EFLAGS_IF) && |
2370 | (svm->vcpu.arch.hflags & HF_GIF_MASK)); | ||
1643 | 2371 | ||
1644 | if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary) | 2372 | if (svm->vcpu.arch.interrupt_window_open && svm->vcpu.arch.irq_summary) |
1645 | /* | 2373 | /* |
@@ -1652,9 +2380,9 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, | |||
1652 | */ | 2380 | */ |
1653 | if (!svm->vcpu.arch.interrupt_window_open && | 2381 | if (!svm->vcpu.arch.interrupt_window_open && |
1654 | (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window)) | 2382 | (svm->vcpu.arch.irq_summary || kvm_run->request_interrupt_window)) |
1655 | control->intercept |= 1ULL << INTERCEPT_VINTR; | 2383 | svm_set_vintr(svm); |
1656 | else | 2384 | else |
1657 | control->intercept &= ~(1ULL << INTERCEPT_VINTR); | 2385 | svm_clear_vintr(svm); |
1658 | } | 2386 | } |
1659 | 2387 | ||
1660 | static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) | 2388 | static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) |
@@ -1662,22 +2390,6 @@ static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr) | |||
1662 | return 0; | 2390 | return 0; |
1663 | } | 2391 | } |
1664 | 2392 | ||
1665 | static void save_db_regs(unsigned long *db_regs) | ||
1666 | { | ||
1667 | asm volatile ("mov %%dr0, %0" : "=r"(db_regs[0])); | ||
1668 | asm volatile ("mov %%dr1, %0" : "=r"(db_regs[1])); | ||
1669 | asm volatile ("mov %%dr2, %0" : "=r"(db_regs[2])); | ||
1670 | asm volatile ("mov %%dr3, %0" : "=r"(db_regs[3])); | ||
1671 | } | ||
1672 | |||
1673 | static void load_db_regs(unsigned long *db_regs) | ||
1674 | { | ||
1675 | asm volatile ("mov %0, %%dr0" : : "r"(db_regs[0])); | ||
1676 | asm volatile ("mov %0, %%dr1" : : "r"(db_regs[1])); | ||
1677 | asm volatile ("mov %0, %%dr2" : : "r"(db_regs[2])); | ||
1678 | asm volatile ("mov %0, %%dr3" : : "r"(db_regs[3])); | ||
1679 | } | ||
1680 | |||
1681 | static void svm_flush_tlb(struct kvm_vcpu *vcpu) | 2393 | static void svm_flush_tlb(struct kvm_vcpu *vcpu) |
1682 | { | 2394 | { |
1683 | force_new_asid(vcpu); | 2395 | force_new_asid(vcpu); |
@@ -1736,19 +2448,12 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
1736 | gs_selector = kvm_read_gs(); | 2448 | gs_selector = kvm_read_gs(); |
1737 | ldt_selector = kvm_read_ldt(); | 2449 | ldt_selector = kvm_read_ldt(); |
1738 | svm->host_cr2 = kvm_read_cr2(); | 2450 | svm->host_cr2 = kvm_read_cr2(); |
1739 | svm->host_dr6 = read_dr6(); | 2451 | if (!is_nested(svm)) |
1740 | svm->host_dr7 = read_dr7(); | 2452 | svm->vmcb->save.cr2 = vcpu->arch.cr2; |
1741 | svm->vmcb->save.cr2 = vcpu->arch.cr2; | ||
1742 | /* required for live migration with NPT */ | 2453 | /* required for live migration with NPT */ |
1743 | if (npt_enabled) | 2454 | if (npt_enabled) |
1744 | svm->vmcb->save.cr3 = vcpu->arch.cr3; | 2455 | svm->vmcb->save.cr3 = vcpu->arch.cr3; |
1745 | 2456 | ||
1746 | if (svm->vmcb->save.dr7 & 0xff) { | ||
1747 | write_dr7(0); | ||
1748 | save_db_regs(svm->host_db_regs); | ||
1749 | load_db_regs(svm->db_regs); | ||
1750 | } | ||
1751 | |||
1752 | clgi(); | 2457 | clgi(); |
1753 | 2458 | ||
1754 | local_irq_enable(); | 2459 | local_irq_enable(); |
@@ -1824,16 +2529,11 @@ static void svm_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
1824 | #endif | 2529 | #endif |
1825 | ); | 2530 | ); |
1826 | 2531 | ||
1827 | if ((svm->vmcb->save.dr7 & 0xff)) | ||
1828 | load_db_regs(svm->host_db_regs); | ||
1829 | |||
1830 | vcpu->arch.cr2 = svm->vmcb->save.cr2; | 2532 | vcpu->arch.cr2 = svm->vmcb->save.cr2; |
1831 | vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; | 2533 | vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; |
1832 | vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; | 2534 | vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; |
1833 | vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; | 2535 | vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; |
1834 | 2536 | ||
1835 | write_dr6(svm->host_dr6); | ||
1836 | write_dr7(svm->host_dr7); | ||
1837 | kvm_write_cr2(svm->host_cr2); | 2537 | kvm_write_cr2(svm->host_cr2); |
1838 | 2538 | ||
1839 | kvm_load_fs(fs_selector); | 2539 | kvm_load_fs(fs_selector); |
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c index 7611af576829..bb481330716f 100644 --- a/arch/x86/kvm/vmx.c +++ b/arch/x86/kvm/vmx.c | |||
@@ -91,6 +91,7 @@ struct vcpu_vmx { | |||
91 | } rmode; | 91 | } rmode; |
92 | int vpid; | 92 | int vpid; |
93 | bool emulation_required; | 93 | bool emulation_required; |
94 | enum emulation_result invalid_state_emulation_result; | ||
94 | 95 | ||
95 | /* Support for vnmi-less CPUs */ | 96 | /* Support for vnmi-less CPUs */ |
96 | int soft_vnmi_blocked; | 97 | int soft_vnmi_blocked; |
@@ -189,21 +190,21 @@ static inline int is_page_fault(u32 intr_info) | |||
189 | { | 190 | { |
190 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | | 191 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | |
191 | INTR_INFO_VALID_MASK)) == | 192 | INTR_INFO_VALID_MASK)) == |
192 | (INTR_TYPE_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK); | 193 | (INTR_TYPE_HARD_EXCEPTION | PF_VECTOR | INTR_INFO_VALID_MASK); |
193 | } | 194 | } |
194 | 195 | ||
195 | static inline int is_no_device(u32 intr_info) | 196 | static inline int is_no_device(u32 intr_info) |
196 | { | 197 | { |
197 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | | 198 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | |
198 | INTR_INFO_VALID_MASK)) == | 199 | INTR_INFO_VALID_MASK)) == |
199 | (INTR_TYPE_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK); | 200 | (INTR_TYPE_HARD_EXCEPTION | NM_VECTOR | INTR_INFO_VALID_MASK); |
200 | } | 201 | } |
201 | 202 | ||
202 | static inline int is_invalid_opcode(u32 intr_info) | 203 | static inline int is_invalid_opcode(u32 intr_info) |
203 | { | 204 | { |
204 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | | 205 | return (intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK | |
205 | INTR_INFO_VALID_MASK)) == | 206 | INTR_INFO_VALID_MASK)) == |
206 | (INTR_TYPE_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK); | 207 | (INTR_TYPE_HARD_EXCEPTION | UD_VECTOR | INTR_INFO_VALID_MASK); |
207 | } | 208 | } |
208 | 209 | ||
209 | static inline int is_external_interrupt(u32 intr_info) | 210 | static inline int is_external_interrupt(u32 intr_info) |
@@ -480,8 +481,13 @@ static void update_exception_bitmap(struct kvm_vcpu *vcpu) | |||
480 | eb = (1u << PF_VECTOR) | (1u << UD_VECTOR); | 481 | eb = (1u << PF_VECTOR) | (1u << UD_VECTOR); |
481 | if (!vcpu->fpu_active) | 482 | if (!vcpu->fpu_active) |
482 | eb |= 1u << NM_VECTOR; | 483 | eb |= 1u << NM_VECTOR; |
483 | if (vcpu->guest_debug.enabled) | 484 | if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { |
484 | eb |= 1u << DB_VECTOR; | 485 | if (vcpu->guest_debug & |
486 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) | ||
487 | eb |= 1u << DB_VECTOR; | ||
488 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) | ||
489 | eb |= 1u << BP_VECTOR; | ||
490 | } | ||
485 | if (vcpu->arch.rmode.active) | 491 | if (vcpu->arch.rmode.active) |
486 | eb = ~0; | 492 | eb = ~0; |
487 | if (vm_need_ept()) | 493 | if (vm_need_ept()) |
@@ -747,29 +753,33 @@ static void vmx_queue_exception(struct kvm_vcpu *vcpu, unsigned nr, | |||
747 | bool has_error_code, u32 error_code) | 753 | bool has_error_code, u32 error_code) |
748 | { | 754 | { |
749 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 755 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
756 | u32 intr_info = nr | INTR_INFO_VALID_MASK; | ||
750 | 757 | ||
751 | if (has_error_code) | 758 | if (has_error_code) { |
752 | vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); | 759 | vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE, error_code); |
760 | intr_info |= INTR_INFO_DELIVER_CODE_MASK; | ||
761 | } | ||
753 | 762 | ||
754 | if (vcpu->arch.rmode.active) { | 763 | if (vcpu->arch.rmode.active) { |
755 | vmx->rmode.irq.pending = true; | 764 | vmx->rmode.irq.pending = true; |
756 | vmx->rmode.irq.vector = nr; | 765 | vmx->rmode.irq.vector = nr; |
757 | vmx->rmode.irq.rip = kvm_rip_read(vcpu); | 766 | vmx->rmode.irq.rip = kvm_rip_read(vcpu); |
758 | if (nr == BP_VECTOR) | 767 | if (nr == BP_VECTOR || nr == OF_VECTOR) |
759 | vmx->rmode.irq.rip++; | 768 | vmx->rmode.irq.rip++; |
760 | vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, | 769 | intr_info |= INTR_TYPE_SOFT_INTR; |
761 | nr | INTR_TYPE_SOFT_INTR | 770 | vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); |
762 | | (has_error_code ? INTR_INFO_DELIVER_CODE_MASK : 0) | ||
763 | | INTR_INFO_VALID_MASK); | ||
764 | vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); | 771 | vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); |
765 | kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1); | 772 | kvm_rip_write(vcpu, vmx->rmode.irq.rip - 1); |
766 | return; | 773 | return; |
767 | } | 774 | } |
768 | 775 | ||
769 | vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, | 776 | if (nr == BP_VECTOR || nr == OF_VECTOR) { |
770 | nr | INTR_TYPE_EXCEPTION | 777 | vmcs_write32(VM_ENTRY_INSTRUCTION_LEN, 1); |
771 | | (has_error_code ? INTR_INFO_DELIVER_CODE_MASK : 0) | 778 | intr_info |= INTR_TYPE_SOFT_EXCEPTION; |
772 | | INTR_INFO_VALID_MASK); | 779 | } else |
780 | intr_info |= INTR_TYPE_HARD_EXCEPTION; | ||
781 | |||
782 | vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, intr_info); | ||
773 | } | 783 | } |
774 | 784 | ||
775 | static bool vmx_exception_injected(struct kvm_vcpu *vcpu) | 785 | static bool vmx_exception_injected(struct kvm_vcpu *vcpu) |
@@ -856,11 +866,8 @@ static u64 guest_read_tsc(void) | |||
856 | * writes 'guest_tsc' into guest's timestamp counter "register" | 866 | * writes 'guest_tsc' into guest's timestamp counter "register" |
857 | * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc | 867 | * guest_tsc = host_tsc + tsc_offset ==> tsc_offset = guest_tsc - host_tsc |
858 | */ | 868 | */ |
859 | static void guest_write_tsc(u64 guest_tsc) | 869 | static void guest_write_tsc(u64 guest_tsc, u64 host_tsc) |
860 | { | 870 | { |
861 | u64 host_tsc; | ||
862 | |||
863 | rdtscll(host_tsc); | ||
864 | vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc); | 871 | vmcs_write64(TSC_OFFSET, guest_tsc - host_tsc); |
865 | } | 872 | } |
866 | 873 | ||
@@ -925,14 +932,15 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) | |||
925 | { | 932 | { |
926 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 933 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
927 | struct kvm_msr_entry *msr; | 934 | struct kvm_msr_entry *msr; |
935 | u64 host_tsc; | ||
928 | int ret = 0; | 936 | int ret = 0; |
929 | 937 | ||
930 | switch (msr_index) { | 938 | switch (msr_index) { |
931 | #ifdef CONFIG_X86_64 | ||
932 | case MSR_EFER: | 939 | case MSR_EFER: |
933 | vmx_load_host_state(vmx); | 940 | vmx_load_host_state(vmx); |
934 | ret = kvm_set_msr_common(vcpu, msr_index, data); | 941 | ret = kvm_set_msr_common(vcpu, msr_index, data); |
935 | break; | 942 | break; |
943 | #ifdef CONFIG_X86_64 | ||
936 | case MSR_FS_BASE: | 944 | case MSR_FS_BASE: |
937 | vmcs_writel(GUEST_FS_BASE, data); | 945 | vmcs_writel(GUEST_FS_BASE, data); |
938 | break; | 946 | break; |
@@ -950,7 +958,8 @@ static int vmx_set_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data) | |||
950 | vmcs_writel(GUEST_SYSENTER_ESP, data); | 958 | vmcs_writel(GUEST_SYSENTER_ESP, data); |
951 | break; | 959 | break; |
952 | case MSR_IA32_TIME_STAMP_COUNTER: | 960 | case MSR_IA32_TIME_STAMP_COUNTER: |
953 | guest_write_tsc(data); | 961 | rdtscll(host_tsc); |
962 | guest_write_tsc(data, host_tsc); | ||
954 | break; | 963 | break; |
955 | case MSR_P6_PERFCTR0: | 964 | case MSR_P6_PERFCTR0: |
956 | case MSR_P6_PERFCTR1: | 965 | case MSR_P6_PERFCTR1: |
@@ -999,40 +1008,28 @@ static void vmx_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg) | |||
999 | } | 1008 | } |
1000 | } | 1009 | } |
1001 | 1010 | ||
1002 | static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_debug_guest *dbg) | 1011 | static int set_guest_debug(struct kvm_vcpu *vcpu, struct kvm_guest_debug *dbg) |
1003 | { | 1012 | { |
1004 | unsigned long dr7 = 0x400; | 1013 | int old_debug = vcpu->guest_debug; |
1005 | int old_singlestep; | 1014 | unsigned long flags; |
1006 | |||
1007 | old_singlestep = vcpu->guest_debug.singlestep; | ||
1008 | |||
1009 | vcpu->guest_debug.enabled = dbg->enabled; | ||
1010 | if (vcpu->guest_debug.enabled) { | ||
1011 | int i; | ||
1012 | 1015 | ||
1013 | dr7 |= 0x200; /* exact */ | 1016 | vcpu->guest_debug = dbg->control; |
1014 | for (i = 0; i < 4; ++i) { | 1017 | if (!(vcpu->guest_debug & KVM_GUESTDBG_ENABLE)) |
1015 | if (!dbg->breakpoints[i].enabled) | 1018 | vcpu->guest_debug = 0; |
1016 | continue; | ||
1017 | vcpu->guest_debug.bp[i] = dbg->breakpoints[i].address; | ||
1018 | dr7 |= 2 << (i*2); /* global enable */ | ||
1019 | dr7 |= 0 << (i*4+16); /* execution breakpoint */ | ||
1020 | } | ||
1021 | 1019 | ||
1022 | vcpu->guest_debug.singlestep = dbg->singlestep; | 1020 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) |
1023 | } else | 1021 | vmcs_writel(GUEST_DR7, dbg->arch.debugreg[7]); |
1024 | vcpu->guest_debug.singlestep = 0; | 1022 | else |
1025 | 1023 | vmcs_writel(GUEST_DR7, vcpu->arch.dr7); | |
1026 | if (old_singlestep && !vcpu->guest_debug.singlestep) { | ||
1027 | unsigned long flags; | ||
1028 | 1024 | ||
1029 | flags = vmcs_readl(GUEST_RFLAGS); | 1025 | flags = vmcs_readl(GUEST_RFLAGS); |
1026 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
1027 | flags |= X86_EFLAGS_TF | X86_EFLAGS_RF; | ||
1028 | else if (old_debug & KVM_GUESTDBG_SINGLESTEP) | ||
1030 | flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | 1029 | flags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); |
1031 | vmcs_writel(GUEST_RFLAGS, flags); | 1030 | vmcs_writel(GUEST_RFLAGS, flags); |
1032 | } | ||
1033 | 1031 | ||
1034 | update_exception_bitmap(vcpu); | 1032 | update_exception_bitmap(vcpu); |
1035 | vmcs_writel(GUEST_DR7, dr7); | ||
1036 | 1033 | ||
1037 | return 0; | 1034 | return 0; |
1038 | } | 1035 | } |
@@ -1433,6 +1430,29 @@ continue_rmode: | |||
1433 | init_rmode(vcpu->kvm); | 1430 | init_rmode(vcpu->kvm); |
1434 | } | 1431 | } |
1435 | 1432 | ||
1433 | static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) | ||
1434 | { | ||
1435 | struct vcpu_vmx *vmx = to_vmx(vcpu); | ||
1436 | struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); | ||
1437 | |||
1438 | vcpu->arch.shadow_efer = efer; | ||
1439 | if (!msr) | ||
1440 | return; | ||
1441 | if (efer & EFER_LMA) { | ||
1442 | vmcs_write32(VM_ENTRY_CONTROLS, | ||
1443 | vmcs_read32(VM_ENTRY_CONTROLS) | | ||
1444 | VM_ENTRY_IA32E_MODE); | ||
1445 | msr->data = efer; | ||
1446 | } else { | ||
1447 | vmcs_write32(VM_ENTRY_CONTROLS, | ||
1448 | vmcs_read32(VM_ENTRY_CONTROLS) & | ||
1449 | ~VM_ENTRY_IA32E_MODE); | ||
1450 | |||
1451 | msr->data = efer & ~EFER_LME; | ||
1452 | } | ||
1453 | setup_msrs(vmx); | ||
1454 | } | ||
1455 | |||
1436 | #ifdef CONFIG_X86_64 | 1456 | #ifdef CONFIG_X86_64 |
1437 | 1457 | ||
1438 | static void enter_lmode(struct kvm_vcpu *vcpu) | 1458 | static void enter_lmode(struct kvm_vcpu *vcpu) |
@@ -1447,13 +1467,8 @@ static void enter_lmode(struct kvm_vcpu *vcpu) | |||
1447 | (guest_tr_ar & ~AR_TYPE_MASK) | 1467 | (guest_tr_ar & ~AR_TYPE_MASK) |
1448 | | AR_TYPE_BUSY_64_TSS); | 1468 | | AR_TYPE_BUSY_64_TSS); |
1449 | } | 1469 | } |
1450 | |||
1451 | vcpu->arch.shadow_efer |= EFER_LMA; | 1470 | vcpu->arch.shadow_efer |= EFER_LMA; |
1452 | 1471 | vmx_set_efer(vcpu, vcpu->arch.shadow_efer); | |
1453 | find_msr_entry(to_vmx(vcpu), MSR_EFER)->data |= EFER_LMA | EFER_LME; | ||
1454 | vmcs_write32(VM_ENTRY_CONTROLS, | ||
1455 | vmcs_read32(VM_ENTRY_CONTROLS) | ||
1456 | | VM_ENTRY_IA32E_MODE); | ||
1457 | } | 1472 | } |
1458 | 1473 | ||
1459 | static void exit_lmode(struct kvm_vcpu *vcpu) | 1474 | static void exit_lmode(struct kvm_vcpu *vcpu) |
@@ -1612,30 +1627,6 @@ static void vmx_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) | |||
1612 | vmcs_writel(GUEST_CR4, hw_cr4); | 1627 | vmcs_writel(GUEST_CR4, hw_cr4); |
1613 | } | 1628 | } |
1614 | 1629 | ||
1615 | static void vmx_set_efer(struct kvm_vcpu *vcpu, u64 efer) | ||
1616 | { | ||
1617 | struct vcpu_vmx *vmx = to_vmx(vcpu); | ||
1618 | struct kvm_msr_entry *msr = find_msr_entry(vmx, MSR_EFER); | ||
1619 | |||
1620 | vcpu->arch.shadow_efer = efer; | ||
1621 | if (!msr) | ||
1622 | return; | ||
1623 | if (efer & EFER_LMA) { | ||
1624 | vmcs_write32(VM_ENTRY_CONTROLS, | ||
1625 | vmcs_read32(VM_ENTRY_CONTROLS) | | ||
1626 | VM_ENTRY_IA32E_MODE); | ||
1627 | msr->data = efer; | ||
1628 | |||
1629 | } else { | ||
1630 | vmcs_write32(VM_ENTRY_CONTROLS, | ||
1631 | vmcs_read32(VM_ENTRY_CONTROLS) & | ||
1632 | ~VM_ENTRY_IA32E_MODE); | ||
1633 | |||
1634 | msr->data = efer & ~EFER_LME; | ||
1635 | } | ||
1636 | setup_msrs(vmx); | ||
1637 | } | ||
1638 | |||
1639 | static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) | 1630 | static u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) |
1640 | { | 1631 | { |
1641 | struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; | 1632 | struct kvm_vmx_segment_field *sf = &kvm_vmx_segment_fields[seg]; |
@@ -1653,7 +1644,7 @@ static void vmx_get_segment(struct kvm_vcpu *vcpu, | |||
1653 | var->limit = vmcs_read32(sf->limit); | 1644 | var->limit = vmcs_read32(sf->limit); |
1654 | var->selector = vmcs_read16(sf->selector); | 1645 | var->selector = vmcs_read16(sf->selector); |
1655 | ar = vmcs_read32(sf->ar_bytes); | 1646 | ar = vmcs_read32(sf->ar_bytes); |
1656 | if (ar & AR_UNUSABLE_MASK) | 1647 | if ((ar & AR_UNUSABLE_MASK) && !emulate_invalid_guest_state) |
1657 | ar = 0; | 1648 | ar = 0; |
1658 | var->type = ar & 15; | 1649 | var->type = ar & 15; |
1659 | var->s = (ar >> 4) & 1; | 1650 | var->s = (ar >> 4) & 1; |
@@ -1788,14 +1779,16 @@ static bool code_segment_valid(struct kvm_vcpu *vcpu) | |||
1788 | vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); | 1779 | vmx_get_segment(vcpu, &cs, VCPU_SREG_CS); |
1789 | cs_rpl = cs.selector & SELECTOR_RPL_MASK; | 1780 | cs_rpl = cs.selector & SELECTOR_RPL_MASK; |
1790 | 1781 | ||
1782 | if (cs.unusable) | ||
1783 | return false; | ||
1791 | if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK)) | 1784 | if (~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_ACCESSES_MASK)) |
1792 | return false; | 1785 | return false; |
1793 | if (!cs.s) | 1786 | if (!cs.s) |
1794 | return false; | 1787 | return false; |
1795 | if (!(~cs.type & (AR_TYPE_CODE_MASK|AR_TYPE_WRITEABLE_MASK))) { | 1788 | if (cs.type & AR_TYPE_WRITEABLE_MASK) { |
1796 | if (cs.dpl > cs_rpl) | 1789 | if (cs.dpl > cs_rpl) |
1797 | return false; | 1790 | return false; |
1798 | } else if (cs.type & AR_TYPE_CODE_MASK) { | 1791 | } else { |
1799 | if (cs.dpl != cs_rpl) | 1792 | if (cs.dpl != cs_rpl) |
1800 | return false; | 1793 | return false; |
1801 | } | 1794 | } |
@@ -1814,7 +1807,9 @@ static bool stack_segment_valid(struct kvm_vcpu *vcpu) | |||
1814 | vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); | 1807 | vmx_get_segment(vcpu, &ss, VCPU_SREG_SS); |
1815 | ss_rpl = ss.selector & SELECTOR_RPL_MASK; | 1808 | ss_rpl = ss.selector & SELECTOR_RPL_MASK; |
1816 | 1809 | ||
1817 | if ((ss.type != 3) || (ss.type != 7)) | 1810 | if (ss.unusable) |
1811 | return true; | ||
1812 | if (ss.type != 3 && ss.type != 7) | ||
1818 | return false; | 1813 | return false; |
1819 | if (!ss.s) | 1814 | if (!ss.s) |
1820 | return false; | 1815 | return false; |
@@ -1834,6 +1829,8 @@ static bool data_segment_valid(struct kvm_vcpu *vcpu, int seg) | |||
1834 | vmx_get_segment(vcpu, &var, seg); | 1829 | vmx_get_segment(vcpu, &var, seg); |
1835 | rpl = var.selector & SELECTOR_RPL_MASK; | 1830 | rpl = var.selector & SELECTOR_RPL_MASK; |
1836 | 1831 | ||
1832 | if (var.unusable) | ||
1833 | return true; | ||
1837 | if (!var.s) | 1834 | if (!var.s) |
1838 | return false; | 1835 | return false; |
1839 | if (!var.present) | 1836 | if (!var.present) |
@@ -1855,9 +1852,11 @@ static bool tr_valid(struct kvm_vcpu *vcpu) | |||
1855 | 1852 | ||
1856 | vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); | 1853 | vmx_get_segment(vcpu, &tr, VCPU_SREG_TR); |
1857 | 1854 | ||
1855 | if (tr.unusable) | ||
1856 | return false; | ||
1858 | if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */ | 1857 | if (tr.selector & SELECTOR_TI_MASK) /* TI = 1 */ |
1859 | return false; | 1858 | return false; |
1860 | if ((tr.type != 3) || (tr.type != 11)) /* TODO: Check if guest is in IA32e mode */ | 1859 | if (tr.type != 3 && tr.type != 11) /* TODO: Check if guest is in IA32e mode */ |
1861 | return false; | 1860 | return false; |
1862 | if (!tr.present) | 1861 | if (!tr.present) |
1863 | return false; | 1862 | return false; |
@@ -1871,6 +1870,8 @@ static bool ldtr_valid(struct kvm_vcpu *vcpu) | |||
1871 | 1870 | ||
1872 | vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); | 1871 | vmx_get_segment(vcpu, &ldtr, VCPU_SREG_LDTR); |
1873 | 1872 | ||
1873 | if (ldtr.unusable) | ||
1874 | return true; | ||
1874 | if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */ | 1875 | if (ldtr.selector & SELECTOR_TI_MASK) /* TI = 1 */ |
1875 | return false; | 1876 | return false; |
1876 | if (ldtr.type != 2) | 1877 | if (ldtr.type != 2) |
@@ -2112,7 +2113,7 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) | |||
2112 | { | 2113 | { |
2113 | u32 host_sysenter_cs, msr_low, msr_high; | 2114 | u32 host_sysenter_cs, msr_low, msr_high; |
2114 | u32 junk; | 2115 | u32 junk; |
2115 | u64 host_pat; | 2116 | u64 host_pat, tsc_this, tsc_base; |
2116 | unsigned long a; | 2117 | unsigned long a; |
2117 | struct descriptor_table dt; | 2118 | struct descriptor_table dt; |
2118 | int i; | 2119 | int i; |
@@ -2240,6 +2241,12 @@ static int vmx_vcpu_setup(struct vcpu_vmx *vmx) | |||
2240 | vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); | 2241 | vmcs_writel(CR0_GUEST_HOST_MASK, ~0UL); |
2241 | vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); | 2242 | vmcs_writel(CR4_GUEST_HOST_MASK, KVM_GUEST_CR4_MASK); |
2242 | 2243 | ||
2244 | tsc_base = vmx->vcpu.kvm->arch.vm_init_tsc; | ||
2245 | rdtscll(tsc_this); | ||
2246 | if (tsc_this < vmx->vcpu.kvm->arch.vm_init_tsc) | ||
2247 | tsc_base = tsc_this; | ||
2248 | |||
2249 | guest_write_tsc(0, tsc_base); | ||
2243 | 2250 | ||
2244 | return 0; | 2251 | return 0; |
2245 | } | 2252 | } |
@@ -2319,7 +2326,6 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) | |||
2319 | kvm_rip_write(vcpu, 0); | 2326 | kvm_rip_write(vcpu, 0); |
2320 | kvm_register_write(vcpu, VCPU_REGS_RSP, 0); | 2327 | kvm_register_write(vcpu, VCPU_REGS_RSP, 0); |
2321 | 2328 | ||
2322 | /* todo: dr0 = dr1 = dr2 = dr3 = 0; dr6 = 0xffff0ff0 */ | ||
2323 | vmcs_writel(GUEST_DR7, 0x400); | 2329 | vmcs_writel(GUEST_DR7, 0x400); |
2324 | 2330 | ||
2325 | vmcs_writel(GUEST_GDTR_BASE, 0); | 2331 | vmcs_writel(GUEST_GDTR_BASE, 0); |
@@ -2332,8 +2338,6 @@ static int vmx_vcpu_reset(struct kvm_vcpu *vcpu) | |||
2332 | vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); | 2338 | vmcs_write32(GUEST_INTERRUPTIBILITY_INFO, 0); |
2333 | vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); | 2339 | vmcs_write32(GUEST_PENDING_DBG_EXCEPTIONS, 0); |
2334 | 2340 | ||
2335 | guest_write_tsc(0); | ||
2336 | |||
2337 | /* Special registers */ | 2341 | /* Special registers */ |
2338 | vmcs_write64(GUEST_IA32_DEBUGCTL, 0); | 2342 | vmcs_write64(GUEST_IA32_DEBUGCTL, 0); |
2339 | 2343 | ||
@@ -2486,6 +2490,11 @@ static void do_interrupt_requests(struct kvm_vcpu *vcpu, | |||
2486 | { | 2490 | { |
2487 | vmx_update_window_states(vcpu); | 2491 | vmx_update_window_states(vcpu); |
2488 | 2492 | ||
2493 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
2494 | vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, | ||
2495 | GUEST_INTR_STATE_STI | | ||
2496 | GUEST_INTR_STATE_MOV_SS); | ||
2497 | |||
2489 | if (vcpu->arch.nmi_pending && !vcpu->arch.nmi_injected) { | 2498 | if (vcpu->arch.nmi_pending && !vcpu->arch.nmi_injected) { |
2490 | if (vcpu->arch.interrupt.pending) { | 2499 | if (vcpu->arch.interrupt.pending) { |
2491 | enable_nmi_window(vcpu); | 2500 | enable_nmi_window(vcpu); |
@@ -2536,24 +2545,6 @@ static int vmx_set_tss_addr(struct kvm *kvm, unsigned int addr) | |||
2536 | return 0; | 2545 | return 0; |
2537 | } | 2546 | } |
2538 | 2547 | ||
2539 | static void kvm_guest_debug_pre(struct kvm_vcpu *vcpu) | ||
2540 | { | ||
2541 | struct kvm_guest_debug *dbg = &vcpu->guest_debug; | ||
2542 | |||
2543 | set_debugreg(dbg->bp[0], 0); | ||
2544 | set_debugreg(dbg->bp[1], 1); | ||
2545 | set_debugreg(dbg->bp[2], 2); | ||
2546 | set_debugreg(dbg->bp[3], 3); | ||
2547 | |||
2548 | if (dbg->singlestep) { | ||
2549 | unsigned long flags; | ||
2550 | |||
2551 | flags = vmcs_readl(GUEST_RFLAGS); | ||
2552 | flags |= X86_EFLAGS_TF | X86_EFLAGS_RF; | ||
2553 | vmcs_writel(GUEST_RFLAGS, flags); | ||
2554 | } | ||
2555 | } | ||
2556 | |||
2557 | static int handle_rmode_exception(struct kvm_vcpu *vcpu, | 2548 | static int handle_rmode_exception(struct kvm_vcpu *vcpu, |
2558 | int vec, u32 err_code) | 2549 | int vec, u32 err_code) |
2559 | { | 2550 | { |
@@ -2570,9 +2561,17 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, | |||
2570 | * the required debugging infrastructure rework. | 2561 | * the required debugging infrastructure rework. |
2571 | */ | 2562 | */ |
2572 | switch (vec) { | 2563 | switch (vec) { |
2573 | case DE_VECTOR: | ||
2574 | case DB_VECTOR: | 2564 | case DB_VECTOR: |
2565 | if (vcpu->guest_debug & | ||
2566 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) | ||
2567 | return 0; | ||
2568 | kvm_queue_exception(vcpu, vec); | ||
2569 | return 1; | ||
2575 | case BP_VECTOR: | 2570 | case BP_VECTOR: |
2571 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) | ||
2572 | return 0; | ||
2573 | /* fall through */ | ||
2574 | case DE_VECTOR: | ||
2576 | case OF_VECTOR: | 2575 | case OF_VECTOR: |
2577 | case BR_VECTOR: | 2576 | case BR_VECTOR: |
2578 | case UD_VECTOR: | 2577 | case UD_VECTOR: |
@@ -2589,8 +2588,8 @@ static int handle_rmode_exception(struct kvm_vcpu *vcpu, | |||
2589 | static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2588 | static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
2590 | { | 2589 | { |
2591 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 2590 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
2592 | u32 intr_info, error_code; | 2591 | u32 intr_info, ex_no, error_code; |
2593 | unsigned long cr2, rip; | 2592 | unsigned long cr2, rip, dr6; |
2594 | u32 vect_info; | 2593 | u32 vect_info; |
2595 | enum emulation_result er; | 2594 | enum emulation_result er; |
2596 | 2595 | ||
@@ -2649,14 +2648,30 @@ static int handle_exception(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2649 | return 1; | 2648 | return 1; |
2650 | } | 2649 | } |
2651 | 2650 | ||
2652 | if ((intr_info & (INTR_INFO_INTR_TYPE_MASK | INTR_INFO_VECTOR_MASK)) == | 2651 | ex_no = intr_info & INTR_INFO_VECTOR_MASK; |
2653 | (INTR_TYPE_EXCEPTION | 1)) { | 2652 | switch (ex_no) { |
2653 | case DB_VECTOR: | ||
2654 | dr6 = vmcs_readl(EXIT_QUALIFICATION); | ||
2655 | if (!(vcpu->guest_debug & | ||
2656 | (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))) { | ||
2657 | vcpu->arch.dr6 = dr6 | DR6_FIXED_1; | ||
2658 | kvm_queue_exception(vcpu, DB_VECTOR); | ||
2659 | return 1; | ||
2660 | } | ||
2661 | kvm_run->debug.arch.dr6 = dr6 | DR6_FIXED_1; | ||
2662 | kvm_run->debug.arch.dr7 = vmcs_readl(GUEST_DR7); | ||
2663 | /* fall through */ | ||
2664 | case BP_VECTOR: | ||
2654 | kvm_run->exit_reason = KVM_EXIT_DEBUG; | 2665 | kvm_run->exit_reason = KVM_EXIT_DEBUG; |
2655 | return 0; | 2666 | kvm_run->debug.arch.pc = vmcs_readl(GUEST_CS_BASE) + rip; |
2667 | kvm_run->debug.arch.exception = ex_no; | ||
2668 | break; | ||
2669 | default: | ||
2670 | kvm_run->exit_reason = KVM_EXIT_EXCEPTION; | ||
2671 | kvm_run->ex.exception = ex_no; | ||
2672 | kvm_run->ex.error_code = error_code; | ||
2673 | break; | ||
2656 | } | 2674 | } |
2657 | kvm_run->exit_reason = KVM_EXIT_EXCEPTION; | ||
2658 | kvm_run->ex.exception = intr_info & INTR_INFO_VECTOR_MASK; | ||
2659 | kvm_run->ex.error_code = error_code; | ||
2660 | return 0; | 2675 | return 0; |
2661 | } | 2676 | } |
2662 | 2677 | ||
@@ -2677,7 +2692,7 @@ static int handle_triple_fault(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2677 | static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 2692 | static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
2678 | { | 2693 | { |
2679 | unsigned long exit_qualification; | 2694 | unsigned long exit_qualification; |
2680 | int size, down, in, string, rep; | 2695 | int size, in, string; |
2681 | unsigned port; | 2696 | unsigned port; |
2682 | 2697 | ||
2683 | ++vcpu->stat.io_exits; | 2698 | ++vcpu->stat.io_exits; |
@@ -2693,8 +2708,6 @@ static int handle_io(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2693 | 2708 | ||
2694 | size = (exit_qualification & 7) + 1; | 2709 | size = (exit_qualification & 7) + 1; |
2695 | in = (exit_qualification & 8) != 0; | 2710 | in = (exit_qualification & 8) != 0; |
2696 | down = (vmcs_readl(GUEST_RFLAGS) & X86_EFLAGS_DF) != 0; | ||
2697 | rep = (exit_qualification & 32) != 0; | ||
2698 | port = exit_qualification >> 16; | 2711 | port = exit_qualification >> 16; |
2699 | 2712 | ||
2700 | skip_emulated_instruction(vcpu); | 2713 | skip_emulated_instruction(vcpu); |
@@ -2795,21 +2808,44 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2795 | unsigned long val; | 2808 | unsigned long val; |
2796 | int dr, reg; | 2809 | int dr, reg; |
2797 | 2810 | ||
2798 | /* | 2811 | dr = vmcs_readl(GUEST_DR7); |
2799 | * FIXME: this code assumes the host is debugging the guest. | 2812 | if (dr & DR7_GD) { |
2800 | * need to deal with guest debugging itself too. | 2813 | /* |
2801 | */ | 2814 | * As the vm-exit takes precedence over the debug trap, we |
2815 | * need to emulate the latter, either for the host or the | ||
2816 | * guest debugging itself. | ||
2817 | */ | ||
2818 | if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP) { | ||
2819 | kvm_run->debug.arch.dr6 = vcpu->arch.dr6; | ||
2820 | kvm_run->debug.arch.dr7 = dr; | ||
2821 | kvm_run->debug.arch.pc = | ||
2822 | vmcs_readl(GUEST_CS_BASE) + | ||
2823 | vmcs_readl(GUEST_RIP); | ||
2824 | kvm_run->debug.arch.exception = DB_VECTOR; | ||
2825 | kvm_run->exit_reason = KVM_EXIT_DEBUG; | ||
2826 | return 0; | ||
2827 | } else { | ||
2828 | vcpu->arch.dr7 &= ~DR7_GD; | ||
2829 | vcpu->arch.dr6 |= DR6_BD; | ||
2830 | vmcs_writel(GUEST_DR7, vcpu->arch.dr7); | ||
2831 | kvm_queue_exception(vcpu, DB_VECTOR); | ||
2832 | return 1; | ||
2833 | } | ||
2834 | } | ||
2835 | |||
2802 | exit_qualification = vmcs_readl(EXIT_QUALIFICATION); | 2836 | exit_qualification = vmcs_readl(EXIT_QUALIFICATION); |
2803 | dr = exit_qualification & 7; | 2837 | dr = exit_qualification & DEBUG_REG_ACCESS_NUM; |
2804 | reg = (exit_qualification >> 8) & 15; | 2838 | reg = DEBUG_REG_ACCESS_REG(exit_qualification); |
2805 | if (exit_qualification & 16) { | 2839 | if (exit_qualification & TYPE_MOV_FROM_DR) { |
2806 | /* mov from dr */ | ||
2807 | switch (dr) { | 2840 | switch (dr) { |
2841 | case 0 ... 3: | ||
2842 | val = vcpu->arch.db[dr]; | ||
2843 | break; | ||
2808 | case 6: | 2844 | case 6: |
2809 | val = 0xffff0ff0; | 2845 | val = vcpu->arch.dr6; |
2810 | break; | 2846 | break; |
2811 | case 7: | 2847 | case 7: |
2812 | val = 0x400; | 2848 | val = vcpu->arch.dr7; |
2813 | break; | 2849 | break; |
2814 | default: | 2850 | default: |
2815 | val = 0; | 2851 | val = 0; |
@@ -2817,7 +2853,38 @@ static int handle_dr(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2817 | kvm_register_write(vcpu, reg, val); | 2853 | kvm_register_write(vcpu, reg, val); |
2818 | KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler); | 2854 | KVMTRACE_2D(DR_READ, vcpu, (u32)dr, (u32)val, handler); |
2819 | } else { | 2855 | } else { |
2820 | /* mov to dr */ | 2856 | val = vcpu->arch.regs[reg]; |
2857 | switch (dr) { | ||
2858 | case 0 ... 3: | ||
2859 | vcpu->arch.db[dr] = val; | ||
2860 | if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) | ||
2861 | vcpu->arch.eff_db[dr] = val; | ||
2862 | break; | ||
2863 | case 4 ... 5: | ||
2864 | if (vcpu->arch.cr4 & X86_CR4_DE) | ||
2865 | kvm_queue_exception(vcpu, UD_VECTOR); | ||
2866 | break; | ||
2867 | case 6: | ||
2868 | if (val & 0xffffffff00000000ULL) { | ||
2869 | kvm_queue_exception(vcpu, GP_VECTOR); | ||
2870 | break; | ||
2871 | } | ||
2872 | vcpu->arch.dr6 = (val & DR6_VOLATILE) | DR6_FIXED_1; | ||
2873 | break; | ||
2874 | case 7: | ||
2875 | if (val & 0xffffffff00000000ULL) { | ||
2876 | kvm_queue_exception(vcpu, GP_VECTOR); | ||
2877 | break; | ||
2878 | } | ||
2879 | vcpu->arch.dr7 = (val & DR7_VOLATILE) | DR7_FIXED_1; | ||
2880 | if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)) { | ||
2881 | vmcs_writel(GUEST_DR7, vcpu->arch.dr7); | ||
2882 | vcpu->arch.switch_db_regs = | ||
2883 | (val & DR7_BP_EN_MASK); | ||
2884 | } | ||
2885 | break; | ||
2886 | } | ||
2887 | KVMTRACE_2D(DR_WRITE, vcpu, (u32)dr, (u32)val, handler); | ||
2821 | } | 2888 | } |
2822 | skip_emulated_instruction(vcpu); | 2889 | skip_emulated_instruction(vcpu); |
2823 | return 1; | 2890 | return 1; |
@@ -2968,17 +3035,25 @@ static int handle_task_switch(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2968 | } | 3035 | } |
2969 | tss_selector = exit_qualification; | 3036 | tss_selector = exit_qualification; |
2970 | 3037 | ||
2971 | return kvm_task_switch(vcpu, tss_selector, reason); | 3038 | if (!kvm_task_switch(vcpu, tss_selector, reason)) |
3039 | return 0; | ||
3040 | |||
3041 | /* clear all local breakpoint enable flags */ | ||
3042 | vmcs_writel(GUEST_DR7, vmcs_readl(GUEST_DR7) & ~55); | ||
3043 | |||
3044 | /* | ||
3045 | * TODO: What about debug traps on tss switch? | ||
3046 | * Are we supposed to inject them and update dr6? | ||
3047 | */ | ||
3048 | |||
3049 | return 1; | ||
2972 | } | 3050 | } |
2973 | 3051 | ||
2974 | static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3052 | static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
2975 | { | 3053 | { |
2976 | u64 exit_qualification; | 3054 | u64 exit_qualification; |
2977 | enum emulation_result er; | ||
2978 | gpa_t gpa; | 3055 | gpa_t gpa; |
2979 | unsigned long hva; | ||
2980 | int gla_validity; | 3056 | int gla_validity; |
2981 | int r; | ||
2982 | 3057 | ||
2983 | exit_qualification = vmcs_read64(EXIT_QUALIFICATION); | 3058 | exit_qualification = vmcs_read64(EXIT_QUALIFICATION); |
2984 | 3059 | ||
@@ -3001,32 +3076,7 @@ static int handle_ept_violation(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3001 | } | 3076 | } |
3002 | 3077 | ||
3003 | gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); | 3078 | gpa = vmcs_read64(GUEST_PHYSICAL_ADDRESS); |
3004 | hva = gfn_to_hva(vcpu->kvm, gpa >> PAGE_SHIFT); | 3079 | return kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0); |
3005 | if (!kvm_is_error_hva(hva)) { | ||
3006 | r = kvm_mmu_page_fault(vcpu, gpa & PAGE_MASK, 0); | ||
3007 | if (r < 0) { | ||
3008 | printk(KERN_ERR "EPT: Not enough memory!\n"); | ||
3009 | return -ENOMEM; | ||
3010 | } | ||
3011 | return 1; | ||
3012 | } else { | ||
3013 | /* must be MMIO */ | ||
3014 | er = emulate_instruction(vcpu, kvm_run, 0, 0, 0); | ||
3015 | |||
3016 | if (er == EMULATE_FAIL) { | ||
3017 | printk(KERN_ERR | ||
3018 | "EPT: Fail to handle EPT violation vmexit!er is %d\n", | ||
3019 | er); | ||
3020 | printk(KERN_ERR "EPT: GPA: 0x%lx, GVA: 0x%lx\n", | ||
3021 | (long unsigned int)vmcs_read64(GUEST_PHYSICAL_ADDRESS), | ||
3022 | (long unsigned int)vmcs_read64(GUEST_LINEAR_ADDRESS)); | ||
3023 | printk(KERN_ERR "EPT: Exit qualification is 0x%lx\n", | ||
3024 | (long unsigned int)exit_qualification); | ||
3025 | return -ENOTSUPP; | ||
3026 | } else if (er == EMULATE_DO_MMIO) | ||
3027 | return 0; | ||
3028 | } | ||
3029 | return 1; | ||
3030 | } | 3080 | } |
3031 | 3081 | ||
3032 | static int handle_nmi_window(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | 3082 | static int handle_nmi_window(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) |
@@ -3046,7 +3096,7 @@ static void handle_invalid_guest_state(struct kvm_vcpu *vcpu, | |||
3046 | struct kvm_run *kvm_run) | 3096 | struct kvm_run *kvm_run) |
3047 | { | 3097 | { |
3048 | struct vcpu_vmx *vmx = to_vmx(vcpu); | 3098 | struct vcpu_vmx *vmx = to_vmx(vcpu); |
3049 | int err; | 3099 | enum emulation_result err = EMULATE_DONE; |
3050 | 3100 | ||
3051 | preempt_enable(); | 3101 | preempt_enable(); |
3052 | local_irq_enable(); | 3102 | local_irq_enable(); |
@@ -3071,10 +3121,7 @@ static void handle_invalid_guest_state(struct kvm_vcpu *vcpu, | |||
3071 | local_irq_disable(); | 3121 | local_irq_disable(); |
3072 | preempt_disable(); | 3122 | preempt_disable(); |
3073 | 3123 | ||
3074 | /* Guest state should be valid now except if we need to | 3124 | vmx->invalid_state_emulation_result = err; |
3075 | * emulate an MMIO */ | ||
3076 | if (guest_state_valid(vcpu)) | ||
3077 | vmx->emulation_required = 0; | ||
3078 | } | 3125 | } |
3079 | 3126 | ||
3080 | /* | 3127 | /* |
@@ -3123,8 +3170,11 @@ static int kvm_handle_exit(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) | |||
3123 | 3170 | ||
3124 | /* If we need to emulate an MMIO from handle_invalid_guest_state | 3171 | /* If we need to emulate an MMIO from handle_invalid_guest_state |
3125 | * we just return 0 */ | 3172 | * we just return 0 */ |
3126 | if (vmx->emulation_required && emulate_invalid_guest_state) | 3173 | if (vmx->emulation_required && emulate_invalid_guest_state) { |
3127 | return 0; | 3174 | if (guest_state_valid(vcpu)) |
3175 | vmx->emulation_required = 0; | ||
3176 | return vmx->invalid_state_emulation_result != EMULATE_DO_MMIO; | ||
3177 | } | ||
3128 | 3178 | ||
3129 | /* Access CR3 don't cause VMExit in paging mode, so we need | 3179 | /* Access CR3 don't cause VMExit in paging mode, so we need |
3130 | * to sync with guest real CR3. */ | 3180 | * to sync with guest real CR3. */ |
@@ -3238,7 +3288,8 @@ static void vmx_complete_interrupts(struct vcpu_vmx *vmx) | |||
3238 | vmx->vcpu.arch.nmi_injected = false; | 3288 | vmx->vcpu.arch.nmi_injected = false; |
3239 | } | 3289 | } |
3240 | kvm_clear_exception_queue(&vmx->vcpu); | 3290 | kvm_clear_exception_queue(&vmx->vcpu); |
3241 | if (idtv_info_valid && type == INTR_TYPE_EXCEPTION) { | 3291 | if (idtv_info_valid && (type == INTR_TYPE_HARD_EXCEPTION || |
3292 | type == INTR_TYPE_SOFT_EXCEPTION)) { | ||
3242 | if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { | 3293 | if (idt_vectoring_info & VECTORING_INFO_DELIVER_CODE_MASK) { |
3243 | error = vmcs_read32(IDT_VECTORING_ERROR_CODE); | 3294 | error = vmcs_read32(IDT_VECTORING_ERROR_CODE); |
3244 | kvm_queue_exception_e(&vmx->vcpu, vector, error); | 3295 | kvm_queue_exception_e(&vmx->vcpu, vector, error); |
@@ -3259,6 +3310,11 @@ static void vmx_intr_assist(struct kvm_vcpu *vcpu) | |||
3259 | 3310 | ||
3260 | vmx_update_window_states(vcpu); | 3311 | vmx_update_window_states(vcpu); |
3261 | 3312 | ||
3313 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) | ||
3314 | vmcs_clear_bits(GUEST_INTERRUPTIBILITY_INFO, | ||
3315 | GUEST_INTR_STATE_STI | | ||
3316 | GUEST_INTR_STATE_MOV_SS); | ||
3317 | |||
3262 | if (vcpu->arch.nmi_pending && !vcpu->arch.nmi_injected) { | 3318 | if (vcpu->arch.nmi_pending && !vcpu->arch.nmi_injected) { |
3263 | if (vcpu->arch.interrupt.pending) { | 3319 | if (vcpu->arch.interrupt.pending) { |
3264 | enable_nmi_window(vcpu); | 3320 | enable_nmi_window(vcpu); |
@@ -3347,6 +3403,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3347 | */ | 3403 | */ |
3348 | vmcs_writel(HOST_CR0, read_cr0()); | 3404 | vmcs_writel(HOST_CR0, read_cr0()); |
3349 | 3405 | ||
3406 | set_debugreg(vcpu->arch.dr6, 6); | ||
3407 | |||
3350 | asm( | 3408 | asm( |
3351 | /* Store host registers */ | 3409 | /* Store host registers */ |
3352 | "push %%"R"dx; push %%"R"bp;" | 3410 | "push %%"R"dx; push %%"R"bp;" |
@@ -3441,6 +3499,8 @@ static void vmx_vcpu_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3441 | vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)); | 3499 | vcpu->arch.regs_avail = ~((1 << VCPU_REGS_RIP) | (1 << VCPU_REGS_RSP)); |
3442 | vcpu->arch.regs_dirty = 0; | 3500 | vcpu->arch.regs_dirty = 0; |
3443 | 3501 | ||
3502 | get_debugreg(vcpu->arch.dr6, 6); | ||
3503 | |||
3444 | vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); | 3504 | vmx->idt_vectoring_info = vmcs_read32(IDT_VECTORING_INFO_FIELD); |
3445 | if (vmx->rmode.irq.pending) | 3505 | if (vmx->rmode.irq.pending) |
3446 | fixup_rmode_irq(vmx); | 3506 | fixup_rmode_irq(vmx); |
@@ -3595,7 +3655,6 @@ static struct kvm_x86_ops vmx_x86_ops = { | |||
3595 | .vcpu_put = vmx_vcpu_put, | 3655 | .vcpu_put = vmx_vcpu_put, |
3596 | 3656 | ||
3597 | .set_guest_debug = set_guest_debug, | 3657 | .set_guest_debug = set_guest_debug, |
3598 | .guest_debug_pre = kvm_guest_debug_pre, | ||
3599 | .get_msr = vmx_get_msr, | 3658 | .get_msr = vmx_get_msr, |
3600 | .set_msr = vmx_set_msr, | 3659 | .set_msr = vmx_set_msr, |
3601 | .get_segment_base = vmx_get_segment_base, | 3660 | .get_segment_base = vmx_get_segment_base, |
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 758b7a155ae9..8ca100a9ecac 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c | |||
@@ -36,6 +36,7 @@ | |||
36 | #include <linux/highmem.h> | 36 | #include <linux/highmem.h> |
37 | #include <linux/iommu.h> | 37 | #include <linux/iommu.h> |
38 | #include <linux/intel-iommu.h> | 38 | #include <linux/intel-iommu.h> |
39 | #include <linux/cpufreq.h> | ||
39 | 40 | ||
40 | #include <asm/uaccess.h> | 41 | #include <asm/uaccess.h> |
41 | #include <asm/msr.h> | 42 | #include <asm/msr.h> |
@@ -69,6 +70,8 @@ static u64 __read_mostly efer_reserved_bits = 0xfffffffffffffffeULL; | |||
69 | 70 | ||
70 | static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | 71 | static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, |
71 | struct kvm_cpuid_entry2 __user *entries); | 72 | struct kvm_cpuid_entry2 __user *entries); |
73 | struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu, | ||
74 | u32 function, u32 index); | ||
72 | 75 | ||
73 | struct kvm_x86_ops *kvm_x86_ops; | 76 | struct kvm_x86_ops *kvm_x86_ops; |
74 | EXPORT_SYMBOL_GPL(kvm_x86_ops); | 77 | EXPORT_SYMBOL_GPL(kvm_x86_ops); |
@@ -173,6 +176,7 @@ void kvm_inject_page_fault(struct kvm_vcpu *vcpu, unsigned long addr, | |||
173 | u32 error_code) | 176 | u32 error_code) |
174 | { | 177 | { |
175 | ++vcpu->stat.pf_guest; | 178 | ++vcpu->stat.pf_guest; |
179 | |||
176 | if (vcpu->arch.exception.pending) { | 180 | if (vcpu->arch.exception.pending) { |
177 | if (vcpu->arch.exception.nr == PF_VECTOR) { | 181 | if (vcpu->arch.exception.nr == PF_VECTOR) { |
178 | printk(KERN_DEBUG "kvm: inject_page_fault:" | 182 | printk(KERN_DEBUG "kvm: inject_page_fault:" |
@@ -361,6 +365,7 @@ void kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) | |||
361 | } | 365 | } |
362 | kvm_x86_ops->set_cr4(vcpu, cr4); | 366 | kvm_x86_ops->set_cr4(vcpu, cr4); |
363 | vcpu->arch.cr4 = cr4; | 367 | vcpu->arch.cr4 = cr4; |
368 | vcpu->arch.mmu.base_role.cr4_pge = (cr4 & X86_CR4_PGE) && !tdp_enabled; | ||
364 | kvm_mmu_sync_global(vcpu); | 369 | kvm_mmu_sync_global(vcpu); |
365 | kvm_mmu_reset_context(vcpu); | 370 | kvm_mmu_reset_context(vcpu); |
366 | } | 371 | } |
@@ -442,6 +447,11 @@ unsigned long kvm_get_cr8(struct kvm_vcpu *vcpu) | |||
442 | } | 447 | } |
443 | EXPORT_SYMBOL_GPL(kvm_get_cr8); | 448 | EXPORT_SYMBOL_GPL(kvm_get_cr8); |
444 | 449 | ||
450 | static inline u32 bit(int bitno) | ||
451 | { | ||
452 | return 1 << (bitno & 31); | ||
453 | } | ||
454 | |||
445 | /* | 455 | /* |
446 | * List of msr numbers which we expose to userspace through KVM_GET_MSRS | 456 | * List of msr numbers which we expose to userspace through KVM_GET_MSRS |
447 | * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. | 457 | * and KVM_SET_MSRS, and KVM_GET_MSR_INDEX_LIST. |
@@ -456,7 +466,7 @@ static u32 msrs_to_save[] = { | |||
456 | MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, | 466 | MSR_CSTAR, MSR_KERNEL_GS_BASE, MSR_SYSCALL_MASK, MSR_LSTAR, |
457 | #endif | 467 | #endif |
458 | MSR_IA32_TIME_STAMP_COUNTER, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, | 468 | MSR_IA32_TIME_STAMP_COUNTER, MSR_KVM_SYSTEM_TIME, MSR_KVM_WALL_CLOCK, |
459 | MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT | 469 | MSR_IA32_PERF_STATUS, MSR_IA32_CR_PAT, MSR_VM_HSAVE_PA |
460 | }; | 470 | }; |
461 | 471 | ||
462 | static unsigned num_msrs_to_save; | 472 | static unsigned num_msrs_to_save; |
@@ -481,6 +491,28 @@ static void set_efer(struct kvm_vcpu *vcpu, u64 efer) | |||
481 | return; | 491 | return; |
482 | } | 492 | } |
483 | 493 | ||
494 | if (efer & EFER_FFXSR) { | ||
495 | struct kvm_cpuid_entry2 *feat; | ||
496 | |||
497 | feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); | ||
498 | if (!feat || !(feat->edx & bit(X86_FEATURE_FXSR_OPT))) { | ||
499 | printk(KERN_DEBUG "set_efer: #GP, enable FFXSR w/o CPUID capability\n"); | ||
500 | kvm_inject_gp(vcpu, 0); | ||
501 | return; | ||
502 | } | ||
503 | } | ||
504 | |||
505 | if (efer & EFER_SVME) { | ||
506 | struct kvm_cpuid_entry2 *feat; | ||
507 | |||
508 | feat = kvm_find_cpuid_entry(vcpu, 0x80000001, 0); | ||
509 | if (!feat || !(feat->ecx & bit(X86_FEATURE_SVM))) { | ||
510 | printk(KERN_DEBUG "set_efer: #GP, enable SVM w/o SVM\n"); | ||
511 | kvm_inject_gp(vcpu, 0); | ||
512 | return; | ||
513 | } | ||
514 | } | ||
515 | |||
484 | kvm_x86_ops->set_efer(vcpu, efer); | 516 | kvm_x86_ops->set_efer(vcpu, efer); |
485 | 517 | ||
486 | efer &= ~EFER_LMA; | 518 | efer &= ~EFER_LMA; |
@@ -586,6 +618,8 @@ static void kvm_set_time_scale(uint32_t tsc_khz, struct pvclock_vcpu_time_info * | |||
586 | hv_clock->tsc_to_system_mul); | 618 | hv_clock->tsc_to_system_mul); |
587 | } | 619 | } |
588 | 620 | ||
621 | static DEFINE_PER_CPU(unsigned long, cpu_tsc_khz); | ||
622 | |||
589 | static void kvm_write_guest_time(struct kvm_vcpu *v) | 623 | static void kvm_write_guest_time(struct kvm_vcpu *v) |
590 | { | 624 | { |
591 | struct timespec ts; | 625 | struct timespec ts; |
@@ -596,9 +630,9 @@ static void kvm_write_guest_time(struct kvm_vcpu *v) | |||
596 | if ((!vcpu->time_page)) | 630 | if ((!vcpu->time_page)) |
597 | return; | 631 | return; |
598 | 632 | ||
599 | if (unlikely(vcpu->hv_clock_tsc_khz != tsc_khz)) { | 633 | if (unlikely(vcpu->hv_clock_tsc_khz != __get_cpu_var(cpu_tsc_khz))) { |
600 | kvm_set_time_scale(tsc_khz, &vcpu->hv_clock); | 634 | kvm_set_time_scale(__get_cpu_var(cpu_tsc_khz), &vcpu->hv_clock); |
601 | vcpu->hv_clock_tsc_khz = tsc_khz; | 635 | vcpu->hv_clock_tsc_khz = __get_cpu_var(cpu_tsc_khz); |
602 | } | 636 | } |
603 | 637 | ||
604 | /* Keep irq disabled to prevent changes to the clock */ | 638 | /* Keep irq disabled to prevent changes to the clock */ |
@@ -629,6 +663,16 @@ static void kvm_write_guest_time(struct kvm_vcpu *v) | |||
629 | mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT); | 663 | mark_page_dirty(v->kvm, vcpu->time >> PAGE_SHIFT); |
630 | } | 664 | } |
631 | 665 | ||
666 | static int kvm_request_guest_time_update(struct kvm_vcpu *v) | ||
667 | { | ||
668 | struct kvm_vcpu_arch *vcpu = &v->arch; | ||
669 | |||
670 | if (!vcpu->time_page) | ||
671 | return 0; | ||
672 | set_bit(KVM_REQ_KVMCLOCK_UPDATE, &v->requests); | ||
673 | return 1; | ||
674 | } | ||
675 | |||
632 | static bool msr_mtrr_valid(unsigned msr) | 676 | static bool msr_mtrr_valid(unsigned msr) |
633 | { | 677 | { |
634 | switch (msr) { | 678 | switch (msr) { |
@@ -722,6 +766,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
722 | break; | 766 | break; |
723 | case MSR_IA32_UCODE_REV: | 767 | case MSR_IA32_UCODE_REV: |
724 | case MSR_IA32_UCODE_WRITE: | 768 | case MSR_IA32_UCODE_WRITE: |
769 | case MSR_VM_HSAVE_PA: | ||
725 | break; | 770 | break; |
726 | case 0x200 ... 0x2ff: | 771 | case 0x200 ... 0x2ff: |
727 | return set_msr_mtrr(vcpu, msr, data); | 772 | return set_msr_mtrr(vcpu, msr, data); |
@@ -758,7 +803,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 data) | |||
758 | vcpu->arch.time_page = NULL; | 803 | vcpu->arch.time_page = NULL; |
759 | } | 804 | } |
760 | 805 | ||
761 | kvm_write_guest_time(vcpu); | 806 | kvm_request_guest_time_update(vcpu); |
762 | break; | 807 | break; |
763 | } | 808 | } |
764 | default: | 809 | default: |
@@ -843,6 +888,7 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata) | |||
843 | case MSR_IA32_LASTBRANCHTOIP: | 888 | case MSR_IA32_LASTBRANCHTOIP: |
844 | case MSR_IA32_LASTINTFROMIP: | 889 | case MSR_IA32_LASTINTFROMIP: |
845 | case MSR_IA32_LASTINTTOIP: | 890 | case MSR_IA32_LASTINTTOIP: |
891 | case MSR_VM_HSAVE_PA: | ||
846 | data = 0; | 892 | data = 0; |
847 | break; | 893 | break; |
848 | case MSR_MTRRcap: | 894 | case MSR_MTRRcap: |
@@ -967,10 +1013,13 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
967 | case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: | 1013 | case KVM_CAP_MMU_SHADOW_CACHE_CONTROL: |
968 | case KVM_CAP_SET_TSS_ADDR: | 1014 | case KVM_CAP_SET_TSS_ADDR: |
969 | case KVM_CAP_EXT_CPUID: | 1015 | case KVM_CAP_EXT_CPUID: |
1016 | case KVM_CAP_CLOCKSOURCE: | ||
970 | case KVM_CAP_PIT: | 1017 | case KVM_CAP_PIT: |
971 | case KVM_CAP_NOP_IO_DELAY: | 1018 | case KVM_CAP_NOP_IO_DELAY: |
972 | case KVM_CAP_MP_STATE: | 1019 | case KVM_CAP_MP_STATE: |
973 | case KVM_CAP_SYNC_MMU: | 1020 | case KVM_CAP_SYNC_MMU: |
1021 | case KVM_CAP_REINJECT_CONTROL: | ||
1022 | case KVM_CAP_IRQ_INJECT_STATUS: | ||
974 | r = 1; | 1023 | r = 1; |
975 | break; | 1024 | break; |
976 | case KVM_CAP_COALESCED_MMIO: | 1025 | case KVM_CAP_COALESCED_MMIO: |
@@ -991,9 +1040,6 @@ int kvm_dev_ioctl_check_extension(long ext) | |||
991 | case KVM_CAP_IOMMU: | 1040 | case KVM_CAP_IOMMU: |
992 | r = iommu_found(); | 1041 | r = iommu_found(); |
993 | break; | 1042 | break; |
994 | case KVM_CAP_CLOCKSOURCE: | ||
995 | r = boot_cpu_has(X86_FEATURE_CONSTANT_TSC); | ||
996 | break; | ||
997 | default: | 1043 | default: |
998 | r = 0; | 1044 | r = 0; |
999 | break; | 1045 | break; |
@@ -1044,7 +1090,7 @@ long kvm_arch_dev_ioctl(struct file *filp, | |||
1044 | if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) | 1090 | if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) |
1045 | goto out; | 1091 | goto out; |
1046 | r = kvm_dev_ioctl_get_supported_cpuid(&cpuid, | 1092 | r = kvm_dev_ioctl_get_supported_cpuid(&cpuid, |
1047 | cpuid_arg->entries); | 1093 | cpuid_arg->entries); |
1048 | if (r) | 1094 | if (r) |
1049 | goto out; | 1095 | goto out; |
1050 | 1096 | ||
@@ -1064,7 +1110,7 @@ out: | |||
1064 | void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) | 1110 | void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) |
1065 | { | 1111 | { |
1066 | kvm_x86_ops->vcpu_load(vcpu, cpu); | 1112 | kvm_x86_ops->vcpu_load(vcpu, cpu); |
1067 | kvm_write_guest_time(vcpu); | 1113 | kvm_request_guest_time_update(vcpu); |
1068 | } | 1114 | } |
1069 | 1115 | ||
1070 | void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) | 1116 | void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) |
@@ -1142,8 +1188,8 @@ out: | |||
1142 | } | 1188 | } |
1143 | 1189 | ||
1144 | static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, | 1190 | static int kvm_vcpu_ioctl_set_cpuid2(struct kvm_vcpu *vcpu, |
1145 | struct kvm_cpuid2 *cpuid, | 1191 | struct kvm_cpuid2 *cpuid, |
1146 | struct kvm_cpuid_entry2 __user *entries) | 1192 | struct kvm_cpuid_entry2 __user *entries) |
1147 | { | 1193 | { |
1148 | int r; | 1194 | int r; |
1149 | 1195 | ||
@@ -1162,8 +1208,8 @@ out: | |||
1162 | } | 1208 | } |
1163 | 1209 | ||
1164 | static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, | 1210 | static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, |
1165 | struct kvm_cpuid2 *cpuid, | 1211 | struct kvm_cpuid2 *cpuid, |
1166 | struct kvm_cpuid_entry2 __user *entries) | 1212 | struct kvm_cpuid_entry2 __user *entries) |
1167 | { | 1213 | { |
1168 | int r; | 1214 | int r; |
1169 | 1215 | ||
@@ -1172,7 +1218,7 @@ static int kvm_vcpu_ioctl_get_cpuid2(struct kvm_vcpu *vcpu, | |||
1172 | goto out; | 1218 | goto out; |
1173 | r = -EFAULT; | 1219 | r = -EFAULT; |
1174 | if (copy_to_user(entries, &vcpu->arch.cpuid_entries, | 1220 | if (copy_to_user(entries, &vcpu->arch.cpuid_entries, |
1175 | vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) | 1221 | vcpu->arch.cpuid_nent * sizeof(struct kvm_cpuid_entry2))) |
1176 | goto out; | 1222 | goto out; |
1177 | return 0; | 1223 | return 0; |
1178 | 1224 | ||
@@ -1181,18 +1227,13 @@ out: | |||
1181 | return r; | 1227 | return r; |
1182 | } | 1228 | } |
1183 | 1229 | ||
1184 | static inline u32 bit(int bitno) | ||
1185 | { | ||
1186 | return 1 << (bitno & 31); | ||
1187 | } | ||
1188 | |||
1189 | static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, | 1230 | static void do_cpuid_1_ent(struct kvm_cpuid_entry2 *entry, u32 function, |
1190 | u32 index) | 1231 | u32 index) |
1191 | { | 1232 | { |
1192 | entry->function = function; | 1233 | entry->function = function; |
1193 | entry->index = index; | 1234 | entry->index = index; |
1194 | cpuid_count(entry->function, entry->index, | 1235 | cpuid_count(entry->function, entry->index, |
1195 | &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); | 1236 | &entry->eax, &entry->ebx, &entry->ecx, &entry->edx); |
1196 | entry->flags = 0; | 1237 | entry->flags = 0; |
1197 | } | 1238 | } |
1198 | 1239 | ||
@@ -1222,15 +1263,17 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, | |||
1222 | #ifdef CONFIG_X86_64 | 1263 | #ifdef CONFIG_X86_64 |
1223 | bit(X86_FEATURE_LM) | | 1264 | bit(X86_FEATURE_LM) | |
1224 | #endif | 1265 | #endif |
1266 | bit(X86_FEATURE_FXSR_OPT) | | ||
1225 | bit(X86_FEATURE_MMXEXT) | | 1267 | bit(X86_FEATURE_MMXEXT) | |
1226 | bit(X86_FEATURE_3DNOWEXT) | | 1268 | bit(X86_FEATURE_3DNOWEXT) | |
1227 | bit(X86_FEATURE_3DNOW); | 1269 | bit(X86_FEATURE_3DNOW); |
1228 | const u32 kvm_supported_word3_x86_features = | 1270 | const u32 kvm_supported_word3_x86_features = |
1229 | bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16); | 1271 | bit(X86_FEATURE_XMM3) | bit(X86_FEATURE_CX16); |
1230 | const u32 kvm_supported_word6_x86_features = | 1272 | const u32 kvm_supported_word6_x86_features = |
1231 | bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY); | 1273 | bit(X86_FEATURE_LAHF_LM) | bit(X86_FEATURE_CMP_LEGACY) | |
1274 | bit(X86_FEATURE_SVM); | ||
1232 | 1275 | ||
1233 | /* all func 2 cpuid_count() should be called on the same cpu */ | 1276 | /* all calls to cpuid_count() should be made on the same cpu */ |
1234 | get_cpu(); | 1277 | get_cpu(); |
1235 | do_cpuid_1_ent(entry, function, index); | 1278 | do_cpuid_1_ent(entry, function, index); |
1236 | ++*nent; | 1279 | ++*nent; |
@@ -1304,7 +1347,7 @@ static void do_cpuid_ent(struct kvm_cpuid_entry2 *entry, u32 function, | |||
1304 | } | 1347 | } |
1305 | 1348 | ||
1306 | static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | 1349 | static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, |
1307 | struct kvm_cpuid_entry2 __user *entries) | 1350 | struct kvm_cpuid_entry2 __user *entries) |
1308 | { | 1351 | { |
1309 | struct kvm_cpuid_entry2 *cpuid_entries; | 1352 | struct kvm_cpuid_entry2 *cpuid_entries; |
1310 | int limit, nent = 0, r = -E2BIG; | 1353 | int limit, nent = 0, r = -E2BIG; |
@@ -1321,7 +1364,7 @@ static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | |||
1321 | limit = cpuid_entries[0].eax; | 1364 | limit = cpuid_entries[0].eax; |
1322 | for (func = 1; func <= limit && nent < cpuid->nent; ++func) | 1365 | for (func = 1; func <= limit && nent < cpuid->nent; ++func) |
1323 | do_cpuid_ent(&cpuid_entries[nent], func, 0, | 1366 | do_cpuid_ent(&cpuid_entries[nent], func, 0, |
1324 | &nent, cpuid->nent); | 1367 | &nent, cpuid->nent); |
1325 | r = -E2BIG; | 1368 | r = -E2BIG; |
1326 | if (nent >= cpuid->nent) | 1369 | if (nent >= cpuid->nent) |
1327 | goto out_free; | 1370 | goto out_free; |
@@ -1330,10 +1373,10 @@ static int kvm_dev_ioctl_get_supported_cpuid(struct kvm_cpuid2 *cpuid, | |||
1330 | limit = cpuid_entries[nent - 1].eax; | 1373 | limit = cpuid_entries[nent - 1].eax; |
1331 | for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) | 1374 | for (func = 0x80000001; func <= limit && nent < cpuid->nent; ++func) |
1332 | do_cpuid_ent(&cpuid_entries[nent], func, 0, | 1375 | do_cpuid_ent(&cpuid_entries[nent], func, 0, |
1333 | &nent, cpuid->nent); | 1376 | &nent, cpuid->nent); |
1334 | r = -EFAULT; | 1377 | r = -EFAULT; |
1335 | if (copy_to_user(entries, cpuid_entries, | 1378 | if (copy_to_user(entries, cpuid_entries, |
1336 | nent * sizeof(struct kvm_cpuid_entry2))) | 1379 | nent * sizeof(struct kvm_cpuid_entry2))) |
1337 | goto out_free; | 1380 | goto out_free; |
1338 | cpuid->nent = nent; | 1381 | cpuid->nent = nent; |
1339 | r = 0; | 1382 | r = 0; |
@@ -1477,7 +1520,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1477 | if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) | 1520 | if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) |
1478 | goto out; | 1521 | goto out; |
1479 | r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, | 1522 | r = kvm_vcpu_ioctl_set_cpuid2(vcpu, &cpuid, |
1480 | cpuid_arg->entries); | 1523 | cpuid_arg->entries); |
1481 | if (r) | 1524 | if (r) |
1482 | goto out; | 1525 | goto out; |
1483 | break; | 1526 | break; |
@@ -1490,7 +1533,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, | |||
1490 | if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) | 1533 | if (copy_from_user(&cpuid, cpuid_arg, sizeof cpuid)) |
1491 | goto out; | 1534 | goto out; |
1492 | r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, | 1535 | r = kvm_vcpu_ioctl_get_cpuid2(vcpu, &cpuid, |
1493 | cpuid_arg->entries); | 1536 | cpuid_arg->entries); |
1494 | if (r) | 1537 | if (r) |
1495 | goto out; | 1538 | goto out; |
1496 | r = -EFAULT; | 1539 | r = -EFAULT; |
@@ -1710,6 +1753,15 @@ static int kvm_vm_ioctl_set_pit(struct kvm *kvm, struct kvm_pit_state *ps) | |||
1710 | return r; | 1753 | return r; |
1711 | } | 1754 | } |
1712 | 1755 | ||
1756 | static int kvm_vm_ioctl_reinject(struct kvm *kvm, | ||
1757 | struct kvm_reinject_control *control) | ||
1758 | { | ||
1759 | if (!kvm->arch.vpit) | ||
1760 | return -ENXIO; | ||
1761 | kvm->arch.vpit->pit_state.pit_timer.reinject = control->pit_reinject; | ||
1762 | return 0; | ||
1763 | } | ||
1764 | |||
1713 | /* | 1765 | /* |
1714 | * Get (and clear) the dirty memory log for a memory slot. | 1766 | * Get (and clear) the dirty memory log for a memory slot. |
1715 | */ | 1767 | */ |
@@ -1807,13 +1859,26 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1807 | } | 1859 | } |
1808 | } else | 1860 | } else |
1809 | goto out; | 1861 | goto out; |
1862 | r = kvm_setup_default_irq_routing(kvm); | ||
1863 | if (r) { | ||
1864 | kfree(kvm->arch.vpic); | ||
1865 | kfree(kvm->arch.vioapic); | ||
1866 | goto out; | ||
1867 | } | ||
1810 | break; | 1868 | break; |
1811 | case KVM_CREATE_PIT: | 1869 | case KVM_CREATE_PIT: |
1870 | mutex_lock(&kvm->lock); | ||
1871 | r = -EEXIST; | ||
1872 | if (kvm->arch.vpit) | ||
1873 | goto create_pit_unlock; | ||
1812 | r = -ENOMEM; | 1874 | r = -ENOMEM; |
1813 | kvm->arch.vpit = kvm_create_pit(kvm); | 1875 | kvm->arch.vpit = kvm_create_pit(kvm); |
1814 | if (kvm->arch.vpit) | 1876 | if (kvm->arch.vpit) |
1815 | r = 0; | 1877 | r = 0; |
1878 | create_pit_unlock: | ||
1879 | mutex_unlock(&kvm->lock); | ||
1816 | break; | 1880 | break; |
1881 | case KVM_IRQ_LINE_STATUS: | ||
1817 | case KVM_IRQ_LINE: { | 1882 | case KVM_IRQ_LINE: { |
1818 | struct kvm_irq_level irq_event; | 1883 | struct kvm_irq_level irq_event; |
1819 | 1884 | ||
@@ -1821,10 +1886,17 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1821 | if (copy_from_user(&irq_event, argp, sizeof irq_event)) | 1886 | if (copy_from_user(&irq_event, argp, sizeof irq_event)) |
1822 | goto out; | 1887 | goto out; |
1823 | if (irqchip_in_kernel(kvm)) { | 1888 | if (irqchip_in_kernel(kvm)) { |
1889 | __s32 status; | ||
1824 | mutex_lock(&kvm->lock); | 1890 | mutex_lock(&kvm->lock); |
1825 | kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, | 1891 | status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID, |
1826 | irq_event.irq, irq_event.level); | 1892 | irq_event.irq, irq_event.level); |
1827 | mutex_unlock(&kvm->lock); | 1893 | mutex_unlock(&kvm->lock); |
1894 | if (ioctl == KVM_IRQ_LINE_STATUS) { | ||
1895 | irq_event.status = status; | ||
1896 | if (copy_to_user(argp, &irq_event, | ||
1897 | sizeof irq_event)) | ||
1898 | goto out; | ||
1899 | } | ||
1828 | r = 0; | 1900 | r = 0; |
1829 | } | 1901 | } |
1830 | break; | 1902 | break; |
@@ -1907,6 +1979,17 @@ long kvm_arch_vm_ioctl(struct file *filp, | |||
1907 | r = 0; | 1979 | r = 0; |
1908 | break; | 1980 | break; |
1909 | } | 1981 | } |
1982 | case KVM_REINJECT_CONTROL: { | ||
1983 | struct kvm_reinject_control control; | ||
1984 | r = -EFAULT; | ||
1985 | if (copy_from_user(&control, argp, sizeof(control))) | ||
1986 | goto out; | ||
1987 | r = kvm_vm_ioctl_reinject(kvm, &control); | ||
1988 | if (r) | ||
1989 | goto out; | ||
1990 | r = 0; | ||
1991 | break; | ||
1992 | } | ||
1910 | default: | 1993 | default: |
1911 | ; | 1994 | ; |
1912 | } | 1995 | } |
@@ -1960,10 +2043,38 @@ static struct kvm_io_device *vcpu_find_mmio_dev(struct kvm_vcpu *vcpu, | |||
1960 | return dev; | 2043 | return dev; |
1961 | } | 2044 | } |
1962 | 2045 | ||
1963 | int emulator_read_std(unsigned long addr, | 2046 | static int kvm_read_guest_virt(gva_t addr, void *val, unsigned int bytes, |
1964 | void *val, | 2047 | struct kvm_vcpu *vcpu) |
1965 | unsigned int bytes, | 2048 | { |
1966 | struct kvm_vcpu *vcpu) | 2049 | void *data = val; |
2050 | int r = X86EMUL_CONTINUE; | ||
2051 | |||
2052 | while (bytes) { | ||
2053 | gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); | ||
2054 | unsigned offset = addr & (PAGE_SIZE-1); | ||
2055 | unsigned toread = min(bytes, (unsigned)PAGE_SIZE - offset); | ||
2056 | int ret; | ||
2057 | |||
2058 | if (gpa == UNMAPPED_GVA) { | ||
2059 | r = X86EMUL_PROPAGATE_FAULT; | ||
2060 | goto out; | ||
2061 | } | ||
2062 | ret = kvm_read_guest(vcpu->kvm, gpa, data, toread); | ||
2063 | if (ret < 0) { | ||
2064 | r = X86EMUL_UNHANDLEABLE; | ||
2065 | goto out; | ||
2066 | } | ||
2067 | |||
2068 | bytes -= toread; | ||
2069 | data += toread; | ||
2070 | addr += toread; | ||
2071 | } | ||
2072 | out: | ||
2073 | return r; | ||
2074 | } | ||
2075 | |||
2076 | static int kvm_write_guest_virt(gva_t addr, void *val, unsigned int bytes, | ||
2077 | struct kvm_vcpu *vcpu) | ||
1967 | { | 2078 | { |
1968 | void *data = val; | 2079 | void *data = val; |
1969 | int r = X86EMUL_CONTINUE; | 2080 | int r = X86EMUL_CONTINUE; |
@@ -1971,27 +2082,27 @@ int emulator_read_std(unsigned long addr, | |||
1971 | while (bytes) { | 2082 | while (bytes) { |
1972 | gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); | 2083 | gpa_t gpa = vcpu->arch.mmu.gva_to_gpa(vcpu, addr); |
1973 | unsigned offset = addr & (PAGE_SIZE-1); | 2084 | unsigned offset = addr & (PAGE_SIZE-1); |
1974 | unsigned tocopy = min(bytes, (unsigned)PAGE_SIZE - offset); | 2085 | unsigned towrite = min(bytes, (unsigned)PAGE_SIZE - offset); |
1975 | int ret; | 2086 | int ret; |
1976 | 2087 | ||
1977 | if (gpa == UNMAPPED_GVA) { | 2088 | if (gpa == UNMAPPED_GVA) { |
1978 | r = X86EMUL_PROPAGATE_FAULT; | 2089 | r = X86EMUL_PROPAGATE_FAULT; |
1979 | goto out; | 2090 | goto out; |
1980 | } | 2091 | } |
1981 | ret = kvm_read_guest(vcpu->kvm, gpa, data, tocopy); | 2092 | ret = kvm_write_guest(vcpu->kvm, gpa, data, towrite); |
1982 | if (ret < 0) { | 2093 | if (ret < 0) { |
1983 | r = X86EMUL_UNHANDLEABLE; | 2094 | r = X86EMUL_UNHANDLEABLE; |
1984 | goto out; | 2095 | goto out; |
1985 | } | 2096 | } |
1986 | 2097 | ||
1987 | bytes -= tocopy; | 2098 | bytes -= towrite; |
1988 | data += tocopy; | 2099 | data += towrite; |
1989 | addr += tocopy; | 2100 | addr += towrite; |
1990 | } | 2101 | } |
1991 | out: | 2102 | out: |
1992 | return r; | 2103 | return r; |
1993 | } | 2104 | } |
1994 | EXPORT_SYMBOL_GPL(emulator_read_std); | 2105 | |
1995 | 2106 | ||
1996 | static int emulator_read_emulated(unsigned long addr, | 2107 | static int emulator_read_emulated(unsigned long addr, |
1997 | void *val, | 2108 | void *val, |
@@ -2013,8 +2124,8 @@ static int emulator_read_emulated(unsigned long addr, | |||
2013 | if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) | 2124 | if ((gpa & PAGE_MASK) == APIC_DEFAULT_PHYS_BASE) |
2014 | goto mmio; | 2125 | goto mmio; |
2015 | 2126 | ||
2016 | if (emulator_read_std(addr, val, bytes, vcpu) | 2127 | if (kvm_read_guest_virt(addr, val, bytes, vcpu) |
2017 | == X86EMUL_CONTINUE) | 2128 | == X86EMUL_CONTINUE) |
2018 | return X86EMUL_CONTINUE; | 2129 | return X86EMUL_CONTINUE; |
2019 | if (gpa == UNMAPPED_GVA) | 2130 | if (gpa == UNMAPPED_GVA) |
2020 | return X86EMUL_PROPAGATE_FAULT; | 2131 | return X86EMUL_PROPAGATE_FAULT; |
@@ -2217,7 +2328,7 @@ void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) | |||
2217 | 2328 | ||
2218 | rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); | 2329 | rip_linear = rip + get_segment_base(vcpu, VCPU_SREG_CS); |
2219 | 2330 | ||
2220 | emulator_read_std(rip_linear, (void *)opcodes, 4, vcpu); | 2331 | kvm_read_guest_virt(rip_linear, (void *)opcodes, 4, vcpu); |
2221 | 2332 | ||
2222 | printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", | 2333 | printk(KERN_ERR "emulation failed (%s) rip %lx %02x %02x %02x %02x\n", |
2223 | context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); | 2334 | context, rip, opcodes[0], opcodes[1], opcodes[2], opcodes[3]); |
@@ -2225,7 +2336,7 @@ void kvm_report_emulation_failure(struct kvm_vcpu *vcpu, const char *context) | |||
2225 | EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); | 2336 | EXPORT_SYMBOL_GPL(kvm_report_emulation_failure); |
2226 | 2337 | ||
2227 | static struct x86_emulate_ops emulate_ops = { | 2338 | static struct x86_emulate_ops emulate_ops = { |
2228 | .read_std = emulator_read_std, | 2339 | .read_std = kvm_read_guest_virt, |
2229 | .read_emulated = emulator_read_emulated, | 2340 | .read_emulated = emulator_read_emulated, |
2230 | .write_emulated = emulator_write_emulated, | 2341 | .write_emulated = emulator_write_emulated, |
2231 | .cmpxchg_emulated = emulator_cmpxchg_emulated, | 2342 | .cmpxchg_emulated = emulator_cmpxchg_emulated, |
@@ -2327,40 +2438,19 @@ int emulate_instruction(struct kvm_vcpu *vcpu, | |||
2327 | } | 2438 | } |
2328 | EXPORT_SYMBOL_GPL(emulate_instruction); | 2439 | EXPORT_SYMBOL_GPL(emulate_instruction); |
2329 | 2440 | ||
2330 | static void free_pio_guest_pages(struct kvm_vcpu *vcpu) | ||
2331 | { | ||
2332 | int i; | ||
2333 | |||
2334 | for (i = 0; i < ARRAY_SIZE(vcpu->arch.pio.guest_pages); ++i) | ||
2335 | if (vcpu->arch.pio.guest_pages[i]) { | ||
2336 | kvm_release_page_dirty(vcpu->arch.pio.guest_pages[i]); | ||
2337 | vcpu->arch.pio.guest_pages[i] = NULL; | ||
2338 | } | ||
2339 | } | ||
2340 | |||
2341 | static int pio_copy_data(struct kvm_vcpu *vcpu) | 2441 | static int pio_copy_data(struct kvm_vcpu *vcpu) |
2342 | { | 2442 | { |
2343 | void *p = vcpu->arch.pio_data; | 2443 | void *p = vcpu->arch.pio_data; |
2344 | void *q; | 2444 | gva_t q = vcpu->arch.pio.guest_gva; |
2345 | unsigned bytes; | 2445 | unsigned bytes; |
2346 | int nr_pages = vcpu->arch.pio.guest_pages[1] ? 2 : 1; | 2446 | int ret; |
2347 | 2447 | ||
2348 | q = vmap(vcpu->arch.pio.guest_pages, nr_pages, VM_READ|VM_WRITE, | ||
2349 | PAGE_KERNEL); | ||
2350 | if (!q) { | ||
2351 | free_pio_guest_pages(vcpu); | ||
2352 | return -ENOMEM; | ||
2353 | } | ||
2354 | q += vcpu->arch.pio.guest_page_offset; | ||
2355 | bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count; | 2448 | bytes = vcpu->arch.pio.size * vcpu->arch.pio.cur_count; |
2356 | if (vcpu->arch.pio.in) | 2449 | if (vcpu->arch.pio.in) |
2357 | memcpy(q, p, bytes); | 2450 | ret = kvm_write_guest_virt(q, p, bytes, vcpu); |
2358 | else | 2451 | else |
2359 | memcpy(p, q, bytes); | 2452 | ret = kvm_read_guest_virt(q, p, bytes, vcpu); |
2360 | q -= vcpu->arch.pio.guest_page_offset; | 2453 | return ret; |
2361 | vunmap(q); | ||
2362 | free_pio_guest_pages(vcpu); | ||
2363 | return 0; | ||
2364 | } | 2454 | } |
2365 | 2455 | ||
2366 | int complete_pio(struct kvm_vcpu *vcpu) | 2456 | int complete_pio(struct kvm_vcpu *vcpu) |
@@ -2471,7 +2561,6 @@ int kvm_emulate_pio(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2471 | vcpu->arch.pio.in = in; | 2561 | vcpu->arch.pio.in = in; |
2472 | vcpu->arch.pio.string = 0; | 2562 | vcpu->arch.pio.string = 0; |
2473 | vcpu->arch.pio.down = 0; | 2563 | vcpu->arch.pio.down = 0; |
2474 | vcpu->arch.pio.guest_page_offset = 0; | ||
2475 | vcpu->arch.pio.rep = 0; | 2564 | vcpu->arch.pio.rep = 0; |
2476 | 2565 | ||
2477 | if (vcpu->run->io.direction == KVM_EXIT_IO_IN) | 2566 | if (vcpu->run->io.direction == KVM_EXIT_IO_IN) |
@@ -2499,9 +2588,7 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2499 | gva_t address, int rep, unsigned port) | 2588 | gva_t address, int rep, unsigned port) |
2500 | { | 2589 | { |
2501 | unsigned now, in_page; | 2590 | unsigned now, in_page; |
2502 | int i, ret = 0; | 2591 | int ret = 0; |
2503 | int nr_pages = 1; | ||
2504 | struct page *page; | ||
2505 | struct kvm_io_device *pio_dev; | 2592 | struct kvm_io_device *pio_dev; |
2506 | 2593 | ||
2507 | vcpu->run->exit_reason = KVM_EXIT_IO; | 2594 | vcpu->run->exit_reason = KVM_EXIT_IO; |
@@ -2513,7 +2600,6 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2513 | vcpu->arch.pio.in = in; | 2600 | vcpu->arch.pio.in = in; |
2514 | vcpu->arch.pio.string = 1; | 2601 | vcpu->arch.pio.string = 1; |
2515 | vcpu->arch.pio.down = down; | 2602 | vcpu->arch.pio.down = down; |
2516 | vcpu->arch.pio.guest_page_offset = offset_in_page(address); | ||
2517 | vcpu->arch.pio.rep = rep; | 2603 | vcpu->arch.pio.rep = rep; |
2518 | 2604 | ||
2519 | if (vcpu->run->io.direction == KVM_EXIT_IO_IN) | 2605 | if (vcpu->run->io.direction == KVM_EXIT_IO_IN) |
@@ -2533,15 +2619,8 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2533 | else | 2619 | else |
2534 | in_page = offset_in_page(address) + size; | 2620 | in_page = offset_in_page(address) + size; |
2535 | now = min(count, (unsigned long)in_page / size); | 2621 | now = min(count, (unsigned long)in_page / size); |
2536 | if (!now) { | 2622 | if (!now) |
2537 | /* | ||
2538 | * String I/O straddles page boundary. Pin two guest pages | ||
2539 | * so that we satisfy atomicity constraints. Do just one | ||
2540 | * transaction to avoid complexity. | ||
2541 | */ | ||
2542 | nr_pages = 2; | ||
2543 | now = 1; | 2623 | now = 1; |
2544 | } | ||
2545 | if (down) { | 2624 | if (down) { |
2546 | /* | 2625 | /* |
2547 | * String I/O in reverse. Yuck. Kill the guest, fix later. | 2626 | * String I/O in reverse. Yuck. Kill the guest, fix later. |
@@ -2556,15 +2635,7 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2556 | if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count) | 2635 | if (vcpu->arch.pio.cur_count == vcpu->arch.pio.count) |
2557 | kvm_x86_ops->skip_emulated_instruction(vcpu); | 2636 | kvm_x86_ops->skip_emulated_instruction(vcpu); |
2558 | 2637 | ||
2559 | for (i = 0; i < nr_pages; ++i) { | 2638 | vcpu->arch.pio.guest_gva = address; |
2560 | page = gva_to_page(vcpu, address + i * PAGE_SIZE); | ||
2561 | vcpu->arch.pio.guest_pages[i] = page; | ||
2562 | if (!page) { | ||
2563 | kvm_inject_gp(vcpu, 0); | ||
2564 | free_pio_guest_pages(vcpu); | ||
2565 | return 1; | ||
2566 | } | ||
2567 | } | ||
2568 | 2639 | ||
2569 | pio_dev = vcpu_find_pio_dev(vcpu, port, | 2640 | pio_dev = vcpu_find_pio_dev(vcpu, port, |
2570 | vcpu->arch.pio.cur_count, | 2641 | vcpu->arch.pio.cur_count, |
@@ -2572,7 +2643,11 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2572 | if (!vcpu->arch.pio.in) { | 2643 | if (!vcpu->arch.pio.in) { |
2573 | /* string PIO write */ | 2644 | /* string PIO write */ |
2574 | ret = pio_copy_data(vcpu); | 2645 | ret = pio_copy_data(vcpu); |
2575 | if (ret >= 0 && pio_dev) { | 2646 | if (ret == X86EMUL_PROPAGATE_FAULT) { |
2647 | kvm_inject_gp(vcpu, 0); | ||
2648 | return 1; | ||
2649 | } | ||
2650 | if (ret == 0 && pio_dev) { | ||
2576 | pio_string_write(pio_dev, vcpu); | 2651 | pio_string_write(pio_dev, vcpu); |
2577 | complete_pio(vcpu); | 2652 | complete_pio(vcpu); |
2578 | if (vcpu->arch.pio.count == 0) | 2653 | if (vcpu->arch.pio.count == 0) |
@@ -2587,9 +2662,72 @@ int kvm_emulate_pio_string(struct kvm_vcpu *vcpu, struct kvm_run *run, int in, | |||
2587 | } | 2662 | } |
2588 | EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); | 2663 | EXPORT_SYMBOL_GPL(kvm_emulate_pio_string); |
2589 | 2664 | ||
2665 | static void bounce_off(void *info) | ||
2666 | { | ||
2667 | /* nothing */ | ||
2668 | } | ||
2669 | |||
2670 | static unsigned int ref_freq; | ||
2671 | static unsigned long tsc_khz_ref; | ||
2672 | |||
2673 | static int kvmclock_cpufreq_notifier(struct notifier_block *nb, unsigned long val, | ||
2674 | void *data) | ||
2675 | { | ||
2676 | struct cpufreq_freqs *freq = data; | ||
2677 | struct kvm *kvm; | ||
2678 | struct kvm_vcpu *vcpu; | ||
2679 | int i, send_ipi = 0; | ||
2680 | |||
2681 | if (!ref_freq) | ||
2682 | ref_freq = freq->old; | ||
2683 | |||
2684 | if (val == CPUFREQ_PRECHANGE && freq->old > freq->new) | ||
2685 | return 0; | ||
2686 | if (val == CPUFREQ_POSTCHANGE && freq->old < freq->new) | ||
2687 | return 0; | ||
2688 | per_cpu(cpu_tsc_khz, freq->cpu) = cpufreq_scale(tsc_khz_ref, ref_freq, freq->new); | ||
2689 | |||
2690 | spin_lock(&kvm_lock); | ||
2691 | list_for_each_entry(kvm, &vm_list, vm_list) { | ||
2692 | for (i = 0; i < KVM_MAX_VCPUS; ++i) { | ||
2693 | vcpu = kvm->vcpus[i]; | ||
2694 | if (!vcpu) | ||
2695 | continue; | ||
2696 | if (vcpu->cpu != freq->cpu) | ||
2697 | continue; | ||
2698 | if (!kvm_request_guest_time_update(vcpu)) | ||
2699 | continue; | ||
2700 | if (vcpu->cpu != smp_processor_id()) | ||
2701 | send_ipi++; | ||
2702 | } | ||
2703 | } | ||
2704 | spin_unlock(&kvm_lock); | ||
2705 | |||
2706 | if (freq->old < freq->new && send_ipi) { | ||
2707 | /* | ||
2708 | * We upscale the frequency. Must make the guest | ||
2709 | * doesn't see old kvmclock values while running with | ||
2710 | * the new frequency, otherwise we risk the guest sees | ||
2711 | * time go backwards. | ||
2712 | * | ||
2713 | * In case we update the frequency for another cpu | ||
2714 | * (which might be in guest context) send an interrupt | ||
2715 | * to kick the cpu out of guest context. Next time | ||
2716 | * guest context is entered kvmclock will be updated, | ||
2717 | * so the guest will not see stale values. | ||
2718 | */ | ||
2719 | smp_call_function_single(freq->cpu, bounce_off, NULL, 1); | ||
2720 | } | ||
2721 | return 0; | ||
2722 | } | ||
2723 | |||
2724 | static struct notifier_block kvmclock_cpufreq_notifier_block = { | ||
2725 | .notifier_call = kvmclock_cpufreq_notifier | ||
2726 | }; | ||
2727 | |||
2590 | int kvm_arch_init(void *opaque) | 2728 | int kvm_arch_init(void *opaque) |
2591 | { | 2729 | { |
2592 | int r; | 2730 | int r, cpu; |
2593 | struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; | 2731 | struct kvm_x86_ops *ops = (struct kvm_x86_ops *)opaque; |
2594 | 2732 | ||
2595 | if (kvm_x86_ops) { | 2733 | if (kvm_x86_ops) { |
@@ -2620,6 +2758,15 @@ int kvm_arch_init(void *opaque) | |||
2620 | kvm_mmu_set_base_ptes(PT_PRESENT_MASK); | 2758 | kvm_mmu_set_base_ptes(PT_PRESENT_MASK); |
2621 | kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, | 2759 | kvm_mmu_set_mask_ptes(PT_USER_MASK, PT_ACCESSED_MASK, |
2622 | PT_DIRTY_MASK, PT64_NX_MASK, 0, 0); | 2760 | PT_DIRTY_MASK, PT64_NX_MASK, 0, 0); |
2761 | |||
2762 | for_each_possible_cpu(cpu) | ||
2763 | per_cpu(cpu_tsc_khz, cpu) = tsc_khz; | ||
2764 | if (!boot_cpu_has(X86_FEATURE_CONSTANT_TSC)) { | ||
2765 | tsc_khz_ref = tsc_khz; | ||
2766 | cpufreq_register_notifier(&kvmclock_cpufreq_notifier_block, | ||
2767 | CPUFREQ_TRANSITION_NOTIFIER); | ||
2768 | } | ||
2769 | |||
2623 | return 0; | 2770 | return 0; |
2624 | 2771 | ||
2625 | out: | 2772 | out: |
@@ -2827,25 +2974,20 @@ static int is_matching_cpuid_entry(struct kvm_cpuid_entry2 *e, | |||
2827 | if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) | 2974 | if ((e->flags & KVM_CPUID_FLAG_SIGNIFCANT_INDEX) && e->index != index) |
2828 | return 0; | 2975 | return 0; |
2829 | if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && | 2976 | if ((e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) && |
2830 | !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) | 2977 | !(e->flags & KVM_CPUID_FLAG_STATE_READ_NEXT)) |
2831 | return 0; | 2978 | return 0; |
2832 | return 1; | 2979 | return 1; |
2833 | } | 2980 | } |
2834 | 2981 | ||
2835 | void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) | 2982 | struct kvm_cpuid_entry2 *kvm_find_cpuid_entry(struct kvm_vcpu *vcpu, |
2983 | u32 function, u32 index) | ||
2836 | { | 2984 | { |
2837 | int i; | 2985 | int i; |
2838 | u32 function, index; | 2986 | struct kvm_cpuid_entry2 *best = NULL; |
2839 | struct kvm_cpuid_entry2 *e, *best; | ||
2840 | 2987 | ||
2841 | function = kvm_register_read(vcpu, VCPU_REGS_RAX); | ||
2842 | index = kvm_register_read(vcpu, VCPU_REGS_RCX); | ||
2843 | kvm_register_write(vcpu, VCPU_REGS_RAX, 0); | ||
2844 | kvm_register_write(vcpu, VCPU_REGS_RBX, 0); | ||
2845 | kvm_register_write(vcpu, VCPU_REGS_RCX, 0); | ||
2846 | kvm_register_write(vcpu, VCPU_REGS_RDX, 0); | ||
2847 | best = NULL; | ||
2848 | for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { | 2988 | for (i = 0; i < vcpu->arch.cpuid_nent; ++i) { |
2989 | struct kvm_cpuid_entry2 *e; | ||
2990 | |||
2849 | e = &vcpu->arch.cpuid_entries[i]; | 2991 | e = &vcpu->arch.cpuid_entries[i]; |
2850 | if (is_matching_cpuid_entry(e, function, index)) { | 2992 | if (is_matching_cpuid_entry(e, function, index)) { |
2851 | if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) | 2993 | if (e->flags & KVM_CPUID_FLAG_STATEFUL_FUNC) |
@@ -2860,6 +3002,21 @@ void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) | |||
2860 | if (!best || e->function > best->function) | 3002 | if (!best || e->function > best->function) |
2861 | best = e; | 3003 | best = e; |
2862 | } | 3004 | } |
3005 | return best; | ||
3006 | } | ||
3007 | |||
3008 | void kvm_emulate_cpuid(struct kvm_vcpu *vcpu) | ||
3009 | { | ||
3010 | u32 function, index; | ||
3011 | struct kvm_cpuid_entry2 *best; | ||
3012 | |||
3013 | function = kvm_register_read(vcpu, VCPU_REGS_RAX); | ||
3014 | index = kvm_register_read(vcpu, VCPU_REGS_RCX); | ||
3015 | kvm_register_write(vcpu, VCPU_REGS_RAX, 0); | ||
3016 | kvm_register_write(vcpu, VCPU_REGS_RBX, 0); | ||
3017 | kvm_register_write(vcpu, VCPU_REGS_RCX, 0); | ||
3018 | kvm_register_write(vcpu, VCPU_REGS_RDX, 0); | ||
3019 | best = kvm_find_cpuid_entry(vcpu, function, index); | ||
2863 | if (best) { | 3020 | if (best) { |
2864 | kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax); | 3021 | kvm_register_write(vcpu, VCPU_REGS_RAX, best->eax); |
2865 | kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx); | 3022 | kvm_register_write(vcpu, VCPU_REGS_RBX, best->ebx); |
@@ -2945,6 +3102,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2945 | if (vcpu->requests) { | 3102 | if (vcpu->requests) { |
2946 | if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests)) | 3103 | if (test_and_clear_bit(KVM_REQ_MIGRATE_TIMER, &vcpu->requests)) |
2947 | __kvm_migrate_timers(vcpu); | 3104 | __kvm_migrate_timers(vcpu); |
3105 | if (test_and_clear_bit(KVM_REQ_KVMCLOCK_UPDATE, &vcpu->requests)) | ||
3106 | kvm_write_guest_time(vcpu); | ||
2948 | if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests)) | 3107 | if (test_and_clear_bit(KVM_REQ_MMU_SYNC, &vcpu->requests)) |
2949 | kvm_mmu_sync_roots(vcpu); | 3108 | kvm_mmu_sync_roots(vcpu); |
2950 | if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) | 3109 | if (test_and_clear_bit(KVM_REQ_TLB_FLUSH, &vcpu->requests)) |
@@ -2979,9 +3138,6 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
2979 | goto out; | 3138 | goto out; |
2980 | } | 3139 | } |
2981 | 3140 | ||
2982 | if (vcpu->guest_debug.enabled) | ||
2983 | kvm_x86_ops->guest_debug_pre(vcpu); | ||
2984 | |||
2985 | vcpu->guest_mode = 1; | 3141 | vcpu->guest_mode = 1; |
2986 | /* | 3142 | /* |
2987 | * Make sure that guest_mode assignment won't happen after | 3143 | * Make sure that guest_mode assignment won't happen after |
@@ -3002,10 +3158,34 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) | |||
3002 | 3158 | ||
3003 | kvm_guest_enter(); | 3159 | kvm_guest_enter(); |
3004 | 3160 | ||
3161 | get_debugreg(vcpu->arch.host_dr6, 6); | ||
3162 | get_debugreg(vcpu->arch.host_dr7, 7); | ||
3163 | if (unlikely(vcpu->arch.switch_db_regs)) { | ||
3164 | get_debugreg(vcpu->arch.host_db[0], 0); | ||
3165 | get_debugreg(vcpu->arch.host_db[1], 1); | ||
3166 | get_debugreg(vcpu->arch.host_db[2], 2); | ||
3167 | get_debugreg(vcpu->arch.host_db[3], 3); | ||
3168 | |||
3169 | set_debugreg(0, 7); | ||
3170 | set_debugreg(vcpu->arch.eff_db[0], 0); | ||
3171 | set_debugreg(vcpu->arch.eff_db[1], 1); | ||
3172 | set_debugreg(vcpu->arch.eff_db[2], 2); | ||
3173 | set_debugreg(vcpu->arch.eff_db[3], 3); | ||
3174 | } | ||
3005 | 3175 | ||
3006 | KVMTRACE_0D(VMENTRY, vcpu, entryexit); | 3176 | KVMTRACE_0D(VMENTRY, vcpu, entryexit); |
3007 | kvm_x86_ops->run(vcpu, kvm_run); | 3177 | kvm_x86_ops->run(vcpu, kvm_run); |
3008 | 3178 | ||
3179 | if (unlikely(vcpu->arch.switch_db_regs)) { | ||
3180 | set_debugreg(0, 7); | ||
3181 | set_debugreg(vcpu->arch.host_db[0], 0); | ||
3182 | set_debugreg(vcpu->arch.host_db[1], 1); | ||
3183 | set_debugreg(vcpu->arch.host_db[2], 2); | ||
3184 | set_debugreg(vcpu->arch.host_db[3], 3); | ||
3185 | } | ||
3186 | set_debugreg(vcpu->arch.host_dr6, 6); | ||
3187 | set_debugreg(vcpu->arch.host_dr7, 7); | ||
3188 | |||
3009 | vcpu->guest_mode = 0; | 3189 | vcpu->guest_mode = 0; |
3010 | local_irq_enable(); | 3190 | local_irq_enable(); |
3011 | 3191 | ||
@@ -3192,7 +3372,7 @@ int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) | |||
3192 | /* | 3372 | /* |
3193 | * Don't leak debug flags in case they were set for guest debugging | 3373 | * Don't leak debug flags in case they were set for guest debugging |
3194 | */ | 3374 | */ |
3195 | if (vcpu->guest_debug.enabled && vcpu->guest_debug.singlestep) | 3375 | if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP) |
3196 | regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); | 3376 | regs->rflags &= ~(X86_EFLAGS_TF | X86_EFLAGS_RF); |
3197 | 3377 | ||
3198 | vcpu_put(vcpu); | 3378 | vcpu_put(vcpu); |
@@ -3811,15 +3991,32 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, | |||
3811 | return 0; | 3991 | return 0; |
3812 | } | 3992 | } |
3813 | 3993 | ||
3814 | int kvm_arch_vcpu_ioctl_debug_guest(struct kvm_vcpu *vcpu, | 3994 | int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, |
3815 | struct kvm_debug_guest *dbg) | 3995 | struct kvm_guest_debug *dbg) |
3816 | { | 3996 | { |
3817 | int r; | 3997 | int i, r; |
3818 | 3998 | ||
3819 | vcpu_load(vcpu); | 3999 | vcpu_load(vcpu); |
3820 | 4000 | ||
4001 | if ((dbg->control & (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP)) == | ||
4002 | (KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP)) { | ||
4003 | for (i = 0; i < KVM_NR_DB_REGS; ++i) | ||
4004 | vcpu->arch.eff_db[i] = dbg->arch.debugreg[i]; | ||
4005 | vcpu->arch.switch_db_regs = | ||
4006 | (dbg->arch.debugreg[7] & DR7_BP_EN_MASK); | ||
4007 | } else { | ||
4008 | for (i = 0; i < KVM_NR_DB_REGS; i++) | ||
4009 | vcpu->arch.eff_db[i] = vcpu->arch.db[i]; | ||
4010 | vcpu->arch.switch_db_regs = (vcpu->arch.dr7 & DR7_BP_EN_MASK); | ||
4011 | } | ||
4012 | |||
3821 | r = kvm_x86_ops->set_guest_debug(vcpu, dbg); | 4013 | r = kvm_x86_ops->set_guest_debug(vcpu, dbg); |
3822 | 4014 | ||
4015 | if (dbg->control & KVM_GUESTDBG_INJECT_DB) | ||
4016 | kvm_queue_exception(vcpu, DB_VECTOR); | ||
4017 | else if (dbg->control & KVM_GUESTDBG_INJECT_BP) | ||
4018 | kvm_queue_exception(vcpu, BP_VECTOR); | ||
4019 | |||
3823 | vcpu_put(vcpu); | 4020 | vcpu_put(vcpu); |
3824 | 4021 | ||
3825 | return r; | 4022 | return r; |
@@ -4007,6 +4204,11 @@ int kvm_arch_vcpu_reset(struct kvm_vcpu *vcpu) | |||
4007 | vcpu->arch.nmi_pending = false; | 4204 | vcpu->arch.nmi_pending = false; |
4008 | vcpu->arch.nmi_injected = false; | 4205 | vcpu->arch.nmi_injected = false; |
4009 | 4206 | ||
4207 | vcpu->arch.switch_db_regs = 0; | ||
4208 | memset(vcpu->arch.db, 0, sizeof(vcpu->arch.db)); | ||
4209 | vcpu->arch.dr6 = DR6_FIXED_1; | ||
4210 | vcpu->arch.dr7 = DR7_FIXED_1; | ||
4211 | |||
4010 | return kvm_x86_ops->vcpu_reset(vcpu); | 4212 | return kvm_x86_ops->vcpu_reset(vcpu); |
4011 | } | 4213 | } |
4012 | 4214 | ||
@@ -4100,6 +4302,8 @@ struct kvm *kvm_arch_create_vm(void) | |||
4100 | /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ | 4302 | /* Reserve bit 0 of irq_sources_bitmap for userspace irq source */ |
4101 | set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); | 4303 | set_bit(KVM_USERSPACE_IRQ_SOURCE_ID, &kvm->arch.irq_sources_bitmap); |
4102 | 4304 | ||
4305 | rdtscll(kvm->arch.vm_init_tsc); | ||
4306 | |||
4103 | return kvm; | 4307 | return kvm; |
4104 | } | 4308 | } |
4105 | 4309 | ||
diff --git a/arch/x86/kvm/x86_emulate.c b/arch/x86/kvm/x86_emulate.c index d174db7a3370..ca91749d2083 100644 --- a/arch/x86/kvm/x86_emulate.c +++ b/arch/x86/kvm/x86_emulate.c | |||
@@ -178,7 +178,7 @@ static u32 opcode_table[256] = { | |||
178 | 0, ImplicitOps | Stack, 0, 0, | 178 | 0, ImplicitOps | Stack, 0, 0, |
179 | ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, | 179 | ByteOp | DstMem | SrcImm | ModRM | Mov, DstMem | SrcImm | ModRM | Mov, |
180 | /* 0xC8 - 0xCF */ | 180 | /* 0xC8 - 0xCF */ |
181 | 0, 0, 0, 0, 0, 0, 0, 0, | 181 | 0, 0, 0, ImplicitOps | Stack, 0, 0, 0, 0, |
182 | /* 0xD0 - 0xD7 */ | 182 | /* 0xD0 - 0xD7 */ |
183 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, | 183 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, |
184 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, | 184 | ByteOp | DstMem | SrcImplicit | ModRM, DstMem | SrcImplicit | ModRM, |
@@ -1136,18 +1136,19 @@ static inline void emulate_push(struct x86_emulate_ctxt *ctxt) | |||
1136 | } | 1136 | } |
1137 | 1137 | ||
1138 | static int emulate_pop(struct x86_emulate_ctxt *ctxt, | 1138 | static int emulate_pop(struct x86_emulate_ctxt *ctxt, |
1139 | struct x86_emulate_ops *ops) | 1139 | struct x86_emulate_ops *ops, |
1140 | void *dest, int len) | ||
1140 | { | 1141 | { |
1141 | struct decode_cache *c = &ctxt->decode; | 1142 | struct decode_cache *c = &ctxt->decode; |
1142 | int rc; | 1143 | int rc; |
1143 | 1144 | ||
1144 | rc = ops->read_emulated(register_address(c, ss_base(ctxt), | 1145 | rc = ops->read_emulated(register_address(c, ss_base(ctxt), |
1145 | c->regs[VCPU_REGS_RSP]), | 1146 | c->regs[VCPU_REGS_RSP]), |
1146 | &c->src.val, c->src.bytes, ctxt->vcpu); | 1147 | dest, len, ctxt->vcpu); |
1147 | if (rc != 0) | 1148 | if (rc != 0) |
1148 | return rc; | 1149 | return rc; |
1149 | 1150 | ||
1150 | register_address_increment(c, &c->regs[VCPU_REGS_RSP], c->src.bytes); | 1151 | register_address_increment(c, &c->regs[VCPU_REGS_RSP], len); |
1151 | return rc; | 1152 | return rc; |
1152 | } | 1153 | } |
1153 | 1154 | ||
@@ -1157,11 +1158,9 @@ static inline int emulate_grp1a(struct x86_emulate_ctxt *ctxt, | |||
1157 | struct decode_cache *c = &ctxt->decode; | 1158 | struct decode_cache *c = &ctxt->decode; |
1158 | int rc; | 1159 | int rc; |
1159 | 1160 | ||
1160 | c->src.bytes = c->dst.bytes; | 1161 | rc = emulate_pop(ctxt, ops, &c->dst.val, c->dst.bytes); |
1161 | rc = emulate_pop(ctxt, ops); | ||
1162 | if (rc != 0) | 1162 | if (rc != 0) |
1163 | return rc; | 1163 | return rc; |
1164 | c->dst.val = c->src.val; | ||
1165 | return 0; | 1164 | return 0; |
1166 | } | 1165 | } |
1167 | 1166 | ||
@@ -1279,6 +1278,25 @@ static inline int emulate_grp9(struct x86_emulate_ctxt *ctxt, | |||
1279 | return 0; | 1278 | return 0; |
1280 | } | 1279 | } |
1281 | 1280 | ||
1281 | static int emulate_ret_far(struct x86_emulate_ctxt *ctxt, | ||
1282 | struct x86_emulate_ops *ops) | ||
1283 | { | ||
1284 | struct decode_cache *c = &ctxt->decode; | ||
1285 | int rc; | ||
1286 | unsigned long cs; | ||
1287 | |||
1288 | rc = emulate_pop(ctxt, ops, &c->eip, c->op_bytes); | ||
1289 | if (rc) | ||
1290 | return rc; | ||
1291 | if (c->op_bytes == 4) | ||
1292 | c->eip = (u32)c->eip; | ||
1293 | rc = emulate_pop(ctxt, ops, &cs, c->op_bytes); | ||
1294 | if (rc) | ||
1295 | return rc; | ||
1296 | rc = kvm_load_segment_descriptor(ctxt->vcpu, (u16)cs, 1, VCPU_SREG_CS); | ||
1297 | return rc; | ||
1298 | } | ||
1299 | |||
1282 | static inline int writeback(struct x86_emulate_ctxt *ctxt, | 1300 | static inline int writeback(struct x86_emulate_ctxt *ctxt, |
1283 | struct x86_emulate_ops *ops) | 1301 | struct x86_emulate_ops *ops) |
1284 | { | 1302 | { |
@@ -1467,11 +1485,9 @@ special_insn: | |||
1467 | break; | 1485 | break; |
1468 | case 0x58 ... 0x5f: /* pop reg */ | 1486 | case 0x58 ... 0x5f: /* pop reg */ |
1469 | pop_instruction: | 1487 | pop_instruction: |
1470 | c->src.bytes = c->op_bytes; | 1488 | rc = emulate_pop(ctxt, ops, &c->dst.val, c->op_bytes); |
1471 | rc = emulate_pop(ctxt, ops); | ||
1472 | if (rc != 0) | 1489 | if (rc != 0) |
1473 | goto done; | 1490 | goto done; |
1474 | c->dst.val = c->src.val; | ||
1475 | break; | 1491 | break; |
1476 | case 0x63: /* movsxd */ | 1492 | case 0x63: /* movsxd */ |
1477 | if (ctxt->mode != X86EMUL_MODE_PROT64) | 1493 | if (ctxt->mode != X86EMUL_MODE_PROT64) |
@@ -1738,6 +1754,11 @@ special_insn: | |||
1738 | mov: | 1754 | mov: |
1739 | c->dst.val = c->src.val; | 1755 | c->dst.val = c->src.val; |
1740 | break; | 1756 | break; |
1757 | case 0xcb: /* ret far */ | ||
1758 | rc = emulate_ret_far(ctxt, ops); | ||
1759 | if (rc) | ||
1760 | goto done; | ||
1761 | break; | ||
1741 | case 0xd0 ... 0xd1: /* Grp2 */ | 1762 | case 0xd0 ... 0xd1: /* Grp2 */ |
1742 | c->src.val = 1; | 1763 | c->src.val = 1; |
1743 | emulate_grp2(ctxt); | 1764 | emulate_grp2(ctxt); |
@@ -1908,11 +1929,16 @@ twobyte_insn: | |||
1908 | c->dst.type = OP_NONE; | 1929 | c->dst.type = OP_NONE; |
1909 | break; | 1930 | break; |
1910 | case 3: /* lidt/vmmcall */ | 1931 | case 3: /* lidt/vmmcall */ |
1911 | if (c->modrm_mod == 3 && c->modrm_rm == 1) { | 1932 | if (c->modrm_mod == 3) { |
1912 | rc = kvm_fix_hypercall(ctxt->vcpu); | 1933 | switch (c->modrm_rm) { |
1913 | if (rc) | 1934 | case 1: |
1914 | goto done; | 1935 | rc = kvm_fix_hypercall(ctxt->vcpu); |
1915 | kvm_emulate_hypercall(ctxt->vcpu); | 1936 | if (rc) |
1937 | goto done; | ||
1938 | break; | ||
1939 | default: | ||
1940 | goto cannot_emulate; | ||
1941 | } | ||
1916 | } else { | 1942 | } else { |
1917 | rc = read_descriptor(ctxt, ops, c->src.ptr, | 1943 | rc = read_descriptor(ctxt, ops, c->src.ptr, |
1918 | &size, &address, | 1944 | &size, &address, |
diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c index 90e44a10e68a..e94a11e42f98 100644 --- a/arch/x86/lguest/boot.c +++ b/arch/x86/lguest/boot.c | |||
@@ -107,7 +107,7 @@ static void async_hcall(unsigned long call, unsigned long arg1, | |||
107 | local_irq_save(flags); | 107 | local_irq_save(flags); |
108 | if (lguest_data.hcall_status[next_call] != 0xFF) { | 108 | if (lguest_data.hcall_status[next_call] != 0xFF) { |
109 | /* Table full, so do normal hcall which will flush table. */ | 109 | /* Table full, so do normal hcall which will flush table. */ |
110 | hcall(call, arg1, arg2, arg3); | 110 | kvm_hypercall3(call, arg1, arg2, arg3); |
111 | } else { | 111 | } else { |
112 | lguest_data.hcalls[next_call].arg0 = call; | 112 | lguest_data.hcalls[next_call].arg0 = call; |
113 | lguest_data.hcalls[next_call].arg1 = arg1; | 113 | lguest_data.hcalls[next_call].arg1 = arg1; |
@@ -134,13 +134,32 @@ static void async_hcall(unsigned long call, unsigned long arg1, | |||
134 | * | 134 | * |
135 | * So, when we're in lazy mode, we call async_hcall() to store the call for | 135 | * So, when we're in lazy mode, we call async_hcall() to store the call for |
136 | * future processing: */ | 136 | * future processing: */ |
137 | static void lazy_hcall(unsigned long call, | 137 | static void lazy_hcall1(unsigned long call, |
138 | unsigned long arg1) | ||
139 | { | ||
140 | if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) | ||
141 | kvm_hypercall1(call, arg1); | ||
142 | else | ||
143 | async_hcall(call, arg1, 0, 0); | ||
144 | } | ||
145 | |||
146 | static void lazy_hcall2(unsigned long call, | ||
147 | unsigned long arg1, | ||
148 | unsigned long arg2) | ||
149 | { | ||
150 | if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) | ||
151 | kvm_hypercall2(call, arg1, arg2); | ||
152 | else | ||
153 | async_hcall(call, arg1, arg2, 0); | ||
154 | } | ||
155 | |||
156 | static void lazy_hcall3(unsigned long call, | ||
138 | unsigned long arg1, | 157 | unsigned long arg1, |
139 | unsigned long arg2, | 158 | unsigned long arg2, |
140 | unsigned long arg3) | 159 | unsigned long arg3) |
141 | { | 160 | { |
142 | if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) | 161 | if (paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) |
143 | hcall(call, arg1, arg2, arg3); | 162 | kvm_hypercall3(call, arg1, arg2, arg3); |
144 | else | 163 | else |
145 | async_hcall(call, arg1, arg2, arg3); | 164 | async_hcall(call, arg1, arg2, arg3); |
146 | } | 165 | } |
@@ -150,7 +169,7 @@ static void lazy_hcall(unsigned long call, | |||
150 | static void lguest_leave_lazy_mode(void) | 169 | static void lguest_leave_lazy_mode(void) |
151 | { | 170 | { |
152 | paravirt_leave_lazy(paravirt_get_lazy_mode()); | 171 | paravirt_leave_lazy(paravirt_get_lazy_mode()); |
153 | hcall(LHCALL_FLUSH_ASYNC, 0, 0, 0); | 172 | kvm_hypercall0(LHCALL_FLUSH_ASYNC); |
154 | } | 173 | } |
155 | 174 | ||
156 | /*G:033 | 175 | /*G:033 |
@@ -229,7 +248,7 @@ static void lguest_write_idt_entry(gate_desc *dt, | |||
229 | /* Keep the local copy up to date. */ | 248 | /* Keep the local copy up to date. */ |
230 | native_write_idt_entry(dt, entrynum, g); | 249 | native_write_idt_entry(dt, entrynum, g); |
231 | /* Tell Host about this new entry. */ | 250 | /* Tell Host about this new entry. */ |
232 | hcall(LHCALL_LOAD_IDT_ENTRY, entrynum, desc[0], desc[1]); | 251 | kvm_hypercall3(LHCALL_LOAD_IDT_ENTRY, entrynum, desc[0], desc[1]); |
233 | } | 252 | } |
234 | 253 | ||
235 | /* Changing to a different IDT is very rare: we keep the IDT up-to-date every | 254 | /* Changing to a different IDT is very rare: we keep the IDT up-to-date every |
@@ -241,7 +260,7 @@ static void lguest_load_idt(const struct desc_ptr *desc) | |||
241 | struct desc_struct *idt = (void *)desc->address; | 260 | struct desc_struct *idt = (void *)desc->address; |
242 | 261 | ||
243 | for (i = 0; i < (desc->size+1)/8; i++) | 262 | for (i = 0; i < (desc->size+1)/8; i++) |
244 | hcall(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b); | 263 | kvm_hypercall3(LHCALL_LOAD_IDT_ENTRY, i, idt[i].a, idt[i].b); |
245 | } | 264 | } |
246 | 265 | ||
247 | /* | 266 | /* |
@@ -261,8 +280,8 @@ static void lguest_load_idt(const struct desc_ptr *desc) | |||
261 | */ | 280 | */ |
262 | static void lguest_load_gdt(const struct desc_ptr *desc) | 281 | static void lguest_load_gdt(const struct desc_ptr *desc) |
263 | { | 282 | { |
264 | BUG_ON((desc->size+1)/8 != GDT_ENTRIES); | 283 | BUG_ON((desc->size + 1) / 8 != GDT_ENTRIES); |
265 | hcall(LHCALL_LOAD_GDT, __pa(desc->address), GDT_ENTRIES, 0); | 284 | kvm_hypercall2(LHCALL_LOAD_GDT, __pa(desc->address), GDT_ENTRIES); |
266 | } | 285 | } |
267 | 286 | ||
268 | /* For a single GDT entry which changes, we do the lazy thing: alter our GDT, | 287 | /* For a single GDT entry which changes, we do the lazy thing: alter our GDT, |
@@ -272,7 +291,7 @@ static void lguest_write_gdt_entry(struct desc_struct *dt, int entrynum, | |||
272 | const void *desc, int type) | 291 | const void *desc, int type) |
273 | { | 292 | { |
274 | native_write_gdt_entry(dt, entrynum, desc, type); | 293 | native_write_gdt_entry(dt, entrynum, desc, type); |
275 | hcall(LHCALL_LOAD_GDT, __pa(dt), GDT_ENTRIES, 0); | 294 | kvm_hypercall2(LHCALL_LOAD_GDT, __pa(dt), GDT_ENTRIES); |
276 | } | 295 | } |
277 | 296 | ||
278 | /* OK, I lied. There are three "thread local storage" GDT entries which change | 297 | /* OK, I lied. There are three "thread local storage" GDT entries which change |
@@ -284,7 +303,7 @@ static void lguest_load_tls(struct thread_struct *t, unsigned int cpu) | |||
284 | * can't handle us removing entries we're currently using. So we clear | 303 | * can't handle us removing entries we're currently using. So we clear |
285 | * the GS register here: if it's needed it'll be reloaded anyway. */ | 304 | * the GS register here: if it's needed it'll be reloaded anyway. */ |
286 | lazy_load_gs(0); | 305 | lazy_load_gs(0); |
287 | lazy_hcall(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu, 0); | 306 | lazy_hcall2(LHCALL_LOAD_TLS, __pa(&t->tls_array), cpu); |
288 | } | 307 | } |
289 | 308 | ||
290 | /*G:038 That's enough excitement for now, back to ploughing through each of | 309 | /*G:038 That's enough excitement for now, back to ploughing through each of |
@@ -382,7 +401,7 @@ static void lguest_cpuid(unsigned int *ax, unsigned int *bx, | |||
382 | static unsigned long current_cr0; | 401 | static unsigned long current_cr0; |
383 | static void lguest_write_cr0(unsigned long val) | 402 | static void lguest_write_cr0(unsigned long val) |
384 | { | 403 | { |
385 | lazy_hcall(LHCALL_TS, val & X86_CR0_TS, 0, 0); | 404 | lazy_hcall1(LHCALL_TS, val & X86_CR0_TS); |
386 | current_cr0 = val; | 405 | current_cr0 = val; |
387 | } | 406 | } |
388 | 407 | ||
@@ -396,7 +415,7 @@ static unsigned long lguest_read_cr0(void) | |||
396 | * the vowels have been optimized out. */ | 415 | * the vowels have been optimized out. */ |
397 | static void lguest_clts(void) | 416 | static void lguest_clts(void) |
398 | { | 417 | { |
399 | lazy_hcall(LHCALL_TS, 0, 0, 0); | 418 | lazy_hcall1(LHCALL_TS, 0); |
400 | current_cr0 &= ~X86_CR0_TS; | 419 | current_cr0 &= ~X86_CR0_TS; |
401 | } | 420 | } |
402 | 421 | ||
@@ -418,7 +437,7 @@ static bool cr3_changed = false; | |||
418 | static void lguest_write_cr3(unsigned long cr3) | 437 | static void lguest_write_cr3(unsigned long cr3) |
419 | { | 438 | { |
420 | lguest_data.pgdir = cr3; | 439 | lguest_data.pgdir = cr3; |
421 | lazy_hcall(LHCALL_NEW_PGTABLE, cr3, 0, 0); | 440 | lazy_hcall1(LHCALL_NEW_PGTABLE, cr3); |
422 | cr3_changed = true; | 441 | cr3_changed = true; |
423 | } | 442 | } |
424 | 443 | ||
@@ -490,11 +509,17 @@ static void lguest_write_cr4(unsigned long val) | |||
490 | * into a process' address space. We set the entry then tell the Host the | 509 | * into a process' address space. We set the entry then tell the Host the |
491 | * toplevel and address this corresponds to. The Guest uses one pagetable per | 510 | * toplevel and address this corresponds to. The Guest uses one pagetable per |
492 | * process, so we need to tell the Host which one we're changing (mm->pgd). */ | 511 | * process, so we need to tell the Host which one we're changing (mm->pgd). */ |
512 | static void lguest_pte_update(struct mm_struct *mm, unsigned long addr, | ||
513 | pte_t *ptep) | ||
514 | { | ||
515 | lazy_hcall3(LHCALL_SET_PTE, __pa(mm->pgd), addr, ptep->pte_low); | ||
516 | } | ||
517 | |||
493 | static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, | 518 | static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, |
494 | pte_t *ptep, pte_t pteval) | 519 | pte_t *ptep, pte_t pteval) |
495 | { | 520 | { |
496 | *ptep = pteval; | 521 | *ptep = pteval; |
497 | lazy_hcall(LHCALL_SET_PTE, __pa(mm->pgd), addr, pteval.pte_low); | 522 | lguest_pte_update(mm, addr, ptep); |
498 | } | 523 | } |
499 | 524 | ||
500 | /* The Guest calls this to set a top-level entry. Again, we set the entry then | 525 | /* The Guest calls this to set a top-level entry. Again, we set the entry then |
@@ -503,8 +528,8 @@ static void lguest_set_pte_at(struct mm_struct *mm, unsigned long addr, | |||
503 | static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) | 528 | static void lguest_set_pmd(pmd_t *pmdp, pmd_t pmdval) |
504 | { | 529 | { |
505 | *pmdp = pmdval; | 530 | *pmdp = pmdval; |
506 | lazy_hcall(LHCALL_SET_PMD, __pa(pmdp)&PAGE_MASK, | 531 | lazy_hcall2(LHCALL_SET_PMD, __pa(pmdp) & PAGE_MASK, |
507 | (__pa(pmdp)&(PAGE_SIZE-1))/4, 0); | 532 | (__pa(pmdp) & (PAGE_SIZE - 1)) / 4); |
508 | } | 533 | } |
509 | 534 | ||
510 | /* There are a couple of legacy places where the kernel sets a PTE, but we | 535 | /* There are a couple of legacy places where the kernel sets a PTE, but we |
@@ -520,7 +545,7 @@ static void lguest_set_pte(pte_t *ptep, pte_t pteval) | |||
520 | { | 545 | { |
521 | *ptep = pteval; | 546 | *ptep = pteval; |
522 | if (cr3_changed) | 547 | if (cr3_changed) |
523 | lazy_hcall(LHCALL_FLUSH_TLB, 1, 0, 0); | 548 | lazy_hcall1(LHCALL_FLUSH_TLB, 1); |
524 | } | 549 | } |
525 | 550 | ||
526 | /* Unfortunately for Lguest, the pv_mmu_ops for page tables were based on | 551 | /* Unfortunately for Lguest, the pv_mmu_ops for page tables were based on |
@@ -536,7 +561,7 @@ static void lguest_set_pte(pte_t *ptep, pte_t pteval) | |||
536 | static void lguest_flush_tlb_single(unsigned long addr) | 561 | static void lguest_flush_tlb_single(unsigned long addr) |
537 | { | 562 | { |
538 | /* Simply set it to zero: if it was not, it will fault back in. */ | 563 | /* Simply set it to zero: if it was not, it will fault back in. */ |
539 | lazy_hcall(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0); | 564 | lazy_hcall3(LHCALL_SET_PTE, lguest_data.pgdir, addr, 0); |
540 | } | 565 | } |
541 | 566 | ||
542 | /* This is what happens after the Guest has removed a large number of entries. | 567 | /* This is what happens after the Guest has removed a large number of entries. |
@@ -544,7 +569,7 @@ static void lguest_flush_tlb_single(unsigned long addr) | |||
544 | * have changed, ie. virtual addresses below PAGE_OFFSET. */ | 569 | * have changed, ie. virtual addresses below PAGE_OFFSET. */ |
545 | static void lguest_flush_tlb_user(void) | 570 | static void lguest_flush_tlb_user(void) |
546 | { | 571 | { |
547 | lazy_hcall(LHCALL_FLUSH_TLB, 0, 0, 0); | 572 | lazy_hcall1(LHCALL_FLUSH_TLB, 0); |
548 | } | 573 | } |
549 | 574 | ||
550 | /* This is called when the kernel page tables have changed. That's not very | 575 | /* This is called when the kernel page tables have changed. That's not very |
@@ -552,7 +577,7 @@ static void lguest_flush_tlb_user(void) | |||
552 | * slow), so it's worth separating this from the user flushing above. */ | 577 | * slow), so it's worth separating this from the user flushing above. */ |
553 | static void lguest_flush_tlb_kernel(void) | 578 | static void lguest_flush_tlb_kernel(void) |
554 | { | 579 | { |
555 | lazy_hcall(LHCALL_FLUSH_TLB, 1, 0, 0); | 580 | lazy_hcall1(LHCALL_FLUSH_TLB, 1); |
556 | } | 581 | } |
557 | 582 | ||
558 | /* | 583 | /* |
@@ -689,7 +714,7 @@ static int lguest_clockevent_set_next_event(unsigned long delta, | |||
689 | } | 714 | } |
690 | 715 | ||
691 | /* Please wake us this far in the future. */ | 716 | /* Please wake us this far in the future. */ |
692 | hcall(LHCALL_SET_CLOCKEVENT, delta, 0, 0); | 717 | kvm_hypercall1(LHCALL_SET_CLOCKEVENT, delta); |
693 | return 0; | 718 | return 0; |
694 | } | 719 | } |
695 | 720 | ||
@@ -700,7 +725,7 @@ static void lguest_clockevent_set_mode(enum clock_event_mode mode, | |||
700 | case CLOCK_EVT_MODE_UNUSED: | 725 | case CLOCK_EVT_MODE_UNUSED: |
701 | case CLOCK_EVT_MODE_SHUTDOWN: | 726 | case CLOCK_EVT_MODE_SHUTDOWN: |
702 | /* A 0 argument shuts the clock down. */ | 727 | /* A 0 argument shuts the clock down. */ |
703 | hcall(LHCALL_SET_CLOCKEVENT, 0, 0, 0); | 728 | kvm_hypercall0(LHCALL_SET_CLOCKEVENT); |
704 | break; | 729 | break; |
705 | case CLOCK_EVT_MODE_ONESHOT: | 730 | case CLOCK_EVT_MODE_ONESHOT: |
706 | /* This is what we expect. */ | 731 | /* This is what we expect. */ |
@@ -775,8 +800,8 @@ static void lguest_time_init(void) | |||
775 | static void lguest_load_sp0(struct tss_struct *tss, | 800 | static void lguest_load_sp0(struct tss_struct *tss, |
776 | struct thread_struct *thread) | 801 | struct thread_struct *thread) |
777 | { | 802 | { |
778 | lazy_hcall(LHCALL_SET_STACK, __KERNEL_DS|0x1, thread->sp0, | 803 | lazy_hcall3(LHCALL_SET_STACK, __KERNEL_DS | 0x1, thread->sp0, |
779 | THREAD_SIZE/PAGE_SIZE); | 804 | THREAD_SIZE / PAGE_SIZE); |
780 | } | 805 | } |
781 | 806 | ||
782 | /* Let's just say, I wouldn't do debugging under a Guest. */ | 807 | /* Let's just say, I wouldn't do debugging under a Guest. */ |
@@ -849,7 +874,7 @@ static void set_lguest_basic_apic_ops(void) | |||
849 | /* STOP! Until an interrupt comes in. */ | 874 | /* STOP! Until an interrupt comes in. */ |
850 | static void lguest_safe_halt(void) | 875 | static void lguest_safe_halt(void) |
851 | { | 876 | { |
852 | hcall(LHCALL_HALT, 0, 0, 0); | 877 | kvm_hypercall0(LHCALL_HALT); |
853 | } | 878 | } |
854 | 879 | ||
855 | /* The SHUTDOWN hypercall takes a string to describe what's happening, and | 880 | /* The SHUTDOWN hypercall takes a string to describe what's happening, and |
@@ -859,7 +884,8 @@ static void lguest_safe_halt(void) | |||
859 | * rather than virtual addresses, so we use __pa() here. */ | 884 | * rather than virtual addresses, so we use __pa() here. */ |
860 | static void lguest_power_off(void) | 885 | static void lguest_power_off(void) |
861 | { | 886 | { |
862 | hcall(LHCALL_SHUTDOWN, __pa("Power down"), LGUEST_SHUTDOWN_POWEROFF, 0); | 887 | kvm_hypercall2(LHCALL_SHUTDOWN, __pa("Power down"), |
888 | LGUEST_SHUTDOWN_POWEROFF); | ||
863 | } | 889 | } |
864 | 890 | ||
865 | /* | 891 | /* |
@@ -869,7 +895,7 @@ static void lguest_power_off(void) | |||
869 | */ | 895 | */ |
870 | static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p) | 896 | static int lguest_panic(struct notifier_block *nb, unsigned long l, void *p) |
871 | { | 897 | { |
872 | hcall(LHCALL_SHUTDOWN, __pa(p), LGUEST_SHUTDOWN_POWEROFF, 0); | 898 | kvm_hypercall2(LHCALL_SHUTDOWN, __pa(p), LGUEST_SHUTDOWN_POWEROFF); |
873 | /* The hcall won't return, but to keep gcc happy, we're "done". */ | 899 | /* The hcall won't return, but to keep gcc happy, we're "done". */ |
874 | return NOTIFY_DONE; | 900 | return NOTIFY_DONE; |
875 | } | 901 | } |
@@ -910,7 +936,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count) | |||
910 | len = sizeof(scratch) - 1; | 936 | len = sizeof(scratch) - 1; |
911 | scratch[len] = '\0'; | 937 | scratch[len] = '\0'; |
912 | memcpy(scratch, buf, len); | 938 | memcpy(scratch, buf, len); |
913 | hcall(LHCALL_NOTIFY, __pa(scratch), 0, 0); | 939 | kvm_hypercall1(LHCALL_NOTIFY, __pa(scratch)); |
914 | 940 | ||
915 | /* This routine returns the number of bytes actually written. */ | 941 | /* This routine returns the number of bytes actually written. */ |
916 | return len; | 942 | return len; |
@@ -920,7 +946,7 @@ static __init int early_put_chars(u32 vtermno, const char *buf, int count) | |||
920 | * Launcher to reboot us. */ | 946 | * Launcher to reboot us. */ |
921 | static void lguest_restart(char *reason) | 947 | static void lguest_restart(char *reason) |
922 | { | 948 | { |
923 | hcall(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART, 0); | 949 | kvm_hypercall2(LHCALL_SHUTDOWN, __pa(reason), LGUEST_SHUTDOWN_RESTART); |
924 | } | 950 | } |
925 | 951 | ||
926 | /*G:050 | 952 | /*G:050 |
@@ -1040,6 +1066,8 @@ __init void lguest_init(void) | |||
1040 | pv_mmu_ops.read_cr3 = lguest_read_cr3; | 1066 | pv_mmu_ops.read_cr3 = lguest_read_cr3; |
1041 | pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu; | 1067 | pv_mmu_ops.lazy_mode.enter = paravirt_enter_lazy_mmu; |
1042 | pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mode; | 1068 | pv_mmu_ops.lazy_mode.leave = lguest_leave_lazy_mode; |
1069 | pv_mmu_ops.pte_update = lguest_pte_update; | ||
1070 | pv_mmu_ops.pte_update_defer = lguest_pte_update; | ||
1043 | 1071 | ||
1044 | #ifdef CONFIG_X86_LOCAL_APIC | 1072 | #ifdef CONFIG_X86_LOCAL_APIC |
1045 | /* apic read/write intercepts */ | 1073 | /* apic read/write intercepts */ |
diff --git a/arch/x86/lguest/i386_head.S b/arch/x86/lguest/i386_head.S index 10b9bd35a8ff..f79541989471 100644 --- a/arch/x86/lguest/i386_head.S +++ b/arch/x86/lguest/i386_head.S | |||
@@ -27,8 +27,8 @@ ENTRY(lguest_entry) | |||
27 | /* We make the "initialization" hypercall now to tell the Host about | 27 | /* We make the "initialization" hypercall now to tell the Host about |
28 | * us, and also find out where it put our page tables. */ | 28 | * us, and also find out where it put our page tables. */ |
29 | movl $LHCALL_LGUEST_INIT, %eax | 29 | movl $LHCALL_LGUEST_INIT, %eax |
30 | movl $lguest_data - __PAGE_OFFSET, %edx | 30 | movl $lguest_data - __PAGE_OFFSET, %ebx |
31 | int $LGUEST_TRAP_ENTRY | 31 | .byte 0x0f,0x01,0xc1 /* KVM_HYPERCALL */ |
32 | 32 | ||
33 | /* Set up the initial stack so we can run C code. */ | 33 | /* Set up the initial stack so we can run C code. */ |
34 | movl $(init_thread_union+THREAD_SIZE),%esp | 34 | movl $(init_thread_union+THREAD_SIZE),%esp |
diff --git a/arch/x86/mm/highmem_32.c b/arch/x86/mm/highmem_32.c index 522db5e3d0bf..5bc5d1688c1c 100644 --- a/arch/x86/mm/highmem_32.c +++ b/arch/x86/mm/highmem_32.c | |||
@@ -19,49 +19,6 @@ void kunmap(struct page *page) | |||
19 | kunmap_high(page); | 19 | kunmap_high(page); |
20 | } | 20 | } |
21 | 21 | ||
22 | static void debug_kmap_atomic_prot(enum km_type type) | ||
23 | { | ||
24 | #ifdef CONFIG_DEBUG_HIGHMEM | ||
25 | static unsigned warn_count = 10; | ||
26 | |||
27 | if (unlikely(warn_count == 0)) | ||
28 | return; | ||
29 | |||
30 | if (unlikely(in_interrupt())) { | ||
31 | if (in_irq()) { | ||
32 | if (type != KM_IRQ0 && type != KM_IRQ1 && | ||
33 | type != KM_BIO_SRC_IRQ && type != KM_BIO_DST_IRQ && | ||
34 | type != KM_BOUNCE_READ) { | ||
35 | WARN_ON(1); | ||
36 | warn_count--; | ||
37 | } | ||
38 | } else if (!irqs_disabled()) { /* softirq */ | ||
39 | if (type != KM_IRQ0 && type != KM_IRQ1 && | ||
40 | type != KM_SOFTIRQ0 && type != KM_SOFTIRQ1 && | ||
41 | type != KM_SKB_SUNRPC_DATA && | ||
42 | type != KM_SKB_DATA_SOFTIRQ && | ||
43 | type != KM_BOUNCE_READ) { | ||
44 | WARN_ON(1); | ||
45 | warn_count--; | ||
46 | } | ||
47 | } | ||
48 | } | ||
49 | |||
50 | if (type == KM_IRQ0 || type == KM_IRQ1 || type == KM_BOUNCE_READ || | ||
51 | type == KM_BIO_SRC_IRQ || type == KM_BIO_DST_IRQ) { | ||
52 | if (!irqs_disabled()) { | ||
53 | WARN_ON(1); | ||
54 | warn_count--; | ||
55 | } | ||
56 | } else if (type == KM_SOFTIRQ0 || type == KM_SOFTIRQ1) { | ||
57 | if (irq_count() == 0 && !irqs_disabled()) { | ||
58 | WARN_ON(1); | ||
59 | warn_count--; | ||
60 | } | ||
61 | } | ||
62 | #endif | ||
63 | } | ||
64 | |||
65 | /* | 22 | /* |
66 | * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because | 23 | * kmap_atomic/kunmap_atomic is significantly faster than kmap/kunmap because |
67 | * no global lock is needed and because the kmap code must perform a global TLB | 24 | * no global lock is needed and because the kmap code must perform a global TLB |
@@ -81,8 +38,9 @@ void *kmap_atomic_prot(struct page *page, enum km_type type, pgprot_t prot) | |||
81 | if (!PageHighMem(page)) | 38 | if (!PageHighMem(page)) |
82 | return page_address(page); | 39 | return page_address(page); |
83 | 40 | ||
84 | debug_kmap_atomic_prot(type); | 41 | debug_kmap_atomic(type); |
85 | 42 | ||
43 | debug_kmap_atomic(type); | ||
86 | idx = type + KM_TYPE_NR*smp_processor_id(); | 44 | idx = type + KM_TYPE_NR*smp_processor_id(); |
87 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); | 45 | vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); |
88 | BUG_ON(!pte_none(*(kmap_pte-idx))); | 46 | BUG_ON(!pte_none(*(kmap_pte-idx))); |
diff --git a/arch/x86/mm/iomap_32.c b/arch/x86/mm/iomap_32.c index 699c9b2895ae..bff0c9032f8c 100644 --- a/arch/x86/mm/iomap_32.c +++ b/arch/x86/mm/iomap_32.c | |||
@@ -19,6 +19,7 @@ | |||
19 | #include <asm/iomap.h> | 19 | #include <asm/iomap.h> |
20 | #include <asm/pat.h> | 20 | #include <asm/pat.h> |
21 | #include <linux/module.h> | 21 | #include <linux/module.h> |
22 | #include <linux/highmem.h> | ||
22 | 23 | ||
23 | int is_io_mapping_possible(resource_size_t base, unsigned long size) | 24 | int is_io_mapping_possible(resource_size_t base, unsigned long size) |
24 | { | 25 | { |
@@ -71,6 +72,7 @@ iounmap_atomic(void *kvaddr, enum km_type type) | |||
71 | unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; | 72 | unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; |
72 | enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); | 73 | enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); |
73 | 74 | ||
75 | debug_kmap_atomic(type); | ||
74 | /* | 76 | /* |
75 | * Force other mappings to Oops if they'll try to access this pte | 77 | * Force other mappings to Oops if they'll try to access this pte |
76 | * without first remap it. Keeping stale mappings around is a bad idea | 78 | * without first remap it. Keeping stale mappings around is a bad idea |
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index 55e127f71ed9..0dfa09d69e80 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c | |||
@@ -487,12 +487,7 @@ static int __init early_ioremap_debug_setup(char *str) | |||
487 | early_param("early_ioremap_debug", early_ioremap_debug_setup); | 487 | early_param("early_ioremap_debug", early_ioremap_debug_setup); |
488 | 488 | ||
489 | static __initdata int after_paging_init; | 489 | static __initdata int after_paging_init; |
490 | #define __FIXADDR_TOP (-PAGE_SIZE) | 490 | static pte_t bm_pte[PAGE_SIZE/sizeof(pte_t)] __page_aligned_bss; |
491 | static pte_t bm_pte[(__fix_to_virt(FIX_DBGP_BASE) | ||
492 | ^ __fix_to_virt(FIX_BTMAP_BEGIN)) >> PMD_SHIFT | ||
493 | ? PAGE_SIZE / sizeof(pte_t) : 0] __page_aligned_bss; | ||
494 | #undef __FIXADDR_TOP | ||
495 | static __initdata pte_t *bm_ptep; | ||
496 | 491 | ||
497 | static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) | 492 | static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) |
498 | { | 493 | { |
@@ -507,8 +502,6 @@ static inline pmd_t * __init early_ioremap_pmd(unsigned long addr) | |||
507 | 502 | ||
508 | static inline pte_t * __init early_ioremap_pte(unsigned long addr) | 503 | static inline pte_t * __init early_ioremap_pte(unsigned long addr) |
509 | { | 504 | { |
510 | if (!sizeof(bm_pte)) | ||
511 | return &bm_ptep[pte_index(addr)]; | ||
512 | return &bm_pte[pte_index(addr)]; | 505 | return &bm_pte[pte_index(addr)]; |
513 | } | 506 | } |
514 | 507 | ||
@@ -523,17 +516,11 @@ void __init early_ioremap_init(void) | |||
523 | printk(KERN_INFO "early_ioremap_init()\n"); | 516 | printk(KERN_INFO "early_ioremap_init()\n"); |
524 | 517 | ||
525 | for (i = 0; i < FIX_BTMAPS_SLOTS; i++) | 518 | for (i = 0; i < FIX_BTMAPS_SLOTS; i++) |
526 | slot_virt[i] = fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i); | 519 | slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i); |
527 | 520 | ||
528 | pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); | 521 | pmd = early_ioremap_pmd(fix_to_virt(FIX_BTMAP_BEGIN)); |
529 | if (sizeof(bm_pte)) { | 522 | memset(bm_pte, 0, sizeof(bm_pte)); |
530 | memset(bm_pte, 0, sizeof(bm_pte)); | 523 | pmd_populate_kernel(&init_mm, pmd, bm_pte); |
531 | pmd_populate_kernel(&init_mm, pmd, bm_pte); | ||
532 | } else { | ||
533 | bm_ptep = pte_offset_kernel(pmd, 0); | ||
534 | if (early_ioremap_debug) | ||
535 | printk(KERN_INFO "bm_ptep=%p\n", bm_ptep); | ||
536 | } | ||
537 | 524 | ||
538 | /* | 525 | /* |
539 | * The boot-ioremap range spans multiple pmds, for which | 526 | * The boot-ioremap range spans multiple pmds, for which |
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 1280565670e4..d71e1b636ce6 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c | |||
@@ -34,6 +34,7 @@ struct cpa_data { | |||
34 | unsigned long pfn; | 34 | unsigned long pfn; |
35 | unsigned force_split : 1; | 35 | unsigned force_split : 1; |
36 | int curpage; | 36 | int curpage; |
37 | struct page **pages; | ||
37 | }; | 38 | }; |
38 | 39 | ||
39 | /* | 40 | /* |
@@ -46,6 +47,7 @@ static DEFINE_SPINLOCK(cpa_lock); | |||
46 | 47 | ||
47 | #define CPA_FLUSHTLB 1 | 48 | #define CPA_FLUSHTLB 1 |
48 | #define CPA_ARRAY 2 | 49 | #define CPA_ARRAY 2 |
50 | #define CPA_PAGES_ARRAY 4 | ||
49 | 51 | ||
50 | #ifdef CONFIG_PROC_FS | 52 | #ifdef CONFIG_PROC_FS |
51 | static unsigned long direct_pages_count[PG_LEVEL_NUM]; | 53 | static unsigned long direct_pages_count[PG_LEVEL_NUM]; |
@@ -202,10 +204,10 @@ static void cpa_flush_range(unsigned long start, int numpages, int cache) | |||
202 | } | 204 | } |
203 | } | 205 | } |
204 | 206 | ||
205 | static void cpa_flush_array(unsigned long *start, int numpages, int cache) | 207 | static void cpa_flush_array(unsigned long *start, int numpages, int cache, |
208 | int in_flags, struct page **pages) | ||
206 | { | 209 | { |
207 | unsigned int i, level; | 210 | unsigned int i, level; |
208 | unsigned long *addr; | ||
209 | 211 | ||
210 | BUG_ON(irqs_disabled()); | 212 | BUG_ON(irqs_disabled()); |
211 | 213 | ||
@@ -226,14 +228,22 @@ static void cpa_flush_array(unsigned long *start, int numpages, int cache) | |||
226 | * will cause all other CPUs to flush the same | 228 | * will cause all other CPUs to flush the same |
227 | * cachelines: | 229 | * cachelines: |
228 | */ | 230 | */ |
229 | for (i = 0, addr = start; i < numpages; i++, addr++) { | 231 | for (i = 0; i < numpages; i++) { |
230 | pte_t *pte = lookup_address(*addr, &level); | 232 | unsigned long addr; |
233 | pte_t *pte; | ||
234 | |||
235 | if (in_flags & CPA_PAGES_ARRAY) | ||
236 | addr = (unsigned long)page_address(pages[i]); | ||
237 | else | ||
238 | addr = start[i]; | ||
239 | |||
240 | pte = lookup_address(addr, &level); | ||
231 | 241 | ||
232 | /* | 242 | /* |
233 | * Only flush present addresses: | 243 | * Only flush present addresses: |
234 | */ | 244 | */ |
235 | if (pte && (pte_val(*pte) & _PAGE_PRESENT)) | 245 | if (pte && (pte_val(*pte) & _PAGE_PRESENT)) |
236 | clflush_cache_range((void *) *addr, PAGE_SIZE); | 246 | clflush_cache_range((void *)addr, PAGE_SIZE); |
237 | } | 247 | } |
238 | } | 248 | } |
239 | 249 | ||
@@ -585,7 +595,9 @@ static int __change_page_attr(struct cpa_data *cpa, int primary) | |||
585 | unsigned int level; | 595 | unsigned int level; |
586 | pte_t *kpte, old_pte; | 596 | pte_t *kpte, old_pte; |
587 | 597 | ||
588 | if (cpa->flags & CPA_ARRAY) | 598 | if (cpa->flags & CPA_PAGES_ARRAY) |
599 | address = (unsigned long)page_address(cpa->pages[cpa->curpage]); | ||
600 | else if (cpa->flags & CPA_ARRAY) | ||
589 | address = cpa->vaddr[cpa->curpage]; | 601 | address = cpa->vaddr[cpa->curpage]; |
590 | else | 602 | else |
591 | address = *cpa->vaddr; | 603 | address = *cpa->vaddr; |
@@ -688,7 +700,9 @@ static int cpa_process_alias(struct cpa_data *cpa) | |||
688 | * No need to redo, when the primary call touched the direct | 700 | * No need to redo, when the primary call touched the direct |
689 | * mapping already: | 701 | * mapping already: |
690 | */ | 702 | */ |
691 | if (cpa->flags & CPA_ARRAY) | 703 | if (cpa->flags & CPA_PAGES_ARRAY) |
704 | vaddr = (unsigned long)page_address(cpa->pages[cpa->curpage]); | ||
705 | else if (cpa->flags & CPA_ARRAY) | ||
692 | vaddr = cpa->vaddr[cpa->curpage]; | 706 | vaddr = cpa->vaddr[cpa->curpage]; |
693 | else | 707 | else |
694 | vaddr = *cpa->vaddr; | 708 | vaddr = *cpa->vaddr; |
@@ -699,7 +713,7 @@ static int cpa_process_alias(struct cpa_data *cpa) | |||
699 | alias_cpa = *cpa; | 713 | alias_cpa = *cpa; |
700 | temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT); | 714 | temp_cpa_vaddr = (unsigned long) __va(cpa->pfn << PAGE_SHIFT); |
701 | alias_cpa.vaddr = &temp_cpa_vaddr; | 715 | alias_cpa.vaddr = &temp_cpa_vaddr; |
702 | alias_cpa.flags &= ~CPA_ARRAY; | 716 | alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY); |
703 | 717 | ||
704 | 718 | ||
705 | ret = __change_page_attr_set_clr(&alias_cpa, 0); | 719 | ret = __change_page_attr_set_clr(&alias_cpa, 0); |
@@ -725,7 +739,7 @@ static int cpa_process_alias(struct cpa_data *cpa) | |||
725 | alias_cpa = *cpa; | 739 | alias_cpa = *cpa; |
726 | temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base; | 740 | temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + __START_KERNEL_map - phys_base; |
727 | alias_cpa.vaddr = &temp_cpa_vaddr; | 741 | alias_cpa.vaddr = &temp_cpa_vaddr; |
728 | alias_cpa.flags &= ~CPA_ARRAY; | 742 | alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY); |
729 | 743 | ||
730 | /* | 744 | /* |
731 | * The high mapping range is imprecise, so ignore the return value. | 745 | * The high mapping range is imprecise, so ignore the return value. |
@@ -746,7 +760,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) | |||
746 | */ | 760 | */ |
747 | cpa->numpages = numpages; | 761 | cpa->numpages = numpages; |
748 | /* for array changes, we can't use large page */ | 762 | /* for array changes, we can't use large page */ |
749 | if (cpa->flags & CPA_ARRAY) | 763 | if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) |
750 | cpa->numpages = 1; | 764 | cpa->numpages = 1; |
751 | 765 | ||
752 | if (!debug_pagealloc) | 766 | if (!debug_pagealloc) |
@@ -770,7 +784,7 @@ static int __change_page_attr_set_clr(struct cpa_data *cpa, int checkalias) | |||
770 | */ | 784 | */ |
771 | BUG_ON(cpa->numpages > numpages); | 785 | BUG_ON(cpa->numpages > numpages); |
772 | numpages -= cpa->numpages; | 786 | numpages -= cpa->numpages; |
773 | if (cpa->flags & CPA_ARRAY) | 787 | if (cpa->flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) |
774 | cpa->curpage++; | 788 | cpa->curpage++; |
775 | else | 789 | else |
776 | *cpa->vaddr += cpa->numpages * PAGE_SIZE; | 790 | *cpa->vaddr += cpa->numpages * PAGE_SIZE; |
@@ -787,7 +801,8 @@ static inline int cache_attr(pgprot_t attr) | |||
787 | 801 | ||
788 | static int change_page_attr_set_clr(unsigned long *addr, int numpages, | 802 | static int change_page_attr_set_clr(unsigned long *addr, int numpages, |
789 | pgprot_t mask_set, pgprot_t mask_clr, | 803 | pgprot_t mask_set, pgprot_t mask_clr, |
790 | int force_split, int array) | 804 | int force_split, int in_flag, |
805 | struct page **pages) | ||
791 | { | 806 | { |
792 | struct cpa_data cpa; | 807 | struct cpa_data cpa; |
793 | int ret, cache, checkalias; | 808 | int ret, cache, checkalias; |
@@ -802,15 +817,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, | |||
802 | return 0; | 817 | return 0; |
803 | 818 | ||
804 | /* Ensure we are PAGE_SIZE aligned */ | 819 | /* Ensure we are PAGE_SIZE aligned */ |
805 | if (!array) { | 820 | if (in_flag & CPA_ARRAY) { |
806 | if (*addr & ~PAGE_MASK) { | ||
807 | *addr &= PAGE_MASK; | ||
808 | /* | ||
809 | * People should not be passing in unaligned addresses: | ||
810 | */ | ||
811 | WARN_ON_ONCE(1); | ||
812 | } | ||
813 | } else { | ||
814 | int i; | 821 | int i; |
815 | for (i = 0; i < numpages; i++) { | 822 | for (i = 0; i < numpages; i++) { |
816 | if (addr[i] & ~PAGE_MASK) { | 823 | if (addr[i] & ~PAGE_MASK) { |
@@ -818,6 +825,18 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, | |||
818 | WARN_ON_ONCE(1); | 825 | WARN_ON_ONCE(1); |
819 | } | 826 | } |
820 | } | 827 | } |
828 | } else if (!(in_flag & CPA_PAGES_ARRAY)) { | ||
829 | /* | ||
830 | * in_flag of CPA_PAGES_ARRAY implies it is aligned. | ||
831 | * No need to cehck in that case | ||
832 | */ | ||
833 | if (*addr & ~PAGE_MASK) { | ||
834 | *addr &= PAGE_MASK; | ||
835 | /* | ||
836 | * People should not be passing in unaligned addresses: | ||
837 | */ | ||
838 | WARN_ON_ONCE(1); | ||
839 | } | ||
821 | } | 840 | } |
822 | 841 | ||
823 | /* Must avoid aliasing mappings in the highmem code */ | 842 | /* Must avoid aliasing mappings in the highmem code */ |
@@ -833,6 +852,7 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, | |||
833 | arch_flush_lazy_mmu_mode(); | 852 | arch_flush_lazy_mmu_mode(); |
834 | 853 | ||
835 | cpa.vaddr = addr; | 854 | cpa.vaddr = addr; |
855 | cpa.pages = pages; | ||
836 | cpa.numpages = numpages; | 856 | cpa.numpages = numpages; |
837 | cpa.mask_set = mask_set; | 857 | cpa.mask_set = mask_set; |
838 | cpa.mask_clr = mask_clr; | 858 | cpa.mask_clr = mask_clr; |
@@ -840,8 +860,8 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, | |||
840 | cpa.curpage = 0; | 860 | cpa.curpage = 0; |
841 | cpa.force_split = force_split; | 861 | cpa.force_split = force_split; |
842 | 862 | ||
843 | if (array) | 863 | if (in_flag & (CPA_ARRAY | CPA_PAGES_ARRAY)) |
844 | cpa.flags |= CPA_ARRAY; | 864 | cpa.flags |= in_flag; |
845 | 865 | ||
846 | /* No alias checking for _NX bit modifications */ | 866 | /* No alias checking for _NX bit modifications */ |
847 | checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX; | 867 | checkalias = (pgprot_val(mask_set) | pgprot_val(mask_clr)) != _PAGE_NX; |
@@ -867,9 +887,10 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, | |||
867 | * wbindv): | 887 | * wbindv): |
868 | */ | 888 | */ |
869 | if (!ret && cpu_has_clflush) { | 889 | if (!ret && cpu_has_clflush) { |
870 | if (cpa.flags & CPA_ARRAY) | 890 | if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { |
871 | cpa_flush_array(addr, numpages, cache); | 891 | cpa_flush_array(addr, numpages, cache, |
872 | else | 892 | cpa.flags, pages); |
893 | } else | ||
873 | cpa_flush_range(*addr, numpages, cache); | 894 | cpa_flush_range(*addr, numpages, cache); |
874 | } else | 895 | } else |
875 | cpa_flush_all(cache); | 896 | cpa_flush_all(cache); |
@@ -889,14 +910,28 @@ static inline int change_page_attr_set(unsigned long *addr, int numpages, | |||
889 | pgprot_t mask, int array) | 910 | pgprot_t mask, int array) |
890 | { | 911 | { |
891 | return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0, | 912 | return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0, |
892 | array); | 913 | (array ? CPA_ARRAY : 0), NULL); |
893 | } | 914 | } |
894 | 915 | ||
895 | static inline int change_page_attr_clear(unsigned long *addr, int numpages, | 916 | static inline int change_page_attr_clear(unsigned long *addr, int numpages, |
896 | pgprot_t mask, int array) | 917 | pgprot_t mask, int array) |
897 | { | 918 | { |
898 | return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0, | 919 | return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0, |
899 | array); | 920 | (array ? CPA_ARRAY : 0), NULL); |
921 | } | ||
922 | |||
923 | static inline int cpa_set_pages_array(struct page **pages, int numpages, | ||
924 | pgprot_t mask) | ||
925 | { | ||
926 | return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0, | ||
927 | CPA_PAGES_ARRAY, pages); | ||
928 | } | ||
929 | |||
930 | static inline int cpa_clear_pages_array(struct page **pages, int numpages, | ||
931 | pgprot_t mask) | ||
932 | { | ||
933 | return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0, | ||
934 | CPA_PAGES_ARRAY, pages); | ||
900 | } | 935 | } |
901 | 936 | ||
902 | int _set_memory_uc(unsigned long addr, int numpages) | 937 | int _set_memory_uc(unsigned long addr, int numpages) |
@@ -1044,7 +1079,7 @@ int set_memory_np(unsigned long addr, int numpages) | |||
1044 | int set_memory_4k(unsigned long addr, int numpages) | 1079 | int set_memory_4k(unsigned long addr, int numpages) |
1045 | { | 1080 | { |
1046 | return change_page_attr_set_clr(&addr, numpages, __pgprot(0), | 1081 | return change_page_attr_set_clr(&addr, numpages, __pgprot(0), |
1047 | __pgprot(0), 1, 0); | 1082 | __pgprot(0), 1, 0, NULL); |
1048 | } | 1083 | } |
1049 | 1084 | ||
1050 | int set_pages_uc(struct page *page, int numpages) | 1085 | int set_pages_uc(struct page *page, int numpages) |
@@ -1055,6 +1090,35 @@ int set_pages_uc(struct page *page, int numpages) | |||
1055 | } | 1090 | } |
1056 | EXPORT_SYMBOL(set_pages_uc); | 1091 | EXPORT_SYMBOL(set_pages_uc); |
1057 | 1092 | ||
1093 | int set_pages_array_uc(struct page **pages, int addrinarray) | ||
1094 | { | ||
1095 | unsigned long start; | ||
1096 | unsigned long end; | ||
1097 | int i; | ||
1098 | int free_idx; | ||
1099 | |||
1100 | for (i = 0; i < addrinarray; i++) { | ||
1101 | start = (unsigned long)page_address(pages[i]); | ||
1102 | end = start + PAGE_SIZE; | ||
1103 | if (reserve_memtype(start, end, _PAGE_CACHE_UC_MINUS, NULL)) | ||
1104 | goto err_out; | ||
1105 | } | ||
1106 | |||
1107 | if (cpa_set_pages_array(pages, addrinarray, | ||
1108 | __pgprot(_PAGE_CACHE_UC_MINUS)) == 0) { | ||
1109 | return 0; /* Success */ | ||
1110 | } | ||
1111 | err_out: | ||
1112 | free_idx = i; | ||
1113 | for (i = 0; i < free_idx; i++) { | ||
1114 | start = (unsigned long)page_address(pages[i]); | ||
1115 | end = start + PAGE_SIZE; | ||
1116 | free_memtype(start, end); | ||
1117 | } | ||
1118 | return -EINVAL; | ||
1119 | } | ||
1120 | EXPORT_SYMBOL(set_pages_array_uc); | ||
1121 | |||
1058 | int set_pages_wb(struct page *page, int numpages) | 1122 | int set_pages_wb(struct page *page, int numpages) |
1059 | { | 1123 | { |
1060 | unsigned long addr = (unsigned long)page_address(page); | 1124 | unsigned long addr = (unsigned long)page_address(page); |
@@ -1063,6 +1127,26 @@ int set_pages_wb(struct page *page, int numpages) | |||
1063 | } | 1127 | } |
1064 | EXPORT_SYMBOL(set_pages_wb); | 1128 | EXPORT_SYMBOL(set_pages_wb); |
1065 | 1129 | ||
1130 | int set_pages_array_wb(struct page **pages, int addrinarray) | ||
1131 | { | ||
1132 | int retval; | ||
1133 | unsigned long start; | ||
1134 | unsigned long end; | ||
1135 | int i; | ||
1136 | |||
1137 | retval = cpa_clear_pages_array(pages, addrinarray, | ||
1138 | __pgprot(_PAGE_CACHE_MASK)); | ||
1139 | |||
1140 | for (i = 0; i < addrinarray; i++) { | ||
1141 | start = (unsigned long)page_address(pages[i]); | ||
1142 | end = start + PAGE_SIZE; | ||
1143 | free_memtype(start, end); | ||
1144 | } | ||
1145 | |||
1146 | return retval; | ||
1147 | } | ||
1148 | EXPORT_SYMBOL(set_pages_array_wb); | ||
1149 | |||
1066 | int set_pages_x(struct page *page, int numpages) | 1150 | int set_pages_x(struct page *page, int numpages) |
1067 | { | 1151 | { |
1068 | unsigned long addr = (unsigned long)page_address(page); | 1152 | unsigned long addr = (unsigned long)page_address(page); |
diff --git a/arch/x86/pci/early.c b/arch/x86/pci/early.c index f6adf2c6d751..aaf26ae58cd5 100644 --- a/arch/x86/pci/early.c +++ b/arch/x86/pci/early.c | |||
@@ -69,11 +69,12 @@ void early_dump_pci_device(u8 bus, u8 slot, u8 func) | |||
69 | int j; | 69 | int j; |
70 | u32 val; | 70 | u32 val; |
71 | 71 | ||
72 | printk(KERN_INFO "PCI: %02x:%02x:%02x", bus, slot, func); | 72 | printk(KERN_INFO "pci 0000:%02x:%02x.%d config space:", |
73 | bus, slot, func); | ||
73 | 74 | ||
74 | for (i = 0; i < 256; i += 4) { | 75 | for (i = 0; i < 256; i += 4) { |
75 | if (!(i & 0x0f)) | 76 | if (!(i & 0x0f)) |
76 | printk("\n%04x:",i); | 77 | printk("\n %02x:",i); |
77 | 78 | ||
78 | val = read_pci_config(bus, slot, func, i); | 79 | val = read_pci_config(bus, slot, func, i); |
79 | for (j = 0; j < 4; j++) { | 80 | for (j = 0; j < 4; j++) { |
@@ -96,20 +97,22 @@ void early_dump_pci_devices(void) | |||
96 | for (func = 0; func < 8; func++) { | 97 | for (func = 0; func < 8; func++) { |
97 | u32 class; | 98 | u32 class; |
98 | u8 type; | 99 | u8 type; |
100 | |||
99 | class = read_pci_config(bus, slot, func, | 101 | class = read_pci_config(bus, slot, func, |
100 | PCI_CLASS_REVISION); | 102 | PCI_CLASS_REVISION); |
101 | if (class == 0xffffffff) | 103 | if (class == 0xffffffff) |
102 | break; | 104 | continue; |
103 | 105 | ||
104 | early_dump_pci_device(bus, slot, func); | 106 | early_dump_pci_device(bus, slot, func); |
105 | 107 | ||
106 | /* No multi-function device? */ | 108 | if (func == 0) { |
107 | type = read_pci_config_byte(bus, slot, func, | 109 | type = read_pci_config_byte(bus, slot, |
110 | func, | ||
108 | PCI_HEADER_TYPE); | 111 | PCI_HEADER_TYPE); |
109 | if (!(type & 0x80)) | 112 | if (!(type & 0x80)) |
110 | break; | 113 | break; |
114 | } | ||
111 | } | 115 | } |
112 | } | 116 | } |
113 | } | 117 | } |
114 | } | 118 | } |
115 | |||
diff --git a/arch/x86/pci/fixup.c b/arch/x86/pci/fixup.c index 9c49919e4d1c..6dd89555fbfa 100644 --- a/arch/x86/pci/fixup.c +++ b/arch/x86/pci/fixup.c | |||
@@ -495,26 +495,6 @@ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_SIEMENS, 0x0015, | |||
495 | pci_siemens_interrupt_controller); | 495 | pci_siemens_interrupt_controller); |
496 | 496 | ||
497 | /* | 497 | /* |
498 | * Regular PCI devices have 256 bytes, but AMD Family 10h/11h CPUs have | ||
499 | * 4096 bytes configuration space for each function of their processor | ||
500 | * configuration space. | ||
501 | */ | ||
502 | static void amd_cpu_pci_cfg_space_size(struct pci_dev *dev) | ||
503 | { | ||
504 | dev->cfg_size = pci_cfg_space_size_ext(dev); | ||
505 | } | ||
506 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1200, amd_cpu_pci_cfg_space_size); | ||
507 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1201, amd_cpu_pci_cfg_space_size); | ||
508 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1202, amd_cpu_pci_cfg_space_size); | ||
509 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1203, amd_cpu_pci_cfg_space_size); | ||
510 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1204, amd_cpu_pci_cfg_space_size); | ||
511 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1300, amd_cpu_pci_cfg_space_size); | ||
512 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1301, amd_cpu_pci_cfg_space_size); | ||
513 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1302, amd_cpu_pci_cfg_space_size); | ||
514 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1303, amd_cpu_pci_cfg_space_size); | ||
515 | DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_AMD, 0x1304, amd_cpu_pci_cfg_space_size); | ||
516 | |||
517 | /* | ||
518 | * SB600: Disable BAR1 on device 14.0 to avoid HPET resources from | 498 | * SB600: Disable BAR1 on device 14.0 to avoid HPET resources from |
519 | * confusing the PCI engine: | 499 | * confusing the PCI engine: |
520 | */ | 500 | */ |
diff --git a/arch/x86/pci/i386.c b/arch/x86/pci/i386.c index 5ead808dd70c..f234a37bd428 100644 --- a/arch/x86/pci/i386.c +++ b/arch/x86/pci/i386.c | |||
@@ -319,6 +319,9 @@ int pci_mmap_page_range(struct pci_dev *dev, struct vm_area_struct *vma, | |||
319 | return -EINVAL; | 319 | return -EINVAL; |
320 | } | 320 | } |
321 | flags = new_flags; | 321 | flags = new_flags; |
322 | vma->vm_page_prot = __pgprot( | ||
323 | (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK) | | ||
324 | flags); | ||
322 | } | 325 | } |
323 | 326 | ||
324 | if (((vma->vm_pgoff < max_low_pfn_mapped) || | 327 | if (((vma->vm_pgoff < max_low_pfn_mapped) || |
diff --git a/arch/x86/pci/legacy.c b/arch/x86/pci/legacy.c index f1065b129e9c..4061bb0f267d 100644 --- a/arch/x86/pci/legacy.c +++ b/arch/x86/pci/legacy.c | |||
@@ -50,8 +50,6 @@ static int __init pci_legacy_init(void) | |||
50 | if (pci_root_bus) | 50 | if (pci_root_bus) |
51 | pci_bus_add_devices(pci_root_bus); | 51 | pci_bus_add_devices(pci_root_bus); |
52 | 52 | ||
53 | pcibios_fixup_peer_bridges(); | ||
54 | |||
55 | return 0; | 53 | return 0; |
56 | } | 54 | } |
57 | 55 | ||
@@ -67,6 +65,7 @@ int __init pci_subsys_init(void) | |||
67 | pci_visws_init(); | 65 | pci_visws_init(); |
68 | #endif | 66 | #endif |
69 | pci_legacy_init(); | 67 | pci_legacy_init(); |
68 | pcibios_fixup_peer_bridges(); | ||
70 | pcibios_irq_init(); | 69 | pcibios_irq_init(); |
71 | pcibios_init(); | 70 | pcibios_init(); |
72 | 71 | ||
diff --git a/arch/x86/pci/mmconfig-shared.c b/arch/x86/pci/mmconfig-shared.c index 89bf9242c80a..905bb526b133 100644 --- a/arch/x86/pci/mmconfig-shared.c +++ b/arch/x86/pci/mmconfig-shared.c | |||
@@ -14,6 +14,7 @@ | |||
14 | #include <linux/init.h> | 14 | #include <linux/init.h> |
15 | #include <linux/acpi.h> | 15 | #include <linux/acpi.h> |
16 | #include <linux/bitmap.h> | 16 | #include <linux/bitmap.h> |
17 | #include <linux/sort.h> | ||
17 | #include <asm/e820.h> | 18 | #include <asm/e820.h> |
18 | #include <asm/pci_x86.h> | 19 | #include <asm/pci_x86.h> |
19 | 20 | ||
@@ -24,24 +25,49 @@ | |||
24 | /* Indicate if the mmcfg resources have been placed into the resource table. */ | 25 | /* Indicate if the mmcfg resources have been placed into the resource table. */ |
25 | static int __initdata pci_mmcfg_resources_inserted; | 26 | static int __initdata pci_mmcfg_resources_inserted; |
26 | 27 | ||
28 | static __init int extend_mmcfg(int num) | ||
29 | { | ||
30 | struct acpi_mcfg_allocation *new; | ||
31 | int new_num = pci_mmcfg_config_num + num; | ||
32 | |||
33 | new = kzalloc(sizeof(pci_mmcfg_config[0]) * new_num, GFP_KERNEL); | ||
34 | if (!new) | ||
35 | return -1; | ||
36 | |||
37 | if (pci_mmcfg_config) { | ||
38 | memcpy(new, pci_mmcfg_config, | ||
39 | sizeof(pci_mmcfg_config[0]) * new_num); | ||
40 | kfree(pci_mmcfg_config); | ||
41 | } | ||
42 | pci_mmcfg_config = new; | ||
43 | |||
44 | return 0; | ||
45 | } | ||
46 | |||
47 | static __init void fill_one_mmcfg(u64 addr, int segment, int start, int end) | ||
48 | { | ||
49 | int i = pci_mmcfg_config_num; | ||
50 | |||
51 | pci_mmcfg_config_num++; | ||
52 | pci_mmcfg_config[i].address = addr; | ||
53 | pci_mmcfg_config[i].pci_segment = segment; | ||
54 | pci_mmcfg_config[i].start_bus_number = start; | ||
55 | pci_mmcfg_config[i].end_bus_number = end; | ||
56 | } | ||
57 | |||
27 | static const char __init *pci_mmcfg_e7520(void) | 58 | static const char __init *pci_mmcfg_e7520(void) |
28 | { | 59 | { |
29 | u32 win; | 60 | u32 win; |
30 | raw_pci_ops->read(0, 0, PCI_DEVFN(0, 0), 0xce, 2, &win); | 61 | raw_pci_ops->read(0, 0, PCI_DEVFN(0, 0), 0xce, 2, &win); |
31 | 62 | ||
32 | win = win & 0xf000; | 63 | win = win & 0xf000; |
33 | if(win == 0x0000 || win == 0xf000) | 64 | if (win == 0x0000 || win == 0xf000) |
34 | pci_mmcfg_config_num = 0; | 65 | return NULL; |
35 | else { | 66 | |
36 | pci_mmcfg_config_num = 1; | 67 | if (extend_mmcfg(1) == -1) |
37 | pci_mmcfg_config = kzalloc(sizeof(pci_mmcfg_config[0]), GFP_KERNEL); | 68 | return NULL; |
38 | if (!pci_mmcfg_config) | 69 | |
39 | return NULL; | 70 | fill_one_mmcfg(win << 16, 0, 0, 255); |
40 | pci_mmcfg_config[0].address = win << 16; | ||
41 | pci_mmcfg_config[0].pci_segment = 0; | ||
42 | pci_mmcfg_config[0].start_bus_number = 0; | ||
43 | pci_mmcfg_config[0].end_bus_number = 255; | ||
44 | } | ||
45 | 71 | ||
46 | return "Intel Corporation E7520 Memory Controller Hub"; | 72 | return "Intel Corporation E7520 Memory Controller Hub"; |
47 | } | 73 | } |
@@ -50,13 +76,11 @@ static const char __init *pci_mmcfg_intel_945(void) | |||
50 | { | 76 | { |
51 | u32 pciexbar, mask = 0, len = 0; | 77 | u32 pciexbar, mask = 0, len = 0; |
52 | 78 | ||
53 | pci_mmcfg_config_num = 1; | ||
54 | |||
55 | raw_pci_ops->read(0, 0, PCI_DEVFN(0, 0), 0x48, 4, &pciexbar); | 79 | raw_pci_ops->read(0, 0, PCI_DEVFN(0, 0), 0x48, 4, &pciexbar); |
56 | 80 | ||
57 | /* Enable bit */ | 81 | /* Enable bit */ |
58 | if (!(pciexbar & 1)) | 82 | if (!(pciexbar & 1)) |
59 | pci_mmcfg_config_num = 0; | 83 | return NULL; |
60 | 84 | ||
61 | /* Size bits */ | 85 | /* Size bits */ |
62 | switch ((pciexbar >> 1) & 3) { | 86 | switch ((pciexbar >> 1) & 3) { |
@@ -73,28 +97,23 @@ static const char __init *pci_mmcfg_intel_945(void) | |||
73 | len = 0x04000000U; | 97 | len = 0x04000000U; |
74 | break; | 98 | break; |
75 | default: | 99 | default: |
76 | pci_mmcfg_config_num = 0; | 100 | return NULL; |
77 | } | 101 | } |
78 | 102 | ||
79 | /* Errata #2, things break when not aligned on a 256Mb boundary */ | 103 | /* Errata #2, things break when not aligned on a 256Mb boundary */ |
80 | /* Can only happen in 64M/128M mode */ | 104 | /* Can only happen in 64M/128M mode */ |
81 | 105 | ||
82 | if ((pciexbar & mask) & 0x0fffffffU) | 106 | if ((pciexbar & mask) & 0x0fffffffU) |
83 | pci_mmcfg_config_num = 0; | 107 | return NULL; |
84 | 108 | ||
85 | /* Don't hit the APIC registers and their friends */ | 109 | /* Don't hit the APIC registers and their friends */ |
86 | if ((pciexbar & mask) >= 0xf0000000U) | 110 | if ((pciexbar & mask) >= 0xf0000000U) |
87 | pci_mmcfg_config_num = 0; | 111 | return NULL; |
88 | 112 | ||
89 | if (pci_mmcfg_config_num) { | 113 | if (extend_mmcfg(1) == -1) |
90 | pci_mmcfg_config = kzalloc(sizeof(pci_mmcfg_config[0]), GFP_KERNEL); | 114 | return NULL; |
91 | if (!pci_mmcfg_config) | 115 | |
92 | return NULL; | 116 | fill_one_mmcfg(pciexbar & mask, 0, 0, (len >> 20) - 1); |
93 | pci_mmcfg_config[0].address = pciexbar & mask; | ||
94 | pci_mmcfg_config[0].pci_segment = 0; | ||
95 | pci_mmcfg_config[0].start_bus_number = 0; | ||
96 | pci_mmcfg_config[0].end_bus_number = (len >> 20) - 1; | ||
97 | } | ||
98 | 117 | ||
99 | return "Intel Corporation 945G/GZ/P/PL Express Memory Controller Hub"; | 118 | return "Intel Corporation 945G/GZ/P/PL Express Memory Controller Hub"; |
100 | } | 119 | } |
@@ -138,22 +157,77 @@ static const char __init *pci_mmcfg_amd_fam10h(void) | |||
138 | busnbits = 8; | 157 | busnbits = 8; |
139 | } | 158 | } |
140 | 159 | ||
141 | pci_mmcfg_config_num = (1 << segnbits); | 160 | if (extend_mmcfg(1 << segnbits) == -1) |
142 | pci_mmcfg_config = kzalloc(sizeof(pci_mmcfg_config[0]) * | ||
143 | pci_mmcfg_config_num, GFP_KERNEL); | ||
144 | if (!pci_mmcfg_config) | ||
145 | return NULL; | 161 | return NULL; |
146 | 162 | ||
147 | for (i = 0; i < (1 << segnbits); i++) { | 163 | for (i = 0; i < (1 << segnbits); i++) |
148 | pci_mmcfg_config[i].address = base + (1<<28) * i; | 164 | fill_one_mmcfg(base + (1<<28) * i, i, 0, (1 << busnbits) - 1); |
149 | pci_mmcfg_config[i].pci_segment = i; | ||
150 | pci_mmcfg_config[i].start_bus_number = 0; | ||
151 | pci_mmcfg_config[i].end_bus_number = (1 << busnbits) - 1; | ||
152 | } | ||
153 | 165 | ||
154 | return "AMD Family 10h NB"; | 166 | return "AMD Family 10h NB"; |
155 | } | 167 | } |
156 | 168 | ||
169 | static bool __initdata mcp55_checked; | ||
170 | static const char __init *pci_mmcfg_nvidia_mcp55(void) | ||
171 | { | ||
172 | int bus; | ||
173 | int mcp55_mmconf_found = 0; | ||
174 | |||
175 | static const u32 extcfg_regnum = 0x90; | ||
176 | static const u32 extcfg_regsize = 4; | ||
177 | static const u32 extcfg_enable_mask = 1<<31; | ||
178 | static const u32 extcfg_start_mask = 0xff<<16; | ||
179 | static const int extcfg_start_shift = 16; | ||
180 | static const u32 extcfg_size_mask = 0x3<<28; | ||
181 | static const int extcfg_size_shift = 28; | ||
182 | static const int extcfg_sizebus[] = {0x100, 0x80, 0x40, 0x20}; | ||
183 | static const u32 extcfg_base_mask[] = {0x7ff8, 0x7ffc, 0x7ffe, 0x7fff}; | ||
184 | static const int extcfg_base_lshift = 25; | ||
185 | |||
186 | /* | ||
187 | * do check if amd fam10h already took over | ||
188 | */ | ||
189 | if (!acpi_disabled || pci_mmcfg_config_num || mcp55_checked) | ||
190 | return NULL; | ||
191 | |||
192 | mcp55_checked = true; | ||
193 | for (bus = 0; bus < 256; bus++) { | ||
194 | u64 base; | ||
195 | u32 l, extcfg; | ||
196 | u16 vendor, device; | ||
197 | int start, size_index, end; | ||
198 | |||
199 | raw_pci_ops->read(0, bus, PCI_DEVFN(0, 0), 0, 4, &l); | ||
200 | vendor = l & 0xffff; | ||
201 | device = (l >> 16) & 0xffff; | ||
202 | |||
203 | if (PCI_VENDOR_ID_NVIDIA != vendor || 0x0369 != device) | ||
204 | continue; | ||
205 | |||
206 | raw_pci_ops->read(0, bus, PCI_DEVFN(0, 0), extcfg_regnum, | ||
207 | extcfg_regsize, &extcfg); | ||
208 | |||
209 | if (!(extcfg & extcfg_enable_mask)) | ||
210 | continue; | ||
211 | |||
212 | if (extend_mmcfg(1) == -1) | ||
213 | continue; | ||
214 | |||
215 | size_index = (extcfg & extcfg_size_mask) >> extcfg_size_shift; | ||
216 | base = extcfg & extcfg_base_mask[size_index]; | ||
217 | /* base could > 4G */ | ||
218 | base <<= extcfg_base_lshift; | ||
219 | start = (extcfg & extcfg_start_mask) >> extcfg_start_shift; | ||
220 | end = start + extcfg_sizebus[size_index] - 1; | ||
221 | fill_one_mmcfg(base, 0, start, end); | ||
222 | mcp55_mmconf_found++; | ||
223 | } | ||
224 | |||
225 | if (!mcp55_mmconf_found) | ||
226 | return NULL; | ||
227 | |||
228 | return "nVidia MCP55"; | ||
229 | } | ||
230 | |||
157 | struct pci_mmcfg_hostbridge_probe { | 231 | struct pci_mmcfg_hostbridge_probe { |
158 | u32 bus; | 232 | u32 bus; |
159 | u32 devfn; | 233 | u32 devfn; |
@@ -171,8 +245,52 @@ static struct pci_mmcfg_hostbridge_probe pci_mmcfg_probes[] __initdata = { | |||
171 | 0x1200, pci_mmcfg_amd_fam10h }, | 245 | 0x1200, pci_mmcfg_amd_fam10h }, |
172 | { 0xff, PCI_DEVFN(0, 0), PCI_VENDOR_ID_AMD, | 246 | { 0xff, PCI_DEVFN(0, 0), PCI_VENDOR_ID_AMD, |
173 | 0x1200, pci_mmcfg_amd_fam10h }, | 247 | 0x1200, pci_mmcfg_amd_fam10h }, |
248 | { 0, PCI_DEVFN(0, 0), PCI_VENDOR_ID_NVIDIA, | ||
249 | 0x0369, pci_mmcfg_nvidia_mcp55 }, | ||
174 | }; | 250 | }; |
175 | 251 | ||
252 | static int __init cmp_mmcfg(const void *x1, const void *x2) | ||
253 | { | ||
254 | const typeof(pci_mmcfg_config[0]) *m1 = x1; | ||
255 | const typeof(pci_mmcfg_config[0]) *m2 = x2; | ||
256 | int start1, start2; | ||
257 | |||
258 | start1 = m1->start_bus_number; | ||
259 | start2 = m2->start_bus_number; | ||
260 | |||
261 | return start1 - start2; | ||
262 | } | ||
263 | |||
264 | static void __init pci_mmcfg_check_end_bus_number(void) | ||
265 | { | ||
266 | int i; | ||
267 | typeof(pci_mmcfg_config[0]) *cfg, *cfgx; | ||
268 | |||
269 | /* sort them at first */ | ||
270 | sort(pci_mmcfg_config, pci_mmcfg_config_num, | ||
271 | sizeof(pci_mmcfg_config[0]), cmp_mmcfg, NULL); | ||
272 | |||
273 | /* last one*/ | ||
274 | if (pci_mmcfg_config_num > 0) { | ||
275 | i = pci_mmcfg_config_num - 1; | ||
276 | cfg = &pci_mmcfg_config[i]; | ||
277 | if (cfg->end_bus_number < cfg->start_bus_number) | ||
278 | cfg->end_bus_number = 255; | ||
279 | } | ||
280 | |||
281 | /* don't overlap please */ | ||
282 | for (i = 0; i < pci_mmcfg_config_num - 1; i++) { | ||
283 | cfg = &pci_mmcfg_config[i]; | ||
284 | cfgx = &pci_mmcfg_config[i+1]; | ||
285 | |||
286 | if (cfg->end_bus_number < cfg->start_bus_number) | ||
287 | cfg->end_bus_number = 255; | ||
288 | |||
289 | if (cfg->end_bus_number >= cfgx->start_bus_number) | ||
290 | cfg->end_bus_number = cfgx->start_bus_number - 1; | ||
291 | } | ||
292 | } | ||
293 | |||
176 | static int __init pci_mmcfg_check_hostbridge(void) | 294 | static int __init pci_mmcfg_check_hostbridge(void) |
177 | { | 295 | { |
178 | u32 l; | 296 | u32 l; |
@@ -186,31 +304,33 @@ static int __init pci_mmcfg_check_hostbridge(void) | |||
186 | 304 | ||
187 | pci_mmcfg_config_num = 0; | 305 | pci_mmcfg_config_num = 0; |
188 | pci_mmcfg_config = NULL; | 306 | pci_mmcfg_config = NULL; |
189 | name = NULL; | ||
190 | 307 | ||
191 | for (i = 0; !name && i < ARRAY_SIZE(pci_mmcfg_probes); i++) { | 308 | for (i = 0; i < ARRAY_SIZE(pci_mmcfg_probes); i++) { |
192 | bus = pci_mmcfg_probes[i].bus; | 309 | bus = pci_mmcfg_probes[i].bus; |
193 | devfn = pci_mmcfg_probes[i].devfn; | 310 | devfn = pci_mmcfg_probes[i].devfn; |
194 | raw_pci_ops->read(0, bus, devfn, 0, 4, &l); | 311 | raw_pci_ops->read(0, bus, devfn, 0, 4, &l); |
195 | vendor = l & 0xffff; | 312 | vendor = l & 0xffff; |
196 | device = (l >> 16) & 0xffff; | 313 | device = (l >> 16) & 0xffff; |
197 | 314 | ||
315 | name = NULL; | ||
198 | if (pci_mmcfg_probes[i].vendor == vendor && | 316 | if (pci_mmcfg_probes[i].vendor == vendor && |
199 | pci_mmcfg_probes[i].device == device) | 317 | pci_mmcfg_probes[i].device == device) |
200 | name = pci_mmcfg_probes[i].probe(); | 318 | name = pci_mmcfg_probes[i].probe(); |
201 | } | ||
202 | 319 | ||
203 | if (name) { | 320 | if (name) |
204 | printk(KERN_INFO "PCI: Found %s %s MMCONFIG support.\n", | 321 | printk(KERN_INFO "PCI: Found %s with MMCONFIG support.\n", |
205 | name, pci_mmcfg_config_num ? "with" : "without"); | 322 | name); |
206 | } | 323 | } |
207 | 324 | ||
208 | return name != NULL; | 325 | /* some end_bus_number is crazy, fix it */ |
326 | pci_mmcfg_check_end_bus_number(); | ||
327 | |||
328 | return pci_mmcfg_config_num != 0; | ||
209 | } | 329 | } |
210 | 330 | ||
211 | static void __init pci_mmcfg_insert_resources(void) | 331 | static void __init pci_mmcfg_insert_resources(void) |
212 | { | 332 | { |
213 | #define PCI_MMCFG_RESOURCE_NAME_LEN 19 | 333 | #define PCI_MMCFG_RESOURCE_NAME_LEN 24 |
214 | int i; | 334 | int i; |
215 | struct resource *res; | 335 | struct resource *res; |
216 | char *names; | 336 | char *names; |
@@ -228,9 +348,10 @@ static void __init pci_mmcfg_insert_resources(void) | |||
228 | struct acpi_mcfg_allocation *cfg = &pci_mmcfg_config[i]; | 348 | struct acpi_mcfg_allocation *cfg = &pci_mmcfg_config[i]; |
229 | num_buses = cfg->end_bus_number - cfg->start_bus_number + 1; | 349 | num_buses = cfg->end_bus_number - cfg->start_bus_number + 1; |
230 | res->name = names; | 350 | res->name = names; |
231 | snprintf(names, PCI_MMCFG_RESOURCE_NAME_LEN, "PCI MMCONFIG %u", | 351 | snprintf(names, PCI_MMCFG_RESOURCE_NAME_LEN, |
232 | cfg->pci_segment); | 352 | "PCI MMCONFIG %u [%02x-%02x]", cfg->pci_segment, |
233 | res->start = cfg->address; | 353 | cfg->start_bus_number, cfg->end_bus_number); |
354 | res->start = cfg->address + (cfg->start_bus_number << 20); | ||
234 | res->end = res->start + (num_buses << 20) - 1; | 355 | res->end = res->start + (num_buses << 20) - 1; |
235 | res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; | 356 | res->flags = IORESOURCE_MEM | IORESOURCE_BUSY; |
236 | insert_resource(&iomem_resource, res); | 357 | insert_resource(&iomem_resource, res); |
@@ -354,8 +475,6 @@ static void __init pci_mmcfg_reject_broken(int early) | |||
354 | (pci_mmcfg_config[0].address == 0)) | 475 | (pci_mmcfg_config[0].address == 0)) |
355 | return; | 476 | return; |
356 | 477 | ||
357 | cfg = &pci_mmcfg_config[0]; | ||
358 | |||
359 | for (i = 0; i < pci_mmcfg_config_num; i++) { | 478 | for (i = 0; i < pci_mmcfg_config_num; i++) { |
360 | int valid = 0; | 479 | int valid = 0; |
361 | u64 addr, size; | 480 | u64 addr, size; |
@@ -423,10 +542,10 @@ static void __init __pci_mmcfg_init(int early) | |||
423 | known_bridge = 1; | 542 | known_bridge = 1; |
424 | } | 543 | } |
425 | 544 | ||
426 | if (!known_bridge) { | 545 | if (!known_bridge) |
427 | acpi_table_parse(ACPI_SIG_MCFG, acpi_parse_mcfg); | 546 | acpi_table_parse(ACPI_SIG_MCFG, acpi_parse_mcfg); |
428 | pci_mmcfg_reject_broken(early); | 547 | |
429 | } | 548 | pci_mmcfg_reject_broken(early); |
430 | 549 | ||
431 | if ((pci_mmcfg_config_num == 0) || | 550 | if ((pci_mmcfg_config_num == 0) || |
432 | (pci_mmcfg_config == NULL) || | 551 | (pci_mmcfg_config == NULL) || |
diff --git a/arch/x86/pci/mmconfig_64.c b/arch/x86/pci/mmconfig_64.c index 30007ffc8e11..94349f8b2f96 100644 --- a/arch/x86/pci/mmconfig_64.c +++ b/arch/x86/pci/mmconfig_64.c | |||
@@ -112,13 +112,18 @@ static struct pci_raw_ops pci_mmcfg = { | |||
112 | static void __iomem * __init mcfg_ioremap(struct acpi_mcfg_allocation *cfg) | 112 | static void __iomem * __init mcfg_ioremap(struct acpi_mcfg_allocation *cfg) |
113 | { | 113 | { |
114 | void __iomem *addr; | 114 | void __iomem *addr; |
115 | u32 size; | 115 | u64 start, size; |
116 | 116 | ||
117 | size = (cfg->end_bus_number + 1) << 20; | 117 | start = cfg->start_bus_number; |
118 | addr = ioremap_nocache(cfg->address, size); | 118 | start <<= 20; |
119 | start += cfg->address; | ||
120 | size = cfg->end_bus_number + 1 - cfg->start_bus_number; | ||
121 | size <<= 20; | ||
122 | addr = ioremap_nocache(start, size); | ||
119 | if (addr) { | 123 | if (addr) { |
120 | printk(KERN_INFO "PCI: Using MMCONFIG at %Lx - %Lx\n", | 124 | printk(KERN_INFO "PCI: Using MMCONFIG at %Lx - %Lx\n", |
121 | cfg->address, cfg->address + size - 1); | 125 | start, start + size - 1); |
126 | addr -= cfg->start_bus_number << 20; | ||
122 | } | 127 | } |
123 | return addr; | 128 | return addr; |
124 | } | 129 | } |
@@ -157,7 +162,7 @@ void __init pci_mmcfg_arch_free(void) | |||
157 | 162 | ||
158 | for (i = 0; i < pci_mmcfg_config_num; ++i) { | 163 | for (i = 0; i < pci_mmcfg_config_num; ++i) { |
159 | if (pci_mmcfg_virt[i].virt) { | 164 | if (pci_mmcfg_virt[i].virt) { |
160 | iounmap(pci_mmcfg_virt[i].virt); | 165 | iounmap(pci_mmcfg_virt[i].virt + (pci_mmcfg_virt[i].cfg->start_bus_number << 20)); |
161 | pci_mmcfg_virt[i].virt = NULL; | 166 | pci_mmcfg_virt[i].virt = NULL; |
162 | pci_mmcfg_virt[i].cfg = NULL; | 167 | pci_mmcfg_virt[i].cfg = NULL; |
163 | } | 168 | } |
diff --git a/arch/x86/power/cpu_32.c b/arch/x86/power/cpu_32.c index 274d06082f48..ce702c5b3a2c 100644 --- a/arch/x86/power/cpu_32.c +++ b/arch/x86/power/cpu_32.c | |||
@@ -12,6 +12,7 @@ | |||
12 | #include <asm/mtrr.h> | 12 | #include <asm/mtrr.h> |
13 | #include <asm/mce.h> | 13 | #include <asm/mce.h> |
14 | #include <asm/xcr.h> | 14 | #include <asm/xcr.h> |
15 | #include <asm/suspend.h> | ||
15 | 16 | ||
16 | static struct saved_context saved_context; | 17 | static struct saved_context saved_context; |
17 | 18 | ||
diff --git a/arch/x86/power/cpu_64.c b/arch/x86/power/cpu_64.c index e3b6cf70d62c..5343540f2607 100644 --- a/arch/x86/power/cpu_64.c +++ b/arch/x86/power/cpu_64.c | |||
@@ -15,6 +15,7 @@ | |||
15 | #include <asm/pgtable.h> | 15 | #include <asm/pgtable.h> |
16 | #include <asm/mtrr.h> | 16 | #include <asm/mtrr.h> |
17 | #include <asm/xcr.h> | 17 | #include <asm/xcr.h> |
18 | #include <asm/suspend.h> | ||
18 | 19 | ||
19 | static void fix_processor_context(void); | 20 | static void fix_processor_context(void); |
20 | 21 | ||
diff --git a/arch/x86/power/hibernate_64.c b/arch/x86/power/hibernate_64.c index 6dd000dd7933..65fdc86e923f 100644 --- a/arch/x86/power/hibernate_64.c +++ b/arch/x86/power/hibernate_64.c | |||
@@ -14,6 +14,7 @@ | |||
14 | #include <asm/page.h> | 14 | #include <asm/page.h> |
15 | #include <asm/pgtable.h> | 15 | #include <asm/pgtable.h> |
16 | #include <asm/mtrr.h> | 16 | #include <asm/mtrr.h> |
17 | #include <asm/suspend.h> | ||
17 | 18 | ||
18 | /* References to section boundaries */ | 19 | /* References to section boundaries */ |
19 | extern const void __nosave_begin, __nosave_end; | 20 | extern const void __nosave_begin, __nosave_end; |