aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/mm/pat.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/mm/pat.c')
-rw-r--r--arch/x86/mm/pat.c378
1 files changed, 140 insertions, 238 deletions
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 06b7a1c90fb8..d4585077977a 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -26,11 +26,11 @@
26#include <asm/io.h> 26#include <asm/io.h>
27 27
28#ifdef CONFIG_X86_PAT 28#ifdef CONFIG_X86_PAT
29int __read_mostly pat_wc_enabled = 1; 29int __read_mostly pat_enabled = 1;
30 30
31void __cpuinit pat_disable(char *reason) 31void __cpuinit pat_disable(char *reason)
32{ 32{
33 pat_wc_enabled = 0; 33 pat_enabled = 0;
34 printk(KERN_INFO "%s\n", reason); 34 printk(KERN_INFO "%s\n", reason);
35} 35}
36 36
@@ -42,6 +42,19 @@ static int __init nopat(char *str)
42early_param("nopat", nopat); 42early_param("nopat", nopat);
43#endif 43#endif
44 44
45
46static int debug_enable;
47static int __init pat_debug_setup(char *str)
48{
49 debug_enable = 1;
50 return 0;
51}
52__setup("debugpat", pat_debug_setup);
53
54#define dprintk(fmt, arg...) \
55 do { if (debug_enable) printk(KERN_INFO fmt, ##arg); } while (0)
56
57
45static u64 __read_mostly boot_pat_state; 58static u64 __read_mostly boot_pat_state;
46 59
47enum { 60enum {
@@ -53,24 +66,25 @@ enum {
53 PAT_UC_MINUS = 7, /* UC, but can be overriden by MTRR */ 66 PAT_UC_MINUS = 7, /* UC, but can be overriden by MTRR */
54}; 67};
55 68
56#define PAT(x,y) ((u64)PAT_ ## y << ((x)*8)) 69#define PAT(x, y) ((u64)PAT_ ## y << ((x)*8))
57 70
58void pat_init(void) 71void pat_init(void)
59{ 72{
60 u64 pat; 73 u64 pat;
61 74
62 if (!pat_wc_enabled) 75 if (!pat_enabled)
63 return; 76 return;
64 77
65 /* Paranoia check. */ 78 /* Paranoia check. */
66 if (!cpu_has_pat) { 79 if (!cpu_has_pat && boot_pat_state) {
67 printk(KERN_ERR "PAT enabled, but CPU feature cleared\n");
68 /* 80 /*
69 * Panic if this happens on the secondary CPU, and we 81 * If this happens we are on a secondary CPU, but
70 * switched to PAT on the boot CPU. We have no way to 82 * switched to PAT on the boot CPU. We have no way to
71 * undo PAT. 83 * undo PAT.
72 */ 84 */
73 BUG_ON(boot_pat_state); 85 printk(KERN_ERR "PAT enabled, "
86 "but not supported by secondary CPU\n");
87 BUG();
74 } 88 }
75 89
76 /* Set PWT to Write-Combining. All other bits stay the same */ 90 /* Set PWT to Write-Combining. All other bits stay the same */
@@ -86,8 +100,8 @@ void pat_init(void)
86 * 011 UC _PAGE_CACHE_UC 100 * 011 UC _PAGE_CACHE_UC
87 * PAT bit unused 101 * PAT bit unused
88 */ 102 */
89 pat = PAT(0,WB) | PAT(1,WC) | PAT(2,UC_MINUS) | PAT(3,UC) | 103 pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
90 PAT(4,WB) | PAT(5,WC) | PAT(6,UC_MINUS) | PAT(7,UC); 104 PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
91 105
92 /* Boot CPU check */ 106 /* Boot CPU check */
93 if (!boot_pat_state) 107 if (!boot_pat_state)
@@ -103,11 +117,11 @@ void pat_init(void)
103static char *cattr_name(unsigned long flags) 117static char *cattr_name(unsigned long flags)
104{ 118{
105 switch (flags & _PAGE_CACHE_MASK) { 119 switch (flags & _PAGE_CACHE_MASK) {
106 case _PAGE_CACHE_UC: return "uncached"; 120 case _PAGE_CACHE_UC: return "uncached";
107 case _PAGE_CACHE_UC_MINUS: return "uncached-minus"; 121 case _PAGE_CACHE_UC_MINUS: return "uncached-minus";
108 case _PAGE_CACHE_WB: return "write-back"; 122 case _PAGE_CACHE_WB: return "write-back";
109 case _PAGE_CACHE_WC: return "write-combining"; 123 case _PAGE_CACHE_WC: return "write-combining";
110 default: return "broken"; 124 default: return "broken";
111 } 125 }
112} 126}
113 127
@@ -145,47 +159,50 @@ static DEFINE_SPINLOCK(memtype_lock); /* protects memtype list */
145 * The intersection is based on "Effective Memory Type" tables in IA-32 159 * The intersection is based on "Effective Memory Type" tables in IA-32
146 * SDM vol 3a 160 * SDM vol 3a
147 */ 161 */
148static int pat_x_mtrr_type(u64 start, u64 end, unsigned long prot, 162static unsigned long pat_x_mtrr_type(u64 start, u64 end, unsigned long req_type)
149 unsigned long *ret_prot)
150{ 163{
151 unsigned long pat_type;
152 u8 mtrr_type;
153
154 pat_type = prot & _PAGE_CACHE_MASK;
155 prot &= (~_PAGE_CACHE_MASK);
156
157 /*
158 * We return the PAT request directly for types where PAT takes
159 * precedence with respect to MTRR and for UC_MINUS.
160 * Consistency checks with other PAT requests is done later
161 * while going through memtype list.
162 */
163 if (pat_type == _PAGE_CACHE_WC) {
164 *ret_prot = prot | _PAGE_CACHE_WC;
165 return 0;
166 } else if (pat_type == _PAGE_CACHE_UC_MINUS) {
167 *ret_prot = prot | _PAGE_CACHE_UC_MINUS;
168 return 0;
169 } else if (pat_type == _PAGE_CACHE_UC) {
170 *ret_prot = prot | _PAGE_CACHE_UC;
171 return 0;
172 }
173
174 /* 164 /*
175 * Look for MTRR hint to get the effective type in case where PAT 165 * Look for MTRR hint to get the effective type in case where PAT
176 * request is for WB. 166 * request is for WB.
177 */ 167 */
178 mtrr_type = mtrr_type_lookup(start, end); 168 if (req_type == _PAGE_CACHE_WB) {
169 u8 mtrr_type;
170
171 mtrr_type = mtrr_type_lookup(start, end);
172 if (mtrr_type == MTRR_TYPE_UNCACHABLE)
173 return _PAGE_CACHE_UC;
174 if (mtrr_type == MTRR_TYPE_WRCOMB)
175 return _PAGE_CACHE_WC;
176 }
179 177
180 if (mtrr_type == MTRR_TYPE_UNCACHABLE) { 178 return req_type;
181 *ret_prot = prot | _PAGE_CACHE_UC; 179}
182 } else if (mtrr_type == MTRR_TYPE_WRCOMB) { 180
183 *ret_prot = prot | _PAGE_CACHE_WC; 181static int chk_conflict(struct memtype *new, struct memtype *entry,
184 } else { 182 unsigned long *type)
185 *ret_prot = prot | _PAGE_CACHE_WB; 183{
184 if (new->type != entry->type) {
185 if (type) {
186 new->type = entry->type;
187 *type = entry->type;
188 } else
189 goto conflict;
186 } 190 }
187 191
192 /* check overlaps with more than one entry in the list */
193 list_for_each_entry_continue(entry, &memtype_list, nd) {
194 if (new->end <= entry->start)
195 break;
196 else if (new->type != entry->type)
197 goto conflict;
198 }
188 return 0; 199 return 0;
200
201 conflict:
202 printk(KERN_INFO "%s:%d conflicting memory types "
203 "%Lx-%Lx %s<->%s\n", current->comm, current->pid, new->start,
204 new->end, cattr_name(new->type), cattr_name(entry->type));
205 return -EBUSY;
189} 206}
190 207
191/* 208/*
@@ -198,37 +215,36 @@ static int pat_x_mtrr_type(u64 start, u64 end, unsigned long prot,
198 * req_type will have a special case value '-1', when requester want to inherit 215 * req_type will have a special case value '-1', when requester want to inherit
199 * the memory type from mtrr (if WB), existing PAT, defaulting to UC_MINUS. 216 * the memory type from mtrr (if WB), existing PAT, defaulting to UC_MINUS.
200 * 217 *
201 * If ret_type is NULL, function will return an error if it cannot reserve the 218 * If new_type is NULL, function will return an error if it cannot reserve the
202 * region with req_type. If ret_type is non-null, function will return 219 * region with req_type. If new_type is non-NULL, function will return
203 * available type in ret_type in case of no error. In case of any error 220 * available type in new_type in case of no error. In case of any error
204 * it will return a negative return value. 221 * it will return a negative return value.
205 */ 222 */
206int reserve_memtype(u64 start, u64 end, unsigned long req_type, 223int reserve_memtype(u64 start, u64 end, unsigned long req_type,
207 unsigned long *ret_type) 224 unsigned long *new_type)
208{ 225{
209 struct memtype *new_entry = NULL; 226 struct memtype *new, *entry;
210 struct memtype *parse;
211 unsigned long actual_type; 227 unsigned long actual_type;
228 struct list_head *where;
212 int err = 0; 229 int err = 0;
213 230
214 /* Only track when pat_wc_enabled */ 231 BUG_ON(start >= end); /* end is exclusive */
215 if (!pat_wc_enabled) { 232
233 if (!pat_enabled) {
216 /* This is identical to page table setting without PAT */ 234 /* This is identical to page table setting without PAT */
217 if (ret_type) { 235 if (new_type) {
218 if (req_type == -1) { 236 if (req_type == -1)
219 *ret_type = _PAGE_CACHE_WB; 237 *new_type = _PAGE_CACHE_WB;
220 } else { 238 else
221 *ret_type = req_type; 239 *new_type = req_type & _PAGE_CACHE_MASK;
222 }
223 } 240 }
224 return 0; 241 return 0;
225 } 242 }
226 243
227 /* Low ISA region is always mapped WB in page table. No need to track */ 244 /* Low ISA region is always mapped WB in page table. No need to track */
228 if (start >= ISA_START_ADDRESS && (end - 1) <= ISA_END_ADDRESS) { 245 if (is_ISA_range(start, end - 1)) {
229 if (ret_type) 246 if (new_type)
230 *ret_type = _PAGE_CACHE_WB; 247 *new_type = _PAGE_CACHE_WB;
231
232 return 0; 248 return 0;
233 } 249 }
234 250
@@ -241,206 +257,92 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
241 */ 257 */
242 u8 mtrr_type = mtrr_type_lookup(start, end); 258 u8 mtrr_type = mtrr_type_lookup(start, end);
243 259
244 if (mtrr_type == MTRR_TYPE_WRBACK) { 260 if (mtrr_type == MTRR_TYPE_WRBACK)
245 req_type = _PAGE_CACHE_WB;
246 actual_type = _PAGE_CACHE_WB; 261 actual_type = _PAGE_CACHE_WB;
247 } else { 262 else
248 req_type = _PAGE_CACHE_UC_MINUS;
249 actual_type = _PAGE_CACHE_UC_MINUS; 263 actual_type = _PAGE_CACHE_UC_MINUS;
250 } 264 } else
251 } else { 265 actual_type = pat_x_mtrr_type(start, end,
252 req_type &= _PAGE_CACHE_MASK; 266 req_type & _PAGE_CACHE_MASK);
253 err = pat_x_mtrr_type(start, end, req_type, &actual_type);
254 }
255
256 if (err) {
257 if (ret_type)
258 *ret_type = actual_type;
259
260 return -EINVAL;
261 }
262 267
263 new_entry = kmalloc(sizeof(struct memtype), GFP_KERNEL); 268 new = kmalloc(sizeof(struct memtype), GFP_KERNEL);
264 if (!new_entry) 269 if (!new)
265 return -ENOMEM; 270 return -ENOMEM;
266 271
267 new_entry->start = start; 272 new->start = start;
268 new_entry->end = end; 273 new->end = end;
269 new_entry->type = actual_type; 274 new->type = actual_type;
270 275
271 if (ret_type) 276 if (new_type)
272 *ret_type = actual_type; 277 *new_type = actual_type;
273 278
274 spin_lock(&memtype_lock); 279 spin_lock(&memtype_lock);
275 280
276 /* Search for existing mapping that overlaps the current range */ 281 /* Search for existing mapping that overlaps the current range */
277 list_for_each_entry(parse, &memtype_list, nd) { 282 where = NULL;
278 struct memtype *saved_ptr; 283 list_for_each_entry(entry, &memtype_list, nd) {
279 284 if (end <= entry->start) {
280 if (parse->start >= end) { 285 where = entry->nd.prev;
281 pr_debug("New Entry\n");
282 list_add(&new_entry->nd, parse->nd.prev);
283 new_entry = NULL;
284 break; 286 break;
285 } 287 } else if (start <= entry->start) { /* end > entry->start */
286 288 err = chk_conflict(new, entry, new_type);
287 if (start <= parse->start && end >= parse->start) { 289 if (!err) {
288 if (actual_type != parse->type && ret_type) { 290 dprintk("Overlap at 0x%Lx-0x%Lx\n",
289 actual_type = parse->type; 291 entry->start, entry->end);
290 *ret_type = actual_type; 292 where = entry->nd.prev;
291 new_entry->type = actual_type;
292 }
293
294 if (actual_type != parse->type) {
295 printk(
296 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
297 current->comm, current->pid,
298 start, end,
299 cattr_name(actual_type),
300 cattr_name(parse->type));
301 err = -EBUSY;
302 break;
303 } 293 }
304
305 saved_ptr = parse;
306 /*
307 * Check to see whether the request overlaps more
308 * than one entry in the list
309 */
310 list_for_each_entry_continue(parse, &memtype_list, nd) {
311 if (end <= parse->start) {
312 break;
313 }
314
315 if (actual_type != parse->type) {
316 printk(
317 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
318 current->comm, current->pid,
319 start, end,
320 cattr_name(actual_type),
321 cattr_name(parse->type));
322 err = -EBUSY;
323 break;
324 }
325 }
326
327 if (err) {
328 break;
329 }
330
331 pr_debug("Overlap at 0x%Lx-0x%Lx\n",
332 saved_ptr->start, saved_ptr->end);
333 /* No conflict. Go ahead and add this new entry */
334 list_add(&new_entry->nd, saved_ptr->nd.prev);
335 new_entry = NULL;
336 break; 294 break;
337 } 295 } else if (start < entry->end) { /* start > entry->start */
338 296 err = chk_conflict(new, entry, new_type);
339 if (start < parse->end) { 297 if (!err) {
340 if (actual_type != parse->type && ret_type) { 298 dprintk("Overlap at 0x%Lx-0x%Lx\n",
341 actual_type = parse->type; 299 entry->start, entry->end);
342 *ret_type = actual_type; 300 where = &entry->nd;
343 new_entry->type = actual_type;
344 }
345
346 if (actual_type != parse->type) {
347 printk(
348 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
349 current->comm, current->pid,
350 start, end,
351 cattr_name(actual_type),
352 cattr_name(parse->type));
353 err = -EBUSY;
354 break;
355 }
356
357 saved_ptr = parse;
358 /*
359 * Check to see whether the request overlaps more
360 * than one entry in the list
361 */
362 list_for_each_entry_continue(parse, &memtype_list, nd) {
363 if (end <= parse->start) {
364 break;
365 }
366
367 if (actual_type != parse->type) {
368 printk(
369 KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
370 current->comm, current->pid,
371 start, end,
372 cattr_name(actual_type),
373 cattr_name(parse->type));
374 err = -EBUSY;
375 break;
376 }
377 }
378
379 if (err) {
380 break;
381 } 301 }
382
383 pr_debug(KERN_INFO "Overlap at 0x%Lx-0x%Lx\n",
384 saved_ptr->start, saved_ptr->end);
385 /* No conflict. Go ahead and add this new entry */
386 list_add(&new_entry->nd, &saved_ptr->nd);
387 new_entry = NULL;
388 break; 302 break;
389 } 303 }
390 } 304 }
391 305
392 if (err) { 306 if (err) {
393 printk(KERN_INFO 307 printk(KERN_INFO "reserve_memtype failed 0x%Lx-0x%Lx, "
394 "reserve_memtype failed 0x%Lx-0x%Lx, track %s, req %s\n", 308 "track %s, req %s\n",
395 start, end, cattr_name(new_entry->type), 309 start, end, cattr_name(new->type), cattr_name(req_type));
396 cattr_name(req_type)); 310 kfree(new);
397 kfree(new_entry);
398 spin_unlock(&memtype_lock); 311 spin_unlock(&memtype_lock);
399 return err; 312 return err;
400 } 313 }
401 314
402 if (new_entry) { 315 if (where)
403 /* No conflict. Not yet added to the list. Add to the tail */ 316 list_add(&new->nd, where);
404 list_add_tail(&new_entry->nd, &memtype_list); 317 else
405 pr_debug("New Entry\n"); 318 list_add_tail(&new->nd, &memtype_list);
406 }
407
408 if (ret_type) {
409 pr_debug(
410 "reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s, ret %s\n",
411 start, end, cattr_name(actual_type),
412 cattr_name(req_type), cattr_name(*ret_type));
413 } else {
414 pr_debug(
415 "reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s\n",
416 start, end, cattr_name(actual_type),
417 cattr_name(req_type));
418 }
419 319
420 spin_unlock(&memtype_lock); 320 spin_unlock(&memtype_lock);
321
322 dprintk("reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s, ret %s\n",
323 start, end, cattr_name(new->type), cattr_name(req_type),
324 new_type ? cattr_name(*new_type) : "-");
325
421 return err; 326 return err;
422} 327}
423 328
424int free_memtype(u64 start, u64 end) 329int free_memtype(u64 start, u64 end)
425{ 330{
426 struct memtype *ml; 331 struct memtype *entry;
427 int err = -EINVAL; 332 int err = -EINVAL;
428 333
429 /* Only track when pat_wc_enabled */ 334 if (!pat_enabled)
430 if (!pat_wc_enabled) {
431 return 0; 335 return 0;
432 }
433 336
434 /* Low ISA region is always mapped WB. No need to track */ 337 /* Low ISA region is always mapped WB. No need to track */
435 if (start >= ISA_START_ADDRESS && end <= ISA_END_ADDRESS) { 338 if (is_ISA_range(start, end - 1))
436 return 0; 339 return 0;
437 }
438 340
439 spin_lock(&memtype_lock); 341 spin_lock(&memtype_lock);
440 list_for_each_entry(ml, &memtype_list, nd) { 342 list_for_each_entry(entry, &memtype_list, nd) {
441 if (ml->start == start && ml->end == end) { 343 if (entry->start == start && entry->end == end) {
442 list_del(&ml->nd); 344 list_del(&entry->nd);
443 kfree(ml); 345 kfree(entry);
444 err = 0; 346 err = 0;
445 break; 347 break;
446 } 348 }
@@ -452,7 +354,7 @@ int free_memtype(u64 start, u64 end)
452 current->comm, current->pid, start, end); 354 current->comm, current->pid, start, end);
453 } 355 }
454 356
455 pr_debug("free_memtype request 0x%Lx-0x%Lx\n", start, end); 357 dprintk("free_memtype request 0x%Lx-0x%Lx\n", start, end);
456 return err; 358 return err;
457} 359}
458 360
@@ -521,12 +423,12 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
521 * caching for the high addresses through the KEN pin, but 423 * caching for the high addresses through the KEN pin, but
522 * we maintain the tradition of paranoia in this code. 424 * we maintain the tradition of paranoia in this code.
523 */ 425 */
524 if (!pat_wc_enabled && 426 if (!pat_enabled &&
525 ! ( test_bit(X86_FEATURE_MTRR, boot_cpu_data.x86_capability) || 427 !(boot_cpu_has(X86_FEATURE_MTRR) ||
526 test_bit(X86_FEATURE_K6_MTRR, boot_cpu_data.x86_capability) || 428 boot_cpu_has(X86_FEATURE_K6_MTRR) ||
527 test_bit(X86_FEATURE_CYRIX_ARR, boot_cpu_data.x86_capability) || 429 boot_cpu_has(X86_FEATURE_CYRIX_ARR) ||
528 test_bit(X86_FEATURE_CENTAUR_MCR, boot_cpu_data.x86_capability)) && 430 boot_cpu_has(X86_FEATURE_CENTAUR_MCR)) &&
529 (pfn << PAGE_SHIFT) >= __pa(high_memory)) { 431 (pfn << PAGE_SHIFT) >= __pa(high_memory)) {
530 flags = _PAGE_CACHE_UC; 432 flags = _PAGE_CACHE_UC;
531 } 433 }
532#endif 434#endif
@@ -547,8 +449,9 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
547 if (retval < 0) 449 if (retval < 0)
548 return 0; 450 return 0;
549 451
550 if (pfn <= max_pfn_mapped && 452 if (((pfn < max_low_pfn_mapped) ||
551 ioremap_change_attr((unsigned long)__va(offset), size, flags) < 0) { 453 (pfn >= (1UL<<(32 - PAGE_SHIFT)) && pfn < max_pfn_mapped)) &&
454 ioremap_change_attr((unsigned long)__va(offset), size, flags) < 0) {
552 free_memtype(offset, offset + size); 455 free_memtype(offset, offset + size);
553 printk(KERN_INFO 456 printk(KERN_INFO
554 "%s:%d /dev/mem ioremap_change_attr failed %s for %Lx-%Lx\n", 457 "%s:%d /dev/mem ioremap_change_attr failed %s for %Lx-%Lx\n",
@@ -586,4 +489,3 @@ void unmap_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot)
586 489
587 free_memtype(addr, addr + size); 490 free_memtype(addr, addr + size);
588} 491}
589