diff options
Diffstat (limited to 'arch/x86/mm')
-rw-r--r-- | arch/x86/mm/discontig_32.c | 2 | ||||
-rw-r--r-- | arch/x86/mm/dump_pagetables.c | 4 | ||||
-rw-r--r-- | arch/x86/mm/fault.c | 3 | ||||
-rw-r--r-- | arch/x86/mm/init_32.c | 5 | ||||
-rw-r--r-- | arch/x86/mm/init_64.c | 56 | ||||
-rw-r--r-- | arch/x86/mm/ioremap.c | 14 | ||||
-rw-r--r-- | arch/x86/mm/mmio-mod.c | 4 | ||||
-rw-r--r-- | arch/x86/mm/numa_64.c | 10 | ||||
-rw-r--r-- | arch/x86/mm/pageattr-test.c | 3 | ||||
-rw-r--r-- | arch/x86/mm/pageattr.c | 31 | ||||
-rw-r--r-- | arch/x86/mm/pat.c | 50 | ||||
-rw-r--r-- | arch/x86/mm/pgtable.c | 6 | ||||
-rw-r--r-- | arch/x86/mm/pgtable_32.c | 3 | ||||
-rw-r--r-- | arch/x86/mm/srat_32.c | 12 |
14 files changed, 121 insertions, 82 deletions
diff --git a/arch/x86/mm/discontig_32.c b/arch/x86/mm/discontig_32.c index 62fa440678d8..847c164725f4 100644 --- a/arch/x86/mm/discontig_32.c +++ b/arch/x86/mm/discontig_32.c | |||
@@ -328,7 +328,7 @@ void __init initmem_init(unsigned long start_pfn, | |||
328 | 328 | ||
329 | get_memcfg_numa(); | 329 | get_memcfg_numa(); |
330 | 330 | ||
331 | kva_pages = round_up(calculate_numa_remap_pages(), PTRS_PER_PTE); | 331 | kva_pages = roundup(calculate_numa_remap_pages(), PTRS_PER_PTE); |
332 | 332 | ||
333 | kva_target_pfn = round_down(max_low_pfn - kva_pages, PTRS_PER_PTE); | 333 | kva_target_pfn = round_down(max_low_pfn - kva_pages, PTRS_PER_PTE); |
334 | do { | 334 | do { |
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c index a20d1fa64b4e..e7277cbcfb40 100644 --- a/arch/x86/mm/dump_pagetables.c +++ b/arch/x86/mm/dump_pagetables.c | |||
@@ -148,8 +148,8 @@ static void note_page(struct seq_file *m, struct pg_state *st, | |||
148 | * we have now. "break" is either changing perms, levels or | 148 | * we have now. "break" is either changing perms, levels or |
149 | * address space marker. | 149 | * address space marker. |
150 | */ | 150 | */ |
151 | prot = pgprot_val(new_prot) & ~(PTE_PFN_MASK); | 151 | prot = pgprot_val(new_prot) & PTE_FLAGS_MASK; |
152 | cur = pgprot_val(st->current_prot) & ~(PTE_PFN_MASK); | 152 | cur = pgprot_val(st->current_prot) & PTE_FLAGS_MASK; |
153 | 153 | ||
154 | if (!st->level) { | 154 | if (!st->level) { |
155 | /* First entry */ | 155 | /* First entry */ |
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 455f3fe67b42..8f92cac4e6db 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c | |||
@@ -35,6 +35,7 @@ | |||
35 | #include <asm/tlbflush.h> | 35 | #include <asm/tlbflush.h> |
36 | #include <asm/proto.h> | 36 | #include <asm/proto.h> |
37 | #include <asm-generic/sections.h> | 37 | #include <asm-generic/sections.h> |
38 | #include <asm/traps.h> | ||
38 | 39 | ||
39 | /* | 40 | /* |
40 | * Page fault error code bits | 41 | * Page fault error code bits |
@@ -357,8 +358,6 @@ static int is_errata100(struct pt_regs *regs, unsigned long address) | |||
357 | return 0; | 358 | return 0; |
358 | } | 359 | } |
359 | 360 | ||
360 | void do_invalid_op(struct pt_regs *, unsigned long); | ||
361 | |||
362 | static int is_f00f_bug(struct pt_regs *regs, unsigned long address) | 361 | static int is_f00f_bug(struct pt_regs *regs, unsigned long address) |
363 | { | 362 | { |
364 | #ifdef CONFIG_X86_F00F_BUG | 363 | #ifdef CONFIG_X86_F00F_BUG |
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c index d37f29376b0c..6b9a9358b330 100644 --- a/arch/x86/mm/init_32.c +++ b/arch/x86/mm/init_32.c | |||
@@ -47,6 +47,7 @@ | |||
47 | #include <asm/paravirt.h> | 47 | #include <asm/paravirt.h> |
48 | #include <asm/setup.h> | 48 | #include <asm/setup.h> |
49 | #include <asm/cacheflush.h> | 49 | #include <asm/cacheflush.h> |
50 | #include <asm/smp.h> | ||
50 | 51 | ||
51 | unsigned int __VMALLOC_RESERVE = 128 << 20; | 52 | unsigned int __VMALLOC_RESERVE = 128 << 20; |
52 | 53 | ||
@@ -458,11 +459,7 @@ static void __init pagetable_init(void) | |||
458 | { | 459 | { |
459 | pgd_t *pgd_base = swapper_pg_dir; | 460 | pgd_t *pgd_base = swapper_pg_dir; |
460 | 461 | ||
461 | paravirt_pagetable_setup_start(pgd_base); | ||
462 | |||
463 | permanent_kmaps_init(pgd_base); | 462 | permanent_kmaps_init(pgd_base); |
464 | |||
465 | paravirt_pagetable_setup_done(pgd_base); | ||
466 | } | 463 | } |
467 | 464 | ||
468 | #ifdef CONFIG_ACPI_SLEEP | 465 | #ifdef CONFIG_ACPI_SLEEP |
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c index 129618ca0ea2..770536ebf7e9 100644 --- a/arch/x86/mm/init_64.c +++ b/arch/x86/mm/init_64.c | |||
@@ -60,7 +60,7 @@ static unsigned long dma_reserve __initdata; | |||
60 | 60 | ||
61 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); | 61 | DEFINE_PER_CPU(struct mmu_gather, mmu_gathers); |
62 | 62 | ||
63 | int direct_gbpages __meminitdata | 63 | int direct_gbpages |
64 | #ifdef CONFIG_DIRECT_GBPAGES | 64 | #ifdef CONFIG_DIRECT_GBPAGES |
65 | = 1 | 65 | = 1 |
66 | #endif | 66 | #endif |
@@ -88,7 +88,11 @@ early_param("gbpages", parse_direct_gbpages_on); | |||
88 | 88 | ||
89 | int after_bootmem; | 89 | int after_bootmem; |
90 | 90 | ||
91 | static __init void *spp_getpage(void) | 91 | /* |
92 | * NOTE: This function is marked __ref because it calls __init function | ||
93 | * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0. | ||
94 | */ | ||
95 | static __ref void *spp_getpage(void) | ||
92 | { | 96 | { |
93 | void *ptr; | 97 | void *ptr; |
94 | 98 | ||
@@ -221,7 +225,7 @@ void __init init_extra_mapping_uc(unsigned long phys, unsigned long size) | |||
221 | void __init cleanup_highmap(void) | 225 | void __init cleanup_highmap(void) |
222 | { | 226 | { |
223 | unsigned long vaddr = __START_KERNEL_map; | 227 | unsigned long vaddr = __START_KERNEL_map; |
224 | unsigned long end = round_up((unsigned long)_end, PMD_SIZE) - 1; | 228 | unsigned long end = roundup((unsigned long)_end, PMD_SIZE) - 1; |
225 | pmd_t *pmd = level2_kernel_pgt; | 229 | pmd_t *pmd = level2_kernel_pgt; |
226 | pmd_t *last_pmd = pmd + PTRS_PER_PMD; | 230 | pmd_t *last_pmd = pmd + PTRS_PER_PMD; |
227 | 231 | ||
@@ -237,7 +241,7 @@ static unsigned long __initdata table_start; | |||
237 | static unsigned long __meminitdata table_end; | 241 | static unsigned long __meminitdata table_end; |
238 | static unsigned long __meminitdata table_top; | 242 | static unsigned long __meminitdata table_top; |
239 | 243 | ||
240 | static __meminit void *alloc_low_page(unsigned long *phys) | 244 | static __ref void *alloc_low_page(unsigned long *phys) |
241 | { | 245 | { |
242 | unsigned long pfn = table_end++; | 246 | unsigned long pfn = table_end++; |
243 | void *adr; | 247 | void *adr; |
@@ -258,7 +262,7 @@ static __meminit void *alloc_low_page(unsigned long *phys) | |||
258 | return adr; | 262 | return adr; |
259 | } | 263 | } |
260 | 264 | ||
261 | static __meminit void unmap_low_page(void *adr) | 265 | static __ref void unmap_low_page(void *adr) |
262 | { | 266 | { |
263 | if (after_bootmem) | 267 | if (after_bootmem) |
264 | return; | 268 | return; |
@@ -314,6 +318,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, | |||
314 | { | 318 | { |
315 | unsigned long pages = 0; | 319 | unsigned long pages = 0; |
316 | unsigned long last_map_addr = end; | 320 | unsigned long last_map_addr = end; |
321 | unsigned long start = address; | ||
317 | 322 | ||
318 | int i = pmd_index(address); | 323 | int i = pmd_index(address); |
319 | 324 | ||
@@ -331,16 +336,24 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, | |||
331 | } | 336 | } |
332 | 337 | ||
333 | if (pmd_val(*pmd)) { | 338 | if (pmd_val(*pmd)) { |
334 | if (!pmd_large(*pmd)) | 339 | if (!pmd_large(*pmd)) { |
340 | spin_lock(&init_mm.page_table_lock); | ||
335 | last_map_addr = phys_pte_update(pmd, address, | 341 | last_map_addr = phys_pte_update(pmd, address, |
336 | end); | 342 | end); |
343 | spin_unlock(&init_mm.page_table_lock); | ||
344 | } | ||
345 | /* Count entries we're using from level2_ident_pgt */ | ||
346 | if (start == 0) | ||
347 | pages++; | ||
337 | continue; | 348 | continue; |
338 | } | 349 | } |
339 | 350 | ||
340 | if (page_size_mask & (1<<PG_LEVEL_2M)) { | 351 | if (page_size_mask & (1<<PG_LEVEL_2M)) { |
341 | pages++; | 352 | pages++; |
353 | spin_lock(&init_mm.page_table_lock); | ||
342 | set_pte((pte_t *)pmd, | 354 | set_pte((pte_t *)pmd, |
343 | pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); | 355 | pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); |
356 | spin_unlock(&init_mm.page_table_lock); | ||
344 | last_map_addr = (address & PMD_MASK) + PMD_SIZE; | 357 | last_map_addr = (address & PMD_MASK) + PMD_SIZE; |
345 | continue; | 358 | continue; |
346 | } | 359 | } |
@@ -349,7 +362,9 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end, | |||
349 | last_map_addr = phys_pte_init(pte, address, end); | 362 | last_map_addr = phys_pte_init(pte, address, end); |
350 | unmap_low_page(pte); | 363 | unmap_low_page(pte); |
351 | 364 | ||
365 | spin_lock(&init_mm.page_table_lock); | ||
352 | pmd_populate_kernel(&init_mm, pmd, __va(pte_phys)); | 366 | pmd_populate_kernel(&init_mm, pmd, __va(pte_phys)); |
367 | spin_unlock(&init_mm.page_table_lock); | ||
353 | } | 368 | } |
354 | update_page_count(PG_LEVEL_2M, pages); | 369 | update_page_count(PG_LEVEL_2M, pages); |
355 | return last_map_addr; | 370 | return last_map_addr; |
@@ -362,9 +377,7 @@ phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end, | |||
362 | pmd_t *pmd = pmd_offset(pud, 0); | 377 | pmd_t *pmd = pmd_offset(pud, 0); |
363 | unsigned long last_map_addr; | 378 | unsigned long last_map_addr; |
364 | 379 | ||
365 | spin_lock(&init_mm.page_table_lock); | ||
366 | last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask); | 380 | last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask); |
367 | spin_unlock(&init_mm.page_table_lock); | ||
368 | __flush_tlb_all(); | 381 | __flush_tlb_all(); |
369 | return last_map_addr; | 382 | return last_map_addr; |
370 | } | 383 | } |
@@ -400,20 +413,21 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end, | |||
400 | 413 | ||
401 | if (page_size_mask & (1<<PG_LEVEL_1G)) { | 414 | if (page_size_mask & (1<<PG_LEVEL_1G)) { |
402 | pages++; | 415 | pages++; |
416 | spin_lock(&init_mm.page_table_lock); | ||
403 | set_pte((pte_t *)pud, | 417 | set_pte((pte_t *)pud, |
404 | pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); | 418 | pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE)); |
419 | spin_unlock(&init_mm.page_table_lock); | ||
405 | last_map_addr = (addr & PUD_MASK) + PUD_SIZE; | 420 | last_map_addr = (addr & PUD_MASK) + PUD_SIZE; |
406 | continue; | 421 | continue; |
407 | } | 422 | } |
408 | 423 | ||
409 | pmd = alloc_low_page(&pmd_phys); | 424 | pmd = alloc_low_page(&pmd_phys); |
410 | |||
411 | spin_lock(&init_mm.page_table_lock); | ||
412 | last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask); | 425 | last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask); |
413 | unmap_low_page(pmd); | 426 | unmap_low_page(pmd); |
427 | |||
428 | spin_lock(&init_mm.page_table_lock); | ||
414 | pud_populate(&init_mm, pud, __va(pmd_phys)); | 429 | pud_populate(&init_mm, pud, __va(pmd_phys)); |
415 | spin_unlock(&init_mm.page_table_lock); | 430 | spin_unlock(&init_mm.page_table_lock); |
416 | |||
417 | } | 431 | } |
418 | __flush_tlb_all(); | 432 | __flush_tlb_all(); |
419 | update_page_count(PG_LEVEL_1G, pages); | 433 | update_page_count(PG_LEVEL_1G, pages); |
@@ -437,14 +451,14 @@ static void __init find_early_table_space(unsigned long end) | |||
437 | unsigned long puds, pmds, ptes, tables, start; | 451 | unsigned long puds, pmds, ptes, tables, start; |
438 | 452 | ||
439 | puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; | 453 | puds = (end + PUD_SIZE - 1) >> PUD_SHIFT; |
440 | tables = round_up(puds * sizeof(pud_t), PAGE_SIZE); | 454 | tables = roundup(puds * sizeof(pud_t), PAGE_SIZE); |
441 | if (direct_gbpages) { | 455 | if (direct_gbpages) { |
442 | unsigned long extra; | 456 | unsigned long extra; |
443 | extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT); | 457 | extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT); |
444 | pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT; | 458 | pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT; |
445 | } else | 459 | } else |
446 | pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; | 460 | pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT; |
447 | tables += round_up(pmds * sizeof(pmd_t), PAGE_SIZE); | 461 | tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE); |
448 | 462 | ||
449 | if (cpu_has_pse) { | 463 | if (cpu_has_pse) { |
450 | unsigned long extra; | 464 | unsigned long extra; |
@@ -452,7 +466,7 @@ static void __init find_early_table_space(unsigned long end) | |||
452 | ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; | 466 | ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT; |
453 | } else | 467 | } else |
454 | ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; | 468 | ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT; |
455 | tables += round_up(ptes * sizeof(pte_t), PAGE_SIZE); | 469 | tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE); |
456 | 470 | ||
457 | /* | 471 | /* |
458 | * RED-PEN putting page tables only on node 0 could | 472 | * RED-PEN putting page tables only on node 0 could |
@@ -505,16 +519,14 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start, | |||
505 | continue; | 519 | continue; |
506 | } | 520 | } |
507 | 521 | ||
508 | if (after_bootmem) | 522 | pud = alloc_low_page(&pud_phys); |
509 | pud = pud_offset(pgd, start & PGDIR_MASK); | ||
510 | else | ||
511 | pud = alloc_low_page(&pud_phys); | ||
512 | |||
513 | last_map_addr = phys_pud_init(pud, __pa(start), __pa(next), | 523 | last_map_addr = phys_pud_init(pud, __pa(start), __pa(next), |
514 | page_size_mask); | 524 | page_size_mask); |
515 | unmap_low_page(pud); | 525 | unmap_low_page(pud); |
516 | pgd_populate(&init_mm, pgd_offset_k(start), | 526 | |
517 | __va(pud_phys)); | 527 | spin_lock(&init_mm.page_table_lock); |
528 | pgd_populate(&init_mm, pgd, __va(pud_phys)); | ||
529 | spin_unlock(&init_mm.page_table_lock); | ||
518 | } | 530 | } |
519 | 531 | ||
520 | return last_map_addr; | 532 | return last_map_addr; |
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c index 016f335bbeea..cac6da54203b 100644 --- a/arch/x86/mm/ioremap.c +++ b/arch/x86/mm/ioremap.c | |||
@@ -170,7 +170,7 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr, | |||
170 | phys_addr &= PAGE_MASK; | 170 | phys_addr &= PAGE_MASK; |
171 | size = PAGE_ALIGN(last_addr+1) - phys_addr; | 171 | size = PAGE_ALIGN(last_addr+1) - phys_addr; |
172 | 172 | ||
173 | retval = reserve_memtype(phys_addr, phys_addr + size, | 173 | retval = reserve_memtype(phys_addr, (u64)phys_addr + size, |
174 | prot_val, &new_prot_val); | 174 | prot_val, &new_prot_val); |
175 | if (retval) { | 175 | if (retval) { |
176 | pr_debug("Warning: reserve_memtype returned %d\n", retval); | 176 | pr_debug("Warning: reserve_memtype returned %d\n", retval); |
@@ -421,7 +421,7 @@ void unxlate_dev_mem_ptr(unsigned long phys, void *addr) | |||
421 | return; | 421 | return; |
422 | } | 422 | } |
423 | 423 | ||
424 | int __initdata early_ioremap_debug; | 424 | static int __initdata early_ioremap_debug; |
425 | 425 | ||
426 | static int __init early_ioremap_debug_setup(char *str) | 426 | static int __init early_ioremap_debug_setup(char *str) |
427 | { | 427 | { |
@@ -547,19 +547,17 @@ static inline void __init early_clear_fixmap(enum fixed_addresses idx) | |||
547 | } | 547 | } |
548 | 548 | ||
549 | 549 | ||
550 | int __initdata early_ioremap_nested; | 550 | static int __initdata early_ioremap_nested; |
551 | 551 | ||
552 | static int __init check_early_ioremap_leak(void) | 552 | static int __init check_early_ioremap_leak(void) |
553 | { | 553 | { |
554 | if (!early_ioremap_nested) | 554 | if (!early_ioremap_nested) |
555 | return 0; | 555 | return 0; |
556 | 556 | WARN(1, KERN_WARNING | |
557 | printk(KERN_WARNING | ||
558 | "Debug warning: early ioremap leak of %d areas detected.\n", | 557 | "Debug warning: early ioremap leak of %d areas detected.\n", |
559 | early_ioremap_nested); | 558 | early_ioremap_nested); |
560 | printk(KERN_WARNING | 559 | printk(KERN_WARNING |
561 | "please boot with early_ioremap_debug and report the dmesg.\n"); | 560 | "please boot with early_ioremap_debug and report the dmesg.\n"); |
562 | WARN_ON(1); | ||
563 | 561 | ||
564 | return 1; | 562 | return 1; |
565 | } | 563 | } |
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c index e7397e108beb..635b50e85581 100644 --- a/arch/x86/mm/mmio-mod.c +++ b/arch/x86/mm/mmio-mod.c | |||
@@ -430,7 +430,9 @@ static void enter_uniprocessor(void) | |||
430 | "may miss events.\n"); | 430 | "may miss events.\n"); |
431 | } | 431 | } |
432 | 432 | ||
433 | static void leave_uniprocessor(void) | 433 | /* __ref because leave_uniprocessor calls cpu_up which is __cpuinit, |
434 | but this whole function is ifdefed CONFIG_HOTPLUG_CPU */ | ||
435 | static void __ref leave_uniprocessor(void) | ||
434 | { | 436 | { |
435 | int cpu; | 437 | int cpu; |
436 | int err; | 438 | int err; |
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c index a4dd793d6003..cebcbf152d46 100644 --- a/arch/x86/mm/numa_64.c +++ b/arch/x86/mm/numa_64.c | |||
@@ -79,7 +79,7 @@ static int __init allocate_cachealigned_memnodemap(void) | |||
79 | return 0; | 79 | return 0; |
80 | 80 | ||
81 | addr = 0x8000; | 81 | addr = 0x8000; |
82 | nodemap_size = round_up(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES); | 82 | nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES); |
83 | nodemap_addr = find_e820_area(addr, max_pfn<<PAGE_SHIFT, | 83 | nodemap_addr = find_e820_area(addr, max_pfn<<PAGE_SHIFT, |
84 | nodemap_size, L1_CACHE_BYTES); | 84 | nodemap_size, L1_CACHE_BYTES); |
85 | if (nodemap_addr == -1UL) { | 85 | if (nodemap_addr == -1UL) { |
@@ -176,10 +176,10 @@ void __init setup_node_bootmem(int nodeid, unsigned long start, | |||
176 | unsigned long start_pfn, last_pfn, bootmap_pages, bootmap_size; | 176 | unsigned long start_pfn, last_pfn, bootmap_pages, bootmap_size; |
177 | unsigned long bootmap_start, nodedata_phys; | 177 | unsigned long bootmap_start, nodedata_phys; |
178 | void *bootmap; | 178 | void *bootmap; |
179 | const int pgdat_size = round_up(sizeof(pg_data_t), PAGE_SIZE); | 179 | const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE); |
180 | int nid; | 180 | int nid; |
181 | 181 | ||
182 | start = round_up(start, ZONE_ALIGN); | 182 | start = roundup(start, ZONE_ALIGN); |
183 | 183 | ||
184 | printk(KERN_INFO "Bootmem setup node %d %016lx-%016lx\n", nodeid, | 184 | printk(KERN_INFO "Bootmem setup node %d %016lx-%016lx\n", nodeid, |
185 | start, end); | 185 | start, end); |
@@ -210,9 +210,9 @@ void __init setup_node_bootmem(int nodeid, unsigned long start, | |||
210 | bootmap_pages = bootmem_bootmap_pages(last_pfn - start_pfn); | 210 | bootmap_pages = bootmem_bootmap_pages(last_pfn - start_pfn); |
211 | nid = phys_to_nid(nodedata_phys); | 211 | nid = phys_to_nid(nodedata_phys); |
212 | if (nid == nodeid) | 212 | if (nid == nodeid) |
213 | bootmap_start = round_up(nodedata_phys + pgdat_size, PAGE_SIZE); | 213 | bootmap_start = roundup(nodedata_phys + pgdat_size, PAGE_SIZE); |
214 | else | 214 | else |
215 | bootmap_start = round_up(start, PAGE_SIZE); | 215 | bootmap_start = roundup(start, PAGE_SIZE); |
216 | /* | 216 | /* |
217 | * SMP_CACHE_BYTES could be enough, but init_bootmem_node like | 217 | * SMP_CACHE_BYTES could be enough, but init_bootmem_node like |
218 | * to use that to align to PAGE_SIZE | 218 | * to use that to align to PAGE_SIZE |
diff --git a/arch/x86/mm/pageattr-test.c b/arch/x86/mm/pageattr-test.c index 0dcd42eb94e6..d4aa503caaa2 100644 --- a/arch/x86/mm/pageattr-test.c +++ b/arch/x86/mm/pageattr-test.c | |||
@@ -221,8 +221,7 @@ static int pageattr_test(void) | |||
221 | failed += print_split(&sc); | 221 | failed += print_split(&sc); |
222 | 222 | ||
223 | if (failed) { | 223 | if (failed) { |
224 | printk(KERN_ERR "NOT PASSED. Please report.\n"); | 224 | WARN(1, KERN_ERR "NOT PASSED. Please report.\n"); |
225 | WARN_ON(1); | ||
226 | return -EINVAL; | 225 | return -EINVAL; |
227 | } else { | 226 | } else { |
228 | if (print) | 227 | if (print) |
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index 65c6e46bf059..898fad617abe 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c | |||
@@ -55,13 +55,19 @@ static void split_page_count(int level) | |||
55 | 55 | ||
56 | int arch_report_meminfo(char *page) | 56 | int arch_report_meminfo(char *page) |
57 | { | 57 | { |
58 | int n = sprintf(page, "DirectMap4k: %8lu\n" | 58 | int n = sprintf(page, "DirectMap4k: %8lu kB\n", |
59 | "DirectMap2M: %8lu\n", | 59 | direct_pages_count[PG_LEVEL_4K] << 2); |
60 | direct_pages_count[PG_LEVEL_4K], | 60 | #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) |
61 | direct_pages_count[PG_LEVEL_2M]); | 61 | n += sprintf(page + n, "DirectMap2M: %8lu kB\n", |
62 | direct_pages_count[PG_LEVEL_2M] << 11); | ||
63 | #else | ||
64 | n += sprintf(page + n, "DirectMap4M: %8lu kB\n", | ||
65 | direct_pages_count[PG_LEVEL_2M] << 12); | ||
66 | #endif | ||
62 | #ifdef CONFIG_X86_64 | 67 | #ifdef CONFIG_X86_64 |
63 | n += sprintf(page + n, "DirectMap1G: %8lu\n", | 68 | if (direct_gbpages) |
64 | direct_pages_count[PG_LEVEL_1G]); | 69 | n += sprintf(page + n, "DirectMap1G: %8lu kB\n", |
70 | direct_pages_count[PG_LEVEL_1G] << 20); | ||
65 | #endif | 71 | #endif |
66 | return n; | 72 | return n; |
67 | } | 73 | } |
@@ -78,7 +84,7 @@ static inline unsigned long highmap_start_pfn(void) | |||
78 | 84 | ||
79 | static inline unsigned long highmap_end_pfn(void) | 85 | static inline unsigned long highmap_end_pfn(void) |
80 | { | 86 | { |
81 | return __pa(round_up((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT; | 87 | return __pa(roundup((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT; |
82 | } | 88 | } |
83 | 89 | ||
84 | #endif | 90 | #endif |
@@ -592,10 +598,9 @@ repeat: | |||
592 | if (!pte_val(old_pte)) { | 598 | if (!pte_val(old_pte)) { |
593 | if (!primary) | 599 | if (!primary) |
594 | return 0; | 600 | return 0; |
595 | printk(KERN_WARNING "CPA: called for zero pte. " | 601 | WARN(1, KERN_WARNING "CPA: called for zero pte. " |
596 | "vaddr = %lx cpa->vaddr = %lx\n", address, | 602 | "vaddr = %lx cpa->vaddr = %lx\n", address, |
597 | cpa->vaddr); | 603 | cpa->vaddr); |
598 | WARN_ON(1); | ||
599 | return -EINVAL; | 604 | return -EINVAL; |
600 | } | 605 | } |
601 | 606 | ||
@@ -844,7 +849,7 @@ int set_memory_uc(unsigned long addr, int numpages) | |||
844 | /* | 849 | /* |
845 | * for now UC MINUS. see comments in ioremap_nocache() | 850 | * for now UC MINUS. see comments in ioremap_nocache() |
846 | */ | 851 | */ |
847 | if (reserve_memtype(addr, addr + numpages * PAGE_SIZE, | 852 | if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE, |
848 | _PAGE_CACHE_UC_MINUS, NULL)) | 853 | _PAGE_CACHE_UC_MINUS, NULL)) |
849 | return -EINVAL; | 854 | return -EINVAL; |
850 | 855 | ||
@@ -863,7 +868,7 @@ int set_memory_wc(unsigned long addr, int numpages) | |||
863 | if (!pat_enabled) | 868 | if (!pat_enabled) |
864 | return set_memory_uc(addr, numpages); | 869 | return set_memory_uc(addr, numpages); |
865 | 870 | ||
866 | if (reserve_memtype(addr, addr + numpages * PAGE_SIZE, | 871 | if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE, |
867 | _PAGE_CACHE_WC, NULL)) | 872 | _PAGE_CACHE_WC, NULL)) |
868 | return -EINVAL; | 873 | return -EINVAL; |
869 | 874 | ||
@@ -879,7 +884,7 @@ int _set_memory_wb(unsigned long addr, int numpages) | |||
879 | 884 | ||
880 | int set_memory_wb(unsigned long addr, int numpages) | 885 | int set_memory_wb(unsigned long addr, int numpages) |
881 | { | 886 | { |
882 | free_memtype(addr, addr + numpages * PAGE_SIZE); | 887 | free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE); |
883 | 888 | ||
884 | return _set_memory_wb(addr, numpages); | 889 | return _set_memory_wb(addr, numpages); |
885 | } | 890 | } |
@@ -901,11 +906,13 @@ int set_memory_ro(unsigned long addr, int numpages) | |||
901 | { | 906 | { |
902 | return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_RW)); | 907 | return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_RW)); |
903 | } | 908 | } |
909 | EXPORT_SYMBOL_GPL(set_memory_ro); | ||
904 | 910 | ||
905 | int set_memory_rw(unsigned long addr, int numpages) | 911 | int set_memory_rw(unsigned long addr, int numpages) |
906 | { | 912 | { |
907 | return change_page_attr_set(addr, numpages, __pgprot(_PAGE_RW)); | 913 | return change_page_attr_set(addr, numpages, __pgprot(_PAGE_RW)); |
908 | } | 914 | } |
915 | EXPORT_SYMBOL_GPL(set_memory_rw); | ||
909 | 916 | ||
910 | int set_memory_np(unsigned long addr, int numpages) | 917 | int set_memory_np(unsigned long addr, int numpages) |
911 | { | 918 | { |
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c index 2fe30916d4b6..2a50e0fa64a5 100644 --- a/arch/x86/mm/pat.c +++ b/arch/x86/mm/pat.c | |||
@@ -207,6 +207,9 @@ static int chk_conflict(struct memtype *new, struct memtype *entry, | |||
207 | return -EBUSY; | 207 | return -EBUSY; |
208 | } | 208 | } |
209 | 209 | ||
210 | static struct memtype *cached_entry; | ||
211 | static u64 cached_start; | ||
212 | |||
210 | /* | 213 | /* |
211 | * req_type typically has one of the: | 214 | * req_type typically has one of the: |
212 | * - _PAGE_CACHE_WB | 215 | * - _PAGE_CACHE_WB |
@@ -280,11 +283,17 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type, | |||
280 | 283 | ||
281 | spin_lock(&memtype_lock); | 284 | spin_lock(&memtype_lock); |
282 | 285 | ||
286 | if (cached_entry && start >= cached_start) | ||
287 | entry = cached_entry; | ||
288 | else | ||
289 | entry = list_entry(&memtype_list, struct memtype, nd); | ||
290 | |||
283 | /* Search for existing mapping that overlaps the current range */ | 291 | /* Search for existing mapping that overlaps the current range */ |
284 | where = NULL; | 292 | where = NULL; |
285 | list_for_each_entry(entry, &memtype_list, nd) { | 293 | list_for_each_entry_continue(entry, &memtype_list, nd) { |
286 | if (end <= entry->start) { | 294 | if (end <= entry->start) { |
287 | where = entry->nd.prev; | 295 | where = entry->nd.prev; |
296 | cached_entry = list_entry(where, struct memtype, nd); | ||
288 | break; | 297 | break; |
289 | } else if (start <= entry->start) { /* end > entry->start */ | 298 | } else if (start <= entry->start) { /* end > entry->start */ |
290 | err = chk_conflict(new, entry, new_type); | 299 | err = chk_conflict(new, entry, new_type); |
@@ -292,6 +301,8 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type, | |||
292 | dprintk("Overlap at 0x%Lx-0x%Lx\n", | 301 | dprintk("Overlap at 0x%Lx-0x%Lx\n", |
293 | entry->start, entry->end); | 302 | entry->start, entry->end); |
294 | where = entry->nd.prev; | 303 | where = entry->nd.prev; |
304 | cached_entry = list_entry(where, | ||
305 | struct memtype, nd); | ||
295 | } | 306 | } |
296 | break; | 307 | break; |
297 | } else if (start < entry->end) { /* start > entry->start */ | 308 | } else if (start < entry->end) { /* start > entry->start */ |
@@ -299,7 +310,20 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type, | |||
299 | if (!err) { | 310 | if (!err) { |
300 | dprintk("Overlap at 0x%Lx-0x%Lx\n", | 311 | dprintk("Overlap at 0x%Lx-0x%Lx\n", |
301 | entry->start, entry->end); | 312 | entry->start, entry->end); |
302 | where = &entry->nd; | 313 | cached_entry = list_entry(entry->nd.prev, |
314 | struct memtype, nd); | ||
315 | |||
316 | /* | ||
317 | * Move to right position in the linked | ||
318 | * list to add this new entry | ||
319 | */ | ||
320 | list_for_each_entry_continue(entry, | ||
321 | &memtype_list, nd) { | ||
322 | if (start <= entry->start) { | ||
323 | where = entry->nd.prev; | ||
324 | break; | ||
325 | } | ||
326 | } | ||
303 | } | 327 | } |
304 | break; | 328 | break; |
305 | } | 329 | } |
@@ -314,6 +338,8 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type, | |||
314 | return err; | 338 | return err; |
315 | } | 339 | } |
316 | 340 | ||
341 | cached_start = start; | ||
342 | |||
317 | if (where) | 343 | if (where) |
318 | list_add(&new->nd, where); | 344 | list_add(&new->nd, where); |
319 | else | 345 | else |
@@ -343,6 +369,9 @@ int free_memtype(u64 start, u64 end) | |||
343 | spin_lock(&memtype_lock); | 369 | spin_lock(&memtype_lock); |
344 | list_for_each_entry(entry, &memtype_list, nd) { | 370 | list_for_each_entry(entry, &memtype_list, nd) { |
345 | if (entry->start == start && entry->end == end) { | 371 | if (entry->start == start && entry->end == end) { |
372 | if (cached_entry == entry || cached_start == start) | ||
373 | cached_entry = NULL; | ||
374 | |||
346 | list_del(&entry->nd); | 375 | list_del(&entry->nd); |
347 | kfree(entry); | 376 | kfree(entry); |
348 | err = 0; | 377 | err = 0; |
@@ -361,14 +390,6 @@ int free_memtype(u64 start, u64 end) | |||
361 | } | 390 | } |
362 | 391 | ||
363 | 392 | ||
364 | /* | ||
365 | * /dev/mem mmap interface. The memtype used for mapping varies: | ||
366 | * - Use UC for mappings with O_SYNC flag | ||
367 | * - Without O_SYNC flag, if there is any conflict in reserve_memtype, | ||
368 | * inherit the memtype from existing mapping. | ||
369 | * - Else use UC_MINUS memtype (for backward compatibility with existing | ||
370 | * X drivers. | ||
371 | */ | ||
372 | pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, | 393 | pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, |
373 | unsigned long size, pgprot_t vma_prot) | 394 | unsigned long size, pgprot_t vma_prot) |
374 | { | 395 | { |
@@ -406,14 +427,14 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, | |||
406 | unsigned long size, pgprot_t *vma_prot) | 427 | unsigned long size, pgprot_t *vma_prot) |
407 | { | 428 | { |
408 | u64 offset = ((u64) pfn) << PAGE_SHIFT; | 429 | u64 offset = ((u64) pfn) << PAGE_SHIFT; |
409 | unsigned long flags = _PAGE_CACHE_UC_MINUS; | 430 | unsigned long flags = -1; |
410 | int retval; | 431 | int retval; |
411 | 432 | ||
412 | if (!range_is_allowed(pfn, size)) | 433 | if (!range_is_allowed(pfn, size)) |
413 | return 0; | 434 | return 0; |
414 | 435 | ||
415 | if (file->f_flags & O_SYNC) { | 436 | if (file->f_flags & O_SYNC) { |
416 | flags = _PAGE_CACHE_UC; | 437 | flags = _PAGE_CACHE_UC_MINUS; |
417 | } | 438 | } |
418 | 439 | ||
419 | #ifdef CONFIG_X86_32 | 440 | #ifdef CONFIG_X86_32 |
@@ -436,13 +457,14 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, | |||
436 | #endif | 457 | #endif |
437 | 458 | ||
438 | /* | 459 | /* |
439 | * With O_SYNC, we can only take UC mapping. Fail if we cannot. | 460 | * With O_SYNC, we can only take UC_MINUS mapping. Fail if we cannot. |
461 | * | ||
440 | * Without O_SYNC, we want to get | 462 | * Without O_SYNC, we want to get |
441 | * - WB for WB-able memory and no other conflicting mappings | 463 | * - WB for WB-able memory and no other conflicting mappings |
442 | * - UC_MINUS for non-WB-able memory with no other conflicting mappings | 464 | * - UC_MINUS for non-WB-able memory with no other conflicting mappings |
443 | * - Inherit from confliting mappings otherwise | 465 | * - Inherit from confliting mappings otherwise |
444 | */ | 466 | */ |
445 | if (flags != _PAGE_CACHE_UC_MINUS) { | 467 | if (flags != -1) { |
446 | retval = reserve_memtype(offset, offset + size, flags, NULL); | 468 | retval = reserve_memtype(offset, offset + size, flags, NULL); |
447 | } else { | 469 | } else { |
448 | retval = reserve_memtype(offset, offset + size, -1, &flags); | 470 | retval = reserve_memtype(offset, offset + size, -1, &flags); |
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c index d50302774fe2..86f2ffc43c3d 100644 --- a/arch/x86/mm/pgtable.c +++ b/arch/x86/mm/pgtable.c | |||
@@ -63,10 +63,8 @@ static inline void pgd_list_del(pgd_t *pgd) | |||
63 | #define UNSHARED_PTRS_PER_PGD \ | 63 | #define UNSHARED_PTRS_PER_PGD \ |
64 | (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) | 64 | (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) |
65 | 65 | ||
66 | static void pgd_ctor(void *p) | 66 | static void pgd_ctor(pgd_t *pgd) |
67 | { | 67 | { |
68 | pgd_t *pgd = p; | ||
69 | |||
70 | /* If the pgd points to a shared pagetable level (either the | 68 | /* If the pgd points to a shared pagetable level (either the |
71 | ptes in non-PAE, or shared PMD in PAE), then just copy the | 69 | ptes in non-PAE, or shared PMD in PAE), then just copy the |
72 | references from swapper_pg_dir. */ | 70 | references from swapper_pg_dir. */ |
@@ -87,7 +85,7 @@ static void pgd_ctor(void *p) | |||
87 | pgd_list_add(pgd); | 85 | pgd_list_add(pgd); |
88 | } | 86 | } |
89 | 87 | ||
90 | static void pgd_dtor(void *pgd) | 88 | static void pgd_dtor(pgd_t *pgd) |
91 | { | 89 | { |
92 | unsigned long flags; /* can be called from interrupt context */ | 90 | unsigned long flags; /* can be called from interrupt context */ |
93 | 91 | ||
diff --git a/arch/x86/mm/pgtable_32.c b/arch/x86/mm/pgtable_32.c index cab0abbd1ebe..0951db9ee519 100644 --- a/arch/x86/mm/pgtable_32.c +++ b/arch/x86/mm/pgtable_32.c | |||
@@ -123,7 +123,8 @@ static int __init parse_vmalloc(char *arg) | |||
123 | if (!arg) | 123 | if (!arg) |
124 | return -EINVAL; | 124 | return -EINVAL; |
125 | 125 | ||
126 | __VMALLOC_RESERVE = memparse(arg, &arg); | 126 | /* Add VMALLOC_OFFSET to the parsed value due to vm area guard hole*/ |
127 | __VMALLOC_RESERVE = memparse(arg, &arg) + VMALLOC_OFFSET; | ||
127 | return 0; | 128 | return 0; |
128 | } | 129 | } |
129 | early_param("vmalloc", parse_vmalloc); | 130 | early_param("vmalloc", parse_vmalloc); |
diff --git a/arch/x86/mm/srat_32.c b/arch/x86/mm/srat_32.c index 1eb2973a301c..16ae70fc57e7 100644 --- a/arch/x86/mm/srat_32.c +++ b/arch/x86/mm/srat_32.c | |||
@@ -178,7 +178,7 @@ void acpi_numa_arch_fixup(void) | |||
178 | * start of the node, and that the current "end" address is after | 178 | * start of the node, and that the current "end" address is after |
179 | * the previous one. | 179 | * the previous one. |
180 | */ | 180 | */ |
181 | static __init void node_read_chunk(int nid, struct node_memory_chunk_s *memory_chunk) | 181 | static __init int node_read_chunk(int nid, struct node_memory_chunk_s *memory_chunk) |
182 | { | 182 | { |
183 | /* | 183 | /* |
184 | * Only add present memory as told by the e820. | 184 | * Only add present memory as told by the e820. |
@@ -189,10 +189,10 @@ static __init void node_read_chunk(int nid, struct node_memory_chunk_s *memory_c | |||
189 | if (memory_chunk->start_pfn >= max_pfn) { | 189 | if (memory_chunk->start_pfn >= max_pfn) { |
190 | printk(KERN_INFO "Ignoring SRAT pfns: %08lx - %08lx\n", | 190 | printk(KERN_INFO "Ignoring SRAT pfns: %08lx - %08lx\n", |
191 | memory_chunk->start_pfn, memory_chunk->end_pfn); | 191 | memory_chunk->start_pfn, memory_chunk->end_pfn); |
192 | return; | 192 | return -1; |
193 | } | 193 | } |
194 | if (memory_chunk->nid != nid) | 194 | if (memory_chunk->nid != nid) |
195 | return; | 195 | return -1; |
196 | 196 | ||
197 | if (!node_has_online_mem(nid)) | 197 | if (!node_has_online_mem(nid)) |
198 | node_start_pfn[nid] = memory_chunk->start_pfn; | 198 | node_start_pfn[nid] = memory_chunk->start_pfn; |
@@ -202,6 +202,8 @@ static __init void node_read_chunk(int nid, struct node_memory_chunk_s *memory_c | |||
202 | 202 | ||
203 | if (node_end_pfn[nid] < memory_chunk->end_pfn) | 203 | if (node_end_pfn[nid] < memory_chunk->end_pfn) |
204 | node_end_pfn[nid] = memory_chunk->end_pfn; | 204 | node_end_pfn[nid] = memory_chunk->end_pfn; |
205 | |||
206 | return 0; | ||
205 | } | 207 | } |
206 | 208 | ||
207 | int __init get_memcfg_from_srat(void) | 209 | int __init get_memcfg_from_srat(void) |
@@ -259,7 +261,9 @@ int __init get_memcfg_from_srat(void) | |||
259 | printk(KERN_DEBUG | 261 | printk(KERN_DEBUG |
260 | "chunk %d nid %d start_pfn %08lx end_pfn %08lx\n", | 262 | "chunk %d nid %d start_pfn %08lx end_pfn %08lx\n", |
261 | j, chunk->nid, chunk->start_pfn, chunk->end_pfn); | 263 | j, chunk->nid, chunk->start_pfn, chunk->end_pfn); |
262 | node_read_chunk(chunk->nid, chunk); | 264 | if (node_read_chunk(chunk->nid, chunk)) |
265 | continue; | ||
266 | |||
263 | e820_register_active_regions(chunk->nid, chunk->start_pfn, | 267 | e820_register_active_regions(chunk->nid, chunk->start_pfn, |
264 | min(chunk->end_pfn, max_pfn)); | 268 | min(chunk->end_pfn, max_pfn)); |
265 | } | 269 | } |