11 files changed, 86 insertions, 64 deletions

diff --git a/arch/x86/mm/discontig_32.c b/arch/x86/mm/discontig_32.c
index 62fa440678d..847c164725f 100644
--- a/arch/x86/mm/discontig_32.c
+++ b/arch/x86/mm/discontig_32.c
@@ -328,7 +328,7 @@ void __init initmem_init(unsigned long start_pfn,
 
         get_memcfg_numa();
 
-        kva_pages = round_up(calculate_numa_remap_pages(), PTRS_PER_PTE);
+        kva_pages = roundup(calculate_numa_remap_pages(), PTRS_PER_PTE);
 
         kva_target_pfn = round_down(max_low_pfn - kva_pages, PTRS_PER_PTE);
         do {
diff --git a/arch/x86/mm/dump_pagetables.c b/arch/x86/mm/dump_pagetables.c
index a20d1fa64b4..e7277cbcfb4 100644
--- a/arch/x86/mm/dump_pagetables.c
+++ b/arch/x86/mm/dump_pagetables.c
@@ -148,8 +148,8 @@ static void note_page(struct seq_file *m, struct pg_state *st,
          * we have now. "break" is either changing perms, levels or
          * address space marker.
          */
-        prot = pgprot_val(new_prot) & ~(PTE_PFN_MASK);
-        cur = pgprot_val(st->current_prot) & ~(PTE_PFN_MASK);
+        prot = pgprot_val(new_prot) & PTE_FLAGS_MASK;
+        cur = pgprot_val(st->current_prot) & PTE_FLAGS_MASK;
 
         if (!st->level) {
                 /* First entry */
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 455f3fe67b4..8f92cac4e6d 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -35,6 +35,7 @@
 #include <asm/tlbflush.h>
 #include <asm/proto.h>
 #include <asm-generic/sections.h>
+#include <asm/traps.h>
 
 /*
  * Page fault error code bits
@@ -357,8 +358,6 @@ static int is_errata100(struct pt_regs *regs, unsigned long address)
         return 0;
 }
 
-void do_invalid_op(struct pt_regs *, unsigned long);
-
 static int is_f00f_bug(struct pt_regs *regs, unsigned long address)
 {
 #ifdef CONFIG_X86_F00F_BUG
diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index d37f29376b0..6b9a9358b33 100644
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -47,6 +47,7 @@
 #include <asm/paravirt.h>
 #include <asm/setup.h>
 #include <asm/cacheflush.h>
+#include <asm/smp.h>
 
 unsigned int __VMALLOC_RESERVE = 128 << 20;
 
@@ -458,11 +459,7 @@ static void __init pagetable_init(void)
 {
         pgd_t *pgd_base = swapper_pg_dir;
 
-        paravirt_pagetable_setup_start(pgd_base);
-
         permanent_kmaps_init(pgd_base);
-
-        paravirt_pagetable_setup_done(pgd_base);
 }
 
 #ifdef CONFIG_ACPI_SLEEP
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index a87ea0e4b3d..770536ebf7e 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -225,7 +225,7 @@ void __init init_extra_mapping_uc(unsigned long phys, unsigned long size)
 void __init cleanup_highmap(void)
 {
         unsigned long vaddr = __START_KERNEL_map;
-        unsigned long end = round_up((unsigned long)_end, PMD_SIZE) - 1;
+        unsigned long end = roundup((unsigned long)_end, PMD_SIZE) - 1;
         pmd_t *pmd = level2_kernel_pgt;
         pmd_t *last_pmd = pmd + PTRS_PER_PMD;
 
@@ -241,7 +241,7 @@ static unsigned long __initdata table_start;
 static unsigned long __meminitdata table_end;
 static unsigned long __meminitdata table_top;
 
-static __meminit void *alloc_low_page(unsigned long *phys)
+static __ref void *alloc_low_page(unsigned long *phys)
 {
         unsigned long pfn = table_end++;
         void *adr;
@@ -262,7 +262,7 @@ static __meminit void *alloc_low_page(unsigned long *phys)
         return adr;
 }
 
-static __meminit void unmap_low_page(void *adr)
+static __ref void unmap_low_page(void *adr)
 {
         if (after_bootmem)
                 return;
@@ -336,9 +336,12 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                 }
 
                 if (pmd_val(*pmd)) {
-                        if (!pmd_large(*pmd))
+                        if (!pmd_large(*pmd)) {
+                                spin_lock(&init_mm.page_table_lock);
                                 last_map_addr = phys_pte_update(pmd, address,
-                                                                 end);
+                                                                end);
+                                spin_unlock(&init_mm.page_table_lock);
+                        }
                         /* Count entries we're using from level2_ident_pgt */
                         if (start == 0)
                                 pages++;
@@ -347,8 +350,10 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
 
                 if (page_size_mask & (1<<PG_LEVEL_2M)) {
                         pages++;
+                        spin_lock(&init_mm.page_table_lock);
                         set_pte((pte_t *)pmd,
                                 pfn_pte(address >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
+                        spin_unlock(&init_mm.page_table_lock);
                         last_map_addr = (address & PMD_MASK) + PMD_SIZE;
                         continue;
                 }
@@ -357,7 +362,9 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long address, unsigned long end,
                 last_map_addr = phys_pte_init(pte, address, end);
                 unmap_low_page(pte);
 
+                spin_lock(&init_mm.page_table_lock);
                 pmd_populate_kernel(&init_mm, pmd, __va(pte_phys));
+                spin_unlock(&init_mm.page_table_lock);
         }
         update_page_count(PG_LEVEL_2M, pages);
         return last_map_addr;
@@ -370,9 +377,7 @@ phys_pmd_update(pud_t *pud, unsigned long address, unsigned long end,
         pmd_t *pmd = pmd_offset(pud, 0);
         unsigned long last_map_addr;
 
-        spin_lock(&init_mm.page_table_lock);
         last_map_addr = phys_pmd_init(pmd, address, end, page_size_mask);
-        spin_unlock(&init_mm.page_table_lock);
         __flush_tlb_all();
         return last_map_addr;
 }
@@ -408,20 +413,21 @@ phys_pud_init(pud_t *pud_page, unsigned long addr, unsigned long end,
 
                 if (page_size_mask & (1<<PG_LEVEL_1G)) {
                         pages++;
+                        spin_lock(&init_mm.page_table_lock);
                         set_pte((pte_t *)pud,
                                 pfn_pte(addr >> PAGE_SHIFT, PAGE_KERNEL_LARGE));
+                        spin_unlock(&init_mm.page_table_lock);
                         last_map_addr = (addr & PUD_MASK) + PUD_SIZE;
                         continue;
                 }
 
                 pmd = alloc_low_page(&pmd_phys);
-
-                spin_lock(&init_mm.page_table_lock);
                 last_map_addr = phys_pmd_init(pmd, addr, end, page_size_mask);
                 unmap_low_page(pmd);
+
+                spin_lock(&init_mm.page_table_lock);
                 pud_populate(&init_mm, pud, __va(pmd_phys));
                 spin_unlock(&init_mm.page_table_lock);
-
         }
         __flush_tlb_all();
         update_page_count(PG_LEVEL_1G, pages);
@@ -445,14 +451,14 @@ static void __init find_early_table_space(unsigned long end)
         unsigned long puds, pmds, ptes, tables, start;
 
         puds = (end + PUD_SIZE - 1) >> PUD_SHIFT;
-        tables = round_up(puds * sizeof(pud_t), PAGE_SIZE);
+        tables = roundup(puds * sizeof(pud_t), PAGE_SIZE);
         if (direct_gbpages) {
                 unsigned long extra;
                 extra = end - ((end>>PUD_SHIFT) << PUD_SHIFT);
                 pmds = (extra + PMD_SIZE - 1) >> PMD_SHIFT;
         } else
                 pmds = (end + PMD_SIZE - 1) >> PMD_SHIFT;
-        tables += round_up(pmds * sizeof(pmd_t), PAGE_SIZE);
+        tables += roundup(pmds * sizeof(pmd_t), PAGE_SIZE);
 
         if (cpu_has_pse) {
                 unsigned long extra;
@@ -460,7 +466,7 @@ static void __init find_early_table_space(unsigned long end)
                 ptes = (extra + PAGE_SIZE - 1) >> PAGE_SHIFT;
         } else
                 ptes = (end + PAGE_SIZE - 1) >> PAGE_SHIFT;
-        tables += round_up(ptes * sizeof(pte_t), PAGE_SIZE);
+        tables += roundup(ptes * sizeof(pte_t), PAGE_SIZE);
 
         /*
          * RED-PEN putting page tables only on node 0 could
@@ -513,16 +519,14 @@ static unsigned long __init kernel_physical_mapping_init(unsigned long start,
                         continue;
                 }
 
-                if (after_bootmem)
-                        pud = pud_offset(pgd, start & PGDIR_MASK);
-                else
-                        pud = alloc_low_page(&pud_phys);
-
+                pud = alloc_low_page(&pud_phys);
                 last_map_addr = phys_pud_init(pud, __pa(start), __pa(next),
                                                  page_size_mask);
                 unmap_low_page(pud);
-                pgd_populate(&init_mm, pgd_offset_k(start),
-                             __va(pud_phys));
+
+                spin_lock(&init_mm.page_table_lock);
+                pgd_populate(&init_mm, pgd, __va(pud_phys));
+                spin_unlock(&init_mm.page_table_lock);
         }
 
         return last_map_addr;
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 6ba6f889c79..cac6da54203 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -421,7 +421,7 @@ void unxlate_dev_mem_ptr(unsigned long phys, void *addr)
         return;
 }
 
-int __initdata early_ioremap_debug;
+static int __initdata early_ioremap_debug;
 
 static int __init early_ioremap_debug_setup(char *str)
 {
@@ -547,19 +547,17 @@ static inline void __init early_clear_fixmap(enum fixed_addresses idx)
 }
 
 
-int __initdata early_ioremap_nested;
+static int __initdata early_ioremap_nested;
 
 static int __init check_early_ioremap_leak(void)
 {
         if (!early_ioremap_nested)
                 return 0;
-
-        printk(KERN_WARNING
+        WARN(1, KERN_WARNING
                "Debug warning: early ioremap leak of %d areas detected.\n",
-               early_ioremap_nested);
+                early_ioremap_nested);
         printk(KERN_WARNING
-               "please boot with early_ioremap_debug and report the dmesg.\n");
-        WARN_ON(1);
+                "please boot with early_ioremap_debug and report the dmesg.\n");
 
         return 1;
 }
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
index e7397e108be..635b50e8558 100644
--- a/arch/x86/mm/mmio-mod.c
+++ b/arch/x86/mm/mmio-mod.c
@@ -430,7 +430,9 @@ static void enter_uniprocessor(void)
                                                 "may miss events.\n");
 }
 
-static void leave_uniprocessor(void)
+/* __ref because leave_uniprocessor calls cpu_up which is __cpuinit,
+   but this whole function is ifdefed CONFIG_HOTPLUG_CPU */
+static void __ref leave_uniprocessor(void)
 {
         int cpu;
         int err;
diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c
index a4dd793d600..cebcbf152d4 100644
--- a/arch/x86/mm/numa_64.c
+++ b/arch/x86/mm/numa_64.c
@@ -79,7 +79,7 @@ static int __init allocate_cachealigned_memnodemap(void)
                 return 0;
 
         addr = 0x8000;
-        nodemap_size = round_up(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
+        nodemap_size = roundup(sizeof(s16) * memnodemapsize, L1_CACHE_BYTES);
         nodemap_addr = find_e820_area(addr, max_pfn<<PAGE_SHIFT,
                                       nodemap_size, L1_CACHE_BYTES);
         if (nodemap_addr == -1UL) {
@@ -176,10 +176,10 @@ void __init setup_node_bootmem(int nodeid, unsigned long start,
         unsigned long start_pfn, last_pfn, bootmap_pages, bootmap_size;
         unsigned long bootmap_start, nodedata_phys;
         void *bootmap;
-        const int pgdat_size = round_up(sizeof(pg_data_t), PAGE_SIZE);
+        const int pgdat_size = roundup(sizeof(pg_data_t), PAGE_SIZE);
         int nid;
 
-        start = round_up(start, ZONE_ALIGN);
+        start = roundup(start, ZONE_ALIGN);
 
         printk(KERN_INFO "Bootmem setup node %d %016lx-%016lx\n", nodeid,
                start, end);
@@ -210,9 +210,9 @@ void __init setup_node_bootmem(int nodeid, unsigned long start,
         bootmap_pages = bootmem_bootmap_pages(last_pfn - start_pfn);
         nid = phys_to_nid(nodedata_phys);
         if (nid == nodeid)
-                bootmap_start = round_up(nodedata_phys + pgdat_size, PAGE_SIZE);
+                bootmap_start = roundup(nodedata_phys + pgdat_size, PAGE_SIZE);
         else
-                bootmap_start = round_up(start, PAGE_SIZE);
+                bootmap_start = roundup(start, PAGE_SIZE);
         /*
          * SMP_CACHE_BYTES could be enough, but init_bootmem_node like
          * to use that to align to PAGE_SIZE
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index f5f5154ea11..898fad617ab 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -84,7 +84,7 @@ static inline unsigned long highmap_start_pfn(void)
 
 static inline unsigned long highmap_end_pfn(void)
 {
-        return __pa(round_up((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT;
+        return __pa(roundup((unsigned long)_end, PMD_SIZE)) >> PAGE_SHIFT;
 }
 
 #endif
@@ -849,7 +849,7 @@ int set_memory_uc(unsigned long addr, int numpages)
         /*
          * for now UC MINUS. see comments in ioremap_nocache()
          */
-        if (reserve_memtype(addr, addr + numpages * PAGE_SIZE,
+        if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
                             _PAGE_CACHE_UC_MINUS, NULL))
                 return -EINVAL;
 
@@ -868,7 +868,7 @@ int set_memory_wc(unsigned long addr, int numpages)
         if (!pat_enabled)
                 return set_memory_uc(addr, numpages);
 
-        if (reserve_memtype(addr, addr + numpages * PAGE_SIZE,
+        if (reserve_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE,
                 _PAGE_CACHE_WC, NULL))
                 return -EINVAL;
 
@@ -884,7 +884,7 @@ int _set_memory_wb(unsigned long addr, int numpages)
 
 int set_memory_wb(unsigned long addr, int numpages)
 {
-        free_memtype(addr, addr + numpages * PAGE_SIZE);
+        free_memtype(__pa(addr), __pa(addr) + numpages * PAGE_SIZE);
 
         return _set_memory_wb(addr, numpages);
 }
@@ -906,11 +906,13 @@ int set_memory_ro(unsigned long addr, int numpages)
 {
         return change_page_attr_clear(addr, numpages, __pgprot(_PAGE_RW));
 }
+EXPORT_SYMBOL_GPL(set_memory_ro);
 
 int set_memory_rw(unsigned long addr, int numpages)
 {
         return change_page_attr_set(addr, numpages, __pgprot(_PAGE_RW));
 }
+EXPORT_SYMBOL_GPL(set_memory_rw);
 
 int set_memory_np(unsigned long addr, int numpages)
 {
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index 2fe30916d4b..2a50e0fa64a 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -207,6 +207,9 @@ static int chk_conflict(struct memtype *new, struct memtype *entry,
         return -EBUSY;
 }
 
+static struct memtype *cached_entry;
+static u64 cached_start;
+
 /*
  * req_type typically has one of the:
  * - _PAGE_CACHE_WB
@@ -280,11 +283,17 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
 
         spin_lock(&memtype_lock);
 
+        if (cached_entry && start >= cached_start)
+                entry = cached_entry;
+        else
+                entry = list_entry(&memtype_list, struct memtype, nd);
+
         /* Search for existing mapping that overlaps the current range */
         where = NULL;
-        list_for_each_entry(entry, &memtype_list, nd) {
+        list_for_each_entry_continue(entry, &memtype_list, nd) {
                 if (end <= entry->start) {
                         where = entry->nd.prev;
+                        cached_entry = list_entry(where, struct memtype, nd);
                         break;
                 } else if (start <= entry->start) { /* end > entry->start */
                         err = chk_conflict(new, entry, new_type);
@@ -292,6 +301,8 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                                 dprintk("Overlap at 0x%Lx-0x%Lx\n",
                                         entry->start, entry->end);
                                 where = entry->nd.prev;
+                                cached_entry = list_entry(where,
+                                                        struct memtype, nd);
                         }
                         break;
                 } else if (start < entry->end) { /* start > entry->start */
@@ -299,7 +310,20 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                         if (!err) {
                                 dprintk("Overlap at 0x%Lx-0x%Lx\n",
                                         entry->start, entry->end);
-                                where = &entry->nd;
+                                cached_entry = list_entry(entry->nd.prev,
+                                                        struct memtype, nd);
+
+                                /*
+                                 * Move to right position in the linked
+                                 * list to add this new entry
+                                 */
+                                list_for_each_entry_continue(entry,
+                                                        &memtype_list, nd) {
+                                        if (start <= entry->start) {
+                                                where = entry->nd.prev;
+                                                break;
+                                        }
+                                }
                         }
                         break;
                 }
@@ -314,6 +338,8 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                 return err;
         }
 
+        cached_start = start;
+
         if (where)
                 list_add(&new->nd, where);
         else
@@ -343,6 +369,9 @@ int free_memtype(u64 start, u64 end)
         spin_lock(&memtype_lock);
         list_for_each_entry(entry, &memtype_list, nd) {
                 if (entry->start == start && entry->end == end) {
+                        if (cached_entry == entry || cached_start == start)
+                                cached_entry = NULL;
+
                         list_del(&entry->nd);
                         kfree(entry);
                         err = 0;
@@ -361,14 +390,6 @@ int free_memtype(u64 start, u64 end)
 }
 
 
-/*
- * /dev/mem mmap interface. The memtype used for mapping varies:
- * - Use UC for mappings with O_SYNC flag
- * - Without O_SYNC flag, if there is any conflict in reserve_memtype,
- *   inherit the memtype from existing mapping.
- * - Else use UC_MINUS memtype (for backward compatibility with existing
- *   X drivers.
- */
 pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                                 unsigned long size, pgprot_t vma_prot)
 {
@@ -406,14 +427,14 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
                                 unsigned long size, pgprot_t *vma_prot)
 {
         u64 offset = ((u64) pfn) << PAGE_SHIFT;
-        unsigned long flags = _PAGE_CACHE_UC_MINUS;
+        unsigned long flags = -1;
         int retval;
 
         if (!range_is_allowed(pfn, size))
                 return 0;
 
         if (file->f_flags & O_SYNC) {
-                flags = _PAGE_CACHE_UC;
+                flags = _PAGE_CACHE_UC_MINUS;
         }
 
 #ifdef CONFIG_X86_32
@@ -436,13 +457,14 @@ int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
 #endif
 
         /*
-         * With O_SYNC, we can only take UC mapping. Fail if we cannot.
+         * With O_SYNC, we can only take UC_MINUS mapping. Fail if we cannot.
+         *
          * Without O_SYNC, we want to get
          * - WB for WB-able memory and no other conflicting mappings
          * - UC_MINUS for non-WB-able memory with no other conflicting mappings
          * - Inherit from confliting mappings otherwise
          */
-        if (flags != _PAGE_CACHE_UC_MINUS) {
+        if (flags != -1) {
                 retval = reserve_memtype(offset, offset + size, flags, NULL);
         } else {
                 retval = reserve_memtype(offset, offset + size, -1, &flags);
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index d50302774fe..86f2ffc43c3 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -63,10 +63,8 @@ static inline void pgd_list_del(pgd_t *pgd)
 #define UNSHARED_PTRS_PER_PGD                           \
         (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
 
-static void pgd_ctor(void *p)
+static void pgd_ctor(pgd_t *pgd)
 {
-        pgd_t *pgd = p;
-
         /* If the pgd points to a shared pagetable level (either the
            ptes in non-PAE, or shared PMD in PAE), then just copy the
            references from swapper_pg_dir. */
@@ -87,7 +85,7 @@ static void pgd_ctor(void *p)
                 pgd_list_add(pgd);
 }
 
-static void pgd_dtor(void *pgd)
+static void pgd_dtor(pgd_t *pgd)
 {
         unsigned long flags; /* can be called from interrupt context */