1 files changed, 227 insertions, 89 deletions

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index ff0aa74afaa..a44d56e38bd 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -44,8 +44,10 @@
 
 #include <asm/pgtable.h>
 #include <asm/tlbflush.h>
+#include <asm/fixmap.h>
 #include <asm/mmu_context.h>
 #include <asm/paravirt.h>
+#include <asm/linkage.h>
 
 #include <asm/xen/hypercall.h>
 #include <asm/xen/hypervisor.h>
@@ -56,26 +58,29 @@
 #include "multicalls.h"
 #include "mmu.h"
 
+/*
+ * Just beyond the highest usermode address.  STACK_TOP_MAX has a
+ * redzone above it, so round it up to a PGD boundary.
+ */
+#define USER_LIMIT      ((STACK_TOP_MAX + PGDIR_SIZE - 1) & PGDIR_MASK)
+
+
 #define P2M_ENTRIES_PER_PAGE    (PAGE_SIZE / sizeof(unsigned long))
 #define TOP_ENTRIES             (MAX_DOMAIN_PAGES / P2M_ENTRIES_PER_PAGE)
 
 /* Placeholder for holes in the address space */
-static unsigned long p2m_missing[P2M_ENTRIES_PER_PAGE]
-        __attribute__((section(".data.page_aligned"))) =
+static unsigned long p2m_missing[P2M_ENTRIES_PER_PAGE] __page_aligned_data =
                 { [ 0 ... P2M_ENTRIES_PER_PAGE-1 ] = ~0UL };
 
  /* Array of pointers to pages containing p2m entries */
-static unsigned long *p2m_top[TOP_ENTRIES]
-        __attribute__((section(".data.page_aligned"))) =
+static unsigned long *p2m_top[TOP_ENTRIES] __page_aligned_data =
                 { [ 0 ... TOP_ENTRIES - 1] = &p2m_missing[0] };
 
 /* Arrays of p2m arrays expressed in mfns used for save/restore */
-static unsigned long p2m_top_mfn[TOP_ENTRIES]
-        __attribute__((section(".bss.page_aligned")));
+static unsigned long p2m_top_mfn[TOP_ENTRIES] __page_aligned_bss;
 
-static unsigned long p2m_top_mfn_list[
-                        PAGE_ALIGN(TOP_ENTRIES / P2M_ENTRIES_PER_PAGE)]
-        __attribute__((section(".bss.page_aligned")));
+static unsigned long p2m_top_mfn_list[TOP_ENTRIES / P2M_ENTRIES_PER_PAGE]
+        __page_aligned_bss;
 
 static inline unsigned p2m_top_index(unsigned long pfn)
 {
@@ -181,15 +186,16 @@ void set_phys_to_machine(unsigned long pfn, unsigned long mfn)
         p2m_top[topidx][idx] = mfn;
 }
 
-xmaddr_t arbitrary_virt_to_machine(unsigned long address)
+xmaddr_t arbitrary_virt_to_machine(void *vaddr)
 {
+        unsigned long address = (unsigned long)vaddr;
         unsigned int level;
         pte_t *pte = lookup_address(address, &level);
         unsigned offset = address & ~PAGE_MASK;
 
         BUG_ON(pte == NULL);
 
-        return XMADDR((pte_mfn(*pte) << PAGE_SHIFT) + offset);
+        return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset);
 }
 
 void make_lowmem_page_readonly(void *vaddr)
@@ -256,7 +262,8 @@ void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
 
         xen_mc_batch();
 
-        u.ptr = virt_to_machine(ptr).maddr;
+        /* ptr may be ioremapped for 64-bit pagetable setup */
+        u.ptr = arbitrary_virt_to_machine(ptr).maddr;
         u.val = pmd_val_ma(val);
         extend_mmu_update(&u);
 
@@ -283,35 +290,7 @@ void xen_set_pmd(pmd_t *ptr, pmd_t val)
  */
 void set_pte_mfn(unsigned long vaddr, unsigned long mfn, pgprot_t flags)
 {
-        pgd_t *pgd;
-        pud_t *pud;
-        pmd_t *pmd;
-        pte_t *pte;
-
-        pgd = swapper_pg_dir + pgd_index(vaddr);
-        if (pgd_none(*pgd)) {
-                BUG();
-                return;
-        }
-        pud = pud_offset(pgd, vaddr);
-        if (pud_none(*pud)) {
-                BUG();
-                return;
-        }
-        pmd = pmd_offset(pud, vaddr);
-        if (pmd_none(*pmd)) {
-                BUG();
-                return;
-        }
-        pte = pte_offset_kernel(pmd, vaddr);
-        /* <mfn,flags> stored as-is, to permit clearing entries */
-        xen_set_pte(pte, mfn_pte(mfn, flags));
-
-        /*
-         * It's enough to flush this one mapping.
-         * (PGE mappings get flushed as well)
-         */
-        __flush_tlb_one(vaddr);
+        set_pte_vaddr(vaddr, mfn_pte(mfn, flags));
 }
 
 void xen_set_pte_at(struct mm_struct *mm, unsigned long addr,
@@ -418,7 +397,8 @@ void xen_set_pud_hyper(pud_t *ptr, pud_t val)
 
         xen_mc_batch();
 
-        u.ptr = virt_to_machine(ptr).maddr;
+        /* ptr may be ioremapped for 64-bit pagetable setup */
+        u.ptr = arbitrary_virt_to_machine(ptr).maddr;
         u.val = pud_val_ma(val);
         extend_mmu_update(&u);
 
@@ -441,14 +421,19 @@ void xen_set_pud(pud_t *ptr, pud_t val)
 
 void xen_set_pte(pte_t *ptep, pte_t pte)
 {
+#ifdef CONFIG_X86_PAE
         ptep->pte_high = pte.pte_high;
         smp_wmb();
         ptep->pte_low = pte.pte_low;
+#else
+        *ptep = pte;
+#endif
 }
 
+#ifdef CONFIG_X86_PAE
 void xen_set_pte_atomic(pte_t *ptep, pte_t pte)
 {
-        set_64bit((u64 *)ptep, pte_val_ma(pte));
+        set_64bit((u64 *)ptep, native_pte_val(pte));
 }
 
 void xen_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
@@ -462,6 +447,7 @@ void xen_pmd_clear(pmd_t *pmdp)
 {
         set_pmd(pmdp, __pmd(0));
 }
+#endif  /* CONFIG_X86_PAE */
 
 pmd_t xen_make_pmd(pmdval_t pmd)
 {
@@ -469,78 +455,189 @@ pmd_t xen_make_pmd(pmdval_t pmd)
         return native_make_pmd(pmd);
 }
 
+#if PAGETABLE_LEVELS == 4
+pudval_t xen_pud_val(pud_t pud)
+{
+        return pte_mfn_to_pfn(pud.pud);
+}
+
+pud_t xen_make_pud(pudval_t pud)
+{
+        pud = pte_pfn_to_mfn(pud);
+
+        return native_make_pud(pud);
+}
+
+pgd_t *xen_get_user_pgd(pgd_t *pgd)
+{
+        pgd_t *pgd_page = (pgd_t *)(((unsigned long)pgd) & PAGE_MASK);
+        unsigned offset = pgd - pgd_page;
+        pgd_t *user_ptr = NULL;
+
+        if (offset < pgd_index(USER_LIMIT)) {
+                struct page *page = virt_to_page(pgd_page);
+                user_ptr = (pgd_t *)page->private;
+                if (user_ptr)
+                        user_ptr += offset;
+        }
+
+        return user_ptr;
+}
+
+static void __xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
+{
+        struct mmu_update u;
+
+        u.ptr = virt_to_machine(ptr).maddr;
+        u.val = pgd_val_ma(val);
+        extend_mmu_update(&u);
+}
+
+/*
+ * Raw hypercall-based set_pgd, intended for in early boot before
+ * there's a page structure.  This implies:
+ *  1. The only existing pagetable is the kernel's
+ *  2. It is always pinned
+ *  3. It has no user pagetable attached to it
+ */
+void __init xen_set_pgd_hyper(pgd_t *ptr, pgd_t val)
+{
+        preempt_disable();
+
+        xen_mc_batch();
+
+        __xen_set_pgd_hyper(ptr, val);
+
+        xen_mc_issue(PARAVIRT_LAZY_MMU);
+
+        preempt_enable();
+}
+
+void xen_set_pgd(pgd_t *ptr, pgd_t val)
+{
+        pgd_t *user_ptr = xen_get_user_pgd(ptr);
+
+        /* If page is not pinned, we can just update the entry
+           directly */
+        if (!page_pinned(ptr)) {
+                *ptr = val;
+                if (user_ptr) {
+                        WARN_ON(page_pinned(user_ptr));
+                        *user_ptr = val;
+                }
+                return;
+        }
+
+        /* If it's pinned, then we can at least batch the kernel and
+           user updates together. */
+        xen_mc_batch();
+
+        __xen_set_pgd_hyper(ptr, val);
+        if (user_ptr)
+                __xen_set_pgd_hyper(user_ptr, val);
+
+        xen_mc_issue(PARAVIRT_LAZY_MMU);
+}
+#endif  /* PAGETABLE_LEVELS == 4 */
+
 /*
-  (Yet another) pagetable walker.  This one is intended for pinning a
-  pagetable.  This means that it walks a pagetable and calls the
-  callback function on each page it finds making up the page table,
-  at every level.  It walks the entire pagetable, but it only bothers
-  pinning pte pages which are below pte_limit.  In the normal case
-  this will be TASK_SIZE, but at boot we need to pin up to
-  FIXADDR_TOP.  But the important bit is that we don't pin beyond
-  there, because then we start getting into Xen's ptes.
-*/
-static int pgd_walk(pgd_t *pgd_base, int (*func)(struct page *, enum pt_level),
+ * (Yet another) pagetable walker.  This one is intended for pinning a
+ * pagetable.  This means that it walks a pagetable and calls the
+ * callback function on each page it finds making up the page table,
+ * at every level.  It walks the entire pagetable, but it only bothers
+ * pinning pte pages which are below limit.  In the normal case this
+ * will be STACK_TOP_MAX, but at boot we need to pin up to
+ * FIXADDR_TOP.
+ *
+ * For 32-bit the important bit is that we don't pin beyond there,
+ * because then we start getting into Xen's ptes.
+ *
+ * For 64-bit, we must skip the Xen hole in the middle of the address
+ * space, just after the big x86-64 virtual hole.
+ */
+static int pgd_walk(pgd_t *pgd, int (*func)(struct page *, enum pt_level),
                     unsigned long limit)
 {
-        pgd_t *pgd = pgd_base;
         int flush = 0;
-        unsigned long addr = 0;
-        unsigned long pgd_next;
+        unsigned hole_low, hole_high;
+        unsigned pgdidx_limit, pudidx_limit, pmdidx_limit;
+        unsigned pgdidx, pudidx, pmdidx;
 
-        BUG_ON(limit > FIXADDR_TOP);
+        /* The limit is the last byte to be touched */
+        limit--;
+        BUG_ON(limit >= FIXADDR_TOP);
 
         if (xen_feature(XENFEAT_auto_translated_physmap))
                 return 0;
 
-        for (; addr != FIXADDR_TOP; pgd++, addr = pgd_next) {
+        /*
+         * 64-bit has a great big hole in the middle of the address
+         * space, which contains the Xen mappings.  On 32-bit these
+         * will end up making a zero-sized hole and so is a no-op.
+         */
+        hole_low = pgd_index(USER_LIMIT);
+        hole_high = pgd_index(PAGE_OFFSET);
+
+        pgdidx_limit = pgd_index(limit);
+#if PTRS_PER_PUD > 1
+        pudidx_limit = pud_index(limit);
+#else
+        pudidx_limit = 0;
+#endif
+#if PTRS_PER_PMD > 1
+        pmdidx_limit = pmd_index(limit);
+#else
+        pmdidx_limit = 0;
+#endif
+
+        flush |= (*func)(virt_to_page(pgd), PT_PGD);
+
+        for (pgdidx = 0; pgdidx <= pgdidx_limit; pgdidx++) {
                 pud_t *pud;
-                unsigned long pud_limit, pud_next;
 
-                pgd_next = pud_limit = pgd_addr_end(addr, FIXADDR_TOP);
+                if (pgdidx >= hole_low && pgdidx < hole_high)
+                        continue;
 
-                if (!pgd_val(*pgd))
+                if (!pgd_val(pgd[pgdidx]))
                         continue;
 
-                pud = pud_offset(pgd, 0);
+                pud = pud_offset(&pgd[pgdidx], 0);
 
                 if (PTRS_PER_PUD > 1) /* not folded */
                         flush |= (*func)(virt_to_page(pud), PT_PUD);
 
-                for (; addr != pud_limit; pud++, addr = pud_next) {
+                for (pudidx = 0; pudidx < PTRS_PER_PUD; pudidx++) {
                         pmd_t *pmd;
-                        unsigned long pmd_limit;
 
-                        pud_next = pud_addr_end(addr, pud_limit);
-
-                        if (pud_next < limit)
-                                pmd_limit = pud_next;
-                        else
-                                pmd_limit = limit;
+                        if (pgdidx == pgdidx_limit &&
+                            pudidx > pudidx_limit)
+                                goto out;
 
-                        if (pud_none(*pud))
+                        if (pud_none(pud[pudidx]))
                                 continue;
 
-                        pmd = pmd_offset(pud, 0);
+                        pmd = pmd_offset(&pud[pudidx], 0);
 
                         if (PTRS_PER_PMD > 1) /* not folded */
                                 flush |= (*func)(virt_to_page(pmd), PT_PMD);
 
-                        for (; addr != pmd_limit; pmd++) {
-                                addr += (PAGE_SIZE * PTRS_PER_PTE);
-                                if ((pmd_limit-1) < (addr-1)) {
-                                        addr = pmd_limit;
-                                        break;
-                                }
+                        for (pmdidx = 0; pmdidx < PTRS_PER_PMD; pmdidx++) {
+                                struct page *pte;
+
+                                if (pgdidx == pgdidx_limit &&
+                                    pudidx == pudidx_limit &&
+                                    pmdidx > pmdidx_limit)
+                                        goto out;
 
-                                if (pmd_none(*pmd))
+                                if (pmd_none(pmd[pmdidx]))
                                         continue;
 
-                                flush |= (*func)(pmd_page(*pmd), PT_PTE);
+                                pte = pmd_page(pmd[pmdidx]);
+                                flush |= (*func)(pte, PT_PTE);
                         }
                 }
         }
-
-        flush |= (*func)(virt_to_page(pgd_base), PT_PGD);
+out:
 
         return flush;
 }
@@ -622,14 +719,31 @@ void xen_pgd_pin(pgd_t *pgd)
 {
         xen_mc_batch();
 
-        if (pgd_walk(pgd, pin_page, TASK_SIZE)) {
+        if (pgd_walk(pgd, pin_page, USER_LIMIT)) {
                 /* re-enable interrupts for kmap_flush_unused */
                 xen_mc_issue(0);
                 kmap_flush_unused();
                 xen_mc_batch();
         }
 
+#ifdef CONFIG_X86_64
+        {
+                pgd_t *user_pgd = xen_get_user_pgd(pgd);
+
+                xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(pgd)));
+
+                if (user_pgd) {
+                        pin_page(virt_to_page(user_pgd), PT_PGD);
+                        xen_do_pin(MMUEXT_PIN_L4_TABLE, PFN_DOWN(__pa(user_pgd)));
+                }
+        }
+#else /* CONFIG_X86_32 */
+#ifdef CONFIG_X86_PAE
+        /* Need to make sure unshared kernel PMD is pinnable */
+        pin_page(virt_to_page(pgd_page(pgd[pgd_index(TASK_SIZE)])), PT_PMD);
+#endif
         xen_do_pin(MMUEXT_PIN_L3_TABLE, PFN_DOWN(__pa(pgd)));
+#endif /* CONFIG_X86_64 */
         xen_mc_issue(0);
 }
 
@@ -656,9 +770,11 @@ void xen_mm_pin_all(void)
         spin_unlock_irqrestore(&pgd_lock, flags);
 }
 
-/* The init_mm pagetable is really pinned as soon as its created, but
-   that's before we have page structures to store the bits.  So do all
-   the book-keeping now. */
+/*
+ * The init_mm pagetable is really pinned as soon as its created, but
+ * that's before we have page structures to store the bits.  So do all
+ * the book-keeping now.
+ */
 static __init int mark_pinned(struct page *page, enum pt_level level)
 {
         SetPagePinned(page);
@@ -708,7 +824,23 @@ static void xen_pgd_unpin(pgd_t *pgd)
 
         xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));
 
-        pgd_walk(pgd, unpin_page, TASK_SIZE);
+#ifdef CONFIG_X86_64
+        {
+                pgd_t *user_pgd = xen_get_user_pgd(pgd);
+
+                if (user_pgd) {
+                        xen_do_pin(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(user_pgd)));
+                        unpin_page(virt_to_page(user_pgd), PT_PGD);
+                }
+        }
+#endif
+
+#ifdef CONFIG_X86_PAE
+        /* Need to make sure unshared kernel PMD is unpinned */
+        pin_page(virt_to_page(pgd_page(pgd[pgd_index(TASK_SIZE)])), PT_PMD);
+#endif
+
+        pgd_walk(pgd, unpin_page, USER_LIMIT);
 
         xen_mc_issue(0);
 }
@@ -727,7 +859,6 @@ void xen_mm_unpin_all(void)
         list_for_each_entry(page, &pgd_list, lru) {
                 if (PageSavePinned(page)) {
                         BUG_ON(!PagePinned(page));
-                        printk("unpinning pinned %p\n", page_address(page));
                         xen_pgd_unpin((pgd_t *)page_address(page));
                         ClearPageSavePinned(page);
                 }
@@ -757,8 +888,15 @@ void xen_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm)
 static void drop_other_mm_ref(void *info)
 {
         struct mm_struct *mm = info;
+        struct mm_struct *active_mm;
+
+#ifdef CONFIG_X86_64
+        active_mm = read_pda(active_mm);
+#else
+        active_mm = __get_cpu_var(cpu_tlbstate).active_mm;
+#endif
 
-        if (__get_cpu_var(cpu_tlbstate).active_mm == mm)
+        if (active_mm == mm)
                 leave_mm(smp_processor_id());
 
         /* If this cpu still has a stale cr3 reference, then make sure