1 files changed, 61 insertions, 36 deletions

diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h
index c3b58d473a55..e16359f81a40 100644
--- a/include/asm-i386/pgtable.h
+++ b/include/asm-i386/pgtable.h
@@ -159,6 +159,7 @@ void paging_init(void);
 
 extern unsigned long long __PAGE_KERNEL, __PAGE_KERNEL_EXEC;
 #define __PAGE_KERNEL_RO                (__PAGE_KERNEL & ~_PAGE_RW)
+#define __PAGE_KERNEL_RX                (__PAGE_KERNEL_EXEC & ~_PAGE_RW)
 #define __PAGE_KERNEL_NOCACHE           (__PAGE_KERNEL | _PAGE_PCD)
 #define __PAGE_KERNEL_LARGE             (__PAGE_KERNEL | _PAGE_PSE)
 #define __PAGE_KERNEL_LARGE_EXEC        (__PAGE_KERNEL_EXEC | _PAGE_PSE)
@@ -166,6 +167,7 @@ extern unsigned long long __PAGE_KERNEL, __PAGE_KERNEL_EXEC;
 #define PAGE_KERNEL             __pgprot(__PAGE_KERNEL)
 #define PAGE_KERNEL_RO          __pgprot(__PAGE_KERNEL_RO)
 #define PAGE_KERNEL_EXEC        __pgprot(__PAGE_KERNEL_EXEC)
+#define PAGE_KERNEL_RX          __pgprot(__PAGE_KERNEL_RX)
 #define PAGE_KERNEL_NOCACHE     __pgprot(__PAGE_KERNEL_NOCACHE)
 #define PAGE_KERNEL_LARGE       __pgprot(__PAGE_KERNEL_LARGE)
 #define PAGE_KERNEL_LARGE_EXEC  __pgprot(__PAGE_KERNEL_LARGE_EXEC)
@@ -241,6 +243,8 @@ static inline pte_t pte_mkyoung(pte_t pte)	{ (pte).pte_low |= _PAGE_ACCESSED; re
 static inline pte_t pte_mkwrite(pte_t pte)      { (pte).pte_low |= _PAGE_RW; return pte; }
 static inline pte_t pte_mkhuge(pte_t pte)       { (pte).pte_low |= _PAGE_PSE; return pte; }
 
+extern void vmalloc_sync_all(void);
+
 #ifdef CONFIG_X86_PAE
 # include <asm/pgtable-3level.h>
 #else
@@ -263,9 +267,18 @@ static inline pte_t pte_mkhuge(pte_t pte)	{ (pte).pte_low |= _PAGE_PSE; return p
  */
 #define pte_update(mm, addr, ptep)              do { } while (0)
 #define pte_update_defer(mm, addr, ptep)        do { } while (0)
-#define paravirt_map_pt_hook(slot, va, pfn)     do { } while (0)
 #endif
 
+/* local pte updates need not use xchg for locking */
+static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep)
+{
+        pte_t res = *ptep;
+
+        /* Pure native function needs no input for mm, addr */
+        native_pte_clear(NULL, 0, ptep);
+        return res;
+}
+
 /*
  * We only update the dirty/accessed state if we set
  * the dirty bit by hand in the kernel, since the hardware
@@ -283,12 +296,25 @@ do {									\
         }                                                               \
 } while (0)
 
-/*
- * We don't actually have these, but we want to advertise them so that
- * we can encompass the flush here.
- */
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_DIRTY
+#define ptep_test_and_clear_dirty(vma, addr, ptep) ({                   \
+        int ret = 0;                                                    \
+        if (pte_dirty(*ptep))                                           \
+                ret = test_and_clear_bit(_PAGE_BIT_DIRTY, &ptep->pte_low); \
+        if (ret)                                                        \
+                pte_update_defer(vma->vm_mm, addr, ptep);               \
+        ret;                                                            \
+})
+
 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+#define ptep_test_and_clear_young(vma, addr, ptep) ({                   \
+        int ret = 0;                                                    \
+        if (pte_young(*ptep))                                           \
+                ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, &ptep->pte_low); \
+        if (ret)                                                        \
+                pte_update_defer(vma->vm_mm, addr, ptep);               \
+        ret;                                                            \
+})
 
 /*
  * Rules for using ptep_establish: the pte MUST be a user pte, and
@@ -305,12 +331,9 @@ do {									\
 #define ptep_clear_flush_dirty(vma, address, ptep)                      \
 ({                                                                      \
         int __dirty;                                                    \
-        __dirty = pte_dirty(*(ptep));                                   \
-        if (__dirty) {                                                  \
-                clear_bit(_PAGE_BIT_DIRTY, &(ptep)->pte_low);           \
-                pte_update_defer((vma)->vm_mm, (address), (ptep));      \
+        __dirty = ptep_test_and_clear_dirty((vma), (address), (ptep));  \
+        if (__dirty)                                                    \
                 flush_tlb_page(vma, address);                           \
-        }                                                               \
         __dirty;                                                        \
 })
 
@@ -318,19 +341,16 @@ do {									\
 #define ptep_clear_flush_young(vma, address, ptep)                      \
 ({                                                                      \
         int __young;                                                    \
-        __young = pte_young(*(ptep));                                   \
-        if (__young) {                                                  \
-                clear_bit(_PAGE_BIT_ACCESSED, &(ptep)->pte_low);        \
-                pte_update_defer((vma)->vm_mm, (address), (ptep));      \
+        __young = ptep_test_and_clear_young((vma), (address), (ptep));  \
+        if (__young)                                                    \
                 flush_tlb_page(vma, address);                           \
-        }                                                               \
         __young;                                                        \
 })
 
 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR
 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
 {
-        pte_t pte = raw_ptep_get_and_clear(ptep);
+        pte_t pte = native_ptep_get_and_clear(ptep);
         pte_update(mm, addr, ptep);
         return pte;
 }
@@ -340,8 +360,11 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long
 {
         pte_t pte;
         if (full) {
-                pte = *ptep;
-                pte_clear(mm, addr, ptep);
+                /*
+                 * Full address destruction in progress; paravirt does not
+                 * care about updates and native needs no locking
+                 */
+                pte = native_local_ptep_get_and_clear(ptep);
         } else {
                 pte = ptep_get_and_clear(mm, addr, ptep);
         }
@@ -470,24 +493,10 @@ extern pte_t *lookup_address(unsigned long address);
 #endif
 
 #if defined(CONFIG_HIGHPTE)
-#define pte_offset_map(dir, address)                            \
-({                                                              \
-        pte_t *__ptep;                                          \
-        unsigned pfn = pmd_val(*(dir)) >> PAGE_SHIFT;           \
-        __ptep = (pte_t *)kmap_atomic(pfn_to_page(pfn),KM_PTE0);\
-        paravirt_map_pt_hook(KM_PTE0,__ptep, pfn);              \
-        __ptep = __ptep + pte_index(address);                   \
-        __ptep;                                                 \
-})
-#define pte_offset_map_nested(dir, address)                     \
-({                                                              \
-        pte_t *__ptep;                                          \
-        unsigned pfn = pmd_val(*(dir)) >> PAGE_SHIFT;           \
-        __ptep = (pte_t *)kmap_atomic(pfn_to_page(pfn),KM_PTE1);\
-        paravirt_map_pt_hook(KM_PTE1,__ptep, pfn);              \
-        __ptep = __ptep + pte_index(address);                   \
-        __ptep;                                                 \
-})
+#define pte_offset_map(dir, address) \
+        ((pte_t *)kmap_atomic_pte(pmd_page(*(dir)),KM_PTE0) + pte_index(address))
+#define pte_offset_map_nested(dir, address) \
+        ((pte_t *)kmap_atomic_pte(pmd_page(*(dir)),KM_PTE1) + pte_index(address))
 #define pte_unmap(pte) kunmap_atomic(pte, KM_PTE0)
 #define pte_unmap_nested(pte) kunmap_atomic(pte, KM_PTE1)
 #else
@@ -510,6 +519,22 @@ do {									\
  * tables contain all the necessary information.
  */
 #define update_mmu_cache(vma,address,pte) do { } while (0)
+
+void native_pagetable_setup_start(pgd_t *base);
+void native_pagetable_setup_done(pgd_t *base);
+
+#ifndef CONFIG_PARAVIRT
+static inline void paravirt_pagetable_setup_start(pgd_t *base)
+{
+        native_pagetable_setup_start(base);
+}
+
+static inline void paravirt_pagetable_setup_done(pgd_t *base)
+{
+        native_pagetable_setup_done(base);
+}
+#endif  /* !CONFIG_PARAVIRT */
+
 #endif /* !__ASSEMBLY__ */
 
 #ifdef CONFIG_FLATMEM