17 files changed, 88 insertions, 62 deletions

diff --git a/arch/frv/kernel/futex.c b/arch/frv/kernel/futex.c
index eae874a970c6..53dc5ed1ebda 100644
--- a/arch/frv/kernel/futex.c
+++ b/arch/frv/kernel/futex.c
@@ -200,7 +200,7 @@ int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
         if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -223,7 +223,7 @@ int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
                 break;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/arch/i386/mm/highmem.c b/arch/i386/mm/highmem.c
index f9f647cdbc7b..178bbfe6cbac 100644
--- a/arch/i386/mm/highmem.c
+++ b/arch/i386/mm/highmem.c
@@ -32,7 +32,7 @@ void *kmap_atomic(struct page *page, enum km_type type)
         unsigned long vaddr;
 
         /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-        inc_preempt_count();
+        pagefault_disable();
         if (!PageHighMem(page))
                 return page_address(page);
 
@@ -52,8 +52,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
 
 #ifdef CONFIG_DEBUG_HIGHMEM
         if (vaddr >= PAGE_OFFSET && vaddr < (unsigned long)high_memory) {
-                dec_preempt_count();
-                preempt_check_resched();
+                pagefault_enable();
                 return;
         }
 
@@ -68,8 +67,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
          */
         kpte_clear_flush(kmap_pte-idx, vaddr);
 
-        dec_preempt_count();
-        preempt_check_resched();
+        pagefault_enable();
 }
 
 /* This is the same as kmap_atomic() but can map memory that doesn't
@@ -80,7 +78,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
         enum fixed_addresses idx;
         unsigned long vaddr;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         idx = type + KM_TYPE_NR*smp_processor_id();
         vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
diff --git a/arch/mips/mm/highmem.c b/arch/mips/mm/highmem.c
index 99ebf3ccc222..675502ada5a2 100644
--- a/arch/mips/mm/highmem.c
+++ b/arch/mips/mm/highmem.c
@@ -39,7 +39,7 @@ void *__kmap_atomic(struct page *page, enum km_type type)
         unsigned long vaddr;
 
         /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-        inc_preempt_count();
+        pagefault_disable();
         if (!PageHighMem(page))
                 return page_address(page);
 
@@ -62,8 +62,7 @@ void __kunmap_atomic(void *kvaddr, enum km_type type)
         enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id();
 
         if (vaddr < FIXADDR_START) { // FIXME
-                dec_preempt_count();
-                preempt_check_resched();
+                pagefault_enable();
                 return;
         }
 
@@ -78,8 +77,7 @@ void __kunmap_atomic(void *kvaddr, enum km_type type)
         local_flush_tlb_one(vaddr);
 #endif
 
-        dec_preempt_count();
-        preempt_check_resched();
+        pagefault_enable();
 }
 
 #ifndef CONFIG_LIMITED_DMA
@@ -92,7 +90,7 @@ void *kmap_atomic_pfn(unsigned long pfn, enum km_type type)
         enum fixed_addresses idx;
         unsigned long vaddr;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         idx = type + KM_TYPE_NR*smp_processor_id();
         vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
diff --git a/arch/s390/lib/uaccess_std.c b/arch/s390/lib/uaccess_std.c
index 2d549ed2e113..bbaca66fa293 100644
--- a/arch/s390/lib/uaccess_std.c
+++ b/arch/s390/lib/uaccess_std.c
@@ -11,7 +11,7 @@
 
 #include <linux/errno.h>
 #include <linux/mm.h>
-#include <asm/uaccess.h>
+#include <linux/uaccess.h>
 #include <asm/futex.h>
 
 #ifndef __s390x__
@@ -258,7 +258,7 @@ int futex_atomic_op(int op, int __user *uaddr, int oparg, int *old)
 {
         int oldval = 0, newval, ret;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -284,7 +284,7 @@ int futex_atomic_op(int op, int __user *uaddr, int oparg, int *old)
         default:
                 ret = -ENOSYS;
         }
-        dec_preempt_count();
+        pagefault_enable();
         *old = oldval;
         return ret;
 }
diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c
index 4d8ed9c65182..01fc6c254292 100644
--- a/arch/sparc/mm/highmem.c
+++ b/arch/sparc/mm/highmem.c
@@ -35,7 +35,7 @@ void *kmap_atomic(struct page *page, enum km_type type)
         unsigned long vaddr;
 
         /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-        inc_preempt_count();
+        pagefault_disable();
         if (!PageHighMem(page))
                 return page_address(page);
 
@@ -70,8 +70,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
         unsigned long idx = type + KM_TYPE_NR*smp_processor_id();
 
         if (vaddr < FIXADDR_START) { // FIXME
-                dec_preempt_count();
-                preempt_check_resched();
+                pagefault_enable();
                 return;
         }
 
@@ -97,8 +96,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type)
 #endif
 #endif
 
-        dec_preempt_count();
-        preempt_check_resched();
+        pagefault_enable();
 }
 
 /* We may be fed a pagetable here by ptep_to_xxx and others. */
diff --git a/include/asm-frv/highmem.h b/include/asm-frv/highmem.h
index 0f390f41f816..ff4d6cdeb152 100644
--- a/include/asm-frv/highmem.h
+++ b/include/asm-frv/highmem.h
@@ -115,7 +115,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type)
 {
         unsigned long paddr;
 
-        inc_preempt_count();
+        pagefault_disable();
         paddr = page_to_phys(page);
 
         switch (type) {
@@ -170,8 +170,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type)
         default:
                 BUG();
         }
-        dec_preempt_count();
-        preempt_check_resched();
+        pagefault_enable();
 }
 
 #endif /* !__ASSEMBLY__ */
diff --git a/include/asm-generic/futex.h b/include/asm-generic/futex.h
index df893c160318..f422df0956a2 100644
--- a/include/asm-generic/futex.h
+++ b/include/asm-generic/futex.h
@@ -21,7 +21,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -33,7 +33,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-i386/futex.h b/include/asm-i386/futex.h
index 946d97cfea23..438ef0ec7101 100644
--- a/include/asm-i386/futex.h
+++ b/include/asm-i386/futex.h
@@ -56,7 +56,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         if (op == FUTEX_OP_SET)
                 __futex_atomic_op1("xchgl %0, %2", ret, oldval, uaddr, oparg);
@@ -88,7 +88,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 }
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-ia64/futex.h b/include/asm-ia64/futex.h
index 07d77f3a8cbe..8a98a2654139 100644
--- a/include/asm-ia64/futex.h
+++ b/include/asm-ia64/futex.h
@@ -59,7 +59,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -83,7 +83,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-mips/futex.h b/include/asm-mips/futex.h
index 927a216bd530..47e5679c2353 100644
--- a/include/asm-mips/futex.h
+++ b/include/asm-mips/futex.h
@@ -88,7 +88,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -115,7 +115,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-parisc/futex.h b/include/asm-parisc/futex.h
index d84bbb283fd1..dbee6e60aa81 100644
--- a/include/asm-parisc/futex.h
+++ b/include/asm-parisc/futex.h
@@ -21,7 +21,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -33,7 +33,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-powerpc/futex.h b/include/asm-powerpc/futex.h
index 936422e54891..3f3673fd3ff3 100644
--- a/include/asm-powerpc/futex.h
+++ b/include/asm-powerpc/futex.h
@@ -43,7 +43,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -65,7 +65,7 @@ static inline int futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-ppc/highmem.h b/include/asm-ppc/highmem.h
index 1d2c4ef81c22..f7b21ee302b4 100644
--- a/include/asm-ppc/highmem.h
+++ b/include/asm-ppc/highmem.h
@@ -79,7 +79,7 @@ static inline void *kmap_atomic(struct page *page, enum km_type type)
         unsigned long vaddr;
 
         /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */
-        inc_preempt_count();
+        pagefault_disable();
         if (!PageHighMem(page))
                 return page_address(page);
 
@@ -101,8 +101,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type)
         unsigned int idx = type + KM_TYPE_NR*smp_processor_id();
 
         if (vaddr < KMAP_FIX_BEGIN) { // FIXME
-                dec_preempt_count();
-                preempt_check_resched();
+                pagefault_enable();
                 return;
         }
 
@@ -115,8 +114,7 @@ static inline void kunmap_atomic(void *kvaddr, enum km_type type)
         pte_clear(&init_mm, vaddr, kmap_pte+idx);
         flush_tlb_page(NULL, vaddr);
 #endif
-        dec_preempt_count();
-        preempt_check_resched();
+        pagefault_enable();
 }
 
 static inline struct page *kmap_atomic_to_page(void *ptr)
diff --git a/include/asm-sparc64/futex.h b/include/asm-sparc64/futex.h
index 7392fc4a954e..876312fe82cc 100644
--- a/include/asm-sparc64/futex.h
+++ b/include/asm-sparc64/futex.h
@@ -45,7 +45,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
         if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28))
                 oparg = 1 << oparg;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -67,7 +67,7 @@ static inline int futex_atomic_op_inuser(int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/asm-x86_64/futex.h b/include/asm-x86_64/futex.h
index 9804bf07b092..5cdfb08013c3 100644
--- a/include/asm-x86_64/futex.h
+++ b/include/asm-x86_64/futex.h
@@ -55,7 +55,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
         if (! access_ok (VERIFY_WRITE, uaddr, sizeof(int)))
                 return -EFAULT;
 
-        inc_preempt_count();
+        pagefault_disable();
 
         switch (op) {
         case FUTEX_OP_SET:
@@ -78,7 +78,7 @@ futex_atomic_op_inuser (int encoded_op, int __user *uaddr)
                 ret = -ENOSYS;
         }
 
-        dec_preempt_count();
+        pagefault_enable();
 
         if (!ret) {
                 switch (cmp) {
diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h
index a48d7f11c7be..67918c22339c 100644
--- a/include/linux/uaccess.h
+++ b/include/linux/uaccess.h
@@ -1,8 +1,43 @@
 #ifndef __LINUX_UACCESS_H__
 #define __LINUX_UACCESS_H__
 
+#include <linux/preempt.h>
 #include <asm/uaccess.h>
 
+/*
+ * These routines enable/disable the pagefault handler in that
+ * it will not take any locks and go straight to the fixup table.
+ *
+ * They have great resemblance to the preempt_disable/enable calls
+ * and in fact they are identical; this is because currently there is
+ * no other way to make the pagefault handlers do this. So we do
+ * disable preemption but we don't necessarily care about that.
+ */
+static inline void pagefault_disable(void)
+{
+        inc_preempt_count();
+        /*
+         * make sure to have issued the store before a pagefault
+         * can hit.
+         */
+        barrier();
+}
+
+static inline void pagefault_enable(void)
+{
+        /*
+         * make sure to issue those last loads/stores before enabling
+         * the pagefault handler again.
+         */
+        barrier();
+        dec_preempt_count();
+        /*
+         * make sure we do..
+         */
+        barrier();
+        preempt_check_resched();
+}
+
 #ifndef ARCH_HAS_NOCACHE_UACCESS
 
 static inline unsigned long __copy_from_user_inatomic_nocache(void *to,
@@ -35,9 +70,9 @@ static inline unsigned long __copy_from_user_nocache(void *to,
         ({                                              \
                 long ret;                               \
                                                         \
-                inc_preempt_count();                    \
+                pagefault_disable();                    \
                 ret = __get_user(retval, addr);         \
-                dec_preempt_count();                    \
+                pagefault_enable();                     \
                 ret;                                    \
         })
 
diff --git a/kernel/futex.c b/kernel/futex.c
index 93ef30ba209f..af7b81cbde30 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -282,9 +282,9 @@ static inline int get_futex_value_locked(u32 *dest, u32 __user *from)
 {
         int ret;
 
-        inc_preempt_count();
+        pagefault_disable();
         ret = __copy_from_user_inatomic(dest, from, sizeof(u32));
-        dec_preempt_count();
+        pagefault_enable();
 
         return ret ? -EFAULT : 0;
 }
@@ -585,9 +585,9 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
         if (!(uval & FUTEX_OWNER_DIED)) {
                 newval = FUTEX_WAITERS | new_owner->pid;
 
-                inc_preempt_count();
+                pagefault_disable();
                 curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-                dec_preempt_count();
+                pagefault_enable();
                 if (curval == -EFAULT)
                         return -EFAULT;
                 if (curval != uval)
@@ -618,9 +618,9 @@ static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
          * There is no waiter, so we unlock the futex. The owner died
          * bit has not to be preserved here. We are the owner:
          */
-        inc_preempt_count();
+        pagefault_disable();
         oldval = futex_atomic_cmpxchg_inatomic(uaddr, uval, 0);
-        dec_preempt_count();
+        pagefault_enable();
 
         if (oldval == -EFAULT)
                 return oldval;
@@ -1158,9 +1158,9 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
          */
         newval = current->pid;
 
-        inc_preempt_count();
+        pagefault_disable();
         curval = futex_atomic_cmpxchg_inatomic(uaddr, 0, newval);
-        dec_preempt_count();
+        pagefault_enable();
 
         if (unlikely(curval == -EFAULT))
                 goto uaddr_faulted;
@@ -1183,9 +1183,9 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
         uval = curval;
         newval = uval | FUTEX_WAITERS;
 
-        inc_preempt_count();
+        pagefault_disable();
         curval = futex_atomic_cmpxchg_inatomic(uaddr, uval, newval);
-        dec_preempt_count();
+        pagefault_enable();
 
         if (unlikely(curval == -EFAULT))
                 goto uaddr_faulted;
@@ -1215,10 +1215,10 @@ static int futex_lock_pi(u32 __user *uaddr, int detect, unsigned long sec,
                         newval = current->pid |
                                 FUTEX_OWNER_DIED | FUTEX_WAITERS;
 
-                        inc_preempt_count();
+                        pagefault_disable();
                         curval = futex_atomic_cmpxchg_inatomic(uaddr,
                                                                uval, newval);
-                        dec_preempt_count();
+                        pagefault_enable();
 
                         if (unlikely(curval == -EFAULT))
                                 goto uaddr_faulted;
@@ -1390,9 +1390,9 @@ retry_locked:
          * anyone else up:
          */
         if (!(uval & FUTEX_OWNER_DIED)) {
-                inc_preempt_count();
+                pagefault_disable();
                 uval = futex_atomic_cmpxchg_inatomic(uaddr, current->pid, 0);
-                dec_preempt_count();
+                pagefault_enable();
         }
 
         if (unlikely(uval == -EFAULT))