x86: some lock annotations for user copy paths

copy_to/from_user and all its variants (except the atomic ones) can take a
page fault and perform non-trivial work like taking mmap_sem and entering
the filesyste/pagecache.

Unfortunately, this often escapes lockdep because a common pattern is to
use it to read in some arguments just set up from userspace, or write data
back to a hot buffer. In those cases, it will be unlikely for page reclaim
to get a window in to cause copy_*_user to fault.

With the new might_lock primitives, add some annotations to x86. I don't
know if I caught all possible faulting points (it's a bit of a maze, and I
didn't really look at 32-bit). But this is a starting point.

Boots and runs OK so far.

Signed-off-by: Nick Piggin <npiggin@suse.de>
Acked-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

2 files changed, 10 insertions, 1 deletions

diff --git a/arch/x86/lib/usercopy_32.c b/arch/x86/lib/usercopy_32.c
index 24e60944971a..8eedde2a9cac 100644
--- a/arch/x86/lib/usercopy_32.c
+++ b/arch/x86/lib/usercopy_32.c
@@ -33,6 +33,8 @@ static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned lon
 do {                                                                       \
         int __d0, __d1, __d2;                                              \
         might_sleep();                                                     \
+        if (current->mm)                                                   \
+                might_lock_read(&current->mm->mmap_sem);                   \
         __asm__ __volatile__(                                              \
                 "       testl %1,%1\n"                                     \
                 "       jz 2f\n"                                           \
@@ -120,6 +122,8 @@ EXPORT_SYMBOL(strncpy_from_user);
 do {                                                                    \
         int __d0;                                                       \
         might_sleep();                                                  \
+        if (current->mm)                                                \
+                might_lock_read(&current->mm->mmap_sem);                \
         __asm__ __volatile__(                                           \
                 "0:     rep; stosl\n"                                   \
                 "       movl %2,%0\n"                                   \
@@ -148,7 +152,6 @@ do {									\
 unsigned long
 clear_user(void __user *to, unsigned long n)
 {
-        might_sleep();
         if (access_ok(VERIFY_WRITE, to, n))
                 __do_clear_user(to, n);
         return n;
@@ -191,6 +194,8 @@ long strnlen_user(const char __user *s, long n)
         unsigned long res, tmp;
 
         might_sleep();
+        if (current->mm)
+                might_lock_read(&current->mm->mmap_sem);
 
         __asm__ __volatile__(
                 "       testl %0, %0\n"
diff --git a/arch/x86/lib/usercopy_64.c b/arch/x86/lib/usercopy_64.c
index f4df6e7c718b..847d12945998 100644
--- a/arch/x86/lib/usercopy_64.c
+++ b/arch/x86/lib/usercopy_64.c
@@ -16,6 +16,8 @@
 do {                                                                       \
         long __d0, __d1, __d2;                                             \
         might_sleep();                                                     \
+        if (current->mm)                                                   \
+                might_lock_read(&current->mm->mmap_sem);                   \
         __asm__ __volatile__(                                              \
                 "       testq %1,%1\n"                                     \
                 "       jz 2f\n"                                           \
@@ -65,6 +67,8 @@ unsigned long __clear_user(void __user *addr, unsigned long size)
 {
         long __d0;
         might_sleep();
+        if (current->mm)
+                might_lock_read(&current->mm->mmap_sem);
         /* no memory constraint because it doesn't change any memory gcc knows
            about */
         asm volatile(