mm: gup: use get_user_pages_unlocked within get_user_pages_fast

This allows the get_user_pages_fast slow path to release the mmap_sem
before blocking.

Signed-off-by: Andrea Arcangeli <aarcange@redhat.com>
Reviewed-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Andres Lagar-Cavilla <andreslc@google.com>
Cc: Peter Feiner <pfeiner@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

5 files changed, 12 insertions, 21 deletions

diff --git a/arch/mips/mm/gup.c b/arch/mips/mm/gup.c
index 70795a67a276..349995d19c7f 100644
--- a/arch/mips/mm/gup.c
+++ b/arch/mips/mm/gup.c
@@ -301,11 +301,9 @@ slow_irqon:
         start += nr << PAGE_SHIFT;
         pages += nr;
 
-        down_read(&mm->mmap_sem);
-        ret = get_user_pages(current, mm, start,
-                                (end - start) >> PAGE_SHIFT,
-                                write, 0, pages, NULL);
-        up_read(&mm->mmap_sem);
+        ret = get_user_pages_unlocked(current, mm, start,
+                                      (end - start) >> PAGE_SHIFT,
+                                      write, 0, pages);
 
         /* Have to be a bit careful with return values */
         if (nr > 0) {
diff --git a/arch/s390/mm/gup.c b/arch/s390/mm/gup.c
index 639fce464008..5c586c78ca8d 100644
--- a/arch/s390/mm/gup.c
+++ b/arch/s390/mm/gup.c
@@ -235,10 +235,8 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write,
         /* Try to get the remaining pages with get_user_pages */
         start += nr << PAGE_SHIFT;
         pages += nr;
-        down_read(&mm->mmap_sem);
-        ret = get_user_pages(current, mm, start,
-                             nr_pages - nr, write, 0, pages, NULL);
-        up_read(&mm->mmap_sem);
+        ret = get_user_pages_unlocked(current, mm, start,
+                             nr_pages - nr, write, 0, pages);
         /* Have to be a bit careful with return values */
         if (nr > 0)
                 ret = (ret < 0) ? nr : ret + nr;
diff --git a/arch/sh/mm/gup.c b/arch/sh/mm/gup.c
index 37458f38b220..e15f52a17b6c 100644
--- a/arch/sh/mm/gup.c
+++ b/arch/sh/mm/gup.c
@@ -257,10 +257,8 @@ slow_irqon:
                 start += nr << PAGE_SHIFT;
                 pages += nr;
 
-                down_read(&mm->mmap_sem);
-                ret = get_user_pages(current, mm, start,
-                        (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-                up_read(&mm->mmap_sem);
+                ret = get_user_pages_unlocked(current, mm, start,
+                        (end - start) >> PAGE_SHIFT, write, 0, pages);
 
                 /* Have to be a bit careful with return values */
                 if (nr > 0) {
diff --git a/arch/sparc/mm/gup.c b/arch/sparc/mm/gup.c
index ae6ce383d4df..2e5c4fc2daa9 100644
--- a/arch/sparc/mm/gup.c
+++ b/arch/sparc/mm/gup.c
@@ -249,10 +249,8 @@ slow:
                 start += nr << PAGE_SHIFT;
                 pages += nr;
 
-                down_read(&mm->mmap_sem);
-                ret = get_user_pages(current, mm, start,
-                        (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-                up_read(&mm->mmap_sem);
+                ret = get_user_pages_unlocked(current, mm, start,
+                        (end - start) >> PAGE_SHIFT, write, 0, pages);
 
                 /* Have to be a bit careful with return values */
                 if (nr > 0) {
diff --git a/arch/x86/mm/gup.c b/arch/x86/mm/gup.c
index 224b14235e96..89df70e0caa6 100644
--- a/arch/x86/mm/gup.c
+++ b/arch/x86/mm/gup.c
@@ -388,10 +388,9 @@ slow_irqon:
                 start += nr << PAGE_SHIFT;
                 pages += nr;
 
-                down_read(&mm->mmap_sem);
-                ret = get_user_pages(current, mm, start,
-                        (end - start) >> PAGE_SHIFT, write, 0, pages, NULL);
-                up_read(&mm->mmap_sem);
+                ret = get_user_pages_unlocked(current, mm, start,
+                                              (end - start) >> PAGE_SHIFT,
+                                              write, 0, pages);
 
                 /* Have to be a bit careful with return values */
                 if (nr > 0) {