mm: make mm->pinned_vm an atomic64 counter

Taking a sleeping lock to _only_ increment a variable is quite the
overkill, and pretty much all users do this. Furthermore, some drivers
(ie: infiniband and scif) that need pinned semantics can go to quite
some trouble to actually delay via workqueue (un)accounting for pinned
pages when not possible to acquire it.

By making the counter atomic we no longer need to hold the mmap_sem and
can simply some code around it for pinned_vm users. The counter is 64-bit
such that we need not worry about overflows such as rdma user input
controlled from userspace.

Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Christoph Lameter <cl@linux.com>
Reviewed-by: Daniel Jordan <daniel.m.jordan@oracle.com>
Reviewed-by: Jan Kara <jack@suse.cz>
Signed-off-by: Davidlohr Bueso <dbueso@suse.de>
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

2 files changed, 5 insertions, 5 deletions

diff --git a/kernel/events/core.c b/kernel/events/core.c
index e5ede6918050..29e9f2473656 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -5459,7 +5459,7 @@ static void perf_mmap_close(struct vm_area_struct *vma)
 
                 /* now it's safe to free the pages */
                 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm);
-                vma->vm_mm->pinned_vm -= rb->aux_mmap_locked;
+                atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm);
 
                 /* this has to be the last one */
                 rb_free_aux(rb);
@@ -5532,7 +5532,7 @@ again:
          */
 
         atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
-        vma->vm_mm->pinned_vm -= mmap_locked;
+        atomic64_sub(mmap_locked, &vma->vm_mm->pinned_vm);
         free_uid(mmap_user);
 
 out_put:
@@ -5680,7 +5680,7 @@ accounting:
 
         lock_limit = rlimit(RLIMIT_MEMLOCK);
         lock_limit >>= PAGE_SHIFT;
-        locked = vma->vm_mm->pinned_vm + extra;
+        locked = atomic64_read(&vma->vm_mm->pinned_vm) + extra;
 
         if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
                 !capable(CAP_IPC_LOCK)) {
@@ -5721,7 +5721,7 @@ accounting:
 unlock:
         if (!ret) {
                 atomic_long_add(user_extra, &user->locked_vm);
-                vma->vm_mm->pinned_vm += extra;
+                atomic64_add(extra, &vma->vm_mm->pinned_vm);
 
                 atomic_inc(&event->mmap_count);
         } else if (rb) {
diff --git a/kernel/fork.c b/kernel/fork.c
index b69248e6f0e0..85e08c379a9e 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -981,7 +981,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
         mm_pgtables_bytes_init(mm);
         mm->map_count = 0;
         mm->locked_vm = 0;
-        mm->pinned_vm = 0;
+        atomic64_set(&mm->pinned_vm, 0);
         memset(&mm->rss_stat, 0, sizeof(mm->rss_stat));
         spin_lock_init(&mm->page_table_lock);
         spin_lock_init(&mm->arg_lock);