Patched in Tegra support.

1 files changed, 51 insertions, 81 deletions

diff --git a/mm/mremap.c b/mm/mremap.c
index e1031e1f6a6..506fa44403d 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -41,7 +41,8 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr)
                 return NULL;
 
         pmd = pmd_offset(pud, addr);
-        if (pmd_none(*pmd))
+        split_huge_page_pmd(mm, pmd);
+        if (pmd_none_or_clear_bad(pmd))
                 return NULL;
 
         return pmd;
@@ -64,6 +65,8 @@ static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
                 return NULL;
 
         VM_BUG_ON(pmd_trans_huge(*pmd));
+        if (pmd_none(*pmd) && __pte_alloc(mm, vma, pmd, addr))
+                return NULL;
 
         return pmd;
 }
@@ -71,41 +74,26 @@ static pmd_t *alloc_new_pmd(struct mm_struct *mm, struct vm_area_struct *vma,
 static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
                 unsigned long old_addr, unsigned long old_end,
                 struct vm_area_struct *new_vma, pmd_t *new_pmd,
-                unsigned long new_addr, bool need_rmap_locks)
+                unsigned long new_addr)
 {
         struct address_space *mapping = NULL;
-        struct anon_vma *anon_vma = NULL;
         struct mm_struct *mm = vma->vm_mm;
         pte_t *old_pte, *new_pte, pte;
         spinlock_t *old_ptl, *new_ptl;
+        unsigned long old_start;
 
-        /*
-         * When need_rmap_locks is true, we take the i_mmap_mutex and anon_vma
-         * locks to ensure that rmap will always observe either the old or the
-         * new ptes. This is the easiest way to avoid races with
-         * truncate_pagecache(), page migration, etc...
-         *
-         * When need_rmap_locks is false, we use other ways to avoid
-         * such races:
-         *
-         * - During exec() shift_arg_pages(), we use a specially tagged vma
-         *   which rmap call sites look for using is_vma_temporary_stack().
-         *
-         * - During mremap(), new_vma is often known to be placed after vma
-         *   in rmap traversal order. This ensures rmap will always observe
-         *   either the old pte, or the new pte, or both (the page table locks
-         *   serialize access to individual ptes, but only rmap traversal
-         *   order guarantees that we won't miss both the old and new ptes).
-         */
-        if (need_rmap_locks) {
-                if (vma->vm_file) {
-                        mapping = vma->vm_file->f_mapping;
-                        mutex_lock(&mapping->i_mmap_mutex);
-                }
-                if (vma->anon_vma) {
-                        anon_vma = vma->anon_vma;
-                        anon_vma_lock_write(anon_vma);
-                }
+        old_start = old_addr;
+        mmu_notifier_invalidate_range_start(vma->vm_mm,
+                                            old_start, old_end);
+        if (vma->vm_file) {
+                /*
+                 * Subtle point from Rajesh Venkatasubramanian: before
+                 * moving file-based ptes, we must lock truncate_pagecache
+                 * out, since it might clean the dst vma before the src vma,
+                 * and we propagate stale pages into the dst afterward.
+                 */
+                mapping = vma->vm_file->f_mapping;
+                mutex_lock(&mapping->i_mmap_mutex);
         }
 
         /*
@@ -123,7 +111,7 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
                                    new_pte++, new_addr += PAGE_SIZE) {
                 if (pte_none(*old_pte))
                         continue;
-                pte = ptep_get_and_clear(mm, old_addr, old_pte);
+                pte = ptep_clear_flush(vma, old_addr, old_pte);
                 pte = move_pte(pte, new_vma->vm_page_prot, old_addr, new_addr);
                 set_pte_at(mm, new_addr, new_pte, pte);
         }
@@ -133,75 +121,43 @@ static void move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
                 spin_unlock(new_ptl);
         pte_unmap(new_pte - 1);
         pte_unmap_unlock(old_pte - 1, old_ptl);
-        if (anon_vma)
-                anon_vma_unlock(anon_vma);
         if (mapping)
                 mutex_unlock(&mapping->i_mmap_mutex);
+        mmu_notifier_invalidate_range_end(vma->vm_mm, old_start, old_end);
 }
 
 #define LATENCY_LIMIT   (64 * PAGE_SIZE)
 
 unsigned long move_page_tables(struct vm_area_struct *vma,
                 unsigned long old_addr, struct vm_area_struct *new_vma,
-                unsigned long new_addr, unsigned long len,
-                bool need_rmap_locks)
+                unsigned long new_addr, unsigned long len)
 {
         unsigned long extent, next, old_end;
         pmd_t *old_pmd, *new_pmd;
-        bool need_flush = false;
-        unsigned long mmun_start;       /* For mmu_notifiers */
-        unsigned long mmun_end;         /* For mmu_notifiers */
 
         old_end = old_addr + len;
         flush_cache_range(vma, old_addr, old_end);
 
-        mmun_start = old_addr;
-        mmun_end   = old_end;
-        mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
-
         for (; old_addr < old_end; old_addr += extent, new_addr += extent) {
                 cond_resched();
                 next = (old_addr + PMD_SIZE) & PMD_MASK;
-                /* even if next overflowed, extent below will be ok */
+                if (next - 1 > old_end)
+                        next = old_end;
                 extent = next - old_addr;
-                if (extent > old_end - old_addr)
-                        extent = old_end - old_addr;
                 old_pmd = get_old_pmd(vma->vm_mm, old_addr);
                 if (!old_pmd)
                         continue;
                 new_pmd = alloc_new_pmd(vma->vm_mm, vma, new_addr);
                 if (!new_pmd)
                         break;
-                if (pmd_trans_huge(*old_pmd)) {
-                        int err = 0;
-                        if (extent == HPAGE_PMD_SIZE)
-                                err = move_huge_pmd(vma, new_vma, old_addr,
-                                                    new_addr, old_end,
-                                                    old_pmd, new_pmd);
-                        if (err > 0) {
-                                need_flush = true;
-                                continue;
-                        } else if (!err) {
-                                split_huge_page_pmd(vma, old_addr, old_pmd);
-                        }
-                        VM_BUG_ON(pmd_trans_huge(*old_pmd));
-                }
-                if (pmd_none(*new_pmd) && __pte_alloc(new_vma->vm_mm, new_vma,
-                                                      new_pmd, new_addr))
-                        break;
                 next = (new_addr + PMD_SIZE) & PMD_MASK;
                 if (extent > next - new_addr)
                         extent = next - new_addr;
                 if (extent > LATENCY_LIMIT)
                         extent = LATENCY_LIMIT;
                 move_ptes(vma, old_pmd, old_addr, old_addr + extent,
-                          new_vma, new_pmd, new_addr, need_rmap_locks);
-                need_flush = true;
+                                new_vma, new_pmd, new_addr);
         }
-        if (likely(need_flush))
-                flush_tlb_range(vma, old_end-len, old_addr);
-
-        mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
 
         return len + old_addr - old_end;        /* how much done */
 }
@@ -219,7 +175,6 @@ static unsigned long move_vma(struct vm_area_struct *vma,
         unsigned long hiwater_vm;
         int split = 0;
         int err;
-        bool need_rmap_locks;
 
         /*
          * We'd prefer to avoid failure later on in do_munmap:
@@ -241,21 +196,18 @@ static unsigned long move_vma(struct vm_area_struct *vma,
                 return err;
 
         new_pgoff = vma->vm_pgoff + ((old_addr - vma->vm_start) >> PAGE_SHIFT);
-        new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff,
-                           &need_rmap_locks);
+        new_vma = copy_vma(&vma, new_addr, new_len, new_pgoff);
         if (!new_vma)
                 return -ENOMEM;
 
-        moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
-                                     need_rmap_locks);
+        moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len);
         if (moved_len < old_len) {
                 /*
                  * On error, move entries back from new area to old,
                  * which will succeed since page tables still there,
                  * and then proceed to unmap new area instead of old.
                  */
-                move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
-                                 true);
+                move_page_tables(new_vma, new_addr, vma, old_addr, moved_len);
                 vma = new_vma;
                 old_len = new_len;
                 old_addr = new_addr;
@@ -281,6 +233,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
          * If this were a serious issue, we'd add a flag to do_munmap().
          */
         hiwater_vm = mm->hiwater_vm;
+        mm->total_vm += new_len >> PAGE_SHIFT;
         vm_stat_account(mm, vma->vm_flags, vma->vm_file, new_len>>PAGE_SHIFT);
 
         if (do_munmap(mm, old_addr, old_len) < 0) {
@@ -349,7 +302,7 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
 
         if (vma->vm_flags & VM_ACCOUNT) {
                 unsigned long charged = (new_len - old_len) >> PAGE_SHIFT;
-                if (security_vm_enough_memory_mm(mm, charged))
+                if (security_vm_enough_memory(charged))
                         goto Efault;
                 *p = charged;
         }
@@ -391,6 +344,10 @@ static unsigned long mremap_to(unsigned long addr,
         if ((addr <= new_addr) && (addr+old_len) > new_addr)
                 goto out;
 
+        ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
+        if (ret)
+                goto out;
+
         ret = do_munmap(mm, new_addr, new_len);
         if (ret)
                 goto out;
@@ -448,17 +405,15 @@ static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
  * MREMAP_FIXED option added 5-Dec-1999 by Benjamin LaHaise
  * This option implies MREMAP_MAYMOVE.
  */
-SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
-                unsigned long, new_len, unsigned long, flags,
-                unsigned long, new_addr)
+unsigned long do_mremap(unsigned long addr,
+        unsigned long old_len, unsigned long new_len,
+        unsigned long flags, unsigned long new_addr)
 {
         struct mm_struct *mm = current->mm;
         struct vm_area_struct *vma;
         unsigned long ret = -EINVAL;
         unsigned long charged = 0;
 
-        down_write(&current->mm->mmap_sem);
-
         if (flags & ~(MREMAP_FIXED | MREMAP_MAYMOVE))
                 goto out;
 
@@ -517,6 +472,7 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                                 goto out;
                         }
 
+                        mm->total_vm += pages;
                         vm_stat_account(mm, vma->vm_flags, vma->vm_file, pages);
                         if (vma->vm_flags & VM_LOCKED) {
                                 mm->locked_vm += pages;
@@ -547,11 +503,25 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
                         goto out;
                 }
 
+                ret = security_file_mmap(NULL, 0, 0, 0, new_addr, 1);
+                if (ret)
+                        goto out;
                 ret = move_vma(vma, addr, old_len, new_len, new_addr);
         }
 out:
         if (ret & ~PAGE_MASK)
                 vm_unacct_memory(charged);
+        return ret;
+}
+
+SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
+                unsigned long, new_len, unsigned long, flags,
+                unsigned long, new_addr)
+{
+        unsigned long ret;
+
+        down_write(&current->mm->mmap_sem);
+        ret = do_mremap(addr, old_len, new_len, flags, new_addr);
         up_write(&current->mm->mmap_sem);
         return ret;
 }