1 files changed, 35 insertions, 25 deletions

diff --git a/mm/memory.c b/mm/memory.c
index 09e4b1be7b67..aecb7451965c 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -974,10 +974,13 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
         return addr;
 }
 
-#ifdef CONFIG_PREEMPT
+#if defined(CONFIG_PREEMPT) && !defined(CONFIG_PREEMPT_RT)
 # define ZAP_BLOCK_SIZE (8 * PAGE_SIZE)
 #else
-/* No preempt: go for improved straight-line efficiency */
+/*
+ * No preempt: go for improved straight-line efficiency
+ * on PREEMPT_RT this is not a critical latency-path.
+ */
 # define ZAP_BLOCK_SIZE (1024 * PAGE_SIZE)
 #endif
 
@@ -1007,17 +1010,14 @@ static unsigned long unmap_page_range(struct mmu_gather *tlb,
  * ensure that any thus-far unmapped pages are flushed before unmap_vmas()
  * drops the lock and schedules.
  */
-unsigned long unmap_vmas(struct mmu_gather **tlbp,
+unsigned long unmap_vmas(struct mmu_gather *tlb,
                 struct vm_area_struct *vma, unsigned long start_addr,
                 unsigned long end_addr, unsigned long *nr_accounted,
                 struct zap_details *details)
 {
         long zap_work = ZAP_BLOCK_SIZE;
-        unsigned long tlb_start = 0;    /* For tlb_finish_mmu */
-        int tlb_start_valid = 0;
         unsigned long start = start_addr;
         spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
-        int fullmm = (*tlbp)->fullmm;
         struct mm_struct *mm = vma->vm_mm;
 
         mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
@@ -1038,11 +1038,6 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
                         untrack_pfn_vma(vma, 0, 0);
 
                 while (start != end) {
-                        if (!tlb_start_valid) {
-                                tlb_start = start;
-                                tlb_start_valid = 1;
-                        }
-
                         if (unlikely(is_vm_hugetlb_page(vma))) {
                                 /*
                                  * It is undesirable to test vma->vm_file as it
@@ -1063,7 +1058,7 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
 
                                 start = end;
                         } else
-                                start = unmap_page_range(*tlbp, vma,
+                                start = unmap_page_range(tlb, vma,
                                                 start, end, &zap_work, details);
 
                         if (zap_work > 0) {
@@ -1071,19 +1066,13 @@ unsigned long unmap_vmas(struct mmu_gather **tlbp,
                                 break;
                         }
 
-                        tlb_finish_mmu(*tlbp, tlb_start, start);
-
                         if (need_resched() ||
                                 (i_mmap_lock && spin_needbreak(i_mmap_lock))) {
-                                if (i_mmap_lock) {
-                                        *tlbp = NULL;
+                                if (i_mmap_lock)
                                         goto out;
-                                }
                                 cond_resched();
                         }
 
-                        *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm);
-                        tlb_start_valid = 0;
                         zap_work = ZAP_BLOCK_SIZE;
                 }
         }
@@ -1103,16 +1092,15 @@ unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
                 unsigned long size, struct zap_details *details)
 {
         struct mm_struct *mm = vma->vm_mm;
-        struct mmu_gather *tlb;
+        struct mmu_gather tlb;
         unsigned long end = address + size;
         unsigned long nr_accounted = 0;
 
         lru_add_drain();
-        tlb = tlb_gather_mmu(mm, 0);
+        tlb_gather_mmu(&tlb, mm, 0);
         update_hiwater_rss(mm);
         end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
-        if (tlb)
-                tlb_finish_mmu(tlb, address, end);
+        tlb_finish_mmu(&tlb, address, end);
         return end;
 }
 
@@ -2491,12 +2479,12 @@ int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end)
                 return -ENOSYS;
 
         mutex_lock(&inode->i_mutex);
-        down_write(&inode->i_alloc_sem);
+        anon_down_write(&inode->i_alloc_sem);
         unmap_mapping_range(mapping, offset, (end - offset), 1);
         truncate_inode_pages_range(mapping, offset, end);
         unmap_mapping_range(mapping, offset, (end - offset), 1);
         inode->i_op->truncate_range(inode, offset, end);
-        up_write(&inode->i_alloc_sem);
+        anon_up_write(&inode->i_alloc_sem);
         mutex_unlock(&inode->i_mutex);
 
         return 0;
@@ -3008,6 +2996,28 @@ unlock:
         return 0;
 }
 
+void pagefault_disable(void)
+{
+        current->pagefault_disabled++;
+        /*
+         * make sure to have issued the store before a pagefault
+         * can hit.
+         */
+        barrier();
+}
+EXPORT_SYMBOL(pagefault_disable);
+
+void pagefault_enable(void)
+{
+        /*
+         * make sure to issue those last loads/stores before enabling
+         * the pagefault handler again.
+         */
+        barrier();
+        current->pagefault_disabled--;
+}
+EXPORT_SYMBOL(pagefault_enable);
+
 /*
  * By the time we get here, we already hold the mm semaphore
  */