1 files changed, 71 insertions, 28 deletions

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 7ef0903058ee..3231bf332878 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -1253,18 +1253,22 @@ EXPORT_SYMBOL_GPL(map_vm_area);
 DEFINE_RWLOCK(vmlist_lock);
 struct vm_struct *vmlist;
 
-static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
+static void setup_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
                               unsigned long flags, void *caller)
 {
-        struct vm_struct *tmp, **p;
-
         vm->flags = flags;
         vm->addr = (void *)va->va_start;
         vm->size = va->va_end - va->va_start;
         vm->caller = caller;
         va->private = vm;
         va->flags |= VM_VM_AREA;
+}
 
+static void insert_vmalloc_vmlist(struct vm_struct *vm)
+{
+        struct vm_struct *tmp, **p;
+
+        vm->flags &= ~VM_UNLIST;
         write_lock(&vmlist_lock);
         for (p = &vmlist; (tmp = *p) != NULL; p = &tmp->next) {
                 if (tmp->addr >= vm->addr)
@@ -1275,6 +1279,13 @@ static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
         write_unlock(&vmlist_lock);
 }
 
+static void insert_vmalloc_vm(struct vm_struct *vm, struct vmap_area *va,
+                              unsigned long flags, void *caller)
+{
+        setup_vmalloc_vm(vm, va, flags, caller);
+        insert_vmalloc_vmlist(vm);
+}
+
 static struct vm_struct *__get_vm_area_node(unsigned long size,
                 unsigned long align, unsigned long flags, unsigned long start,
                 unsigned long end, int node, gfp_t gfp_mask, void *caller)
@@ -1313,7 +1324,18 @@ static struct vm_struct *__get_vm_area_node(unsigned long size,
                 return NULL;
         }
 
-        insert_vmalloc_vm(area, va, flags, caller);
+        /*
+         * When this function is called from __vmalloc_node_range,
+         * we do not add vm_struct to vmlist here to avoid
+         * accessing uninitialized members of vm_struct such as
+         * pages and nr_pages fields. They will be set later.
+         * To distinguish it from others, we use a VM_UNLIST flag.
+         */
+        if (flags & VM_UNLIST)
+                setup_vmalloc_vm(area, va, flags, caller);
+        else
+                insert_vmalloc_vm(area, va, flags, caller);
+
         return area;
 }
 
@@ -1381,17 +1403,20 @@ struct vm_struct *remove_vm_area(const void *addr)
         va = find_vmap_area((unsigned long)addr);
         if (va && va->flags & VM_VM_AREA) {
                 struct vm_struct *vm = va->private;
-                struct vm_struct *tmp, **p;
-                /*
-                 * remove from list and disallow access to this vm_struct
-                 * before unmap. (address range confliction is maintained by
-                 * vmap.)
-                 */
-                write_lock(&vmlist_lock);
-                for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
-                        ;
-                *p = tmp->next;
-                write_unlock(&vmlist_lock);
+
+                if (!(vm->flags & VM_UNLIST)) {
+                        struct vm_struct *tmp, **p;
+                        /*
+                         * remove from list and disallow access to
+                         * this vm_struct before unmap. (address range
+                         * confliction is maintained by vmap.)
+                         */
+                        write_lock(&vmlist_lock);
+                        for (p = &vmlist; (tmp = *p) != vm; p = &tmp->next)
+                                ;
+                        *p = tmp->next;
+                        write_unlock(&vmlist_lock);
+                }
 
                 vmap_debug_free_range(va->va_start, va->va_end);
                 free_unmap_vmap_area(va);
@@ -1568,8 +1593,8 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
         return area->addr;
 
 fail:
-        warn_alloc_failed(gfp_mask, order, "vmalloc: allocation failure, "
-                          "allocated %ld of %ld bytes\n",
+        warn_alloc_failed(gfp_mask, order,
+                          "vmalloc: allocation failure, allocated %ld of %ld bytes\n",
                           (area->nr_pages*PAGE_SIZE), area->size);
         vfree(area->addr);
         return NULL;
@@ -1600,17 +1625,22 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
 
         size = PAGE_ALIGN(size);
         if (!size || (size >> PAGE_SHIFT) > totalram_pages)
-                return NULL;
-
-        area = __get_vm_area_node(size, align, VM_ALLOC, start, end, node,
-                                  gfp_mask, caller);
+                goto fail;
 
+        area = __get_vm_area_node(size, align, VM_ALLOC | VM_UNLIST,
+                                  start, end, node, gfp_mask, caller);
         if (!area)
-                return NULL;
+                goto fail;
 
         addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
 
         /*
+         * In this function, newly allocated vm_struct is not added
+         * to vmlist at __get_vm_area_node(). so, it is added here.
+         */
+        insert_vmalloc_vmlist(area);
+
+        /*
          * A ref_count = 3 is needed because the vm_struct and vmap_area
          * structures allocated in the __get_vm_area_node() function contain
          * references to the virtual address of the vmalloc'ed block.
@@ -1618,6 +1648,12 @@ void *__vmalloc_node_range(unsigned long size, unsigned long align,
         kmemleak_alloc(addr, real_size, 3, gfp_mask);
 
         return addr;
+
+fail:
+        warn_alloc_failed(gfp_mask, 0,
+                          "vmalloc: allocation failure: %lu bytes\n",
+                          real_size);
+        return NULL;
 }
 
 /**
@@ -2105,23 +2141,30 @@ void  __attribute__((weak)) vmalloc_sync_all(void)
 
 static int f(pte_t *pte, pgtable_t table, unsigned long addr, void *data)
 {
-        /* apply_to_page_range() does all the hard work. */
+        pte_t ***p = data;
+
+        if (p) {
+                *(*p) = pte;
+                (*p)++;
+        }
         return 0;
 }
 
 /**
  *      alloc_vm_area - allocate a range of kernel address space
  *      @size:          size of the area
+ *      @ptes:          returns the PTEs for the address space
  *
  *      Returns:        NULL on failure, vm_struct on success
  *
  *      This function reserves a range of kernel address space, and
  *      allocates pagetables to map that range.  No actual mappings
- *      are created.  If the kernel address space is not shared
- *      between processes, it syncs the pagetable across all
- *      processes.
+ *      are created.
+ *
+ *      If @ptes is non-NULL, pointers to the PTEs (in init_mm)
+ *      allocated for the VM area are returned.
  */
-struct vm_struct *alloc_vm_area(size_t size)
+struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes)
 {
         struct vm_struct *area;
 
@@ -2135,7 +2178,7 @@ struct vm_struct *alloc_vm_area(size_t size)
          * of kernel virtual address space and mapped into init_mm.
          */
         if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
-                                area->size, f, NULL)) {
+                                size, f, ptes ? &ptes : NULL)) {
                 free_vm_area(area);
                 return NULL;
         }