1 files changed, 126 insertions, 24 deletions

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 520a75980269..f8189a4b3e13 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -23,7 +23,8 @@
 #include <linux/rbtree.h>
 #include <linux/radix-tree.h>
 #include <linux/rcupdate.h>
-#include <linux/bootmem.h>
+#include <linux/pfn.h>
+#include <linux/kmemleak.h>
 
 #include <asm/atomic.h>
 #include <asm/uaccess.h>
@@ -152,8 +153,8 @@ static int vmap_pud_range(pgd_t *pgd, unsigned long addr,
  *
  * Ie. pte at addr+N*PAGE_SIZE shall point to pfn corresponding to pages[N]
  */
-static int vmap_page_range(unsigned long start, unsigned long end,
-                                pgprot_t prot, struct page **pages)
+static int vmap_page_range_noflush(unsigned long start, unsigned long end,
+                                   pgprot_t prot, struct page **pages)
 {
         pgd_t *pgd;
         unsigned long next;
@@ -169,13 +170,22 @@ static int vmap_page_range(unsigned long start, unsigned long end,
                 if (err)
                         break;
         } while (pgd++, addr = next, addr != end);
-        flush_cache_vmap(start, end);
 
         if (unlikely(err))
                 return err;
         return nr;
 }
 
+static int vmap_page_range(unsigned long start, unsigned long end,
+                           pgprot_t prot, struct page **pages)
+{
+        int ret;
+
+        ret = vmap_page_range_noflush(start, end, prot, pages);
+        flush_cache_vmap(start, end);
+        return ret;
+}
+
 static inline int is_vmalloc_or_module_addr(const void *x)
 {
         /*
@@ -392,6 +402,7 @@ overflow:
                         printk(KERN_WARNING
                                 "vmap allocation for size %lu failed: "
                                 "use vmalloc=<size> to increase size.\n", size);
+                kfree(va);
                 return ERR_PTR(-EBUSY);
         }
 
@@ -661,10 +672,7 @@ struct vmap_block {
         DECLARE_BITMAP(alloc_map, VMAP_BBMAP_BITS);
         DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
         union {
-                struct {
-                        struct list_head free_list;
-                        struct list_head dirty_list;
-                };
+                struct list_head free_list;
                 struct rcu_head rcu_head;
         };
 };
@@ -731,7 +739,6 @@ static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
         bitmap_zero(vb->alloc_map, VMAP_BBMAP_BITS);
         bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
         INIT_LIST_HEAD(&vb->free_list);
-        INIT_LIST_HEAD(&vb->dirty_list);
 
         vb_idx = addr_to_vb_idx(va->va_start);
         spin_lock(&vmap_block_tree_lock);
@@ -762,12 +769,7 @@ static void free_vmap_block(struct vmap_block *vb)
         struct vmap_block *tmp;
         unsigned long vb_idx;
 
-        spin_lock(&vb->vbq->lock);
-        if (!list_empty(&vb->free_list))
-                list_del(&vb->free_list);
-        if (!list_empty(&vb->dirty_list))
-                list_del(&vb->dirty_list);
-        spin_unlock(&vb->vbq->lock);
+        BUG_ON(!list_empty(&vb->free_list));
 
         vb_idx = addr_to_vb_idx(vb->va->va_start);
         spin_lock(&vmap_block_tree_lock);
@@ -852,11 +854,7 @@ static void vb_free(const void *addr, unsigned long size)
 
         spin_lock(&vb->lock);
         bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order);
-        if (!vb->dirty) {
-                spin_lock(&vb->vbq->lock);
-                list_add(&vb->dirty_list, &vb->vbq->dirty);
-                spin_unlock(&vb->vbq->lock);
-        }
+
         vb->dirty += 1UL << order;
         if (vb->dirty == VMAP_BBMAP_BITS) {
                 BUG_ON(vb->free || !list_empty(&vb->free_list));
@@ -990,6 +988,32 @@ void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t pro
 }
 EXPORT_SYMBOL(vm_map_ram);
 
+/**
+ * vm_area_register_early - register vmap area early during boot
+ * @vm: vm_struct to register
+ * @align: requested alignment
+ *
+ * This function is used to register kernel vm area before
+ * vmalloc_init() is called.  @vm->size and @vm->flags should contain
+ * proper values on entry and other fields should be zero.  On return,
+ * vm->addr contains the allocated address.
+ *
+ * DO NOT USE THIS FUNCTION UNLESS YOU KNOW WHAT YOU'RE DOING.
+ */
+void __init vm_area_register_early(struct vm_struct *vm, size_t align)
+{
+        static size_t vm_init_off __initdata;
+        unsigned long addr;
+
+        addr = ALIGN(VMALLOC_START + vm_init_off, align);
+        vm_init_off = PFN_ALIGN(addr + vm->size) - VMALLOC_START;
+
+        vm->addr = (void *)addr;
+
+        vm->next = vmlist;
+        vmlist = vm;
+}
+
 void __init vmalloc_init(void)
 {
         struct vmap_area *va;
@@ -1008,7 +1032,7 @@ void __init vmalloc_init(void)
 
         /* Import existing vmlist entries. */
         for (tmp = vmlist; tmp; tmp = tmp->next) {
-                va = alloc_bootmem(sizeof(struct vmap_area));
+                va = kzalloc(sizeof(struct vmap_area), GFP_NOWAIT);
                 va->flags = tmp->flags | VM_VM_AREA;
                 va->va_start = (unsigned long)tmp->addr;
                 va->va_end = va->va_start + tmp->size;
@@ -1017,6 +1041,58 @@ void __init vmalloc_init(void)
         vmap_initialized = true;
 }
 
+/**
+ * map_kernel_range_noflush - map kernel VM area with the specified pages
+ * @addr: start of the VM area to map
+ * @size: size of the VM area to map
+ * @prot: page protection flags to use
+ * @pages: pages to map
+ *
+ * Map PFN_UP(@size) pages at @addr.  The VM area @addr and @size
+ * specify should have been allocated using get_vm_area() and its
+ * friends.
+ *
+ * NOTE:
+ * This function does NOT do any cache flushing.  The caller is
+ * responsible for calling flush_cache_vmap() on to-be-mapped areas
+ * before calling this function.
+ *
+ * RETURNS:
+ * The number of pages mapped on success, -errno on failure.
+ */
+int map_kernel_range_noflush(unsigned long addr, unsigned long size,
+                             pgprot_t prot, struct page **pages)
+{
+        return vmap_page_range_noflush(addr, addr + size, prot, pages);
+}
+
+/**
+ * unmap_kernel_range_noflush - unmap kernel VM area
+ * @addr: start of the VM area to unmap
+ * @size: size of the VM area to unmap
+ *
+ * Unmap PFN_UP(@size) pages at @addr.  The VM area @addr and @size
+ * specify should have been allocated using get_vm_area() and its
+ * friends.
+ *
+ * NOTE:
+ * This function does NOT do any cache flushing.  The caller is
+ * responsible for calling flush_cache_vunmap() on to-be-mapped areas
+ * before calling this function and flush_tlb_kernel_range() after.
+ */
+void unmap_kernel_range_noflush(unsigned long addr, unsigned long size)
+{
+        vunmap_page_range(addr, addr + size);
+}
+
+/**
+ * unmap_kernel_range - unmap kernel VM area and flush cache and TLB
+ * @addr: start of the VM area to unmap
+ * @size: size of the VM area to unmap
+ *
+ * Similar to unmap_kernel_range_noflush() but flushes vcache before
+ * the unmapping and tlb after.
+ */
 void unmap_kernel_range(unsigned long addr, unsigned long size)
 {
         unsigned long end = addr + size;
@@ -1251,6 +1327,9 @@ static void __vunmap(const void *addr, int deallocate_pages)
 void vfree(const void *addr)
 {
         BUG_ON(in_interrupt());
+
+        kmemleak_free(addr);
+
         __vunmap(addr, 1);
 }
 EXPORT_SYMBOL(vfree);
@@ -1267,6 +1346,7 @@ EXPORT_SYMBOL(vfree);
 void vunmap(const void *addr)
 {
         BUG_ON(in_interrupt());
+        might_sleep();
         __vunmap(addr, 0);
 }
 EXPORT_SYMBOL(vunmap);
@@ -1286,6 +1366,8 @@ void *vmap(struct page **pages, unsigned int count,
 {
         struct vm_struct *area;
 
+        might_sleep();
+
         if (count > num_physpages)
                 return NULL;
 
@@ -1360,8 +1442,17 @@ fail:
 
 void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
 {
-        return __vmalloc_area_node(area, gfp_mask, prot, -1,
-                                        __builtin_return_address(0));
+        void *addr = __vmalloc_area_node(area, gfp_mask, prot, -1,
+                                         __builtin_return_address(0));
+
+        /*
+         * A ref_count = 3 is needed because the vm_struct and vmap_area
+         * structures allocated in the __get_vm_area_node() function contain
+         * references to the virtual address of the vmalloc'ed block.
+         */
+        kmemleak_alloc(addr, area->size - PAGE_SIZE, 3, gfp_mask);
+
+        return addr;
 }
 
 /**
@@ -1380,6 +1471,8 @@ static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
                                                 int node, void *caller)
 {
         struct vm_struct *area;
+        void *addr;
+        unsigned long real_size = size;
 
         size = PAGE_ALIGN(size);
         if (!size || (size >> PAGE_SHIFT) > num_physpages)
@@ -1391,7 +1484,16 @@ static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
         if (!area)
                 return NULL;
 
-        return __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+        addr = __vmalloc_area_node(area, gfp_mask, prot, node, caller);
+
+        /*
+         * A ref_count = 3 is needed because the vm_struct and vmap_area
+         * structures allocated in the __get_vm_area_node() function contain
+         * references to the virtual address of the vmalloc'ed block.
+         */
+        kmemleak_alloc(addr, real_size, 3, gfp_mask);
+
+        return addr;
 }
 
 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)