1 files changed, 55 insertions, 40 deletions

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index a5bbdd3b5d67..2faaa2976447 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -765,7 +765,7 @@ struct vmap_block {
         spinlock_t lock;
         struct vmap_area *va;
         unsigned long free, dirty;
-        DECLARE_BITMAP(dirty_map, VMAP_BBMAP_BITS);
+        unsigned long dirty_min, dirty_max; /*< dirty range */
         struct list_head free_list;
         struct rcu_head rcu_head;
         struct list_head purge;
@@ -796,13 +796,31 @@ static unsigned long addr_to_vb_idx(unsigned long addr)
         return addr;
 }
 
-static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
+static void *vmap_block_vaddr(unsigned long va_start, unsigned long pages_off)
+{
+        unsigned long addr;
+
+        addr = va_start + (pages_off << PAGE_SHIFT);
+        BUG_ON(addr_to_vb_idx(addr) != addr_to_vb_idx(va_start));
+        return (void *)addr;
+}
+
+/**
+ * new_vmap_block - allocates new vmap_block and occupies 2^order pages in this
+ *                  block. Of course pages number can't exceed VMAP_BBMAP_BITS
+ * @order:    how many 2^order pages should be occupied in newly allocated block
+ * @gfp_mask: flags for the page level allocator
+ *
+ * Returns: virtual address in a newly allocated block or ERR_PTR(-errno)
+ */
+static void *new_vmap_block(unsigned int order, gfp_t gfp_mask)
 {
         struct vmap_block_queue *vbq;
         struct vmap_block *vb;
         struct vmap_area *va;
         unsigned long vb_idx;
         int node, err;
+        void *vaddr;
 
         node = numa_node_id();
 
@@ -826,11 +844,15 @@ static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
                 return ERR_PTR(err);
         }
 
+        vaddr = vmap_block_vaddr(va->va_start, 0);
         spin_lock_init(&vb->lock);
         vb->va = va;
-        vb->free = VMAP_BBMAP_BITS;
+        /* At least something should be left free */
+        BUG_ON(VMAP_BBMAP_BITS <= (1UL << order));
+        vb->free = VMAP_BBMAP_BITS - (1UL << order);
         vb->dirty = 0;
-        bitmap_zero(vb->dirty_map, VMAP_BBMAP_BITS);
+        vb->dirty_min = VMAP_BBMAP_BITS;
+        vb->dirty_max = 0;
         INIT_LIST_HEAD(&vb->free_list);
 
         vb_idx = addr_to_vb_idx(va->va_start);
@@ -842,11 +864,11 @@ static struct vmap_block *new_vmap_block(gfp_t gfp_mask)
 
         vbq = &get_cpu_var(vmap_block_queue);
         spin_lock(&vbq->lock);
-        list_add_rcu(&vb->free_list, &vbq->free);
+        list_add_tail_rcu(&vb->free_list, &vbq->free);
         spin_unlock(&vbq->lock);
         put_cpu_var(vmap_block_queue);
 
-        return vb;
+        return vaddr;
 }
 
 static void free_vmap_block(struct vmap_block *vb)
@@ -881,7 +903,8 @@ static void purge_fragmented_blocks(int cpu)
                 if (vb->free + vb->dirty == VMAP_BBMAP_BITS && vb->dirty != VMAP_BBMAP_BITS) {
                         vb->free = 0; /* prevent further allocs after releasing lock */
                         vb->dirty = VMAP_BBMAP_BITS; /* prevent purging it again */
-                        bitmap_fill(vb->dirty_map, VMAP_BBMAP_BITS);
+                        vb->dirty_min = 0;
+                        vb->dirty_max = VMAP_BBMAP_BITS;
                         spin_lock(&vbq->lock);
                         list_del_rcu(&vb->free_list);
                         spin_unlock(&vbq->lock);
@@ -910,7 +933,7 @@ static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
 {
         struct vmap_block_queue *vbq;
         struct vmap_block *vb;
-        unsigned long addr = 0;
+        void *vaddr = NULL;
         unsigned int order;
 
         BUG_ON(size & ~PAGE_MASK);
@@ -925,43 +948,38 @@ static void *vb_alloc(unsigned long size, gfp_t gfp_mask)
         }
         order = get_order(size);
 
-again:
         rcu_read_lock();
         vbq = &get_cpu_var(vmap_block_queue);
         list_for_each_entry_rcu(vb, &vbq->free, free_list) {
-                int i;
+                unsigned long pages_off;
 
                 spin_lock(&vb->lock);
-                if (vb->free < 1UL << order)
-                        goto next;
+                if (vb->free < (1UL << order)) {
+                        spin_unlock(&vb->lock);
+                        continue;
+                }
 
-                i = VMAP_BBMAP_BITS - vb->free;
-                addr = vb->va->va_start + (i << PAGE_SHIFT);
-                BUG_ON(addr_to_vb_idx(addr) !=
-                                addr_to_vb_idx(vb->va->va_start));
+                pages_off = VMAP_BBMAP_BITS - vb->free;
+                vaddr = vmap_block_vaddr(vb->va->va_start, pages_off);
                 vb->free -= 1UL << order;
                 if (vb->free == 0) {
                         spin_lock(&vbq->lock);
                         list_del_rcu(&vb->free_list);
                         spin_unlock(&vbq->lock);
                 }
+
                 spin_unlock(&vb->lock);
                 break;
-next:
-                spin_unlock(&vb->lock);
         }
 
         put_cpu_var(vmap_block_queue);
         rcu_read_unlock();
 
-        if (!addr) {
-                vb = new_vmap_block(gfp_mask);
-                if (IS_ERR(vb))
-                        return vb;
-                goto again;
-        }
+        /* Allocate new block if nothing was found */
+        if (!vaddr)
+                vaddr = new_vmap_block(order, gfp_mask);
 
-        return (void *)addr;
+        return vaddr;
 }
 
 static void vb_free(const void *addr, unsigned long size)
@@ -979,6 +997,7 @@ static void vb_free(const void *addr, unsigned long size)
         order = get_order(size);
 
         offset = (unsigned long)addr & (VMAP_BLOCK_SIZE - 1);
+        offset >>= PAGE_SHIFT;
 
         vb_idx = addr_to_vb_idx((unsigned long)addr);
         rcu_read_lock();
@@ -989,7 +1008,10 @@ static void vb_free(const void *addr, unsigned long size)
         vunmap_page_range((unsigned long)addr, (unsigned long)addr + size);
 
         spin_lock(&vb->lock);
-        BUG_ON(bitmap_allocate_region(vb->dirty_map, offset >> PAGE_SHIFT, order));
+
+        /* Expand dirty range */
+        vb->dirty_min = min(vb->dirty_min, offset);
+        vb->dirty_max = max(vb->dirty_max, offset + (1UL << order));
 
         vb->dirty += 1UL << order;
         if (vb->dirty == VMAP_BBMAP_BITS) {
@@ -1028,25 +1050,18 @@ void vm_unmap_aliases(void)
 
                 rcu_read_lock();
                 list_for_each_entry_rcu(vb, &vbq->free, free_list) {
-                        int i, j;
-
                         spin_lock(&vb->lock);
-                        i = find_first_bit(vb->dirty_map, VMAP_BBMAP_BITS);
-                        if (i < VMAP_BBMAP_BITS) {
+                        if (vb->dirty) {
+                                unsigned long va_start = vb->va->va_start;
                                 unsigned long s, e;
 
-                                j = find_last_bit(vb->dirty_map,
-                                                        VMAP_BBMAP_BITS);
-                                j = j + 1; /* need exclusive index */
+                                s = va_start + (vb->dirty_min << PAGE_SHIFT);
+                                e = va_start + (vb->dirty_max << PAGE_SHIFT);
 
-                                s = vb->va->va_start + (i << PAGE_SHIFT);
-                                e = vb->va->va_start + (j << PAGE_SHIFT);
-                                flush = 1;
+                                start = min(s, start);
+                                end   = max(e, end);
 
-                                if (s < start)
-                                        start = s;
-                                if (e > end)
-                                        end = e;
+                                flush = 1;
                         }
                         spin_unlock(&vb->lock);
                 }