1 files changed, 427 insertions, 99 deletions

diff --git a/mm/percpu.c b/mm/percpu.c
index da997f9800bd..014bab65e0ff 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -76,6 +76,10 @@
 
 #define PCPU_SLOT_BASE_SHIFT            5       /* 1-31 shares the same slot */
 #define PCPU_DFL_MAP_ALLOC              16      /* start a map with 16 ents */
+#define PCPU_ATOMIC_MAP_MARGIN_LOW      32
+#define PCPU_ATOMIC_MAP_MARGIN_HIGH     64
+#define PCPU_EMPTY_POP_PAGES_LOW        2
+#define PCPU_EMPTY_POP_PAGES_HIGH       4
 
 #ifdef CONFIG_SMP
 /* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
@@ -102,12 +106,16 @@ struct pcpu_chunk {
         int                     free_size;      /* free bytes in the chunk */
         int                     contig_hint;    /* max contiguous size hint */
         void                    *base_addr;     /* base address of this chunk */
+
         int                     map_used;       /* # of map entries used before the sentry */
         int                     map_alloc;      /* # of map entries allocated */
         int                     *map;           /* allocation map */
+        struct work_struct      map_extend_work;/* async ->map[] extension */
+
         void                    *data;          /* chunk data */
         int                     first_free;     /* no free below this */
         bool                    immutable;      /* no [de]population allowed */
+        int                     nr_populated;   /* # of populated pages */
         unsigned long           populated[];    /* populated bitmap */
 };
 
@@ -151,38 +159,33 @@ static struct pcpu_chunk *pcpu_first_chunk;
 static struct pcpu_chunk *pcpu_reserved_chunk;
 static int pcpu_reserved_chunk_limit;
 
+static DEFINE_SPINLOCK(pcpu_lock);      /* all internal data structures */
+static DEFINE_MUTEX(pcpu_alloc_mutex);  /* chunk create/destroy, [de]pop */
+
+static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
+
 /*
- * Synchronization rules.
- *
- * There are two locks - pcpu_alloc_mutex and pcpu_lock.  The former
- * protects allocation/reclaim paths, chunks, populated bitmap and
- * vmalloc mapping.  The latter is a spinlock and protects the index
- * data structures - chunk slots, chunks and area maps in chunks.
- *
- * During allocation, pcpu_alloc_mutex is kept locked all the time and
- * pcpu_lock is grabbed and released as necessary.  All actual memory
- * allocations are done using GFP_KERNEL with pcpu_lock released.  In
- * general, percpu memory can't be allocated with irq off but
- * irqsave/restore are still used in alloc path so that it can be used
- * from early init path - sched_init() specifically.
- *
- * Free path accesses and alters only the index data structures, so it
- * can be safely called from atomic context.  When memory needs to be
- * returned to the system, free path schedules reclaim_work which
- * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be
- * reclaimed, release both locks and frees the chunks.  Note that it's
- * necessary to grab both locks to remove a chunk from circulation as
- * allocation path might be referencing the chunk with only
- * pcpu_alloc_mutex locked.
+ * The number of empty populated pages, protected by pcpu_lock.  The
+ * reserved chunk doesn't contribute to the count.
  */
-static DEFINE_MUTEX(pcpu_alloc_mutex);  /* protects whole alloc and reclaim */
-static DEFINE_SPINLOCK(pcpu_lock);      /* protects index data structures */
+static int pcpu_nr_empty_pop_pages;
 
-static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
+/*
+ * Balance work is used to populate or destroy chunks asynchronously.  We
+ * try to keep the number of populated free pages between
+ * PCPU_EMPTY_POP_PAGES_LOW and HIGH for atomic allocations and at most one
+ * empty chunk.
+ */
+static void pcpu_balance_workfn(struct work_struct *work);
+static DECLARE_WORK(pcpu_balance_work, pcpu_balance_workfn);
+static bool pcpu_async_enabled __read_mostly;
+static bool pcpu_atomic_alloc_failed;
 
-/* reclaim work to release fully free chunks, scheduled from free path */
-static void pcpu_reclaim(struct work_struct *work);
-static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
+static void pcpu_schedule_balance_work(void)
+{
+        if (pcpu_async_enabled)
+                schedule_work(&pcpu_balance_work);
+}
 
 static bool pcpu_addr_in_first_chunk(void *addr)
 {
@@ -315,6 +318,38 @@ static void pcpu_mem_free(void *ptr, size_t size)
 }
 
 /**
+ * pcpu_count_occupied_pages - count the number of pages an area occupies
+ * @chunk: chunk of interest
+ * @i: index of the area in question
+ *
+ * Count the number of pages chunk's @i'th area occupies.  When the area's
+ * start and/or end address isn't aligned to page boundary, the straddled
+ * page is included in the count iff the rest of the page is free.
+ */
+static int pcpu_count_occupied_pages(struct pcpu_chunk *chunk, int i)
+{
+        int off = chunk->map[i] & ~1;
+        int end = chunk->map[i + 1] & ~1;
+
+        if (!PAGE_ALIGNED(off) && i > 0) {
+                int prev = chunk->map[i - 1];
+
+                if (!(prev & 1) && prev <= round_down(off, PAGE_SIZE))
+                        off = round_down(off, PAGE_SIZE);
+        }
+
+        if (!PAGE_ALIGNED(end) && i + 1 < chunk->map_used) {
+                int next = chunk->map[i + 1];
+                int nend = chunk->map[i + 2] & ~1;
+
+                if (!(next & 1) && nend >= round_up(end, PAGE_SIZE))
+                        end = round_up(end, PAGE_SIZE);
+        }
+
+        return max_t(int, PFN_DOWN(end) - PFN_UP(off), 0);
+}
+
+/**
  * pcpu_chunk_relocate - put chunk in the appropriate chunk slot
  * @chunk: chunk of interest
  * @oslot: the previous slot it was on
@@ -342,9 +377,14 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
 /**
  * pcpu_need_to_extend - determine whether chunk area map needs to be extended
  * @chunk: chunk of interest
+ * @is_atomic: the allocation context
  *
- * Determine whether area map of @chunk needs to be extended to
- * accommodate a new allocation.
+ * Determine whether area map of @chunk needs to be extended.  If
+ * @is_atomic, only the amount necessary for a new allocation is
+ * considered; however, async extension is scheduled if the left amount is
+ * low.  If !@is_atomic, it aims for more empty space.  Combined, this
+ * ensures that the map is likely to have enough available space to
+ * accomodate atomic allocations which can't extend maps directly.
  *
  * CONTEXT:
  * pcpu_lock.
@@ -353,15 +393,26 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
  * New target map allocation length if extension is necessary, 0
  * otherwise.
  */
-static int pcpu_need_to_extend(struct pcpu_chunk *chunk)
+static int pcpu_need_to_extend(struct pcpu_chunk *chunk, bool is_atomic)
 {
-        int new_alloc;
+        int margin, new_alloc;
+
+        if (is_atomic) {
+                margin = 3;
+
+                if (chunk->map_alloc <
+                    chunk->map_used + PCPU_ATOMIC_MAP_MARGIN_LOW &&
+                    pcpu_async_enabled)
+                        schedule_work(&chunk->map_extend_work);
+        } else {
+                margin = PCPU_ATOMIC_MAP_MARGIN_HIGH;
+        }
 
-        if (chunk->map_alloc >= chunk->map_used + 3)
+        if (chunk->map_alloc >= chunk->map_used + margin)
                 return 0;
 
         new_alloc = PCPU_DFL_MAP_ALLOC;
-        while (new_alloc < chunk->map_used + 3)
+        while (new_alloc < chunk->map_used + margin)
                 new_alloc *= 2;
 
         return new_alloc;
@@ -418,11 +469,76 @@ out_unlock:
         return 0;
 }
 
+static void pcpu_map_extend_workfn(struct work_struct *work)
+{
+        struct pcpu_chunk *chunk = container_of(work, struct pcpu_chunk,
+                                                map_extend_work);
+        int new_alloc;
+
+        spin_lock_irq(&pcpu_lock);
+        new_alloc = pcpu_need_to_extend(chunk, false);
+        spin_unlock_irq(&pcpu_lock);
+
+        if (new_alloc)
+                pcpu_extend_area_map(chunk, new_alloc);
+}
+
+/**
+ * pcpu_fit_in_area - try to fit the requested allocation in a candidate area
+ * @chunk: chunk the candidate area belongs to
+ * @off: the offset to the start of the candidate area
+ * @this_size: the size of the candidate area
+ * @size: the size of the target allocation
+ * @align: the alignment of the target allocation
+ * @pop_only: only allocate from already populated region
+ *
+ * We're trying to allocate @size bytes aligned at @align.  @chunk's area
+ * at @off sized @this_size is a candidate.  This function determines
+ * whether the target allocation fits in the candidate area and returns the
+ * number of bytes to pad after @off.  If the target area doesn't fit, -1
+ * is returned.
+ *
+ * If @pop_only is %true, this function only considers the already
+ * populated part of the candidate area.
+ */
+static int pcpu_fit_in_area(struct pcpu_chunk *chunk, int off, int this_size,
+                            int size, int align, bool pop_only)
+{
+        int cand_off = off;
+
+        while (true) {
+                int head = ALIGN(cand_off, align) - off;
+                int page_start, page_end, rs, re;
+
+                if (this_size < head + size)
+                        return -1;
+
+                if (!pop_only)
+                        return head;
+
+                /*
+                 * If the first unpopulated page is beyond the end of the
+                 * allocation, the whole allocation is populated;
+                 * otherwise, retry from the end of the unpopulated area.
+                 */
+                page_start = PFN_DOWN(head + off);
+                page_end = PFN_UP(head + off + size);
+
+                rs = page_start;
+                pcpu_next_unpop(chunk, &rs, &re, PFN_UP(off + this_size));
+                if (rs >= page_end)
+                        return head;
+                cand_off = re * PAGE_SIZE;
+        }
+}
+
 /**
  * pcpu_alloc_area - allocate area from a pcpu_chunk
  * @chunk: chunk of interest
  * @size: wanted size in bytes
  * @align: wanted align
+ * @pop_only: allocate only from the populated area
+ * @occ_pages_p: out param for the number of pages the area occupies
  *
  * Try to allocate @size bytes area aligned at @align from @chunk.
  * Note that this function only allocates the offset.  It doesn't
@@ -437,7 +553,8 @@ out_unlock:
  * Allocated offset in @chunk on success, -1 if no matching area is
  * found.
  */
-static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
+static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align,
+                           bool pop_only, int *occ_pages_p)
 {
         int oslot = pcpu_chunk_slot(chunk);
         int max_contig = 0;
@@ -453,11 +570,11 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
                 if (off & 1)
                         continue;
 
-                /* extra for alignment requirement */
-                head = ALIGN(off, align) - off;
-
                 this_size = (p[1] & ~1) - off;
-                if (this_size < head + size) {
+
+                head = pcpu_fit_in_area(chunk, off, this_size, size, align,
+                                        pop_only);
+                if (head < 0) {
                         if (!seen_free) {
                                 chunk->first_free = i;
                                 seen_free = true;
@@ -526,6 +643,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
                 chunk->free_size -= size;
                 *p |= 1;
 
+                *occ_pages_p = pcpu_count_occupied_pages(chunk, i);
                 pcpu_chunk_relocate(chunk, oslot);
                 return off;
         }
@@ -541,6 +659,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
  * pcpu_free_area - free area to a pcpu_chunk
  * @chunk: chunk of interest
  * @freeme: offset of area to free
+ * @occ_pages_p: out param for the number of pages the area occupies
  *
  * Free area starting from @freeme to @chunk.  Note that this function
  * only modifies the allocation map.  It doesn't depopulate or unmap
@@ -549,7 +668,8 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
  * CONTEXT:
  * pcpu_lock.
  */
-static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
+static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme,
+                           int *occ_pages_p)
 {
         int oslot = pcpu_chunk_slot(chunk);
         int off = 0;
@@ -580,6 +700,8 @@ static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
         *p = off &= ~1;
         chunk->free_size += (p[1] & ~1) - off;
 
+        *occ_pages_p = pcpu_count_occupied_pages(chunk, i);
+
         /* merge with next? */
         if (!(p[1] & 1))
                 to_free++;
@@ -620,6 +742,7 @@ static struct pcpu_chunk *pcpu_alloc_chunk(void)
         chunk->map_used = 1;
 
         INIT_LIST_HEAD(&chunk->list);
+        INIT_WORK(&chunk->map_extend_work, pcpu_map_extend_workfn);
         chunk->free_size = pcpu_unit_size;
         chunk->contig_hint = pcpu_unit_size;
 
@@ -634,6 +757,50 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk)
         pcpu_mem_free(chunk, pcpu_chunk_struct_size);
 }
 
+/**
+ * pcpu_chunk_populated - post-population bookkeeping
+ * @chunk: pcpu_chunk which got populated
+ * @page_start: the start page
+ * @page_end: the end page
+ *
+ * Pages in [@page_start,@page_end) have been populated to @chunk.  Update
+ * the bookkeeping information accordingly.  Must be called after each
+ * successful population.
+ */
+static void pcpu_chunk_populated(struct pcpu_chunk *chunk,
+                                 int page_start, int page_end)
+{
+        int nr = page_end - page_start;
+
+        lockdep_assert_held(&pcpu_lock);
+
+        bitmap_set(chunk->populated, page_start, nr);
+        chunk->nr_populated += nr;
+        pcpu_nr_empty_pop_pages += nr;
+}
+
+/**
+ * pcpu_chunk_depopulated - post-depopulation bookkeeping
+ * @chunk: pcpu_chunk which got depopulated
+ * @page_start: the start page
+ * @page_end: the end page
+ *
+ * Pages in [@page_start,@page_end) have been depopulated from @chunk.
+ * Update the bookkeeping information accordingly.  Must be called after
+ * each successful depopulation.
+ */
+static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,
+                                   int page_start, int page_end)
+{
+        int nr = page_end - page_start;
+
+        lockdep_assert_held(&pcpu_lock);
+
+        bitmap_clear(chunk->populated, page_start, nr);
+        chunk->nr_populated -= nr;
+        pcpu_nr_empty_pop_pages -= nr;
+}
+
 /*
  * Chunk management implementation.
  *
@@ -695,21 +862,23 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
  * @size: size of area to allocate in bytes
  * @align: alignment of area (max PAGE_SIZE)
  * @reserved: allocate from the reserved chunk if available
+ * @gfp: allocation flags
  *
- * Allocate percpu area of @size bytes aligned at @align.
- *
- * CONTEXT:
- * Does GFP_KERNEL allocation.
+ * Allocate percpu area of @size bytes aligned at @align.  If @gfp doesn't
+ * contain %GFP_KERNEL, the allocation is atomic.
  *
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
+static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
+                                 gfp_t gfp)
 {
         static int warn_limit = 10;
         struct pcpu_chunk *chunk;
         const char *err;
-        int slot, off, new_alloc;
+        bool is_atomic = (gfp & GFP_KERNEL) != GFP_KERNEL;
+        int occ_pages = 0;
+        int slot, off, new_alloc, cpu, ret;
         unsigned long flags;
         void __percpu *ptr;
 
@@ -728,7 +897,6 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
                 return NULL;
         }
 
-        mutex_lock(&pcpu_alloc_mutex);
         spin_lock_irqsave(&pcpu_lock, flags);
 
         /* serve reserved allocations from the reserved chunk if available */
@@ -740,16 +908,18 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
                         goto fail_unlock;
                 }
 
-                while ((new_alloc = pcpu_need_to_extend(chunk))) {
+                while ((new_alloc = pcpu_need_to_extend(chunk, is_atomic))) {
                         spin_unlock_irqrestore(&pcpu_lock, flags);
-                        if (pcpu_extend_area_map(chunk, new_alloc) < 0) {
+                        if (is_atomic ||
+                            pcpu_extend_area_map(chunk, new_alloc) < 0) {
                                 err = "failed to extend area map of reserved chunk";
-                                goto fail_unlock_mutex;
+                                goto fail;
                         }
                         spin_lock_irqsave(&pcpu_lock, flags);
                 }
 
-                off = pcpu_alloc_area(chunk, size, align);
+                off = pcpu_alloc_area(chunk, size, align, is_atomic,
+                                      &occ_pages);
                 if (off >= 0)
                         goto area_found;
 
@@ -764,13 +934,15 @@ restart:
                         if (size > chunk->contig_hint)
                                 continue;
 
-                        new_alloc = pcpu_need_to_extend(chunk);
+                        new_alloc = pcpu_need_to_extend(chunk, is_atomic);
                         if (new_alloc) {
+                                if (is_atomic)
+                                        continue;
                                 spin_unlock_irqrestore(&pcpu_lock, flags);
                                 if (pcpu_extend_area_map(chunk,
                                                          new_alloc) < 0) {
                                         err = "failed to extend area map";
-                                        goto fail_unlock_mutex;
+                                        goto fail;
                                 }
                                 spin_lock_irqsave(&pcpu_lock, flags);
                                 /*
@@ -780,74 +952,134 @@ restart:
                                 goto restart;
                         }
 
-                        off = pcpu_alloc_area(chunk, size, align);
+                        off = pcpu_alloc_area(chunk, size, align, is_atomic,
+                                              &occ_pages);
                         if (off >= 0)
                                 goto area_found;
                 }
         }
 
-        /* hmmm... no space left, create a new chunk */
         spin_unlock_irqrestore(&pcpu_lock, flags);
 
-        chunk = pcpu_create_chunk();
-        if (!chunk) {
-                err = "failed to allocate new chunk";
-                goto fail_unlock_mutex;
+        /*
+         * No space left.  Create a new chunk.  We don't want multiple
+         * tasks to create chunks simultaneously.  Serialize and create iff
+         * there's still no empty chunk after grabbing the mutex.
+         */
+        if (is_atomic)
+                goto fail;
+
+        mutex_lock(&pcpu_alloc_mutex);
+
+        if (list_empty(&pcpu_slot[pcpu_nr_slots - 1])) {
+                chunk = pcpu_create_chunk();
+                if (!chunk) {
+                        mutex_unlock(&pcpu_alloc_mutex);
+                        err = "failed to allocate new chunk";
+                        goto fail;
+                }
+
+                spin_lock_irqsave(&pcpu_lock, flags);
+                pcpu_chunk_relocate(chunk, -1);
+        } else {
+                spin_lock_irqsave(&pcpu_lock, flags);
         }
 
-        spin_lock_irqsave(&pcpu_lock, flags);
-        pcpu_chunk_relocate(chunk, -1);
+        mutex_unlock(&pcpu_alloc_mutex);
         goto restart;
 
 area_found:
         spin_unlock_irqrestore(&pcpu_lock, flags);
 
-        /* populate, map and clear the area */
-        if (pcpu_populate_chunk(chunk, off, size)) {
-                spin_lock_irqsave(&pcpu_lock, flags);
-                pcpu_free_area(chunk, off);
-                err = "failed to populate";
-                goto fail_unlock;
+        /* populate if not all pages are already there */
+        if (!is_atomic) {
+                int page_start, page_end, rs, re;
+
+                mutex_lock(&pcpu_alloc_mutex);
+
+                page_start = PFN_DOWN(off);
+                page_end = PFN_UP(off + size);
+
+                pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
+                        WARN_ON(chunk->immutable);
+
+                        ret = pcpu_populate_chunk(chunk, rs, re);
+
+                        spin_lock_irqsave(&pcpu_lock, flags);
+                        if (ret) {
+                                mutex_unlock(&pcpu_alloc_mutex);
+                                pcpu_free_area(chunk, off, &occ_pages);
+                                err = "failed to populate";
+                                goto fail_unlock;
+                        }
+                        pcpu_chunk_populated(chunk, rs, re);
+                        spin_unlock_irqrestore(&pcpu_lock, flags);
+                }
+
+                mutex_unlock(&pcpu_alloc_mutex);
         }
 
-        mutex_unlock(&pcpu_alloc_mutex);
+        if (chunk != pcpu_reserved_chunk)
+                pcpu_nr_empty_pop_pages -= occ_pages;
+
+        if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW)
+                pcpu_schedule_balance_work();
+
+        /* clear the areas and return address relative to base address */
+        for_each_possible_cpu(cpu)
+                memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
 
-        /* return address relative to base address */
         ptr = __addr_to_pcpu_ptr(chunk->base_addr + off);
         kmemleak_alloc_percpu(ptr, size);
         return ptr;
 
 fail_unlock:
         spin_unlock_irqrestore(&pcpu_lock, flags);
-fail_unlock_mutex:
-        mutex_unlock(&pcpu_alloc_mutex);
-        if (warn_limit) {
-                pr_warning("PERCPU: allocation failed, size=%zu align=%zu, "
-                           "%s\n", size, align, err);
+fail:
+        if (!is_atomic && warn_limit) {
+                pr_warning("PERCPU: allocation failed, size=%zu align=%zu atomic=%d, %s\n",
+                           size, align, is_atomic, err);
                 dump_stack();
                 if (!--warn_limit)
                         pr_info("PERCPU: limit reached, disable warning\n");
         }
+        if (is_atomic) {
+                /* see the flag handling in pcpu_blance_workfn() */
+                pcpu_atomic_alloc_failed = true;
+                pcpu_schedule_balance_work();
+        }
         return NULL;
 }
 
 /**
- * __alloc_percpu - allocate dynamic percpu area
+ * __alloc_percpu_gfp - allocate dynamic percpu area
  * @size: size of area to allocate in bytes
  * @align: alignment of area (max PAGE_SIZE)
+ * @gfp: allocation flags
  *
- * Allocate zero-filled percpu area of @size bytes aligned at @align.
- * Might sleep.  Might trigger writeouts.
- *
- * CONTEXT:
- * Does GFP_KERNEL allocation.
+ * Allocate zero-filled percpu area of @size bytes aligned at @align.  If
+ * @gfp doesn't contain %GFP_KERNEL, the allocation doesn't block and can
+ * be called from any context but is a lot more likely to fail.
  *
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
+void __percpu *__alloc_percpu_gfp(size_t size, size_t align, gfp_t gfp)
+{
+        return pcpu_alloc(size, align, false, gfp);
+}
+EXPORT_SYMBOL_GPL(__alloc_percpu_gfp);
+
+/**
+ * __alloc_percpu - allocate dynamic percpu area
+ * @size: size of area to allocate in bytes
+ * @align: alignment of area (max PAGE_SIZE)
+ *
+ * Equivalent to __alloc_percpu_gfp(size, align, %GFP_KERNEL).
+ */
 void __percpu *__alloc_percpu(size_t size, size_t align)
 {
-        return pcpu_alloc(size, align, false);
+        return pcpu_alloc(size, align, false, GFP_KERNEL);
 }
 EXPORT_SYMBOL_GPL(__alloc_percpu);
 
@@ -869,44 +1101,121 @@ EXPORT_SYMBOL_GPL(__alloc_percpu);
  */
 void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
 {
-        return pcpu_alloc(size, align, true);
+        return pcpu_alloc(size, align, true, GFP_KERNEL);
 }
 
 /**
- * pcpu_reclaim - reclaim fully free chunks, workqueue function
+ * pcpu_balance_workfn - manage the amount of free chunks and populated pages
  * @work: unused
  *
  * Reclaim all fully free chunks except for the first one.
- *
- * CONTEXT:
- * workqueue context.
  */
-static void pcpu_reclaim(struct work_struct *work)
+static void pcpu_balance_workfn(struct work_struct *work)
 {
-        LIST_HEAD(todo);
-        struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1];
+        LIST_HEAD(to_free);
+        struct list_head *free_head = &pcpu_slot[pcpu_nr_slots - 1];
         struct pcpu_chunk *chunk, *next;
+        int slot, nr_to_pop, ret;
 
+        /*
+         * There's no reason to keep around multiple unused chunks and VM
+         * areas can be scarce.  Destroy all free chunks except for one.
+         */
         mutex_lock(&pcpu_alloc_mutex);
         spin_lock_irq(&pcpu_lock);
 
-        list_for_each_entry_safe(chunk, next, head, list) {
+        list_for_each_entry_safe(chunk, next, free_head, list) {
                 WARN_ON(chunk->immutable);
 
                 /* spare the first one */
-                if (chunk == list_first_entry(head, struct pcpu_chunk, list))
+                if (chunk == list_first_entry(free_head, struct pcpu_chunk, list))
                         continue;
 
-                list_move(&chunk->list, &todo);
+                list_move(&chunk->list, &to_free);
         }
 
         spin_unlock_irq(&pcpu_lock);
 
-        list_for_each_entry_safe(chunk, next, &todo, list) {
-                pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size);
+        list_for_each_entry_safe(chunk, next, &to_free, list) {
+                int rs, re;
+
+                pcpu_for_each_pop_region(chunk, rs, re, 0, pcpu_unit_pages) {
+                        pcpu_depopulate_chunk(chunk, rs, re);
+                        spin_lock_irq(&pcpu_lock);
+                        pcpu_chunk_depopulated(chunk, rs, re);
+                        spin_unlock_irq(&pcpu_lock);
+                }
                 pcpu_destroy_chunk(chunk);
         }
 
+        /*
+         * Ensure there are certain number of free populated pages for
+         * atomic allocs.  Fill up from the most packed so that atomic
+         * allocs don't increase fragmentation.  If atomic allocation
+         * failed previously, always populate the maximum amount.  This
+         * should prevent atomic allocs larger than PAGE_SIZE from keeping
+         * failing indefinitely; however, large atomic allocs are not
+         * something we support properly and can be highly unreliable and
+         * inefficient.
+         */
+retry_pop:
+        if (pcpu_atomic_alloc_failed) {
+                nr_to_pop = PCPU_EMPTY_POP_PAGES_HIGH;
+                /* best effort anyway, don't worry about synchronization */
+                pcpu_atomic_alloc_failed = false;
+        } else {
+                nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -
+                                  pcpu_nr_empty_pop_pages,
+                                  0, PCPU_EMPTY_POP_PAGES_HIGH);
+        }
+
+        for (slot = pcpu_size_to_slot(PAGE_SIZE); slot < pcpu_nr_slots; slot++) {
+                int nr_unpop = 0, rs, re;
+
+                if (!nr_to_pop)
+                        break;
+
+                spin_lock_irq(&pcpu_lock);
+                list_for_each_entry(chunk, &pcpu_slot[slot], list) {
+                        nr_unpop = pcpu_unit_pages - chunk->nr_populated;
+                        if (nr_unpop)
+                                break;
+                }
+                spin_unlock_irq(&pcpu_lock);
+
+                if (!nr_unpop)
+                        continue;
+
+                /* @chunk can't go away while pcpu_alloc_mutex is held */
+                pcpu_for_each_unpop_region(chunk, rs, re, 0, pcpu_unit_pages) {
+                        int nr = min(re - rs, nr_to_pop);
+
+                        ret = pcpu_populate_chunk(chunk, rs, rs + nr);
+                        if (!ret) {
+                                nr_to_pop -= nr;
+                                spin_lock_irq(&pcpu_lock);
+                                pcpu_chunk_populated(chunk, rs, rs + nr);
+                                spin_unlock_irq(&pcpu_lock);
+                        } else {
+                                nr_to_pop = 0;
+                        }
+
+                        if (!nr_to_pop)
+                                break;
+                }
+        }
+
+        if (nr_to_pop) {
+                /* ran out of chunks to populate, create a new one and retry */
+                chunk = pcpu_create_chunk();
+                if (chunk) {
+                        spin_lock_irq(&pcpu_lock);
+                        pcpu_chunk_relocate(chunk, -1);
+                        spin_unlock_irq(&pcpu_lock);
+                        goto retry_pop;
+                }
+        }
+
         mutex_unlock(&pcpu_alloc_mutex);
 }
 
@@ -924,7 +1233,7 @@ void free_percpu(void __percpu *ptr)
         void *addr;
         struct pcpu_chunk *chunk;
         unsigned long flags;
-        int off;
+        int off, occ_pages;
 
         if (!ptr)
                 return;
@@ -938,7 +1247,10 @@ void free_percpu(void __percpu *ptr)
         chunk = pcpu_chunk_addr_search(addr);
         off = addr - chunk->base_addr;
 
-        pcpu_free_area(chunk, off);
+        pcpu_free_area(chunk, off, &occ_pages);
+
+        if (chunk != pcpu_reserved_chunk)
+                pcpu_nr_empty_pop_pages += occ_pages;
 
         /* if there are more than one fully free chunks, wake up grim reaper */
         if (chunk->free_size == pcpu_unit_size) {
@@ -946,7 +1258,7 @@ void free_percpu(void __percpu *ptr)
 
                 list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list)
                         if (pos != chunk) {
-                                schedule_work(&pcpu_reclaim_work);
+                                pcpu_schedule_balance_work();
                                 break;
                         }
         }
@@ -1336,11 +1648,13 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
          */
         schunk = memblock_virt_alloc(pcpu_chunk_struct_size, 0);
         INIT_LIST_HEAD(&schunk->list);
+        INIT_WORK(&schunk->map_extend_work, pcpu_map_extend_workfn);
         schunk->base_addr = base_addr;
         schunk->map = smap;
         schunk->map_alloc = ARRAY_SIZE(smap);
         schunk->immutable = true;
         bitmap_fill(schunk->populated, pcpu_unit_pages);
+        schunk->nr_populated = pcpu_unit_pages;
 
         if (ai->reserved_size) {
                 schunk->free_size = ai->reserved_size;
@@ -1364,11 +1678,13 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
         if (dyn_size) {
                 dchunk = memblock_virt_alloc(pcpu_chunk_struct_size, 0);
                 INIT_LIST_HEAD(&dchunk->list);
+                INIT_WORK(&dchunk->map_extend_work, pcpu_map_extend_workfn);
                 dchunk->base_addr = base_addr;
                 dchunk->map = dmap;
                 dchunk->map_alloc = ARRAY_SIZE(dmap);
                 dchunk->immutable = true;
                 bitmap_fill(dchunk->populated, pcpu_unit_pages);
+                dchunk->nr_populated = pcpu_unit_pages;
 
                 dchunk->contig_hint = dchunk->free_size = dyn_size;
                 dchunk->map[0] = 1;
@@ -1379,6 +1695,8 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
 
         /* link the first chunk in */
         pcpu_first_chunk = dchunk ?: schunk;
+        pcpu_nr_empty_pop_pages +=
+                pcpu_count_occupied_pages(pcpu_first_chunk, 1);
         pcpu_chunk_relocate(pcpu_first_chunk, -1);
 
         /* we're done */
@@ -1932,8 +2250,6 @@ void __init setup_per_cpu_areas(void)
 
         if (pcpu_setup_first_chunk(ai, fc) < 0)
                 panic("Failed to initialize percpu areas.");
-
-        pcpu_free_alloc_info(ai);
 }
 
 #endif  /* CONFIG_SMP */
@@ -1967,3 +2283,15 @@ void __init percpu_init_late(void)
                 spin_unlock_irqrestore(&pcpu_lock, flags);
         }
 }
+
+/*
+ * Percpu allocator is initialized early during boot when neither slab or
+ * workqueue is available.  Plug async management until everything is up
+ * and running.
+ */
+static int __init percpu_enable_async(void)
+{
+        pcpu_async_enabled = true;
+        return 0;
+}
+subsys_initcall(percpu_enable_async);