Merge branch 'for-3.18' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu

Pull percpu updates from Tejun Heo:
 "A lot of activities on percpu front.  Notable changes are...

   - percpu allocator now can take @gfp.  If @gfp doesn't contain
     GFP_KERNEL, it tries to allocate from what's already available to
     the allocator and a work item tries to keep the reserve around
     certain level so that these atomic allocations usually succeed.

     This will replace the ad-hoc percpu memory pool used by
     blk-throttle and also be used by the planned blkcg support for
     writeback IOs.

     Please note that I noticed a bug in how @gfp is interpreted while
     preparing this pull request and applied the fix 6ae833c7fe0c
     ("percpu: fix how @gfp is interpreted by the percpu allocator")
     just now.

   - percpu_ref now uses longs for percpu and global counters instead of
     ints.  It leads to more sparse packing of the percpu counters on
     64bit machines but the overhead should be negligible and this
     allows using percpu_ref for refcnting pages and in-memory objects
     directly.

   - The switching between percpu and single counter modes of a
     percpu_ref is made independent of putting the base ref and a
     percpu_ref can now optionally be initialized in single or killed
     mode.  This allows avoiding percpu shutdown latency for cases where
     the refcounted objects may be synchronously created and destroyed
     in rapid succession with only a fraction of them reaching fully
     operational status (SCSI probing does this when combined with
     blk-mq support).  It's also planned to be used to implement forced
     single mode to detect underflow more timely for debugging.

  There's a separate branch percpu/for-3.18-consistent-ops which cleans
  up the duplicate percpu accessors.  That branch causes a number of
  conflicts with s390 and other trees.  I'll send a separate pull
  request w/ resolutions once other branches are merged"

* 'for-3.18' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu: (33 commits)
  percpu: fix how @gfp is interpreted by the percpu allocator
  blk-mq, percpu_ref: start q->mq_usage_counter in atomic mode
  percpu_ref: make INIT_ATOMIC and switch_to_atomic() sticky
  percpu_ref: add PERCPU_REF_INIT_* flags
  percpu_ref: decouple switching to percpu mode and reinit
  percpu_ref: decouple switching to atomic mode and killing
  percpu_ref: add PCPU_REF_DEAD
  percpu_ref: rename things to prepare for decoupling percpu/atomic mode switch
  percpu_ref: replace pcpu_ prefix with percpu_
  percpu_ref: minor code and comment updates
  percpu_ref: relocate percpu_ref_reinit()
  Revert "blk-mq, percpu_ref: implement a kludge for SCSI blk-mq stall during probe"
  Revert "percpu: free percpu allocation info for uniprocessor system"
  percpu-refcount: make percpu_ref based on longs instead of ints
  percpu-refcount: improve WARN messages
  percpu: fix locking regression in the failure path of pcpu_alloc()
  percpu-refcount: add @gfp to percpu_ref_init()
  proportions: add @gfp to init functions
  percpu_counter: add @gfp to percpu_counter_init()
  percpu_counter: make percpu_counters_lock irq-safe
  ...

1 files changed, 427 insertions, 99 deletions

diff --git a/mm/percpu.c b/mm/percpu.c
index da997f9800bd..014bab65e0ff 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -76,6 +76,10 @@
 
 #define PCPU_SLOT_BASE_SHIFT            5       /* 1-31 shares the same slot */
 #define PCPU_DFL_MAP_ALLOC              16      /* start a map with 16 ents */
+#define PCPU_ATOMIC_MAP_MARGIN_LOW      32
+#define PCPU_ATOMIC_MAP_MARGIN_HIGH     64
+#define PCPU_EMPTY_POP_PAGES_LOW        2
+#define PCPU_EMPTY_POP_PAGES_HIGH       4
 
 #ifdef CONFIG_SMP
 /* default addr <-> pcpu_ptr mapping, override in asm/percpu.h if necessary */
@@ -102,12 +106,16 @@ struct pcpu_chunk {
         int                     free_size;      /* free bytes in the chunk */
         int                     contig_hint;    /* max contiguous size hint */
         void                    *base_addr;     /* base address of this chunk */
+
         int                     map_used;       /* # of map entries used before the sentry */
         int                     map_alloc;      /* # of map entries allocated */
         int                     *map;           /* allocation map */
+        struct work_struct      map_extend_work;/* async ->map[] extension */
+
         void                    *data;          /* chunk data */
         int                     first_free;     /* no free below this */
         bool                    immutable;      /* no [de]population allowed */
+        int                     nr_populated;   /* # of populated pages */
         unsigned long           populated[];    /* populated bitmap */
 };
 
@@ -151,38 +159,33 @@ static struct pcpu_chunk *pcpu_first_chunk;
 static struct pcpu_chunk *pcpu_reserved_chunk;
 static int pcpu_reserved_chunk_limit;
 
+static DEFINE_SPINLOCK(pcpu_lock);      /* all internal data structures */
+static DEFINE_MUTEX(pcpu_alloc_mutex);  /* chunk create/destroy, [de]pop */
+
+static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
+
 /*
- * Synchronization rules.
- *
- * There are two locks - pcpu_alloc_mutex and pcpu_lock.  The former
- * protects allocation/reclaim paths, chunks, populated bitmap and
- * vmalloc mapping.  The latter is a spinlock and protects the index
- * data structures - chunk slots, chunks and area maps in chunks.
- *
- * During allocation, pcpu_alloc_mutex is kept locked all the time and
- * pcpu_lock is grabbed and released as necessary.  All actual memory
- * allocations are done using GFP_KERNEL with pcpu_lock released.  In
- * general, percpu memory can't be allocated with irq off but
- * irqsave/restore are still used in alloc path so that it can be used
- * from early init path - sched_init() specifically.
- *
- * Free path accesses and alters only the index data structures, so it
- * can be safely called from atomic context.  When memory needs to be
- * returned to the system, free path schedules reclaim_work which
- * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be
- * reclaimed, release both locks and frees the chunks.  Note that it's
- * necessary to grab both locks to remove a chunk from circulation as
- * allocation path might be referencing the chunk with only
- * pcpu_alloc_mutex locked.
+ * The number of empty populated pages, protected by pcpu_lock.  The
+ * reserved chunk doesn't contribute to the count.
  */
-static DEFINE_MUTEX(pcpu_alloc_mutex);  /* protects whole alloc and reclaim */
-static DEFINE_SPINLOCK(pcpu_lock);      /* protects index data structures */
+static int pcpu_nr_empty_pop_pages;
 
-static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
+/*
+ * Balance work is used to populate or destroy chunks asynchronously.  We
+ * try to keep the number of populated free pages between
+ * PCPU_EMPTY_POP_PAGES_LOW and HIGH for atomic allocations and at most one
+ * empty chunk.
+ */
+static void pcpu_balance_workfn(struct work_struct *work);
+static DECLARE_WORK(pcpu_balance_work, pcpu_balance_workfn);
+static bool pcpu_async_enabled __read_mostly;
+static bool pcpu_atomic_alloc_failed;
 
-/* reclaim work to release fully free chunks, scheduled from free path */
-static void pcpu_reclaim(struct work_struct *work);
-static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
+static void pcpu_schedule_balance_work(void)
+{
+        if (pcpu_async_enabled)
+                schedule_work(&pcpu_balance_work);
+}
 
 static bool pcpu_addr_in_first_chunk(void *addr)
 {
@@ -315,6 +318,38 @@ static void pcpu_mem_free(void *ptr, size_t size)
 }
 
 /**
+ * pcpu_count_occupied_pages - count the number of pages an area occupies
+ * @chunk: chunk of interest
+ * @i: index of the area in question
+ *
+ * Count the number of pages chunk's @i'th area occupies.  When the area's
+ * start and/or end address isn't aligned to page boundary, the straddled
+ * page is included in the count iff the rest of the page is free.
+ */
+static int pcpu_count_occupied_pages(struct pcpu_chunk *chunk, int i)
+{
+        int off = chunk->map[i] & ~1;
+        int end = chunk->map[i + 1] & ~1;
+
+        if (!PAGE_ALIGNED(off) && i > 0) {
+                int prev = chunk->map[i - 1];
+
+                if (!(prev & 1) && prev <= round_down(off, PAGE_SIZE))
+                        off = round_down(off, PAGE_SIZE);
+        }
+
+        if (!PAGE_ALIGNED(end) && i + 1 < chunk->map_used) {
+                int next = chunk->map[i + 1];
+                int nend = chunk->map[i + 2] & ~1;
+
+                if (!(next & 1) && nend >= round_up(end, PAGE_SIZE))
+                        end = round_up(end, PAGE_SIZE);
+        }
+
+        return max_t(int, PFN_DOWN(end) - PFN_UP(off), 0);
+}
+
+/**
  * pcpu_chunk_relocate - put chunk in the appropriate chunk slot
  * @chunk: chunk of interest
  * @oslot: the previous slot it was on
@@ -342,9 +377,14 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
 /**
  * pcpu_need_to_extend - determine whether chunk area map needs to be extended
  * @chunk: chunk of interest
+ * @is_atomic: the allocation context
  *
- * Determine whether area map of @chunk needs to be extended to
- * accommodate a new allocation.
+ * Determine whether area map of @chunk needs to be extended.  If
+ * @is_atomic, only the amount necessary for a new allocation is
+ * considered; however, async extension is scheduled if the left amount is
+ * low.  If !@is_atomic, it aims for more empty space.  Combined, this
+ * ensures that the map is likely to have enough available space to
+ * accomodate atomic allocations which can't extend maps directly.
  *
  * CONTEXT:
  * pcpu_lock.
@@ -353,15 +393,26 @@ static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
  * New target map allocation length if extension is necessary, 0
  * otherwise.
  */
-static int pcpu_need_to_extend(struct pcpu_chunk *chunk)
+static int pcpu_need_to_extend(struct pcpu_chunk *chunk, bool is_atomic)
 {
-        int new_alloc;
+        int margin, new_alloc;
+
+        if (is_atomic) {
+                margin = 3;
+
+                if (chunk->map_alloc <
+                    chunk->map_used + PCPU_ATOMIC_MAP_MARGIN_LOW &&
+                    pcpu_async_enabled)
+                        schedule_work(&chunk->map_extend_work);
+        } else {
+                margin = PCPU_ATOMIC_MAP_MARGIN_HIGH;
+        }
 
-        if (chunk->map_alloc >= chunk->map_used + 3)
+        if (chunk->map_alloc >= chunk->map_used + margin)
                 return 0;
 
         new_alloc = PCPU_DFL_MAP_ALLOC;
-        while (new_alloc < chunk->map_used + 3)
+        while (new_alloc < chunk->map_used + margin)
                 new_alloc *= 2;
 
         return new_alloc;
@@ -418,11 +469,76 @@ out_unlock:
         return 0;
 }
 
+static void pcpu_map_extend_workfn(struct work_struct *work)
+{
+        struct pcpu_chunk *chunk = container_of(work, struct pcpu_chunk,
+                                                map_extend_work);
+        int new_alloc;
+
+        spin_lock_irq(&pcpu_lock);
+        new_alloc = pcpu_need_to_extend(chunk, false);
+        spin_unlock_irq(&pcpu_lock);
+
+        if (new_alloc)
+                pcpu_extend_area_map(chunk, new_alloc);
+}
+
+/**
+ * pcpu_fit_in_area - try to fit the requested allocation in a candidate area
+ * @chunk: chunk the candidate area belongs to
+ * @off: the offset to the start of the candidate area
+ * @this_size: the size of the candidate area
+ * @size: the size of the target allocation
+ * @align: the alignment of the target allocation
+ * @pop_only: only allocate from already populated region
+ *
+ * We're trying to allocate @size bytes aligned at @align.  @chunk's area
+ * at @off sized @this_size is a candidate.  This function determines
+ * whether the target allocation fits in the candidate area and returns the
+ * number of bytes to pad after @off.  If the target area doesn't fit, -1
+ * is returned.
+ *
+ * If @pop_only is %true, this function only considers the already
+ * populated part of the candidate area.
+ */
+static int pcpu_fit_in_area(struct pcpu_chunk *chunk, int off, int this_size,
+                            int size, int align, bool pop_only)
+{
+        int cand_off = off;
+
+        while (true) {
+                int head = ALIGN(cand_off, align) - off;
+                int page_start, page_end, rs, re;
+
+                if (this_size < head + size)
+                        return -1;
+
+                if (!pop_only)
+                        return head;
+
+                /*
+                 * If the first unpopulated page is beyond the end of the
+                 * allocation, the whole allocation is populated;
+                 * otherwise, retry from the end of the unpopulated area.
+                 */
+                page_start = PFN_DOWN(head + off);
+                page_end = PFN_UP(head + off + size);
+
+                rs = page_start;
+                pcpu_next_unpop(chunk, &rs, &re, PFN_UP(off + this_size));
+                if (rs >= page_end)
+                        return head;
+                cand_off = re * PAGE_SIZE;
+        }
+}
+
 /**
  * pcpu_alloc_area - allocate area from a pcpu_chunk
  * @chunk: chunk of interest
  * @size: wanted size in bytes
  * @align: wanted align
+ * @pop_only: allocate only from the populated area
+ * @occ_pages_p: out param for the number of pages the area occupies
  *
  * Try to allocate @size bytes area aligned at @align from @chunk.
  * Note that this function only allocates the offset.  It doesn't
@@ -437,7 +553,8 @@ out_unlock:
  * Allocated offset in @chunk on success, -1 if no matching area is
  * found.
  */
-static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
+static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align,
+                           bool pop_only, int *occ_pages_p)
 {
         int oslot = pcpu_chunk_slot(chunk);
         int max_contig = 0;
@@ -453,11 +570,11 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
                 if (off & 1)
                         continue;
 
-                /* extra for alignment requirement */
-                head = ALIGN(off, align) - off;
-
                 this_size = (p[1] & ~1) - off;
-                if (this_size < head + size) {
+
+                head = pcpu_fit_in_area(chunk, off, this_size, size, align,
+                                        pop_only);
+                if (head < 0) {
                         if (!seen_free) {
                                 chunk->first_free = i;
                                 seen_free = true;
@@ -526,6 +643,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
                 chunk->free_size -= size;
                 *p |= 1;
 
+                *occ_pages_p = pcpu_count_occupied_pages(chunk, i);
                 pcpu_chunk_relocate(chunk, oslot);
                 return off;
         }
@@ -541,6 +659,7 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
  * pcpu_free_area - free area to a pcpu_chunk
  * @chunk: chunk of interest
  * @freeme: offset of area to free
+ * @occ_pages_p: out param for the number of pages the area occupies
  *
  * Free area starting from @freeme to @chunk.  Note that this function
  * only modifies the allocation map.  It doesn't depopulate or unmap
@@ -549,7 +668,8 @@ static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
  * CONTEXT:
  * pcpu_lock.
  */
-static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
+static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme,
+                           int *occ_pages_p)
 {
         int oslot = pcpu_chunk_slot(chunk);
         int off = 0;
@@ -580,6 +700,8 @@ static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
         *p = off &= ~1;
         chunk->free_size += (p[1] & ~1) - off;
 
+        *occ_pages_p = pcpu_count_occupied_pages(chunk, i);
+
         /* merge with next? */
         if (!(p[1] & 1))
                 to_free++;
@@ -620,6 +742,7 @@ static struct pcpu_chunk *pcpu_alloc_chunk(void)
         chunk->map_used = 1;
 
         INIT_LIST_HEAD(&chunk->list);
+        INIT_WORK(&chunk->map_extend_work, pcpu_map_extend_workfn);
         chunk->free_size = pcpu_unit_size;
         chunk->contig_hint = pcpu_unit_size;
 
@@ -634,6 +757,50 @@ static void pcpu_free_chunk(struct pcpu_chunk *chunk)
         pcpu_mem_free(chunk, pcpu_chunk_struct_size);
 }
 
+/**
+ * pcpu_chunk_populated - post-population bookkeeping
+ * @chunk: pcpu_chunk which got populated
+ * @page_start: the start page
+ * @page_end: the end page
+ *
+ * Pages in [@page_start,@page_end) have been populated to @chunk.  Update
+ * the bookkeeping information accordingly.  Must be called after each
+ * successful population.
+ */
+static void pcpu_chunk_populated(struct pcpu_chunk *chunk,
+                                 int page_start, int page_end)
+{
+        int nr = page_end - page_start;
+
+        lockdep_assert_held(&pcpu_lock);
+
+        bitmap_set(chunk->populated, page_start, nr);
+        chunk->nr_populated += nr;
+        pcpu_nr_empty_pop_pages += nr;
+}
+
+/**
+ * pcpu_chunk_depopulated - post-depopulation bookkeeping
+ * @chunk: pcpu_chunk which got depopulated
+ * @page_start: the start page
+ * @page_end: the end page
+ *
+ * Pages in [@page_start,@page_end) have been depopulated from @chunk.
+ * Update the bookkeeping information accordingly.  Must be called after
+ * each successful depopulation.
+ */
+static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,
+                                   int page_start, int page_end)
+{
+        int nr = page_end - page_start;
+
+        lockdep_assert_held(&pcpu_lock);
+
+        bitmap_clear(chunk->populated, page_start, nr);
+        chunk->nr_populated -= nr;
+        pcpu_nr_empty_pop_pages -= nr;
+}
+
 /*
  * Chunk management implementation.
  *
@@ -695,21 +862,23 @@ static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
  * @size: size of area to allocate in bytes
  * @align: alignment of area (max PAGE_SIZE)
  * @reserved: allocate from the reserved chunk if available
+ * @gfp: allocation flags
  *
- * Allocate percpu area of @size bytes aligned at @align.
- *
- * CONTEXT:
- * Does GFP_KERNEL allocation.
+ * Allocate percpu area of @size bytes aligned at @align.  If @gfp doesn't
+ * contain %GFP_KERNEL, the allocation is atomic.
  *
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
-static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
+static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved,
+                                 gfp_t gfp)
 {
         static int warn_limit = 10;
         struct pcpu_chunk *chunk;
         const char *err;
-        int slot, off, new_alloc;
+        bool is_atomic = (gfp & GFP_KERNEL) != GFP_KERNEL;
+        int occ_pages = 0;
+        int slot, off, new_alloc, cpu, ret;
         unsigned long flags;
         void __percpu *ptr;
 
@@ -728,7 +897,6 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
                 return NULL;
         }
 
-        mutex_lock(&pcpu_alloc_mutex);
         spin_lock_irqsave(&pcpu_lock, flags);
 
         /* serve reserved allocations from the reserved chunk if available */
@@ -740,16 +908,18 @@ static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
                         goto fail_unlock;
                 }
 
-                while ((new_alloc = pcpu_need_to_extend(chunk))) {
+                while ((new_alloc = pcpu_need_to_extend(chunk, is_atomic))) {
                         spin_unlock_irqrestore(&pcpu_lock, flags);
-                        if (pcpu_extend_area_map(chunk, new_alloc) < 0) {
+                        if (is_atomic ||
+                            pcpu_extend_area_map(chunk, new_alloc) < 0) {
                                 err = "failed to extend area map of reserved chunk";
-                                goto fail_unlock_mutex;
+                                goto fail;
                         }
                         spin_lock_irqsave(&pcpu_lock, flags);
                 }
 
-                off = pcpu_alloc_area(chunk, size, align);
+                off = pcpu_alloc_area(chunk, size, align, is_atomic,
+                                      &occ_pages);
                 if (off >= 0)
                         goto area_found;
 
@@ -764,13 +934,15 @@ restart:
                         if (size > chunk->contig_hint)
                                 continue;
 
-                        new_alloc = pcpu_need_to_extend(chunk);
+                        new_alloc = pcpu_need_to_extend(chunk, is_atomic);
                         if (new_alloc) {
+                                if (is_atomic)
+                                        continue;
                                 spin_unlock_irqrestore(&pcpu_lock, flags);
                                 if (pcpu_extend_area_map(chunk,
                                                          new_alloc) < 0) {
                                         err = "failed to extend area map";
-                                        goto fail_unlock_mutex;
+                                        goto fail;
                                 }
                                 spin_lock_irqsave(&pcpu_lock, flags);
                                 /*
@@ -780,74 +952,134 @@ restart:
                                 goto restart;
                         }
 
-                        off = pcpu_alloc_area(chunk, size, align);
+                        off = pcpu_alloc_area(chunk, size, align, is_atomic,
+                                              &occ_pages);
                         if (off >= 0)
                                 goto area_found;
                 }
         }
 
-        /* hmmm... no space left, create a new chunk */
         spin_unlock_irqrestore(&pcpu_lock, flags);
 
-        chunk = pcpu_create_chunk();
-        if (!chunk) {
-                err = "failed to allocate new chunk";
-                goto fail_unlock_mutex;
+        /*
+         * No space left.  Create a new chunk.  We don't want multiple
+         * tasks to create chunks simultaneously.  Serialize and create iff
+         * there's still no empty chunk after grabbing the mutex.
+         */
+        if (is_atomic)
+                goto fail;
+
+        mutex_lock(&pcpu_alloc_mutex);
+
+        if (list_empty(&pcpu_slot[pcpu_nr_slots - 1])) {
+                chunk = pcpu_create_chunk();
+                if (!chunk) {
+                        mutex_unlock(&pcpu_alloc_mutex);
+                        err = "failed to allocate new chunk";
+                        goto fail;
+                }
+
+                spin_lock_irqsave(&pcpu_lock, flags);
+                pcpu_chunk_relocate(chunk, -1);
+        } else {
+                spin_lock_irqsave(&pcpu_lock, flags);
         }
 
-        spin_lock_irqsave(&pcpu_lock, flags);
-        pcpu_chunk_relocate(chunk, -1);
+        mutex_unlock(&pcpu_alloc_mutex);
         goto restart;
 
 area_found:
         spin_unlock_irqrestore(&pcpu_lock, flags);
 
-        /* populate, map and clear the area */
-        if (pcpu_populate_chunk(chunk, off, size)) {
-                spin_lock_irqsave(&pcpu_lock, flags);
-                pcpu_free_area(chunk, off);
-                err = "failed to populate";
-                goto fail_unlock;
+        /* populate if not all pages are already there */
+        if (!is_atomic) {
+                int page_start, page_end, rs, re;
+
+                mutex_lock(&pcpu_alloc_mutex);
+
+                page_start = PFN_DOWN(off);
+                page_end = PFN_UP(off + size);
+
+                pcpu_for_each_unpop_region(chunk, rs, re, page_start, page_end) {
+                        WARN_ON(chunk->immutable);
+
+                        ret = pcpu_populate_chunk(chunk, rs, re);
+
+                        spin_lock_irqsave(&pcpu_lock, flags);
+                        if (ret) {
+                                mutex_unlock(&pcpu_alloc_mutex);
+                                pcpu_free_area(chunk, off, &occ_pages);
+                                err = "failed to populate";
+                                goto fail_unlock;
+                        }
+                        pcpu_chunk_populated(chunk, rs, re);
+                        spin_unlock_irqrestore(&pcpu_lock, flags);
+                }
+
+                mutex_unlock(&pcpu_alloc_mutex);
         }
 
-        mutex_unlock(&pcpu_alloc_mutex);
+        if (chunk != pcpu_reserved_chunk)
+                pcpu_nr_empty_pop_pages -= occ_pages;
+
+        if (pcpu_nr_empty_pop_pages < PCPU_EMPTY_POP_PAGES_LOW)
+                pcpu_schedule_balance_work();
+
+        /* clear the areas and return address relative to base address */
+        for_each_possible_cpu(cpu)
+                memset((void *)pcpu_chunk_addr(chunk, cpu, 0) + off, 0, size);
 
-        /* return address relative to base address */
         ptr = __addr_to_pcpu_ptr(chunk->base_addr + off);
         kmemleak_alloc_percpu(ptr, size);
         return ptr;
 
 fail_unlock:
         spin_unlock_irqrestore(&pcpu_lock, flags);
-fail_unlock_mutex:
-        mutex_unlock(&pcpu_alloc_mutex);
-        if (warn_limit) {
-                pr_warning("PERCPU: allocation failed, size=%zu align=%zu, "
-                           "%s\n", size, align, err);
+fail:
+        if (!is_atomic && warn_limit) {
+                pr_warning("PERCPU: allocation failed, size=%zu align=%zu atomic=%d, %s\n",
+                           size, align, is_atomic, err);
                 dump_stack();
                 if (!--warn_limit)
                         pr_info("PERCPU: limit reached, disable warning\n");
         }
+        if (is_atomic) {
+                /* see the flag handling in pcpu_blance_workfn() */
+                pcpu_atomic_alloc_failed = true;
+                pcpu_schedule_balance_work();
+        }
         return NULL;
 }
 
 /**
- * __alloc_percpu - allocate dynamic percpu area
+ * __alloc_percpu_gfp - allocate dynamic percpu area
  * @size: size of area to allocate in bytes
  * @align: alignment of area (max PAGE_SIZE)
+ * @gfp: allocation flags
  *
- * Allocate zero-filled percpu area of @size bytes aligned at @align.
- * Might sleep.  Might trigger writeouts.
- *
- * CONTEXT:
- * Does GFP_KERNEL allocation.
+ * Allocate zero-filled percpu area of @size bytes aligned at @align.  If
+ * @gfp doesn't contain %GFP_KERNEL, the allocation doesn't block and can
+ * be called from any context but is a lot more likely to fail.
  *
  * RETURNS:
  * Percpu pointer to the allocated area on success, NULL on failure.
  */
+void __percpu *__alloc_percpu_gfp(size_t size, size_t align, gfp_t gfp)
+{
+        return pcpu_alloc(size, align, false, gfp);
+}
+EXPORT_SYMBOL_GPL(__alloc_percpu_gfp);
+
+/**
+ * __alloc_percpu - allocate dynamic percpu area
+ * @size: size of area to allocate in bytes
+ * @align: alignment of area (max PAGE_SIZE)
+ *
+ * Equivalent to __alloc_percpu_gfp(size, align, %GFP_KERNEL).
+ */
 void __percpu *__alloc_percpu(size_t size, size_t align)
 {
-        return pcpu_alloc(size, align, false);
+        return pcpu_alloc(size, align, false, GFP_KERNEL);
 }
 EXPORT_SYMBOL_GPL(__alloc_percpu);
 
@@ -869,44 +1101,121 @@ EXPORT_SYMBOL_GPL(__alloc_percpu);
  */
 void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
 {
-        return pcpu_alloc(size, align, true);
+        return pcpu_alloc(size, align, true, GFP_KERNEL);
 }
 
 /**
- * pcpu_reclaim - reclaim fully free chunks, workqueue function
+ * pcpu_balance_workfn - manage the amount of free chunks and populated pages
  * @work: unused
  *
  * Reclaim all fully free chunks except for the first one.
- *
- * CONTEXT:
- * workqueue context.
  */
-static void pcpu_reclaim(struct work_struct *work)
+static void pcpu_balance_workfn(struct work_struct *work)
 {
-        LIST_HEAD(todo);
-        struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1];
+        LIST_HEAD(to_free);
+        struct list_head *free_head = &pcpu_slot[pcpu_nr_slots - 1];
         struct pcpu_chunk *chunk, *next;
+        int slot, nr_to_pop, ret;
 
+        /*
+         * There's no reason to keep around multiple unused chunks and VM
+         * areas can be scarce.  Destroy all free chunks except for one.
+         */
         mutex_lock(&pcpu_alloc_mutex);
         spin_lock_irq(&pcpu_lock);
 
-        list_for_each_entry_safe(chunk, next, head, list) {
+        list_for_each_entry_safe(chunk, next, free_head, list) {
                 WARN_ON(chunk->immutable);
 
                 /* spare the first one */
-                if (chunk == list_first_entry(head, struct pcpu_chunk, list))
+                if (chunk == list_first_entry(free_head, struct pcpu_chunk, list))
                         continue;
 
-                list_move(&chunk->list, &todo);
+                list_move(&chunk->list, &to_free);
         }
 
         spin_unlock_irq(&pcpu_lock);
 
-        list_for_each_entry_safe(chunk, next, &todo, list) {
-                pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size);
+        list_for_each_entry_safe(chunk, next, &to_free, list) {
+                int rs, re;
+
+                pcpu_for_each_pop_region(chunk, rs, re, 0, pcpu_unit_pages) {
+                        pcpu_depopulate_chunk(chunk, rs, re);
+                        spin_lock_irq(&pcpu_lock);
+                        pcpu_chunk_depopulated(chunk, rs, re);
+                        spin_unlock_irq(&pcpu_lock);
+                }
                 pcpu_destroy_chunk(chunk);
         }
 
+        /*
+         * Ensure there are certain number of free populated pages for
+         * atomic allocs.  Fill up from the most packed so that atomic
+         * allocs don't increase fragmentation.  If atomic allocation
+         * failed previously, always populate the maximum amount.  This
+         * should prevent atomic allocs larger than PAGE_SIZE from keeping
+         * failing indefinitely; however, large atomic allocs are not
+         * something we support properly and can be highly unreliable and
+         * inefficient.
+         */
+retry_pop:
+        if (pcpu_atomic_alloc_failed) {
+                nr_to_pop = PCPU_EMPTY_POP_PAGES_HIGH;
+                /* best effort anyway, don't worry about synchronization */
+                pcpu_atomic_alloc_failed = false;
+        } else {
+                nr_to_pop = clamp(PCPU_EMPTY_POP_PAGES_HIGH -
+                                  pcpu_nr_empty_pop_pages,
+                                  0, PCPU_EMPTY_POP_PAGES_HIGH);
+        }
+
+        for (slot = pcpu_size_to_slot(PAGE_SIZE); slot < pcpu_nr_slots; slot++) {
+                int nr_unpop = 0, rs, re;
+
+                if (!nr_to_pop)
+                        break;
+
+                spin_lock_irq(&pcpu_lock);
+                list_for_each_entry(chunk, &pcpu_slot[slot], list) {
+                        nr_unpop = pcpu_unit_pages - chunk->nr_populated;
+                        if (nr_unpop)
+                                break;
+                }
+                spin_unlock_irq(&pcpu_lock);
+
+                if (!nr_unpop)
+                        continue;
+
+                /* @chunk can't go away while pcpu_alloc_mutex is held */
+                pcpu_for_each_unpop_region(chunk, rs, re, 0, pcpu_unit_pages) {
+                        int nr = min(re - rs, nr_to_pop);
+
+                        ret = pcpu_populate_chunk(chunk, rs, rs + nr);
+                        if (!ret) {
+                                nr_to_pop -= nr;
+                                spin_lock_irq(&pcpu_lock);
+                                pcpu_chunk_populated(chunk, rs, rs + nr);
+                                spin_unlock_irq(&pcpu_lock);
+                        } else {
+                                nr_to_pop = 0;
+                        }
+
+                        if (!nr_to_pop)
+                                break;
+                }
+        }
+
+        if (nr_to_pop) {
+                /* ran out of chunks to populate, create a new one and retry */
+                chunk = pcpu_create_chunk();
+                if (chunk) {
+                        spin_lock_irq(&pcpu_lock);
+                        pcpu_chunk_relocate(chunk, -1);
+                        spin_unlock_irq(&pcpu_lock);
+                        goto retry_pop;
+                }
+        }
+
         mutex_unlock(&pcpu_alloc_mutex);
 }
 
@@ -924,7 +1233,7 @@ void free_percpu(void __percpu *ptr)
         void *addr;
         struct pcpu_chunk *chunk;
         unsigned long flags;
-        int off;
+        int off, occ_pages;
 
         if (!ptr)
                 return;
@@ -938,7 +1247,10 @@ void free_percpu(void __percpu *ptr)
         chunk = pcpu_chunk_addr_search(addr);
         off = addr - chunk->base_addr;
 
-        pcpu_free_area(chunk, off);
+        pcpu_free_area(chunk, off, &occ_pages);
+
+        if (chunk != pcpu_reserved_chunk)
+                pcpu_nr_empty_pop_pages += occ_pages;
 
         /* if there are more than one fully free chunks, wake up grim reaper */
         if (chunk->free_size == pcpu_unit_size) {
@@ -946,7 +1258,7 @@ void free_percpu(void __percpu *ptr)
 
                 list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list)
                         if (pos != chunk) {
-                                schedule_work(&pcpu_reclaim_work);
+                                pcpu_schedule_balance_work();
                                 break;
                         }
         }
@@ -1336,11 +1648,13 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
          */
         schunk = memblock_virt_alloc(pcpu_chunk_struct_size, 0);
         INIT_LIST_HEAD(&schunk->list);
+        INIT_WORK(&schunk->map_extend_work, pcpu_map_extend_workfn);
         schunk->base_addr = base_addr;
         schunk->map = smap;
         schunk->map_alloc = ARRAY_SIZE(smap);
         schunk->immutable = true;
         bitmap_fill(schunk->populated, pcpu_unit_pages);
+        schunk->nr_populated = pcpu_unit_pages;
 
         if (ai->reserved_size) {
                 schunk->free_size = ai->reserved_size;
@@ -1364,11 +1678,13 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
         if (dyn_size) {
                 dchunk = memblock_virt_alloc(pcpu_chunk_struct_size, 0);
                 INIT_LIST_HEAD(&dchunk->list);
+                INIT_WORK(&dchunk->map_extend_work, pcpu_map_extend_workfn);
                 dchunk->base_addr = base_addr;
                 dchunk->map = dmap;
                 dchunk->map_alloc = ARRAY_SIZE(dmap);
                 dchunk->immutable = true;
                 bitmap_fill(dchunk->populated, pcpu_unit_pages);
+                dchunk->nr_populated = pcpu_unit_pages;
 
                 dchunk->contig_hint = dchunk->free_size = dyn_size;
                 dchunk->map[0] = 1;
@@ -1379,6 +1695,8 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
 
         /* link the first chunk in */
         pcpu_first_chunk = dchunk ?: schunk;
+        pcpu_nr_empty_pop_pages +=
+                pcpu_count_occupied_pages(pcpu_first_chunk, 1);
         pcpu_chunk_relocate(pcpu_first_chunk, -1);
 
         /* we're done */
@@ -1932,8 +2250,6 @@ void __init setup_per_cpu_areas(void)
 
         if (pcpu_setup_first_chunk(ai, fc) < 0)
                 panic("Failed to initialize percpu areas.");
-
-        pcpu_free_alloc_info(ai);
 }
 
 #endif  /* CONFIG_SMP */
@@ -1967,3 +2283,15 @@ void __init percpu_init_late(void)
                 spin_unlock_irqrestore(&pcpu_lock, flags);
         }
 }
+
+/*
+ * Percpu allocator is initialized early during boot when neither slab or
+ * workqueue is available.  Plug async management until everything is up
+ * and running.
+ */
+static int __init percpu_enable_async(void)
+{
+        pcpu_async_enabled = true;
+        return 0;
+}
+subsys_initcall(percpu_enable_async);