1 files changed, 51 insertions, 4 deletions

diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index b645686ef9b6..45e0dc0e09f8 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -365,6 +365,13 @@ static LIST_HEAD(free_vmap_area_list);
  */
 static struct rb_root free_vmap_area_root = RB_ROOT;
 
+/*
+ * Preload a CPU with one object for "no edge" split case. The
+ * aim is to get rid of allocations from the atomic context, thus
+ * to use more permissive allocation masks.
+ */
+static DEFINE_PER_CPU(struct vmap_area *, ne_fit_preload_node);
+
 static __always_inline unsigned long
 va_size(struct vmap_area *va)
 {
@@ -951,9 +958,24 @@ adjust_va_to_fit_type(struct vmap_area *va,
                  *   L V  NVA  V R
                  * |---|-------|---|
                  */
-                lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
-                if (unlikely(!lva))
-                        return -1;
+                lva = __this_cpu_xchg(ne_fit_preload_node, NULL);
+                if (unlikely(!lva)) {
+                        /*
+                         * For percpu allocator we do not do any pre-allocation
+                         * and leave it as it is. The reason is it most likely
+                         * never ends up with NE_FIT_TYPE splitting. In case of
+                         * percpu allocations offsets and sizes are aligned to
+                         * fixed align request, i.e. RE_FIT_TYPE and FL_FIT_TYPE
+                         * are its main fitting cases.
+                         *
+                         * There are a few exceptions though, as an example it is
+                         * a first allocation (early boot up) when we have "one"
+                         * big free space that has to be split.
+                         */
+                        lva = kmem_cache_alloc(vmap_area_cachep, GFP_NOWAIT);
+                        if (!lva)
+                                return -1;
+                }
 
                 /*
                  * Build the remainder.
@@ -1032,7 +1054,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
                                 unsigned long vstart, unsigned long vend,
                                 int node, gfp_t gfp_mask)
 {
-        struct vmap_area *va;
+        struct vmap_area *va, *pva;
         unsigned long addr;
         int purged = 0;
 
@@ -1057,7 +1079,32 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
         kmemleak_scan_area(&va->rb_node, SIZE_MAX, gfp_mask & GFP_RECLAIM_MASK);
 
 retry:
+        /*
+         * Preload this CPU with one extra vmap_area object to ensure
+         * that we have it available when fit type of free area is
+         * NE_FIT_TYPE.
+         *
+         * The preload is done in non-atomic context, thus it allows us
+         * to use more permissive allocation masks to be more stable under
+         * low memory condition and high memory pressure.
+         *
+         * Even if it fails we do not really care about that. Just proceed
+         * as it is. "overflow" path will refill the cache we allocate from.
+         */
+        preempt_disable();
+        if (!__this_cpu_read(ne_fit_preload_node)) {
+                preempt_enable();
+                pva = kmem_cache_alloc_node(vmap_area_cachep, GFP_KERNEL, node);
+                preempt_disable();
+
+                if (__this_cpu_cmpxchg(ne_fit_preload_node, NULL, pva)) {
+                        if (pva)
+                                kmem_cache_free(vmap_area_cachep, pva);
+                }
+        }
+
         spin_lock(&vmap_area_lock);
+        preempt_enable();
 
         /*
          * If an allocation fails, the "vend" address is