1 files changed, 39 insertions, 43 deletions

diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h
index 9f63538928c0..c8668d161dd8 100644
--- a/include/linux/slub_def.h
+++ b/include/linux/slub_def.h
@@ -10,9 +10,8 @@
 #include <linux/gfp.h>
 #include <linux/workqueue.h>
 #include <linux/kobject.h>
-#include <linux/kmemleak.h>
 
-#include <trace/events/kmem.h>
+#include <linux/kmemleak.h>
 
 enum stat_item {
         ALLOC_FASTPATH,         /* Allocation from cpu slab */
@@ -33,10 +32,12 @@ enum stat_item {
         DEACTIVATE_TO_TAIL,     /* Cpu slab was moved to the tail of partials */
         DEACTIVATE_REMOTE_FREES,/* Slab contained remotely freed objects */
         ORDER_FALLBACK,         /* Number of times fallback was necessary */
+        CMPXCHG_DOUBLE_CPU_FAIL,/* Failure of this_cpu_cmpxchg_double */
         NR_SLUB_STAT_ITEMS };
 
 struct kmem_cache_cpu {
-        void **freelist;        /* Pointer to first free per cpu object */
+        void **freelist;        /* Pointer to next available object */
+        unsigned long tid;      /* Globally unique transaction id */
         struct page *page;      /* The slab from which we are allocating */
         int node;               /* The node of the page (or -1 for debug) */
 #ifdef CONFIG_SLUB_STATS
@@ -71,6 +72,7 @@ struct kmem_cache {
         struct kmem_cache_cpu __percpu *cpu_slab;
         /* Used for retriving partial slabs etc */
         unsigned long flags;
+        unsigned long min_partial;
         int size;               /* The size of an object including meta data */
         int objsize;            /* The size of an object without meta data */
         int offset;             /* Free pointer offset. */
@@ -84,10 +86,10 @@ struct kmem_cache {
         void (*ctor)(void *);
         int inuse;              /* Offset to metadata */
         int align;              /* Alignment */
-        unsigned long min_partial;
+        int reserved;           /* Reserved bytes at the end of slabs */
         const char *name;       /* Name (only for display!) */
         struct list_head list;  /* List of slab caches */
-#ifdef CONFIG_SLUB_DEBUG
+#ifdef CONFIG_SYSFS
         struct kobject kobj;    /* For sysfs */
 #endif
 
@@ -96,11 +98,8 @@ struct kmem_cache {
          * Defragmentation by allocating from a remote node.
          */
         int remote_node_defrag_ratio;
-        struct kmem_cache_node *node[MAX_NUMNODES];
-#else
-        /* Avoid an extra cache line for UP */
-        struct kmem_cache_node local_node;
 #endif
+        struct kmem_cache_node *node[MAX_NUMNODES];
 };
 
 /*
@@ -139,19 +138,16 @@ struct kmem_cache {
 
 #ifdef CONFIG_ZONE_DMA
 #define SLUB_DMA __GFP_DMA
-/* Reserve extra caches for potential DMA use */
-#define KMALLOC_CACHES (2 * SLUB_PAGE_SHIFT)
 #else
 /* Disable DMA functionality */
 #define SLUB_DMA (__force gfp_t)0
-#define KMALLOC_CACHES SLUB_PAGE_SHIFT
 #endif
 
 /*
  * We keep the general caches in an array of slab caches that are used for
  * 2^x bytes of allocations.
  */
-extern struct kmem_cache kmalloc_caches[KMALLOC_CACHES];
+extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
 
 /*
  * Sorry that the following has to be that ugly but some versions of GCC
@@ -181,7 +177,8 @@ static __always_inline int kmalloc_index(size_t size)
         if (size <=   4 * 1024) return 12;
 /*
  * The following is only needed to support architectures with a larger page
- * size than 4k.
+ * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page
+ * size we would have to go up to 128k.
  */
         if (size <=   8 * 1024) return 13;
         if (size <=  16 * 1024) return 14;
@@ -192,7 +189,8 @@ static __always_inline int kmalloc_index(size_t size)
         if (size <= 512 * 1024) return 19;
         if (size <= 1024 * 1024) return 20;
         if (size <=  2 * 1024 * 1024) return 21;
-        return -1;
+        BUG();
+        return -1; /* Will never be reached */
 
 /*
  * What we really wanted to do and cannot do because of compiler issues is:
@@ -216,37 +214,46 @@ static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
         if (index == 0)
                 return NULL;
 
-        return &kmalloc_caches[index];
+        return kmalloc_caches[index];
 }
 
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 void *__kmalloc(size_t size, gfp_t flags);
 
+static __always_inline void *
+kmalloc_order(size_t size, gfp_t flags, unsigned int order)
+{
+        void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
+        kmemleak_alloc(ret, size, 1, flags);
+        return ret;
+}
+
 #ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags);
+extern void *
+kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size);
+extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);
 #else
 static __always_inline void *
-kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
+kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
 {
         return kmem_cache_alloc(s, gfpflags);
 }
+
+static __always_inline void *
+kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
+{
+        return kmalloc_order(size, flags, order);
+}
 #endif
 
 static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
 {
         unsigned int order = get_order(size);
-        void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
-
-        kmemleak_alloc(ret, size, 1, flags);
-        trace_kmalloc(_THIS_IP_, ret, size, PAGE_SIZE << order, flags);
-
-        return ret;
+        return kmalloc_order_trace(size, flags, order);
 }
 
 static __always_inline void *kmalloc(size_t size, gfp_t flags)
 {
-        void *ret;
-
         if (__builtin_constant_p(size)) {
                 if (size > SLUB_MAX_SIZE)
                         return kmalloc_large(size, flags);
@@ -257,11 +264,7 @@ static __always_inline void *kmalloc(size_t size, gfp_t flags)
                         if (!s)
                                 return ZERO_SIZE_PTR;
 
-                        ret = kmem_cache_alloc_notrace(s, flags);
-
-                        trace_kmalloc(_THIS_IP_, ret, size, s->size, flags);
-
-                        return ret;
+                        return kmem_cache_alloc_trace(s, flags, size);
                 }
         }
         return __kmalloc(size, flags);
@@ -272,14 +275,14 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node);
 void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
 
 #ifdef CONFIG_TRACING
-extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
+extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
                                            gfp_t gfpflags,
-                                           int node);
+                                           int node, size_t size);
 #else
 static __always_inline void *
-kmem_cache_alloc_node_notrace(struct kmem_cache *s,
+kmem_cache_alloc_node_trace(struct kmem_cache *s,
                               gfp_t gfpflags,
-                              int node)
+                              int node, size_t size)
 {
         return kmem_cache_alloc_node(s, gfpflags, node);
 }
@@ -287,8 +290,6 @@ kmem_cache_alloc_node_notrace(struct kmem_cache *s,
 
 static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
-        void *ret;
-
         if (__builtin_constant_p(size) &&
                 size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) {
                         struct kmem_cache *s = kmalloc_slab(size);
@@ -296,12 +297,7 @@ static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
                 if (!s)
                         return ZERO_SIZE_PTR;
 
-                ret = kmem_cache_alloc_node_notrace(s, flags, node);
-
-                trace_kmalloc_node(_THIS_IP_, ret,
-                                   size, s->size, flags, node);
-
-                return ret;
+                return kmem_cache_alloc_node_trace(s, flags, node, size);
         }
         return __kmalloc_node(size, flags, node);
 }