kmemtrace: SLAB hooks.

This adds hooks for the SLAB allocator, to allow tracing with kmemtrace.

We also convert some inline functions to __always_inline to make sure
_RET_IP_, which expands to __builtin_return_address(0), always works
as expected.

Signed-off-by: Eduard - Gabriel Munteanu <eduard.munteanu@linux360.ro>
Signed-off-by: Pekka Enberg <penberg@cs.helsinki.fi>

1 files changed, 59 insertions, 9 deletions

diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h
index 39c3a5eb8ebe..7555ce99f6d2 100644
--- a/include/linux/slab_def.h
+++ b/include/linux/slab_def.h
@@ -14,6 +14,7 @@
 #include <asm/page.h>           /* kmalloc_sizes.h needs PAGE_SIZE */
 #include <asm/cache.h>          /* kmalloc_sizes.h needs L1_CACHE_BYTES */
 #include <linux/compiler.h>
+#include <linux/kmemtrace.h>
 
 /* Size description struct for general caches. */
 struct cache_sizes {
@@ -28,8 +29,26 @@ extern struct cache_sizes malloc_sizes[];
 void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
 void *__kmalloc(size_t size, gfp_t flags);
 
-static inline void *kmalloc(size_t size, gfp_t flags)
+#ifdef CONFIG_KMEMTRACE
+extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags);
+extern size_t slab_buffer_size(struct kmem_cache *cachep);
+#else
+static __always_inline void *
+kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
 {
+        return kmem_cache_alloc(cachep, flags);
+}
+static inline size_t slab_buffer_size(struct kmem_cache *cachep)
+{
+        return 0;
+}
+#endif
+
+static __always_inline void *kmalloc(size_t size, gfp_t flags)
+{
+        struct kmem_cache *cachep;
+        void *ret;
+
         if (__builtin_constant_p(size)) {
                 int i = 0;
 
@@ -50,10 +69,17 @@ static inline void *kmalloc(size_t size, gfp_t flags)
 found:
 #ifdef CONFIG_ZONE_DMA
                 if (flags & GFP_DMA)
-                        return kmem_cache_alloc(malloc_sizes[i].cs_dmacachep,
-                                                flags);
+                        cachep = malloc_sizes[i].cs_dmacachep;
+                else
 #endif
-                return kmem_cache_alloc(malloc_sizes[i].cs_cachep, flags);
+                        cachep = malloc_sizes[i].cs_cachep;
+
+                ret = kmem_cache_alloc_notrace(cachep, flags);
+
+                kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, ret,
+                                     size, slab_buffer_size(cachep), flags);
+
+                return ret;
         }
         return __kmalloc(size, flags);
 }
@@ -62,8 +88,25 @@ found:
 extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
 extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
 
-static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
+#ifdef CONFIG_KMEMTRACE
+extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
+                                           gfp_t flags,
+                                           int nodeid);
+#else
+static __always_inline void *
+kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
+                              gfp_t flags,
+                              int nodeid)
+{
+        return kmem_cache_alloc_node(cachep, flags, nodeid);
+}
+#endif
+
+static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 {
+        struct kmem_cache *cachep;
+        void *ret;
+
         if (__builtin_constant_p(size)) {
                 int i = 0;
 
@@ -84,11 +127,18 @@ static inline void *kmalloc_node(size_t size, gfp_t flags, int node)
 found:
 #ifdef CONFIG_ZONE_DMA
                 if (flags & GFP_DMA)
-                        return kmem_cache_alloc_node(malloc_sizes[i].cs_dmacachep,
-                                                flags, node);
+                        cachep = malloc_sizes[i].cs_dmacachep;
+                else
 #endif
-                return kmem_cache_alloc_node(malloc_sizes[i].cs_cachep,
-                                                flags, node);
+                        cachep = malloc_sizes[i].cs_cachep;
+
+                ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
+
+                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_,
+                                          ret, size, slab_buffer_size(cachep),
+                                          flags, node);
+
+                return ret;
         }
         return __kmalloc_node(size, flags, node);
 }