11 files changed, 191 insertions, 33 deletions

diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
index 139d5b7b6621..dfdee6a47359 100644
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -31,7 +31,7 @@ static void percpu_depopulate(void *__pdata, int cpu)
  * @__pdata: per-cpu data to depopulate
  * @mask: depopulate per-cpu data for cpu's selected through mask bits
  */
-static void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
+static void __percpu_depopulate_mask(void *__pdata, const cpumask_t *mask)
 {
         int cpu;
         for_each_cpu_mask_nr(cpu, *mask)
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index be68c956a660..493b468a5035 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -284,12 +284,12 @@ static wait_queue_head_t congestion_wqh[2] = {
         };
 
 
-void clear_bdi_congested(struct backing_dev_info *bdi, int rw)
+void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
         enum bdi_state bit;
-        wait_queue_head_t *wqh = &congestion_wqh[rw];
+        wait_queue_head_t *wqh = &congestion_wqh[sync];
 
-        bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
+        bit = sync ? BDI_sync_congested : BDI_async_congested;
         clear_bit(bit, &bdi->state);
         smp_mb__after_clear_bit();
         if (waitqueue_active(wqh))
@@ -297,11 +297,11 @@ void clear_bdi_congested(struct backing_dev_info *bdi, int rw)
 }
 EXPORT_SYMBOL(clear_bdi_congested);
 
-void set_bdi_congested(struct backing_dev_info *bdi, int rw)
+void set_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
         enum bdi_state bit;
 
-        bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
+        bit = sync ? BDI_sync_congested : BDI_async_congested;
         set_bit(bit, &bdi->state);
 }
 EXPORT_SYMBOL(set_bdi_congested);
diff --git a/mm/failslab.c b/mm/failslab.c
index 7c6ea6493f80..9339de5f0a91 100644
--- a/mm/failslab.c
+++ b/mm/failslab.c
@@ -1,4 +1,5 @@
 #include <linux/fault-inject.h>
+#include <linux/gfp.h>
 
 static struct {
         struct fault_attr attr;
diff --git a/mm/filemap.c b/mm/filemap.c
index fc11974f2bee..2e2d38ebda4b 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -513,6 +513,7 @@ int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
         }
         return ret;
 }
+EXPORT_SYMBOL_GPL(add_to_page_cache_lru);
 
 #ifdef CONFIG_NUMA
 struct page *__page_cache_alloc(gfp_t gfp)
@@ -645,6 +646,7 @@ int __lock_page_killable(struct page *page)
         return __wait_on_bit_lock(page_waitqueue(page), &wait,
                                         sync_page_killable, TASK_KILLABLE);
 }
+EXPORT_SYMBOL_GPL(__lock_page_killable);
 
 /**
  * __lock_page_nosync - get a lock on the page, without calling sync_page()
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 3f30189896fd..e2f26991fff1 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2128,7 +2128,7 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
         int n, val;
         int min_val = INT_MAX;
         int best_node = -1;
-        node_to_cpumask_ptr(tmp, 0);
+        const struct cpumask *tmp = cpumask_of_node(0);
 
         /* Use the local node if we haven't already */
         if (!node_isset(node, *used_node_mask)) {
@@ -2149,8 +2149,8 @@ static int find_next_best_node(int node, nodemask_t *used_node_mask)
                 val += (n < node);
 
                 /* Give preference to headless and unused nodes */
-                node_to_cpumask_ptr_next(tmp, n);
-                if (!cpus_empty(*tmp))
+                tmp = cpumask_of_node(n);
+                if (!cpumask_empty(tmp))
                         val += PENALTY_FOR_NODE_WITH_CPUS;
 
                 /* Slight preference for less loaded node */
diff --git a/mm/quicklist.c b/mm/quicklist.c
index 8dbb6805ef35..e66d07d1b4ff 100644
--- a/mm/quicklist.c
+++ b/mm/quicklist.c
@@ -29,7 +29,7 @@ static unsigned long max_pages(unsigned long min_pages)
         int node = numa_node_id();
         struct zone *zones = NODE_DATA(node)->node_zones;
         int num_cpus_on_node;
-        node_to_cpumask_ptr(cpumask_on_node, node);
+        const struct cpumask *cpumask_on_node = cpumask_of_node(node);
 
         node_free_pages =
 #ifdef CONFIG_ZONE_DMA
diff --git a/mm/slab.c b/mm/slab.c
index 208323fd37bc..9a90b00d2f91 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -102,6 +102,7 @@
 #include        <linux/cpu.h>
 #include        <linux/sysctl.h>
 #include        <linux/module.h>
+#include        <trace/kmemtrace.h>
 #include        <linux/rcupdate.h>
 #include        <linux/string.h>
 #include        <linux/uaccess.h>
@@ -568,6 +569,14 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp)
 
 #endif
 
+#ifdef CONFIG_KMEMTRACE
+size_t slab_buffer_size(struct kmem_cache *cachep)
+{
+        return cachep->buffer_size;
+}
+EXPORT_SYMBOL(slab_buffer_size);
+#endif
+
 /*
  * Do not go above this order unless 0 objects fit into the slab.
  */
@@ -1160,7 +1169,7 @@ static void __cpuinit cpuup_canceled(long cpu)
         struct kmem_cache *cachep;
         struct kmem_list3 *l3 = NULL;
         int node = cpu_to_node(cpu);
-        node_to_cpumask_ptr(mask, node);
+        const struct cpumask *mask = cpumask_of_node(node);
 
         list_for_each_entry(cachep, &cache_chain, next) {
                 struct array_cache *nc;
@@ -3554,10 +3563,23 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
  */
 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
-        return __cache_alloc(cachep, flags, __builtin_return_address(0));
+        void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
+
+        trace_kmem_cache_alloc(_RET_IP_, ret,
+                               obj_size(cachep), cachep->buffer_size, flags);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
+{
+        return __cache_alloc(cachep, flags, __builtin_return_address(0));
+}
+EXPORT_SYMBOL(kmem_cache_alloc_notrace);
+#endif
+
 /**
  * kmem_ptr_validate - check if an untrusted pointer might be a slab entry.
  * @cachep: the cache we're checking against
@@ -3602,23 +3624,46 @@ out:
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 {
-        return __cache_alloc_node(cachep, flags, nodeid,
-                        __builtin_return_address(0));
+        void *ret = __cache_alloc_node(cachep, flags, nodeid,
+                                       __builtin_return_address(0));
+
+        trace_kmem_cache_alloc_node(_RET_IP_, ret,
+                                    obj_size(cachep), cachep->buffer_size,
+                                    flags, nodeid);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
+                                    gfp_t flags,
+                                    int nodeid)
+{
+        return __cache_alloc_node(cachep, flags, nodeid,
+                                  __builtin_return_address(0));
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
+#endif
+
 static __always_inline void *
 __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
 {
         struct kmem_cache *cachep;
+        void *ret;
 
         cachep = kmem_find_general_cachep(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
-        return kmem_cache_alloc_node(cachep, flags, node);
+        ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
+
+        trace_kmalloc_node((unsigned long) caller, ret,
+                           size, cachep->buffer_size, flags, node);
+
+        return ret;
 }
 
-#ifdef CONFIG_DEBUG_SLAB
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         return __do_kmalloc_node(size, flags, node,
@@ -3651,6 +3696,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
                                           void *caller)
 {
         struct kmem_cache *cachep;
+        void *ret;
 
         /* If you want to save a few bytes .text space: replace
          * __ with kmem_.
@@ -3660,11 +3706,16 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
         cachep = __find_general_cachep(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
-        return __cache_alloc(cachep, flags, caller);
+        ret = __cache_alloc(cachep, flags, caller);
+
+        trace_kmalloc((unsigned long) caller, ret,
+                      size, cachep->buffer_size, flags);
+
+        return ret;
 }
 
 
-#ifdef CONFIG_DEBUG_SLAB
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         return __do_kmalloc(size, flags, __builtin_return_address(0));
@@ -3703,6 +3754,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
                 debug_check_no_obj_freed(objp, obj_size(cachep));
         __cache_free(cachep, objp);
         local_irq_restore(flags);
+
+        trace_kmem_cache_free(_RET_IP_, objp);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -3720,6 +3773,8 @@ void kfree(const void *objp)
         struct kmem_cache *c;
         unsigned long flags;
 
+        trace_kfree(_RET_IP_, objp);
+
         if (unlikely(ZERO_OR_NULL_PTR(objp)))
                 return;
         local_irq_save(flags);
diff --git a/mm/slob.c b/mm/slob.c
index 7a3411524dac..a2d4ab32198d 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -65,6 +65,7 @@
 #include <linux/module.h>
 #include <linux/rcupdate.h>
 #include <linux/list.h>
+#include <trace/kmemtrace.h>
 #include <asm/atomic.h>
 
 /*
@@ -474,6 +475,7 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
 {
         unsigned int *m;
         int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
+        void *ret;
 
         lockdep_trace_alloc(gfp);
 
@@ -482,12 +484,16 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
                         return ZERO_SIZE_PTR;
 
                 m = slob_alloc(size + align, gfp, align, node);
+
                 if (!m)
                         return NULL;
                 *m = size;
-                return (void *)m + align;
+                ret = (void *)m + align;
+
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, size + align, gfp, node);
         } else {
-                void *ret;
+                unsigned int order = get_order(size);
 
                 ret = slob_new_pages(gfp | __GFP_COMP, get_order(size), node);
                 if (ret) {
@@ -495,8 +501,12 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
                         page = virt_to_page(ret);
                         page->private = size;
                 }
-                return ret;
+
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, PAGE_SIZE << order, gfp, node);
         }
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node);
 
@@ -504,6 +514,8 @@ void kfree(const void *block)
 {
         struct slob_page *sp;
 
+        trace_kfree(_RET_IP_, block);
+
         if (unlikely(ZERO_OR_NULL_PTR(block)))
                 return;
 
@@ -583,10 +595,17 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 {
         void *b;
 
-        if (c->size < PAGE_SIZE)
+        if (c->size < PAGE_SIZE) {
                 b = slob_alloc(c->size, flags, c->align, node);
-        else
+                trace_kmem_cache_alloc_node(_RET_IP_, b, c->size,
+                                            SLOB_UNITS(c->size) * SLOB_UNIT,
+                                            flags, node);
+        } else {
                 b = slob_new_pages(flags, get_order(c->size), node);
+                trace_kmem_cache_alloc_node(_RET_IP_, b, c->size,
+                                            PAGE_SIZE << get_order(c->size),
+                                            flags, node);
+        }
 
         if (c->ctor)
                 c->ctor(b);
@@ -622,6 +641,8 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
         } else {
                 __kmem_cache_free(b, c->size);
         }
+
+        trace_kmem_cache_free(_RET_IP_, b);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
diff --git a/mm/slub.c b/mm/slub.c
index c4ea9158c9fb..7ab54ecbd3f3 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -16,6 +16,7 @@
 #include <linux/slab.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
+#include <trace/kmemtrace.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/mempolicy.h>
@@ -1618,18 +1619,45 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
 
 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
 {
-        return slab_alloc(s, gfpflags, -1, _RET_IP_);
+        void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
+
+        trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
+{
+        return slab_alloc(s, gfpflags, -1, _RET_IP_);
+}
+EXPORT_SYMBOL(kmem_cache_alloc_notrace);
+#endif
+
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
 {
-        return slab_alloc(s, gfpflags, node, _RET_IP_);
+        void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
+
+        trace_kmem_cache_alloc_node(_RET_IP_, ret,
+                                    s->objsize, s->size, gfpflags, node);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 #endif
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
+                                    gfp_t gfpflags,
+                                    int node)
+{
+        return slab_alloc(s, gfpflags, node, _RET_IP_);
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
+#endif
+
 /*
  * Slow patch handling. This may still be called frequently since objects
  * have a longer lifetime than the cpu slabs in most processing loads.
@@ -1737,6 +1765,8 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
         page = virt_to_head_page(x);
 
         slab_free(s, page, x, _RET_IP_);
+
+        trace_kmem_cache_free(_RET_IP_, x);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -2659,6 +2689,7 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, flags);
@@ -2668,7 +2699,11 @@ void *__kmalloc(size_t size, gfp_t flags)
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, flags, -1, _RET_IP_);
+        ret = slab_alloc(s, flags, -1, _RET_IP_);
+
+        trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc);
 
@@ -2687,16 +2722,28 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         struct kmem_cache *s;
+        void *ret;
 
-        if (unlikely(size > SLUB_MAX_SIZE))
-                return kmalloc_large_node(size, flags, node);
+        if (unlikely(size > SLUB_MAX_SIZE)) {
+                ret = kmalloc_large_node(size, flags, node);
+
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, PAGE_SIZE << get_order(size),
+                                   flags, node);
+
+                return ret;
+        }
 
         s = get_slab(size, flags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, flags, node, _RET_IP_);
+        ret = slab_alloc(s, flags, node, _RET_IP_);
+
+        trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node);
 #endif
@@ -2745,6 +2792,8 @@ void kfree(const void *x)
         struct page *page;
         void *object = (void *)x;
 
+        trace_kfree(_RET_IP_, x);
+
         if (unlikely(ZERO_OR_NULL_PTR(x)))
                 return;
 
@@ -3224,6 +3273,7 @@ static struct notifier_block __cpuinitdata slab_notifier = {
 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, gfpflags);
@@ -3233,13 +3283,19 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, gfpflags, -1, caller);
+        ret = slab_alloc(s, gfpflags, -1, caller);
+
+        /* Honor the call site pointer we recieved. */
+        trace_kmalloc(caller, ret, size, s->size, gfpflags);
+
+        return ret;
 }
 
 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
                                         int node, unsigned long caller)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large_node(size, gfpflags, node);
@@ -3249,7 +3305,12 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, gfpflags, node, caller);
+        ret = slab_alloc(s, gfpflags, node, caller);
+
+        /* Honor the call site pointer we recieved. */
+        trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
+
+        return ret;
 }
 
 #ifdef CONFIG_SLUB_DEBUG
diff --git a/mm/util.c b/mm/util.c
index 7c122e49f769..2599e83eea17 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -4,6 +4,7 @@
 #include <linux/module.h>
 #include <linux/err.h>
 #include <linux/sched.h>
+#include <linux/tracepoint.h>
 #include <asm/uaccess.h>
 
 /**
@@ -236,3 +237,18 @@ int __attribute__((weak)) get_user_pages_fast(unsigned long start,
         return ret;
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);
+
+/* Tracepoints definitions. */
+DEFINE_TRACE(kmalloc);
+DEFINE_TRACE(kmem_cache_alloc);
+DEFINE_TRACE(kmalloc_node);
+DEFINE_TRACE(kmem_cache_alloc_node);
+DEFINE_TRACE(kfree);
+DEFINE_TRACE(kmem_cache_free);
+
+EXPORT_TRACEPOINT_SYMBOL(kmalloc);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
+EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
+EXPORT_TRACEPOINT_SYMBOL(kfree);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 425244988bb2..39fdfb14eeaa 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1967,7 +1967,7 @@ static int kswapd(void *p)
         struct reclaim_state reclaim_state = {
                 .reclaimed_slab = 0,
         };
-        node_to_cpumask_ptr(cpumask, pgdat->node_id);
+        const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
 
         lockdep_set_current_reclaim_state(GFP_KERNEL);
 
@@ -2204,7 +2204,9 @@ static int __devinit cpu_callback(struct notifier_block *nfb,
         if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
                 for_each_node_state(nid, N_HIGH_MEMORY) {
                         pg_data_t *pgdat = NODE_DATA(nid);
-                        node_to_cpumask_ptr(mask, pgdat->node_id);
+                        const struct cpumask *mask;
+
+                        mask = cpumask_of_node(pgdat->node_id);
 
                         if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
                                 /* One of our CPUs online: restore mask */