8 files changed, 217 insertions, 36 deletions

diff --git a/mm/mlock.c b/mm/mlock.c
index 037161d61b4e..cbe9e0581b75 100644
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -660,7 +660,7 @@ void *alloc_locked_buffer(size_t size)
         return buffer;
 }
 
-void free_locked_buffer(void *buffer, size_t size)
+void release_locked_buffer(void *buffer, size_t size)
 {
         unsigned long pgsz = PAGE_ALIGN(size) >> PAGE_SHIFT;
 
@@ -670,6 +670,11 @@ void free_locked_buffer(void *buffer, size_t size)
         current->mm->locked_vm -= pgsz;
 
         up_write(&current->mm->mmap_sem);
+}
+
+void free_locked_buffer(void *buffer, size_t size)
+{
+        release_locked_buffer(buffer, size);
 
         kfree(buffer);
 }
diff --git a/mm/page-writeback.c b/mm/page-writeback.c
index 6106a5c7ed44..74dc57c74349 100644
--- a/mm/page-writeback.c
+++ b/mm/page-writeback.c
@@ -240,7 +240,7 @@ void bdi_writeout_inc(struct backing_dev_info *bdi)
 }
 EXPORT_SYMBOL_GPL(bdi_writeout_inc);
 
-static inline void task_dirty_inc(struct task_struct *tsk)
+void task_dirty_inc(struct task_struct *tsk)
 {
         prop_inc_single(&vm_dirties, &tsk->dirties);
 }
@@ -1079,7 +1079,7 @@ continue_unlock:
                 pagevec_release(&pvec);
                 cond_resched();
         }
-        if (!cycled) {
+        if (!cycled && !done) {
                 /*
                  * range_cyclic:
                  * We hit the last page and there is more work to be done: wrap
@@ -1230,6 +1230,7 @@ int __set_page_dirty_nobuffers(struct page *page)
                                 __inc_zone_page_state(page, NR_FILE_DIRTY);
                                 __inc_bdi_stat(mapping->backing_dev_info,
                                                 BDI_RECLAIMABLE);
+                                task_dirty_inc(current);
                                 task_io_account_write(PAGE_CACHE_SIZE);
                         }
                         radix_tree_tag_set(&mapping->page_tree,
@@ -1262,7 +1263,7 @@ EXPORT_SYMBOL(redirty_page_for_writepage);
  * If the mapping doesn't provide a set_page_dirty a_op, then
  * just fall through and assume that it wants buffer_heads.
  */
-static int __set_page_dirty(struct page *page)
+int set_page_dirty(struct page *page)
 {
         struct address_space *mapping = page_mapping(page);
 
@@ -1280,14 +1281,6 @@ static int __set_page_dirty(struct page *page)
         }
         return 0;
 }
-
-int set_page_dirty(struct page *page)
-{
-        int ret = __set_page_dirty(page);
-        if (ret)
-                task_dirty_inc(current);
-        return ret;
-}
 EXPORT_SYMBOL(set_page_dirty);
 
 /*
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 5675b3073854..5c44ed49ca93 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -2989,7 +2989,7 @@ static int __meminit next_active_region_index_in_nid(int index, int nid)
  * was used and there are no special requirements, this is a convenient
  * alternative
  */
-int __meminit early_pfn_to_nid(unsigned long pfn)
+int __meminit __early_pfn_to_nid(unsigned long pfn)
 {
         int i;
 
@@ -3000,10 +3000,33 @@ int __meminit early_pfn_to_nid(unsigned long pfn)
                 if (start_pfn <= pfn && pfn < end_pfn)
                         return early_node_map[i].nid;
         }
+        /* This is a memory hole */
+        return -1;
+}
+#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
+
+int __meminit early_pfn_to_nid(unsigned long pfn)
+{
+        int nid;
 
+        nid = __early_pfn_to_nid(pfn);
+        if (nid >= 0)
+                return nid;
+        /* just returns 0 */
         return 0;
 }
-#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
+
+#ifdef CONFIG_NODES_SPAN_OTHER_NODES
+bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
+{
+        int nid;
+
+        nid = __early_pfn_to_nid(pfn);
+        if (nid >= 0 && nid != node)
+                return false;
+        return true;
+}
+#endif
 
 /* Basic iterator support to walk early_node_map[] */
 #define for_each_active_range_index_in_nid(i, nid) \
diff --git a/mm/page_io.c b/mm/page_io.c
index dc6ce0afbded..3023c475e041 100644
--- a/mm/page_io.c
+++ b/mm/page_io.c
@@ -111,7 +111,7 @@ int swap_writepage(struct page *page, struct writeback_control *wbc)
                 goto out;
         }
         if (wbc->sync_mode == WB_SYNC_ALL)
-                rw |= (1 << BIO_RW_SYNC);
+                rw |= (1 << BIO_RW_SYNCIO) | (1 << BIO_RW_UNPLUG);
         count_vm_event(PSWPOUT);
         set_page_writeback(page);
         unlock_page(page);
diff --git a/mm/slab.c b/mm/slab.c
index ddc41f337d58..aeeb4ecb9428 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -102,6 +102,7 @@
 #include        <linux/cpu.h>
 #include        <linux/sysctl.h>
 #include        <linux/module.h>
+#include        <trace/kmemtrace.h>
 #include        <linux/rcupdate.h>
 #include        <linux/string.h>
 #include        <linux/uaccess.h>
@@ -568,6 +569,14 @@ static void **dbg_userword(struct kmem_cache *cachep, void *objp)
 
 #endif
 
+#ifdef CONFIG_KMEMTRACE
+size_t slab_buffer_size(struct kmem_cache *cachep)
+{
+        return cachep->buffer_size;
+}
+EXPORT_SYMBOL(slab_buffer_size);
+#endif
+
 /*
  * Do not go above this order unless 0 objects fit into the slab.
  */
@@ -3550,10 +3559,23 @@ static inline void __cache_free(struct kmem_cache *cachep, void *objp)
  */
 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
-        return __cache_alloc(cachep, flags, __builtin_return_address(0));
+        void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
+
+        kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
+                             obj_size(cachep), cachep->buffer_size, flags);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags)
+{
+        return __cache_alloc(cachep, flags, __builtin_return_address(0));
+}
+EXPORT_SYMBOL(kmem_cache_alloc_notrace);
+#endif
+
 /**
  * kmem_ptr_validate - check if an untrusted pointer might be a slab entry.
  * @cachep: the cache we're checking against
@@ -3598,23 +3620,47 @@ out:
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 {
-        return __cache_alloc_node(cachep, flags, nodeid,
-                        __builtin_return_address(0));
+        void *ret = __cache_alloc_node(cachep, flags, nodeid,
+                                       __builtin_return_address(0));
+
+        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
+                                  obj_size(cachep), cachep->buffer_size,
+                                  flags, nodeid);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep,
+                                    gfp_t flags,
+                                    int nodeid)
+{
+        return __cache_alloc_node(cachep, flags, nodeid,
+                                  __builtin_return_address(0));
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
+#endif
+
 static __always_inline void *
 __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
 {
         struct kmem_cache *cachep;
+        void *ret;
 
         cachep = kmem_find_general_cachep(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
-        return kmem_cache_alloc_node(cachep, flags, node);
+        ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
+
+        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
+                                  (unsigned long) caller, ret,
+                                  size, cachep->buffer_size, flags, node);
+
+        return ret;
 }
 
-#ifdef CONFIG_DEBUG_SLAB
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         return __do_kmalloc_node(size, flags, node,
@@ -3647,6 +3693,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
                                           void *caller)
 {
         struct kmem_cache *cachep;
+        void *ret;
 
         /* If you want to save a few bytes .text space: replace
          * __ with kmem_.
@@ -3656,11 +3703,17 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
         cachep = __find_general_cachep(size, flags);
         if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                 return cachep;
-        return __cache_alloc(cachep, flags, caller);
+        ret = __cache_alloc(cachep, flags, caller);
+
+        kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC,
+                             (unsigned long) caller, ret,
+                             size, cachep->buffer_size, flags);
+
+        return ret;
 }
 
 
-#ifdef CONFIG_DEBUG_SLAB
+#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_KMEMTRACE)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         return __do_kmalloc(size, flags, __builtin_return_address(0));
@@ -3699,6 +3752,8 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
                 debug_check_no_obj_freed(objp, obj_size(cachep));
         __cache_free(cachep, objp);
         local_irq_restore(flags);
+
+        kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, objp);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -3725,6 +3780,8 @@ void kfree(const void *objp)
         debug_check_no_obj_freed(objp, obj_size(c));
         __cache_free(c, (void *)objp);
         local_irq_restore(flags);
+
+        kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, objp);
 }
 EXPORT_SYMBOL(kfree);
 
@@ -4457,3 +4514,4 @@ size_t ksize(const void *objp)
 
         return obj_size(virt_to_cache(objp));
 }
+EXPORT_SYMBOL(ksize);
diff --git a/mm/slob.c b/mm/slob.c
index bf7e8fc3aed8..f9cc24688232 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -65,6 +65,7 @@
 #include <linux/module.h>
 #include <linux/rcupdate.h>
 #include <linux/list.h>
+#include <trace/kmemtrace.h>
 #include <asm/atomic.h>
 
 /*
@@ -463,27 +464,38 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
 {
         unsigned int *m;
         int align = max(ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
+        void *ret;
 
         if (size < PAGE_SIZE - align) {
                 if (!size)
                         return ZERO_SIZE_PTR;
 
                 m = slob_alloc(size + align, gfp, align, node);
+
                 if (!m)
                         return NULL;
                 *m = size;
-                return (void *)m + align;
+                ret = (void *)m + align;
+
+                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
+                                          _RET_IP_, ret,
+                                          size, size + align, gfp, node);
         } else {
-                void *ret;
+                unsigned int order = get_order(size);
 
-                ret = slob_new_page(gfp | __GFP_COMP, get_order(size), node);
+                ret = slob_new_page(gfp | __GFP_COMP, order, node);
                 if (ret) {
                         struct page *page;
                         page = virt_to_page(ret);
                         page->private = size;
                 }
-                return ret;
+
+                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
+                                          _RET_IP_, ret,
+                                          size, PAGE_SIZE << order, gfp, node);
         }
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node);
 
@@ -501,6 +513,8 @@ void kfree(const void *block)
                 slob_free(m, *m + align);
         } else
                 put_page(&sp->page);
+
+        kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, block);
 }
 EXPORT_SYMBOL(kfree);
 
@@ -521,6 +535,7 @@ size_t ksize(const void *block)
         } else
                 return sp->page.private;
 }
+EXPORT_SYMBOL(ksize);
 
 struct kmem_cache {
         unsigned int size, align;
@@ -569,10 +584,19 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 {
         void *b;
 
-        if (c->size < PAGE_SIZE)
+        if (c->size < PAGE_SIZE) {
                 b = slob_alloc(c->size, flags, c->align, node);
-        else
+                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE,
+                                          _RET_IP_, b, c->size,
+                                          SLOB_UNITS(c->size) * SLOB_UNIT,
+                                          flags, node);
+        } else {
                 b = slob_new_page(flags, get_order(c->size), node);
+                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE,
+                                          _RET_IP_, b, c->size,
+                                          PAGE_SIZE << get_order(c->size),
+                                          flags, node);
+        }
 
         if (c->ctor)
                 c->ctor(b);
@@ -608,6 +632,8 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
         } else {
                 __kmem_cache_free(b, c->size);
         }
+
+        kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, b);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
diff --git a/mm/slub.c b/mm/slub.c
index 5a5e7f5bf799..6de5e07c8850 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -16,6 +16,7 @@
 #include <linux/slab.h>
 #include <linux/proc_fs.h>
 #include <linux/seq_file.h>
+#include <trace/kmemtrace.h>
 #include <linux/cpu.h>
 #include <linux/cpuset.h>
 #include <linux/mempolicy.h>
@@ -1623,18 +1624,46 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
 
 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
 {
-        return slab_alloc(s, gfpflags, -1, _RET_IP_);
+        void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
+
+        kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
+                             s->objsize, s->size, gfpflags);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc);
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
+{
+        return slab_alloc(s, gfpflags, -1, _RET_IP_);
+}
+EXPORT_SYMBOL(kmem_cache_alloc_notrace);
+#endif
+
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
 {
-        return slab_alloc(s, gfpflags, node, _RET_IP_);
+        void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
+
+        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
+                                  s->objsize, s->size, gfpflags, node);
+
+        return ret;
 }
 EXPORT_SYMBOL(kmem_cache_alloc_node);
 #endif
 
+#ifdef CONFIG_KMEMTRACE
+void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
+                                    gfp_t gfpflags,
+                                    int node)
+{
+        return slab_alloc(s, gfpflags, node, _RET_IP_);
+}
+EXPORT_SYMBOL(kmem_cache_alloc_node_notrace);
+#endif
+
 /*
  * Slow patch handling. This may still be called frequently since objects
  * have a longer lifetime than the cpu slabs in most processing loads.
@@ -1742,6 +1771,8 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
         page = virt_to_head_page(x);
 
         slab_free(s, page, x, _RET_IP_);
+
+        kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, x);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -2657,6 +2688,7 @@ static struct kmem_cache *get_slab(size_t size, gfp_t flags)
 void *__kmalloc(size_t size, gfp_t flags)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, flags);
@@ -2666,7 +2698,12 @@ void *__kmalloc(size_t size, gfp_t flags)
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, flags, -1, _RET_IP_);
+        ret = slab_alloc(s, flags, -1, _RET_IP_);
+
+        kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret,
+                             size, s->size, flags);
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc);
 
@@ -2685,16 +2722,30 @@ static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
 void *__kmalloc_node(size_t size, gfp_t flags, int node)
 {
         struct kmem_cache *s;
+        void *ret;
 
-        if (unlikely(size > SLUB_MAX_SIZE))
-                return kmalloc_large_node(size, flags, node);
+        if (unlikely(size > SLUB_MAX_SIZE)) {
+                ret = kmalloc_large_node(size, flags, node);
+
+                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
+                                          _RET_IP_, ret,
+                                          size, PAGE_SIZE << get_order(size),
+                                          flags, node);
+
+                return ret;
+        }
 
         s = get_slab(size, flags);
 
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, flags, node, _RET_IP_);
+        ret = slab_alloc(s, flags, node, _RET_IP_);
+
+        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret,
+                                  size, s->size, flags, node);
+
+        return ret;
 }
 EXPORT_SYMBOL(__kmalloc_node);
 #endif
@@ -2736,6 +2787,7 @@ size_t ksize(const void *object)
          */
         return s->size;
 }
+EXPORT_SYMBOL(ksize);
 
 void kfree(const void *x)
 {
@@ -2752,6 +2804,8 @@ void kfree(const void *x)
                 return;
         }
         slab_free(page->slab, page, object, _RET_IP_);
+
+        kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, x);
 }
 EXPORT_SYMBOL(kfree);
 
@@ -3221,6 +3275,7 @@ static struct notifier_block __cpuinitdata slab_notifier = {
 void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large(size, gfpflags);
@@ -3230,13 +3285,20 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, gfpflags, -1, caller);
+        ret = slab_alloc(s, gfpflags, -1, caller);
+
+        /* Honor the call site pointer we recieved. */
+        kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, caller, ret, size,
+                             s->size, gfpflags);
+
+        return ret;
 }
 
 void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
                                         int node, unsigned long caller)
 {
         struct kmem_cache *s;
+        void *ret;
 
         if (unlikely(size > SLUB_MAX_SIZE))
                 return kmalloc_large_node(size, gfpflags, node);
@@ -3246,7 +3308,13 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
         if (unlikely(ZERO_OR_NULL_PTR(s)))
                 return s;
 
-        return slab_alloc(s, gfpflags, node, caller);
+        ret = slab_alloc(s, gfpflags, node, caller);
+
+        /* Honor the call site pointer we recieved. */
+        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, caller, ret,
+                                  size, s->size, gfpflags, node);
+
+        return ret;
 }
 
 #ifdef CONFIG_SLUB_DEBUG
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 75f49d312e8c..4dd2636d0b92 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -1106,6 +1106,14 @@ struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
 }
 EXPORT_SYMBOL_GPL(__get_vm_area);
 
+struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags,
+                                       unsigned long start, unsigned long end,
+                                       void *caller)
+{
+        return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL,
+                                  caller);
+}
+
 /**
  *      get_vm_area  -  reserve a contiguous kernel virtual area
  *      @size:          size of the area