8 files changed, 97 insertions, 67 deletions

diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c
index 139d5b7b6621..dfdee6a47359 100644
--- a/mm/allocpercpu.c
+++ b/mm/allocpercpu.c
@@ -31,7 +31,7 @@ static void percpu_depopulate(void *__pdata, int cpu)
  * @__pdata: per-cpu data to depopulate
  * @mask: depopulate per-cpu data for cpu's selected through mask bits
  */
-static void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
+static void __percpu_depopulate_mask(void *__pdata, const cpumask_t *mask)
 {
         int cpu;
         for_each_cpu_mask_nr(cpu, *mask)
diff --git a/mm/backing-dev.c b/mm/backing-dev.c
index be68c956a660..493b468a5035 100644
--- a/mm/backing-dev.c
+++ b/mm/backing-dev.c
@@ -284,12 +284,12 @@ static wait_queue_head_t congestion_wqh[2] = {
         };
 
 
-void clear_bdi_congested(struct backing_dev_info *bdi, int rw)
+void clear_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
         enum bdi_state bit;
-        wait_queue_head_t *wqh = &congestion_wqh[rw];
+        wait_queue_head_t *wqh = &congestion_wqh[sync];
 
-        bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
+        bit = sync ? BDI_sync_congested : BDI_async_congested;
         clear_bit(bit, &bdi->state);
         smp_mb__after_clear_bit();
         if (waitqueue_active(wqh))
@@ -297,11 +297,11 @@ void clear_bdi_congested(struct backing_dev_info *bdi, int rw)
 }
 EXPORT_SYMBOL(clear_bdi_congested);
 
-void set_bdi_congested(struct backing_dev_info *bdi, int rw)
+void set_bdi_congested(struct backing_dev_info *bdi, int sync)
 {
         enum bdi_state bit;
 
-        bit = (rw == WRITE) ? BDI_write_congested : BDI_read_congested;
+        bit = sync ? BDI_sync_congested : BDI_async_congested;
         set_bit(bit, &bdi->state);
 }
 EXPORT_SYMBOL(set_bdi_congested);
diff --git a/mm/failslab.c b/mm/failslab.c
index 7c6ea6493f80..9339de5f0a91 100644
--- a/mm/failslab.c
+++ b/mm/failslab.c
@@ -1,4 +1,5 @@
 #include <linux/fault-inject.h>
+#include <linux/gfp.h>
 
 static struct {
         struct fault_attr attr;
diff --git a/mm/pdflush.c b/mm/pdflush.c
index 118905e3d788..f2caf96993f8 100644
--- a/mm/pdflush.c
+++ b/mm/pdflush.c
@@ -58,6 +58,14 @@ static DEFINE_SPINLOCK(pdflush_lock);
 int nr_pdflush_threads = 0;
 
 /*
+ * The max/min number of pdflush threads. R/W by sysctl at
+ * /proc/sys/vm/nr_pdflush_threads_max/min
+ */
+int nr_pdflush_threads_max __read_mostly = MAX_PDFLUSH_THREADS;
+int nr_pdflush_threads_min __read_mostly = MIN_PDFLUSH_THREADS;
+
+
+/*
  * The time at which the pdflush thread pool last went empty
  */
 static unsigned long last_empty_jifs;
@@ -68,7 +76,7 @@ static unsigned long last_empty_jifs;
  * Thread pool management algorithm:
  * 
  * - The minimum and maximum number of pdflush instances are bound
- *   by MIN_PDFLUSH_THREADS and MAX_PDFLUSH_THREADS.
+ *   by nr_pdflush_threads_min and nr_pdflush_threads_max.
  * 
  * - If there have been no idle pdflush instances for 1 second, create
  *   a new one.
@@ -98,7 +106,6 @@ static int __pdflush(struct pdflush_work *my_work)
         INIT_LIST_HEAD(&my_work->list);
 
         spin_lock_irq(&pdflush_lock);
-        nr_pdflush_threads++;
         for ( ; ; ) {
                 struct pdflush_work *pdf;
 
@@ -126,20 +133,25 @@ static int __pdflush(struct pdflush_work *my_work)
 
                 (*my_work->fn)(my_work->arg0);
 
+                spin_lock_irq(&pdflush_lock);
+
                 /*
                  * Thread creation: For how long have there been zero
                  * available threads?
+                 *
+                 * To throttle creation, we reset last_empty_jifs.
                  */
                 if (time_after(jiffies, last_empty_jifs + 1 * HZ)) {
-                        /* unlocked list_empty() test is OK here */
-                        if (list_empty(&pdflush_list)) {
-                                /* unlocked test is OK here */
-                                if (nr_pdflush_threads < MAX_PDFLUSH_THREADS)
-                                        start_one_pdflush_thread();
+                        if (list_empty(&pdflush_list) &&
+                            nr_pdflush_threads < nr_pdflush_threads_max) {
+                                last_empty_jifs = jiffies;
+                                nr_pdflush_threads++;
+                                spin_unlock_irq(&pdflush_lock);
+                                start_one_pdflush_thread();
+                                spin_lock_irq(&pdflush_lock);
                         }
                 }
 
-                spin_lock_irq(&pdflush_lock);
                 my_work->fn = NULL;
 
                 /*
@@ -148,7 +160,7 @@ static int __pdflush(struct pdflush_work *my_work)
                  */
                 if (list_empty(&pdflush_list))
                         continue;
-                if (nr_pdflush_threads <= MIN_PDFLUSH_THREADS)
+                if (nr_pdflush_threads <= nr_pdflush_threads_min)
                         continue;
                 pdf = list_entry(pdflush_list.prev, struct pdflush_work, list);
                 if (time_after(jiffies, pdf->when_i_went_to_sleep + 1 * HZ)) {
@@ -236,14 +248,27 @@ int pdflush_operation(void (*fn)(unsigned long), unsigned long arg0)
 
 static void start_one_pdflush_thread(void)
 {
-        kthread_run(pdflush, NULL, "pdflush");
+        struct task_struct *k;
+
+        k = kthread_run(pdflush, NULL, "pdflush");
+        if (unlikely(IS_ERR(k))) {
+                spin_lock_irq(&pdflush_lock);
+                nr_pdflush_threads--;
+                spin_unlock_irq(&pdflush_lock);
+        }
 }
 
 static int __init pdflush_init(void)
 {
         int i;
 
-        for (i = 0; i < MIN_PDFLUSH_THREADS; i++)
+        /*
+         * Pre-set nr_pdflush_threads...  If we fail to create,
+         * the count will be decremented.
+         */
+        nr_pdflush_threads = nr_pdflush_threads_min;
+
+        for (i = 0; i < nr_pdflush_threads_min; i++)
                 start_one_pdflush_thread();
         return 0;
 }
diff --git a/mm/slab.c b/mm/slab.c
index 4fc1761c6dc8..9a90b00d2f91 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3565,8 +3565,8 @@ void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
         void *ret = __cache_alloc(cachep, flags, __builtin_return_address(0));
 
-        kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
-                             obj_size(cachep), cachep->buffer_size, flags);
+        trace_kmem_cache_alloc(_RET_IP_, ret,
+                               obj_size(cachep), cachep->buffer_size, flags);
 
         return ret;
 }
@@ -3627,9 +3627,9 @@ void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
         void *ret = __cache_alloc_node(cachep, flags, nodeid,
                                        __builtin_return_address(0));
 
-        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
-                                  obj_size(cachep), cachep->buffer_size,
-                                  flags, nodeid);
+        trace_kmem_cache_alloc_node(_RET_IP_, ret,
+                                    obj_size(cachep), cachep->buffer_size,
+                                    flags, nodeid);
 
         return ret;
 }
@@ -3657,9 +3657,8 @@ __do_kmalloc_node(size_t size, gfp_t flags, int node, void *caller)
                 return cachep;
         ret = kmem_cache_alloc_node_notrace(cachep, flags, node);
 
-        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
-                                  (unsigned long) caller, ret,
-                                  size, cachep->buffer_size, flags, node);
+        trace_kmalloc_node((unsigned long) caller, ret,
+                           size, cachep->buffer_size, flags, node);
 
         return ret;
 }
@@ -3709,9 +3708,8 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
                 return cachep;
         ret = __cache_alloc(cachep, flags, caller);
 
-        kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC,
-                             (unsigned long) caller, ret,
-                             size, cachep->buffer_size, flags);
+        trace_kmalloc((unsigned long) caller, ret,
+                      size, cachep->buffer_size, flags);
 
         return ret;
 }
@@ -3757,7 +3755,7 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
         __cache_free(cachep, objp);
         local_irq_restore(flags);
 
-        kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, objp);
+        trace_kmem_cache_free(_RET_IP_, objp);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -3775,6 +3773,8 @@ void kfree(const void *objp)
         struct kmem_cache *c;
         unsigned long flags;
 
+        trace_kfree(_RET_IP_, objp);
+
         if (unlikely(ZERO_OR_NULL_PTR(objp)))
                 return;
         local_irq_save(flags);
@@ -3784,8 +3784,6 @@ void kfree(const void *objp)
         debug_check_no_obj_freed(objp, obj_size(c));
         __cache_free(c, (void *)objp);
         local_irq_restore(flags);
-
-        kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, objp);
 }
 EXPORT_SYMBOL(kfree);
 
diff --git a/mm/slob.c b/mm/slob.c
index 4dd6516447f2..a2d4ab32198d 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -490,9 +490,8 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
                 *m = size;
                 ret = (void *)m + align;
 
-                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
-                                          _RET_IP_, ret,
-                                          size, size + align, gfp, node);
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, size + align, gfp, node);
         } else {
                 unsigned int order = get_order(size);
 
@@ -503,9 +502,8 @@ void *__kmalloc_node(size_t size, gfp_t gfp, int node)
                         page->private = size;
                 }
 
-                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
-                                          _RET_IP_, ret,
-                                          size, PAGE_SIZE << order, gfp, node);
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, PAGE_SIZE << order, gfp, node);
         }
 
         return ret;
@@ -516,6 +514,8 @@ void kfree(const void *block)
 {
         struct slob_page *sp;
 
+        trace_kfree(_RET_IP_, block);
+
         if (unlikely(ZERO_OR_NULL_PTR(block)))
                 return;
 
@@ -526,8 +526,6 @@ void kfree(const void *block)
                 slob_free(m, *m + align);
         } else
                 put_page(&sp->page);
-
-        kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, block);
 }
 EXPORT_SYMBOL(kfree);
 
@@ -599,16 +597,14 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
 
         if (c->size < PAGE_SIZE) {
                 b = slob_alloc(c->size, flags, c->align, node);
-                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE,
-                                          _RET_IP_, b, c->size,
-                                          SLOB_UNITS(c->size) * SLOB_UNIT,
-                                          flags, node);
+                trace_kmem_cache_alloc_node(_RET_IP_, b, c->size,
+                                            SLOB_UNITS(c->size) * SLOB_UNIT,
+                                            flags, node);
         } else {
                 b = slob_new_pages(flags, get_order(c->size), node);
-                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE,
-                                          _RET_IP_, b, c->size,
-                                          PAGE_SIZE << get_order(c->size),
-                                          flags, node);
+                trace_kmem_cache_alloc_node(_RET_IP_, b, c->size,
+                                            PAGE_SIZE << get_order(c->size),
+                                            flags, node);
         }
 
         if (c->ctor)
@@ -646,7 +642,7 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
                 __kmem_cache_free(b, c->size);
         }
 
-        kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, b);
+        trace_kmem_cache_free(_RET_IP_, b);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
diff --git a/mm/slub.c b/mm/slub.c
index 7aaa121d0ea9..7ab54ecbd3f3 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1621,8 +1621,7 @@ void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
 {
         void *ret = slab_alloc(s, gfpflags, -1, _RET_IP_);
 
-        kmemtrace_mark_alloc(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
-                             s->objsize, s->size, gfpflags);
+        trace_kmem_cache_alloc(_RET_IP_, ret, s->objsize, s->size, gfpflags);
 
         return ret;
 }
@@ -1641,8 +1640,8 @@ void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
 {
         void *ret = slab_alloc(s, gfpflags, node, _RET_IP_);
 
-        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_CACHE, _RET_IP_, ret,
-                                  s->objsize, s->size, gfpflags, node);
+        trace_kmem_cache_alloc_node(_RET_IP_, ret,
+                                    s->objsize, s->size, gfpflags, node);
 
         return ret;
 }
@@ -1767,7 +1766,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
 
         slab_free(s, page, x, _RET_IP_);
 
-        kmemtrace_mark_free(KMEMTRACE_TYPE_CACHE, _RET_IP_, x);
+        trace_kmem_cache_free(_RET_IP_, x);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -2702,8 +2701,7 @@ void *__kmalloc(size_t size, gfp_t flags)
 
         ret = slab_alloc(s, flags, -1, _RET_IP_);
 
-        kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret,
-                             size, s->size, flags);
+        trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
 
         return ret;
 }
@@ -2729,10 +2727,9 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
         if (unlikely(size > SLUB_MAX_SIZE)) {
                 ret = kmalloc_large_node(size, flags, node);
 
-                kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC,
-                                          _RET_IP_, ret,
-                                          size, PAGE_SIZE << get_order(size),
-                                          flags, node);
+                trace_kmalloc_node(_RET_IP_, ret,
+                                   size, PAGE_SIZE << get_order(size),
+                                   flags, node);
 
                 return ret;
         }
@@ -2744,8 +2741,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
 
         ret = slab_alloc(s, flags, node, _RET_IP_);
 
-        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, ret,
-                                  size, s->size, flags, node);
+        trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
 
         return ret;
 }
@@ -2796,6 +2792,8 @@ void kfree(const void *x)
         struct page *page;
         void *object = (void *)x;
 
+        trace_kfree(_RET_IP_, x);
+
         if (unlikely(ZERO_OR_NULL_PTR(x)))
                 return;
 
@@ -2806,8 +2804,6 @@ void kfree(const void *x)
                 return;
         }
         slab_free(page->slab, page, object, _RET_IP_);
-
-        kmemtrace_mark_free(KMEMTRACE_TYPE_KMALLOC, _RET_IP_, x);
 }
 EXPORT_SYMBOL(kfree);
 
@@ -3290,8 +3286,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
         ret = slab_alloc(s, gfpflags, -1, caller);
 
         /* Honor the call site pointer we recieved. */
-        kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, caller, ret, size,
-                             s->size, gfpflags);
+        trace_kmalloc(caller, ret, size, s->size, gfpflags);
 
         return ret;
 }
@@ -3313,8 +3308,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
         ret = slab_alloc(s, gfpflags, node, caller);
 
         /* Honor the call site pointer we recieved. */
-        kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, caller, ret,
-                                  size, s->size, gfpflags, node);
+        trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
 
         return ret;
 }
diff --git a/mm/util.c b/mm/util.c
index 7c122e49f769..2599e83eea17 100644
--- a/mm/util.c
+++ b/mm/util.c
@@ -4,6 +4,7 @@
 #include <linux/module.h>
 #include <linux/err.h>
 #include <linux/sched.h>
+#include <linux/tracepoint.h>
 #include <asm/uaccess.h>
 
 /**
@@ -236,3 +237,18 @@ int __attribute__((weak)) get_user_pages_fast(unsigned long start,
         return ret;
 }
 EXPORT_SYMBOL_GPL(get_user_pages_fast);
+
+/* Tracepoints definitions. */
+DEFINE_TRACE(kmalloc);
+DEFINE_TRACE(kmem_cache_alloc);
+DEFINE_TRACE(kmalloc_node);
+DEFINE_TRACE(kmem_cache_alloc_node);
+DEFINE_TRACE(kfree);
+DEFINE_TRACE(kmem_cache_free);
+
+EXPORT_TRACEPOINT_SYMBOL(kmalloc);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
+EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
+EXPORT_TRACEPOINT_SYMBOL(kfree);
+EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);