1 files changed, 84 insertions, 242 deletions

diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index f6e602918dac..03a8d84badad 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -168,6 +168,8 @@ struct kmemleak_object {
 #define OBJECT_REPORTED         (1 << 1)
 /* flag set to not scan the object */
 #define OBJECT_NO_SCAN          (1 << 2)
+/* flag set to fully scan the object when scan_area allocation failed */
+#define OBJECT_FULL_SCAN        (1 << 3)
 
 #define HEX_PREFIX              "    "
 /* number of bytes to print per line; must be 16 or 32 */
@@ -183,6 +185,10 @@ struct kmemleak_object {
 static LIST_HEAD(object_list);
 /* the list of gray-colored objects (see color_gray comment below) */
 static LIST_HEAD(gray_list);
+/* memory pool allocation */
+static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
+static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
+static LIST_HEAD(mem_pool_free_list);
 /* search tree for object boundaries */
 static struct rb_root object_tree_root = RB_ROOT;
 /* rw_lock protecting the access to object_list and object_tree_root */
@@ -193,13 +199,11 @@ static struct kmem_cache *object_cache;
 static struct kmem_cache *scan_area_cache;
 
 /* set if tracing memory operations is enabled */
-static int kmemleak_enabled;
+static int kmemleak_enabled = 1;
 /* same as above but only for the kmemleak_free() callback */
-static int kmemleak_free_enabled;
+static int kmemleak_free_enabled = 1;
 /* set in the late_initcall if there were no errors */
 static int kmemleak_initialized;
-/* enables or disables early logging of the memory operations */
-static int kmemleak_early_log = 1;
 /* set if a kmemleak warning was issued */
 static int kmemleak_warning;
 /* set if a fatal kmemleak error has occurred */
@@ -227,49 +231,6 @@ static bool kmemleak_found_leaks;
 static bool kmemleak_verbose;
 module_param_named(verbose, kmemleak_verbose, bool, 0600);
 
-/*
- * Early object allocation/freeing logging. Kmemleak is initialized after the
- * kernel allocator. However, both the kernel allocator and kmemleak may
- * allocate memory blocks which need to be tracked. Kmemleak defines an
- * arbitrary buffer to hold the allocation/freeing information before it is
- * fully initialized.
- */
-
-/* kmemleak operation type for early logging */
-enum {
-        KMEMLEAK_ALLOC,
-        KMEMLEAK_ALLOC_PERCPU,
-        KMEMLEAK_FREE,
-        KMEMLEAK_FREE_PART,
-        KMEMLEAK_FREE_PERCPU,
-        KMEMLEAK_NOT_LEAK,
-        KMEMLEAK_IGNORE,
-        KMEMLEAK_SCAN_AREA,
-        KMEMLEAK_NO_SCAN,
-        KMEMLEAK_SET_EXCESS_REF
-};
-
-/*
- * Structure holding the information passed to kmemleak callbacks during the
- * early logging.
- */
-struct early_log {
-        int op_type;                    /* kmemleak operation type */
-        int min_count;                  /* minimum reference count */
-        const void *ptr;                /* allocated/freed memory block */
-        union {
-                size_t size;            /* memory block size */
-                unsigned long excess_ref; /* surplus reference passing */
-        };
-        unsigned long trace[MAX_TRACE]; /* stack trace */
-        unsigned int trace_len;         /* stack trace length */
-};
-
-/* early logging buffer and current position */
-static struct early_log
-        early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
-static int crt_early_log __initdata;
-
 static void kmemleak_disable(void);
 
 /*
@@ -450,6 +411,54 @@ static int get_object(struct kmemleak_object *object)
 }
 
 /*
+ * Memory pool allocation and freeing. kmemleak_lock must not be held.
+ */
+static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
+{
+        unsigned long flags;
+        struct kmemleak_object *object;
+
+        /* try the slab allocator first */
+        if (object_cache) {
+                object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
+                if (object)
+                        return object;
+        }
+
+        /* slab allocation failed, try the memory pool */
+        write_lock_irqsave(&kmemleak_lock, flags);
+        object = list_first_entry_or_null(&mem_pool_free_list,
+                                          typeof(*object), object_list);
+        if (object)
+                list_del(&object->object_list);
+        else if (mem_pool_free_count)
+                object = &mem_pool[--mem_pool_free_count];
+        else
+                pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
+        write_unlock_irqrestore(&kmemleak_lock, flags);
+
+        return object;
+}
+
+/*
+ * Return the object to either the slab allocator or the memory pool.
+ */
+static void mem_pool_free(struct kmemleak_object *object)
+{
+        unsigned long flags;
+
+        if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
+                kmem_cache_free(object_cache, object);
+                return;
+        }
+
+        /* add the object to the memory pool free list */
+        write_lock_irqsave(&kmemleak_lock, flags);
+        list_add(&object->object_list, &mem_pool_free_list);
+        write_unlock_irqrestore(&kmemleak_lock, flags);
+}
+
+/*
  * RCU callback to free a kmemleak_object.
  */
 static void free_object_rcu(struct rcu_head *rcu)
@@ -467,7 +476,7 @@ static void free_object_rcu(struct rcu_head *rcu)
                 hlist_del(&area->node);
                 kmem_cache_free(scan_area_cache, area);
         }
-        kmem_cache_free(object_cache, object);
+        mem_pool_free(object);
 }
 
 /*
@@ -485,7 +494,15 @@ static void put_object(struct kmemleak_object *object)
         /* should only get here after delete_object was called */
         WARN_ON(object->flags & OBJECT_ALLOCATED);
 
-        call_rcu(&object->rcu, free_object_rcu);
+        /*
+         * It may be too early for the RCU callbacks, however, there is no
+         * concurrent object_list traversal when !object_cache and all objects
+         * came from the memory pool. Free the object directly.
+         */
+        if (object_cache)
+                call_rcu(&object->rcu, free_object_rcu);
+        else
+                free_object_rcu(&object->rcu);
 }
 
 /*
@@ -550,7 +567,7 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
         struct rb_node **link, *rb_parent;
         unsigned long untagged_ptr;
 
-        object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
+        object = mem_pool_alloc(gfp);
         if (!object) {
                 pr_warn("Cannot allocate a kmemleak_object structure\n");
                 kmemleak_disable();
@@ -689,9 +706,7 @@ static void delete_object_part(unsigned long ptr, size_t size)
         /*
          * Create one or two objects that may result from the memory block
          * split. Note that partial freeing is only done by free_bootmem() and
-         * this happens before kmemleak_init() is called. The path below is
-         * only executed during early log recording in kmemleak_init(), so
-         * GFP_KERNEL is enough.
+         * this happens before kmemleak_init() is called.
          */
         start = object->pointer;
         end = object->pointer + object->size;
@@ -763,7 +778,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
 {
         unsigned long flags;
         struct kmemleak_object *object;
-        struct kmemleak_scan_area *area;
+        struct kmemleak_scan_area *area = NULL;
 
         object = find_and_get_object(ptr, 1);
         if (!object) {
@@ -772,13 +787,16 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
                 return;
         }
 
-        area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
-        if (!area) {
-                pr_warn("Cannot allocate a scan area\n");
-                goto out;
-        }
+        if (scan_area_cache)
+                area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
 
         spin_lock_irqsave(&object->lock, flags);
+        if (!area) {
+                pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
+                /* mark the object for full scan to avoid false positives */
+                object->flags |= OBJECT_FULL_SCAN;
+                goto out_unlock;
+        }
         if (size == SIZE_MAX) {
                 size = object->pointer + object->size - ptr;
         } else if (ptr + size > object->pointer + object->size) {
@@ -795,7 +813,6 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
         hlist_add_head(&area->node, &object->area_list);
 out_unlock:
         spin_unlock_irqrestore(&object->lock, flags);
-out:
         put_object(object);
 }
 
@@ -845,86 +862,6 @@ static void object_no_scan(unsigned long ptr)
         put_object(object);
 }
 
-/*
- * Log an early kmemleak_* call to the early_log buffer. These calls will be
- * processed later once kmemleak is fully initialized.
- */
-static void __init log_early(int op_type, const void *ptr, size_t size,
-                             int min_count)
-{
-        unsigned long flags;
-        struct early_log *log;
-
-        if (kmemleak_error) {
-                /* kmemleak stopped recording, just count the requests */
-                crt_early_log++;
-                return;
-        }
-
-        if (crt_early_log >= ARRAY_SIZE(early_log)) {
-                crt_early_log++;
-                kmemleak_disable();
-                return;
-        }
-
-        /*
-         * There is no need for locking since the kernel is still in UP mode
-         * at this stage. Disabling the IRQs is enough.
-         */
-        local_irq_save(flags);
-        log = &early_log[crt_early_log];
-        log->op_type = op_type;
-        log->ptr = ptr;
-        log->size = size;
-        log->min_count = min_count;
-        log->trace_len = __save_stack_trace(log->trace);
-        crt_early_log++;
-        local_irq_restore(flags);
-}
-
-/*
- * Log an early allocated block and populate the stack trace.
- */
-static void early_alloc(struct early_log *log)
-{
-        struct kmemleak_object *object;
-        unsigned long flags;
-        int i;
-
-        if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
-                return;
-
-        /*
-         * RCU locking needed to ensure object is not freed via put_object().
-         */
-        rcu_read_lock();
-        object = create_object((unsigned long)log->ptr, log->size,
-                               log->min_count, GFP_ATOMIC);
-        if (!object)
-                goto out;
-        spin_lock_irqsave(&object->lock, flags);
-        for (i = 0; i < log->trace_len; i++)
-                object->trace[i] = log->trace[i];
-        object->trace_len = log->trace_len;
-        spin_unlock_irqrestore(&object->lock, flags);
-out:
-        rcu_read_unlock();
-}
-
-/*
- * Log an early allocated block and populate the stack trace.
- */
-static void early_alloc_percpu(struct early_log *log)
-{
-        unsigned int cpu;
-        const void __percpu *ptr = log->ptr;
-
-        for_each_possible_cpu(cpu) {
-                log->ptr = per_cpu_ptr(ptr, cpu);
-                early_alloc(log);
-        }
-}
-
 /**
  * kmemleak_alloc - register a newly allocated object
  * @ptr:        pointer to beginning of the object
@@ -946,8 +883,6 @@ void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
 
         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
                 create_object((unsigned long)ptr, size, min_count, gfp);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
 }
 EXPORT_SYMBOL_GPL(kmemleak_alloc);
 
@@ -975,8 +910,6 @@ void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
                 for_each_possible_cpu(cpu)
                         create_object((unsigned long)per_cpu_ptr(ptr, cpu),
                                       size, 0, gfp);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
 }
 EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
 
@@ -1001,11 +934,6 @@ void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp
                 create_object((unsigned long)area->addr, size, 2, gfp);
                 object_set_excess_ref((unsigned long)area,
                                       (unsigned long)area->addr);
-        } else if (kmemleak_early_log) {
-                log_early(KMEMLEAK_ALLOC, area->addr, size, 2);
-                /* reusing early_log.size for storing area->addr */
-                log_early(KMEMLEAK_SET_EXCESS_REF,
-                          area, (unsigned long)area->addr, 0);
         }
 }
 EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
@@ -1023,8 +951,6 @@ void __ref kmemleak_free(const void *ptr)
 
         if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
                 delete_object_full((unsigned long)ptr);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_FREE, ptr, 0, 0);
 }
 EXPORT_SYMBOL_GPL(kmemleak_free);
 
@@ -1043,8 +969,6 @@ void __ref kmemleak_free_part(const void *ptr, size_t size)
 
         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
                 delete_object_part((unsigned long)ptr, size);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
 }
 EXPORT_SYMBOL_GPL(kmemleak_free_part);
 
@@ -1065,8 +989,6 @@ void __ref kmemleak_free_percpu(const void __percpu *ptr)
                 for_each_possible_cpu(cpu)
                         delete_object_full((unsigned long)per_cpu_ptr(ptr,
                                                                       cpu));
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
 }
 EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
 
@@ -1117,8 +1039,6 @@ void __ref kmemleak_not_leak(const void *ptr)
 
         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
                 make_gray_object((unsigned long)ptr);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
 }
 EXPORT_SYMBOL(kmemleak_not_leak);
 
@@ -1137,8 +1057,6 @@ void __ref kmemleak_ignore(const void *ptr)
 
         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
                 make_black_object((unsigned long)ptr);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
 }
 EXPORT_SYMBOL(kmemleak_ignore);
 
@@ -1159,8 +1077,6 @@ void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
 
         if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
                 add_scan_area((unsigned long)ptr, size, gfp);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
 }
 EXPORT_SYMBOL(kmemleak_scan_area);
 
@@ -1179,8 +1095,6 @@ void __ref kmemleak_no_scan(const void *ptr)
 
         if (kmemleak_enabled && ptr && !IS_ERR(ptr))
                 object_no_scan((unsigned long)ptr);
-        else if (kmemleak_early_log)
-                log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
 }
 EXPORT_SYMBOL(kmemleak_no_scan);
 
@@ -1408,7 +1322,8 @@ static void scan_object(struct kmemleak_object *object)
         if (!(object->flags & OBJECT_ALLOCATED))
                 /* already freed object */
                 goto out;
-        if (hlist_empty(&object->area_list)) {
+        if (hlist_empty(&object->area_list) ||
+            object->flags & OBJECT_FULL_SCAN) {
                 void *start = (void *)object->pointer;
                 void *end = (void *)(object->pointer + object->size);
                 void *next;
@@ -1966,7 +1881,6 @@ static void kmemleak_disable(void)
 
         /* stop any memory operation tracing */
         kmemleak_enabled = 0;
-        kmemleak_early_log = 0;
 
         /* check whether it is too early for a kernel thread */
         if (kmemleak_initialized)
@@ -1994,20 +1908,11 @@ static int __init kmemleak_boot_config(char *str)
 }
 early_param("kmemleak", kmemleak_boot_config);
 
-static void __init print_log_trace(struct early_log *log)
-{
-        pr_notice("Early log backtrace:\n");
-        stack_trace_print(log->trace, log->trace_len, 2);
-}
-
 /*
  * Kmemleak initialization.
  */
 void __init kmemleak_init(void)
 {
-        int i;
-        unsigned long flags;
-
 #ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
         if (!kmemleak_skip_disable) {
                 kmemleak_disable();
@@ -2015,28 +1920,15 @@ void __init kmemleak_init(void)
         }
 #endif
 
+        if (kmemleak_error)
+                return;
+
         jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
         jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
 
         object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
         scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
 
-        if (crt_early_log > ARRAY_SIZE(early_log))
-                pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
-                        crt_early_log);
-
-        /* the kernel is still in UP mode, so disabling the IRQs is enough */
-        local_irq_save(flags);
-        kmemleak_early_log = 0;
-        if (kmemleak_error) {
-                local_irq_restore(flags);
-                return;
-        } else {
-                kmemleak_enabled = 1;
-                kmemleak_free_enabled = 1;
-        }
-        local_irq_restore(flags);
-
         /* register the data/bss sections */
         create_object((unsigned long)_sdata, _edata - _sdata,
                       KMEMLEAK_GREY, GFP_ATOMIC);
@@ -2047,57 +1939,6 @@ void __init kmemleak_init(void)
                 create_object((unsigned long)__start_ro_after_init,
                               __end_ro_after_init - __start_ro_after_init,
                               KMEMLEAK_GREY, GFP_ATOMIC);
-
-        /*
-         * This is the point where tracking allocations is safe. Automatic
-         * scanning is started during the late initcall. Add the early logged
-         * callbacks to the kmemleak infrastructure.
-         */
-        for (i = 0; i < crt_early_log; i++) {
-                struct early_log *log = &early_log[i];
-
-                switch (log->op_type) {
-                case KMEMLEAK_ALLOC:
-                        early_alloc(log);
-                        break;
-                case KMEMLEAK_ALLOC_PERCPU:
-                        early_alloc_percpu(log);
-                        break;
-                case KMEMLEAK_FREE:
-                        kmemleak_free(log->ptr);
-                        break;
-                case KMEMLEAK_FREE_PART:
-                        kmemleak_free_part(log->ptr, log->size);
-                        break;
-                case KMEMLEAK_FREE_PERCPU:
-                        kmemleak_free_percpu(log->ptr);
-                        break;
-                case KMEMLEAK_NOT_LEAK:
-                        kmemleak_not_leak(log->ptr);
-                        break;
-                case KMEMLEAK_IGNORE:
-                        kmemleak_ignore(log->ptr);
-                        break;
-                case KMEMLEAK_SCAN_AREA:
-                        kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
-                        break;
-                case KMEMLEAK_NO_SCAN:
-                        kmemleak_no_scan(log->ptr);
-                        break;
-                case KMEMLEAK_SET_EXCESS_REF:
-                        object_set_excess_ref((unsigned long)log->ptr,
-                                              log->excess_ref);
-                        break;
-                default:
-                        kmemleak_warn("Unknown early log operation: %d\n",
-                                      log->op_type);
-                }
-
-                if (kmemleak_warning) {
-                        print_log_trace(log);
-                        kmemleak_warning = 0;
-                }
-        }
 }
 
 /*
@@ -2126,7 +1967,8 @@ static int __init kmemleak_late_init(void)
                 mutex_unlock(&scan_mutex);
         }
 
-        pr_info("Kernel memory leak detector initialized\n");
+        pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
+                mem_pool_free_count);
 
         return 0;
 }