aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/linux/fault-inject.h9
-rw-r--r--include/linux/slab.h10
-rw-r--r--lib/Kconfig.debug1
-rw-r--r--mm/Makefile1
-rw-r--r--mm/failslab.c59
-rw-r--r--mm/slab.c89
-rw-r--r--mm/slub.c98
7 files changed, 145 insertions, 122 deletions
diff --git a/include/linux/fault-inject.h b/include/linux/fault-inject.h
index 32368c4f0326..06ca9b21dad2 100644
--- a/include/linux/fault-inject.h
+++ b/include/linux/fault-inject.h
@@ -81,4 +81,13 @@ static inline void cleanup_fault_attr_dentries(struct fault_attr *attr)
81 81
82#endif /* CONFIG_FAULT_INJECTION */ 82#endif /* CONFIG_FAULT_INJECTION */
83 83
84#ifdef CONFIG_FAILSLAB
85extern bool should_failslab(size_t size, gfp_t gfpflags);
86#else
87static inline bool should_failslab(size_t size, gfp_t gfpflags)
88{
89 return false;
90}
91#endif /* CONFIG_FAILSLAB */
92
84#endif /* _LINUX_FAULT_INJECT_H */ 93#endif /* _LINUX_FAULT_INJECT_H */
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 000da12b5cf0..f96d13c281e8 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -253,9 +253,9 @@ static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,
253 * request comes from. 253 * request comes from.
254 */ 254 */
255#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB) 255#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB)
256extern void *__kmalloc_track_caller(size_t, gfp_t, void*); 256extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long);
257#define kmalloc_track_caller(size, flags) \ 257#define kmalloc_track_caller(size, flags) \
258 __kmalloc_track_caller(size, flags, __builtin_return_address(0)) 258 __kmalloc_track_caller(size, flags, _RET_IP_)
259#else 259#else
260#define kmalloc_track_caller(size, flags) \ 260#define kmalloc_track_caller(size, flags) \
261 __kmalloc(size, flags) 261 __kmalloc(size, flags)
@@ -271,10 +271,10 @@ extern void *__kmalloc_track_caller(size_t, gfp_t, void*);
271 * allocation request comes from. 271 * allocation request comes from.
272 */ 272 */
273#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB) 273#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB)
274extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, void *); 274extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long);
275#define kmalloc_node_track_caller(size, flags, node) \ 275#define kmalloc_node_track_caller(size, flags, node) \
276 __kmalloc_node_track_caller(size, flags, node, \ 276 __kmalloc_node_track_caller(size, flags, node, \
277 __builtin_return_address(0)) 277 _RET_IP_)
278#else 278#else
279#define kmalloc_node_track_caller(size, flags, node) \ 279#define kmalloc_node_track_caller(size, flags, node) \
280 __kmalloc_node(size, flags, node) 280 __kmalloc_node(size, flags, node)
@@ -285,7 +285,7 @@ extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, void *);
285#define kmalloc_node_track_caller(size, flags, node) \ 285#define kmalloc_node_track_caller(size, flags, node) \
286 kmalloc_track_caller(size, flags) 286 kmalloc_track_caller(size, flags)
287 287
288#endif /* DEBUG_SLAB */ 288#endif /* CONFIG_NUMA */
289 289
290/* 290/*
291 * Shortcuts 291 * Shortcuts
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index eae594cb6ea9..2e75478e9c69 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -730,6 +730,7 @@ config FAULT_INJECTION
730config FAILSLAB 730config FAILSLAB
731 bool "Fault-injection capability for kmalloc" 731 bool "Fault-injection capability for kmalloc"
732 depends on FAULT_INJECTION 732 depends on FAULT_INJECTION
733 depends on SLAB || SLUB
733 help 734 help
734 Provide fault-injection capability for kmalloc. 735 Provide fault-injection capability for kmalloc.
735 736
diff --git a/mm/Makefile b/mm/Makefile
index c06b45a1ff5f..51c27709cc7c 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -28,6 +28,7 @@ obj-$(CONFIG_SLOB) += slob.o
28obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o 28obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
29obj-$(CONFIG_SLAB) += slab.o 29obj-$(CONFIG_SLAB) += slab.o
30obj-$(CONFIG_SLUB) += slub.o 30obj-$(CONFIG_SLUB) += slub.o
31obj-$(CONFIG_FAILSLAB) += failslab.o
31obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o 32obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
32obj-$(CONFIG_FS_XIP) += filemap_xip.o 33obj-$(CONFIG_FS_XIP) += filemap_xip.o
33obj-$(CONFIG_MIGRATION) += migrate.o 34obj-$(CONFIG_MIGRATION) += migrate.o
diff --git a/mm/failslab.c b/mm/failslab.c
new file mode 100644
index 000000000000..7c6ea6493f80
--- /dev/null
+++ b/mm/failslab.c
@@ -0,0 +1,59 @@
1#include <linux/fault-inject.h>
2
3static struct {
4 struct fault_attr attr;
5 u32 ignore_gfp_wait;
6#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
7 struct dentry *ignore_gfp_wait_file;
8#endif
9} failslab = {
10 .attr = FAULT_ATTR_INITIALIZER,
11 .ignore_gfp_wait = 1,
12};
13
14bool should_failslab(size_t size, gfp_t gfpflags)
15{
16 if (gfpflags & __GFP_NOFAIL)
17 return false;
18
19 if (failslab.ignore_gfp_wait && (gfpflags & __GFP_WAIT))
20 return false;
21
22 return should_fail(&failslab.attr, size);
23}
24
25static int __init setup_failslab(char *str)
26{
27 return setup_fault_attr(&failslab.attr, str);
28}
29__setup("failslab=", setup_failslab);
30
31#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
32
33static int __init failslab_debugfs_init(void)
34{
35 mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
36 struct dentry *dir;
37 int err;
38
39 err = init_fault_attr_dentries(&failslab.attr, "failslab");
40 if (err)
41 return err;
42 dir = failslab.attr.dentries.dir;
43
44 failslab.ignore_gfp_wait_file =
45 debugfs_create_bool("ignore-gfp-wait", mode, dir,
46 &failslab.ignore_gfp_wait);
47
48 if (!failslab.ignore_gfp_wait_file) {
49 err = -ENOMEM;
50 debugfs_remove(failslab.ignore_gfp_wait_file);
51 cleanup_fault_attr_dentries(&failslab.attr);
52 }
53
54 return err;
55}
56
57late_initcall(failslab_debugfs_init);
58
59#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
diff --git a/mm/slab.c b/mm/slab.c
index 09187517f9dc..f97e564bdf11 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -2123,6 +2123,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
2123 * 2123 *
2124 * @name must be valid until the cache is destroyed. This implies that 2124 * @name must be valid until the cache is destroyed. This implies that
2125 * the module calling this has to destroy the cache before getting unloaded. 2125 * the module calling this has to destroy the cache before getting unloaded.
2126 * Note that kmem_cache_name() is not guaranteed to return the same pointer,
2127 * therefore applications must manage it themselves.
2126 * 2128 *
2127 * The flags are 2129 * The flags are
2128 * 2130 *
@@ -2609,7 +2611,7 @@ static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
2609 if (OFF_SLAB(cachep)) { 2611 if (OFF_SLAB(cachep)) {
2610 /* Slab management obj is off-slab. */ 2612 /* Slab management obj is off-slab. */
2611 slabp = kmem_cache_alloc_node(cachep->slabp_cache, 2613 slabp = kmem_cache_alloc_node(cachep->slabp_cache,
2612 local_flags & ~GFP_THISNODE, nodeid); 2614 local_flags, nodeid);
2613 if (!slabp) 2615 if (!slabp)
2614 return NULL; 2616 return NULL;
2615 } else { 2617 } else {
@@ -2997,7 +2999,7 @@ retry:
2997 * there must be at least one object available for 2999 * there must be at least one object available for
2998 * allocation. 3000 * allocation.
2999 */ 3001 */
3000 BUG_ON(slabp->inuse < 0 || slabp->inuse >= cachep->num); 3002 BUG_ON(slabp->inuse >= cachep->num);
3001 3003
3002 while (slabp->inuse < cachep->num && batchcount--) { 3004 while (slabp->inuse < cachep->num && batchcount--) {
3003 STATS_INC_ALLOCED(cachep); 3005 STATS_INC_ALLOCED(cachep);
@@ -3106,79 +3108,14 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
3106#define cache_alloc_debugcheck_after(a,b,objp,d) (objp) 3108#define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
3107#endif 3109#endif
3108 3110
3109#ifdef CONFIG_FAILSLAB 3111static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
3110
3111static struct failslab_attr {
3112
3113 struct fault_attr attr;
3114
3115 u32 ignore_gfp_wait;
3116#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3117 struct dentry *ignore_gfp_wait_file;
3118#endif
3119
3120} failslab = {
3121 .attr = FAULT_ATTR_INITIALIZER,
3122 .ignore_gfp_wait = 1,
3123};
3124
3125static int __init setup_failslab(char *str)
3126{
3127 return setup_fault_attr(&failslab.attr, str);
3128}
3129__setup("failslab=", setup_failslab);
3130
3131static int should_failslab(struct kmem_cache *cachep, gfp_t flags)
3132{ 3112{
3133 if (cachep == &cache_cache) 3113 if (cachep == &cache_cache)
3134 return 0; 3114 return false;
3135 if (flags & __GFP_NOFAIL)
3136 return 0;
3137 if (failslab.ignore_gfp_wait && (flags & __GFP_WAIT))
3138 return 0;
3139 3115
3140 return should_fail(&failslab.attr, obj_size(cachep)); 3116 return should_failslab(obj_size(cachep), flags);
3141} 3117}
3142 3118
3143#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
3144
3145static int __init failslab_debugfs(void)
3146{
3147 mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
3148 struct dentry *dir;
3149 int err;
3150
3151 err = init_fault_attr_dentries(&failslab.attr, "failslab");
3152 if (err)
3153 return err;
3154 dir = failslab.attr.dentries.dir;
3155
3156 failslab.ignore_gfp_wait_file =
3157 debugfs_create_bool("ignore-gfp-wait", mode, dir,
3158 &failslab.ignore_gfp_wait);
3159
3160 if (!failslab.ignore_gfp_wait_file) {
3161 err = -ENOMEM;
3162 debugfs_remove(failslab.ignore_gfp_wait_file);
3163 cleanup_fault_attr_dentries(&failslab.attr);
3164 }
3165
3166 return err;
3167}
3168
3169late_initcall(failslab_debugfs);
3170
3171#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
3172
3173#else /* CONFIG_FAILSLAB */
3174
3175static inline int should_failslab(struct kmem_cache *cachep, gfp_t flags)
3176{
3177 return 0;
3178}
3179
3180#endif /* CONFIG_FAILSLAB */
3181
3182static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) 3119static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
3183{ 3120{
3184 void *objp; 3121 void *objp;
@@ -3381,7 +3318,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
3381 unsigned long save_flags; 3318 unsigned long save_flags;
3382 void *ptr; 3319 void *ptr;
3383 3320
3384 if (should_failslab(cachep, flags)) 3321 if (slab_should_failslab(cachep, flags))
3385 return NULL; 3322 return NULL;
3386 3323
3387 cache_alloc_debugcheck_before(cachep, flags); 3324 cache_alloc_debugcheck_before(cachep, flags);
@@ -3457,7 +3394,7 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
3457 unsigned long save_flags; 3394 unsigned long save_flags;
3458 void *objp; 3395 void *objp;
3459 3396
3460 if (should_failslab(cachep, flags)) 3397 if (slab_should_failslab(cachep, flags))
3461 return NULL; 3398 return NULL;
3462 3399
3463 cache_alloc_debugcheck_before(cachep, flags); 3400 cache_alloc_debugcheck_before(cachep, flags);
@@ -3686,9 +3623,9 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
3686EXPORT_SYMBOL(__kmalloc_node); 3623EXPORT_SYMBOL(__kmalloc_node);
3687 3624
3688void *__kmalloc_node_track_caller(size_t size, gfp_t flags, 3625void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
3689 int node, void *caller) 3626 int node, unsigned long caller)
3690{ 3627{
3691 return __do_kmalloc_node(size, flags, node, caller); 3628 return __do_kmalloc_node(size, flags, node, (void *)caller);
3692} 3629}
3693EXPORT_SYMBOL(__kmalloc_node_track_caller); 3630EXPORT_SYMBOL(__kmalloc_node_track_caller);
3694#else 3631#else
@@ -3730,9 +3667,9 @@ void *__kmalloc(size_t size, gfp_t flags)
3730} 3667}
3731EXPORT_SYMBOL(__kmalloc); 3668EXPORT_SYMBOL(__kmalloc);
3732 3669
3733void *__kmalloc_track_caller(size_t size, gfp_t flags, void *caller) 3670void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
3734{ 3671{
3735 return __do_kmalloc(size, flags, caller); 3672 return __do_kmalloc(size, flags, (void *)caller);
3736} 3673}
3737EXPORT_SYMBOL(__kmalloc_track_caller); 3674EXPORT_SYMBOL(__kmalloc_track_caller);
3738 3675
diff --git a/mm/slub.c b/mm/slub.c
index a2cd47d89e0a..6cb7ad107852 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -24,6 +24,7 @@
24#include <linux/kallsyms.h> 24#include <linux/kallsyms.h>
25#include <linux/memory.h> 25#include <linux/memory.h>
26#include <linux/math64.h> 26#include <linux/math64.h>
27#include <linux/fault-inject.h>
27 28
28/* 29/*
29 * Lock order: 30 * Lock order:
@@ -153,6 +154,10 @@
153#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) 154#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
154#endif 155#endif
155 156
157#define OO_SHIFT 16
158#define OO_MASK ((1 << OO_SHIFT) - 1)
159#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
160
156/* Internal SLUB flags */ 161/* Internal SLUB flags */
157#define __OBJECT_POISON 0x80000000 /* Poison object */ 162#define __OBJECT_POISON 0x80000000 /* Poison object */
158#define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */ 163#define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */
@@ -178,7 +183,7 @@ static LIST_HEAD(slab_caches);
178 * Tracking user of a slab. 183 * Tracking user of a slab.
179 */ 184 */
180struct track { 185struct track {
181 void *addr; /* Called from address */ 186 unsigned long addr; /* Called from address */
182 int cpu; /* Was running on cpu */ 187 int cpu; /* Was running on cpu */
183 int pid; /* Pid context */ 188 int pid; /* Pid context */
184 unsigned long when; /* When did the operation occur */ 189 unsigned long when; /* When did the operation occur */
@@ -290,7 +295,7 @@ static inline struct kmem_cache_order_objects oo_make(int order,
290 unsigned long size) 295 unsigned long size)
291{ 296{
292 struct kmem_cache_order_objects x = { 297 struct kmem_cache_order_objects x = {
293 (order << 16) + (PAGE_SIZE << order) / size 298 (order << OO_SHIFT) + (PAGE_SIZE << order) / size
294 }; 299 };
295 300
296 return x; 301 return x;
@@ -298,12 +303,12 @@ static inline struct kmem_cache_order_objects oo_make(int order,
298 303
299static inline int oo_order(struct kmem_cache_order_objects x) 304static inline int oo_order(struct kmem_cache_order_objects x)
300{ 305{
301 return x.x >> 16; 306 return x.x >> OO_SHIFT;
302} 307}
303 308
304static inline int oo_objects(struct kmem_cache_order_objects x) 309static inline int oo_objects(struct kmem_cache_order_objects x)
305{ 310{
306 return x.x & ((1 << 16) - 1); 311 return x.x & OO_MASK;
307} 312}
308 313
309#ifdef CONFIG_SLUB_DEBUG 314#ifdef CONFIG_SLUB_DEBUG
@@ -367,7 +372,7 @@ static struct track *get_track(struct kmem_cache *s, void *object,
367} 372}
368 373
369static void set_track(struct kmem_cache *s, void *object, 374static void set_track(struct kmem_cache *s, void *object,
370 enum track_item alloc, void *addr) 375 enum track_item alloc, unsigned long addr)
371{ 376{
372 struct track *p; 377 struct track *p;
373 378
@@ -391,8 +396,8 @@ static void init_tracking(struct kmem_cache *s, void *object)
391 if (!(s->flags & SLAB_STORE_USER)) 396 if (!(s->flags & SLAB_STORE_USER))
392 return; 397 return;
393 398
394 set_track(s, object, TRACK_FREE, NULL); 399 set_track(s, object, TRACK_FREE, 0UL);
395 set_track(s, object, TRACK_ALLOC, NULL); 400 set_track(s, object, TRACK_ALLOC, 0UL);
396} 401}
397 402
398static void print_track(const char *s, struct track *t) 403static void print_track(const char *s, struct track *t)
@@ -401,7 +406,7 @@ static void print_track(const char *s, struct track *t)
401 return; 406 return;
402 407
403 printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n", 408 printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
404 s, t->addr, jiffies - t->when, t->cpu, t->pid); 409 s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
405} 410}
406 411
407static void print_tracking(struct kmem_cache *s, void *object) 412static void print_tracking(struct kmem_cache *s, void *object)
@@ -692,7 +697,7 @@ static int check_object(struct kmem_cache *s, struct page *page,
692 if (!check_valid_pointer(s, page, get_freepointer(s, p))) { 697 if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
693 object_err(s, page, p, "Freepointer corrupt"); 698 object_err(s, page, p, "Freepointer corrupt");
694 /* 699 /*
695 * No choice but to zap it and thus loose the remainder 700 * No choice but to zap it and thus lose the remainder
696 * of the free objects in this slab. May cause 701 * of the free objects in this slab. May cause
697 * another error because the object count is now wrong. 702 * another error because the object count is now wrong.
698 */ 703 */
@@ -764,8 +769,8 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
764 } 769 }
765 770
766 max_objects = (PAGE_SIZE << compound_order(page)) / s->size; 771 max_objects = (PAGE_SIZE << compound_order(page)) / s->size;
767 if (max_objects > 65535) 772 if (max_objects > MAX_OBJS_PER_PAGE)
768 max_objects = 65535; 773 max_objects = MAX_OBJS_PER_PAGE;
769 774
770 if (page->objects != max_objects) { 775 if (page->objects != max_objects) {
771 slab_err(s, page, "Wrong number of objects. Found %d but " 776 slab_err(s, page, "Wrong number of objects. Found %d but "
@@ -866,7 +871,7 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page,
866} 871}
867 872
868static int alloc_debug_processing(struct kmem_cache *s, struct page *page, 873static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
869 void *object, void *addr) 874 void *object, unsigned long addr)
870{ 875{
871 if (!check_slab(s, page)) 876 if (!check_slab(s, page))
872 goto bad; 877 goto bad;
@@ -906,7 +911,7 @@ bad:
906} 911}
907 912
908static int free_debug_processing(struct kmem_cache *s, struct page *page, 913static int free_debug_processing(struct kmem_cache *s, struct page *page,
909 void *object, void *addr) 914 void *object, unsigned long addr)
910{ 915{
911 if (!check_slab(s, page)) 916 if (!check_slab(s, page))
912 goto fail; 917 goto fail;
@@ -1029,10 +1034,10 @@ static inline void setup_object_debug(struct kmem_cache *s,
1029 struct page *page, void *object) {} 1034 struct page *page, void *object) {}
1030 1035
1031static inline int alloc_debug_processing(struct kmem_cache *s, 1036static inline int alloc_debug_processing(struct kmem_cache *s,
1032 struct page *page, void *object, void *addr) { return 0; } 1037 struct page *page, void *object, unsigned long addr) { return 0; }
1033 1038
1034static inline int free_debug_processing(struct kmem_cache *s, 1039static inline int free_debug_processing(struct kmem_cache *s,
1035 struct page *page, void *object, void *addr) { return 0; } 1040 struct page *page, void *object, unsigned long addr) { return 0; }
1036 1041
1037static inline int slab_pad_check(struct kmem_cache *s, struct page *page) 1042static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
1038 { return 1; } 1043 { return 1; }
@@ -1499,8 +1504,8 @@ static inline int node_match(struct kmem_cache_cpu *c, int node)
1499 * we need to allocate a new slab. This is the slowest path since it involves 1504 * we need to allocate a new slab. This is the slowest path since it involves
1500 * a call to the page allocator and the setup of a new slab. 1505 * a call to the page allocator and the setup of a new slab.
1501 */ 1506 */
1502static void *__slab_alloc(struct kmem_cache *s, 1507static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
1503 gfp_t gfpflags, int node, void *addr, struct kmem_cache_cpu *c) 1508 unsigned long addr, struct kmem_cache_cpu *c)
1504{ 1509{
1505 void **object; 1510 void **object;
1506 struct page *new; 1511 struct page *new;
@@ -1584,13 +1589,18 @@ debug:
1584 * Otherwise we can simply pick the next object from the lockless free list. 1589 * Otherwise we can simply pick the next object from the lockless free list.
1585 */ 1590 */
1586static __always_inline void *slab_alloc(struct kmem_cache *s, 1591static __always_inline void *slab_alloc(struct kmem_cache *s,
1587 gfp_t gfpflags, int node, void *addr) 1592 gfp_t gfpflags, int node, unsigned long addr)
1588{ 1593{
1589 void **object; 1594 void **object;
1590 struct kmem_cache_cpu *c; 1595 struct kmem_cache_cpu *c;
1591 unsigned long flags; 1596 unsigned long flags;
1592 unsigned int objsize; 1597 unsigned int objsize;
1593 1598
1599 might_sleep_if(gfpflags & __GFP_WAIT);
1600
1601 if (should_failslab(s->objsize, gfpflags))
1602 return NULL;
1603
1594 local_irq_save(flags); 1604 local_irq_save(flags);
1595 c = get_cpu_slab(s, smp_processor_id()); 1605 c = get_cpu_slab(s, smp_processor_id());
1596 objsize = c->objsize; 1606 objsize = c->objsize;
@@ -1613,14 +1623,14 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
1613 1623
1614void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) 1624void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
1615{ 1625{
1616 return slab_alloc(s, gfpflags, -1, __builtin_return_address(0)); 1626 return slab_alloc(s, gfpflags, -1, _RET_IP_);
1617} 1627}
1618EXPORT_SYMBOL(kmem_cache_alloc); 1628EXPORT_SYMBOL(kmem_cache_alloc);
1619 1629
1620#ifdef CONFIG_NUMA 1630#ifdef CONFIG_NUMA
1621void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) 1631void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
1622{ 1632{
1623 return slab_alloc(s, gfpflags, node, __builtin_return_address(0)); 1633 return slab_alloc(s, gfpflags, node, _RET_IP_);
1624} 1634}
1625EXPORT_SYMBOL(kmem_cache_alloc_node); 1635EXPORT_SYMBOL(kmem_cache_alloc_node);
1626#endif 1636#endif
@@ -1634,7 +1644,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node);
1634 * handling required then we can return immediately. 1644 * handling required then we can return immediately.
1635 */ 1645 */
1636static void __slab_free(struct kmem_cache *s, struct page *page, 1646static void __slab_free(struct kmem_cache *s, struct page *page,
1637 void *x, void *addr, unsigned int offset) 1647 void *x, unsigned long addr, unsigned int offset)
1638{ 1648{
1639 void *prior; 1649 void *prior;
1640 void **object = (void *)x; 1650 void **object = (void *)x;
@@ -1704,7 +1714,7 @@ debug:
1704 * with all sorts of special processing. 1714 * with all sorts of special processing.
1705 */ 1715 */
1706static __always_inline void slab_free(struct kmem_cache *s, 1716static __always_inline void slab_free(struct kmem_cache *s,
1707 struct page *page, void *x, void *addr) 1717 struct page *page, void *x, unsigned long addr)
1708{ 1718{
1709 void **object = (void *)x; 1719 void **object = (void *)x;
1710 struct kmem_cache_cpu *c; 1720 struct kmem_cache_cpu *c;
@@ -1731,11 +1741,11 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
1731 1741
1732 page = virt_to_head_page(x); 1742 page = virt_to_head_page(x);
1733 1743
1734 slab_free(s, page, x, __builtin_return_address(0)); 1744 slab_free(s, page, x, _RET_IP_);
1735} 1745}
1736EXPORT_SYMBOL(kmem_cache_free); 1746EXPORT_SYMBOL(kmem_cache_free);
1737 1747
1738/* Figure out on which slab object the object resides */ 1748/* Figure out on which slab page the object resides */
1739static struct page *get_object_page(const void *x) 1749static struct page *get_object_page(const void *x)
1740{ 1750{
1741 struct page *page = virt_to_head_page(x); 1751 struct page *page = virt_to_head_page(x);
@@ -1807,8 +1817,8 @@ static inline int slab_order(int size, int min_objects,
1807 int rem; 1817 int rem;
1808 int min_order = slub_min_order; 1818 int min_order = slub_min_order;
1809 1819
1810 if ((PAGE_SIZE << min_order) / size > 65535) 1820 if ((PAGE_SIZE << min_order) / size > MAX_OBJS_PER_PAGE)
1811 return get_order(size * 65535) - 1; 1821 return get_order(size * MAX_OBJS_PER_PAGE) - 1;
1812 1822
1813 for (order = max(min_order, 1823 for (order = max(min_order,
1814 fls(min_objects * size - 1) - PAGE_SHIFT); 1824 fls(min_objects * size - 1) - PAGE_SHIFT);
@@ -2073,8 +2083,7 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
2073 * when allocating for the kmalloc_node_cache. This is used for bootstrapping 2083 * when allocating for the kmalloc_node_cache. This is used for bootstrapping
2074 * memory on a fresh node that has no slab structures yet. 2084 * memory on a fresh node that has no slab structures yet.
2075 */ 2085 */
2076static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags, 2086static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node)
2077 int node)
2078{ 2087{
2079 struct page *page; 2088 struct page *page;
2080 struct kmem_cache_node *n; 2089 struct kmem_cache_node *n;
@@ -2112,7 +2121,6 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
2112 local_irq_save(flags); 2121 local_irq_save(flags);
2113 add_partial(n, page, 0); 2122 add_partial(n, page, 0);
2114 local_irq_restore(flags); 2123 local_irq_restore(flags);
2115 return n;
2116} 2124}
2117 2125
2118static void free_kmem_cache_nodes(struct kmem_cache *s) 2126static void free_kmem_cache_nodes(struct kmem_cache *s)
@@ -2144,8 +2152,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
2144 n = &s->local_node; 2152 n = &s->local_node;
2145 else { 2153 else {
2146 if (slab_state == DOWN) { 2154 if (slab_state == DOWN) {
2147 n = early_kmem_cache_node_alloc(gfpflags, 2155 early_kmem_cache_node_alloc(gfpflags, node);
2148 node);
2149 continue; 2156 continue;
2150 } 2157 }
2151 n = kmem_cache_alloc_node(kmalloc_caches, 2158 n = kmem_cache_alloc_node(kmalloc_caches,
@@ -2659,7 +2666,7 @@ void *__kmalloc(size_t size, gfp_t flags)
2659 if (unlikely(ZERO_OR_NULL_PTR(s))) 2666 if (unlikely(ZERO_OR_NULL_PTR(s)))
2660 return s; 2667 return s;
2661 2668
2662 return slab_alloc(s, flags, -1, __builtin_return_address(0)); 2669 return slab_alloc(s, flags, -1, _RET_IP_);
2663} 2670}
2664EXPORT_SYMBOL(__kmalloc); 2671EXPORT_SYMBOL(__kmalloc);
2665 2672
@@ -2687,7 +2694,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
2687 if (unlikely(ZERO_OR_NULL_PTR(s))) 2694 if (unlikely(ZERO_OR_NULL_PTR(s)))
2688 return s; 2695 return s;
2689 2696
2690 return slab_alloc(s, flags, node, __builtin_return_address(0)); 2697 return slab_alloc(s, flags, node, _RET_IP_);
2691} 2698}
2692EXPORT_SYMBOL(__kmalloc_node); 2699EXPORT_SYMBOL(__kmalloc_node);
2693#endif 2700#endif
@@ -2744,7 +2751,7 @@ void kfree(const void *x)
2744 put_page(page); 2751 put_page(page);
2745 return; 2752 return;
2746 } 2753 }
2747 slab_free(page->slab, page, object, __builtin_return_address(0)); 2754 slab_free(page->slab, page, object, _RET_IP_);
2748} 2755}
2749EXPORT_SYMBOL(kfree); 2756EXPORT_SYMBOL(kfree);
2750 2757
@@ -3123,8 +3130,12 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
3123 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); 3130 s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
3124 up_write(&slub_lock); 3131 up_write(&slub_lock);
3125 3132
3126 if (sysfs_slab_alias(s, name)) 3133 if (sysfs_slab_alias(s, name)) {
3134 down_write(&slub_lock);
3135 s->refcount--;
3136 up_write(&slub_lock);
3127 goto err; 3137 goto err;
3138 }
3128 return s; 3139 return s;
3129 } 3140 }
3130 3141
@@ -3134,8 +3145,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
3134 size, align, flags, ctor)) { 3145 size, align, flags, ctor)) {
3135 list_add(&s->list, &slab_caches); 3146 list_add(&s->list, &slab_caches);
3136 up_write(&slub_lock); 3147 up_write(&slub_lock);
3137 if (sysfs_slab_add(s)) 3148 if (sysfs_slab_add(s)) {
3149 down_write(&slub_lock);
3150 list_del(&s->list);
3151 up_write(&slub_lock);
3152 kfree(s);
3138 goto err; 3153 goto err;
3154 }
3139 return s; 3155 return s;
3140 } 3156 }
3141 kfree(s); 3157 kfree(s);
@@ -3202,7 +3218,7 @@ static struct notifier_block __cpuinitdata slab_notifier = {
3202 3218
3203#endif 3219#endif
3204 3220
3205void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller) 3221void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
3206{ 3222{
3207 struct kmem_cache *s; 3223 struct kmem_cache *s;
3208 3224
@@ -3218,7 +3234,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller)
3218} 3234}
3219 3235
3220void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, 3236void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
3221 int node, void *caller) 3237 int node, unsigned long caller)
3222{ 3238{
3223 struct kmem_cache *s; 3239 struct kmem_cache *s;
3224 3240
@@ -3429,7 +3445,7 @@ static void resiliency_test(void) {};
3429 3445
3430struct location { 3446struct location {
3431 unsigned long count; 3447 unsigned long count;
3432 void *addr; 3448 unsigned long addr;
3433 long long sum_time; 3449 long long sum_time;
3434 long min_time; 3450 long min_time;
3435 long max_time; 3451 long max_time;
@@ -3477,7 +3493,7 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
3477{ 3493{
3478 long start, end, pos; 3494 long start, end, pos;
3479 struct location *l; 3495 struct location *l;
3480 void *caddr; 3496 unsigned long caddr;
3481 unsigned long age = jiffies - track->when; 3497 unsigned long age = jiffies - track->when;
3482 3498
3483 start = -1; 3499 start = -1;
@@ -4345,7 +4361,7 @@ static void sysfs_slab_remove(struct kmem_cache *s)
4345 4361
4346/* 4362/*
4347 * Need to buffer aliases during bootup until sysfs becomes 4363 * Need to buffer aliases during bootup until sysfs becomes
4348 * available lest we loose that information. 4364 * available lest we lose that information.
4349 */ 4365 */
4350struct saved_alias { 4366struct saved_alias {
4351 struct kmem_cache *s; 4367 struct kmem_cache *s;