diff options
-rw-r--r-- | mm/slab.c | 964 |
1 files changed, 500 insertions, 464 deletions
@@ -130,7 +130,6 @@ | |||
130 | #define FORCED_DEBUG 0 | 130 | #define FORCED_DEBUG 0 |
131 | #endif | 131 | #endif |
132 | 132 | ||
133 | |||
134 | /* Shouldn't this be in a header file somewhere? */ | 133 | /* Shouldn't this be in a header file somewhere? */ |
135 | #define BYTES_PER_WORD sizeof(void *) | 134 | #define BYTES_PER_WORD sizeof(void *) |
136 | 135 | ||
@@ -217,12 +216,12 @@ static unsigned long offslab_limit; | |||
217 | * Slabs are chained into three list: fully used, partial, fully free slabs. | 216 | * Slabs are chained into three list: fully used, partial, fully free slabs. |
218 | */ | 217 | */ |
219 | struct slab { | 218 | struct slab { |
220 | struct list_head list; | 219 | struct list_head list; |
221 | unsigned long colouroff; | 220 | unsigned long colouroff; |
222 | void *s_mem; /* including colour offset */ | 221 | void *s_mem; /* including colour offset */ |
223 | unsigned int inuse; /* num of objs active in slab */ | 222 | unsigned int inuse; /* num of objs active in slab */ |
224 | kmem_bufctl_t free; | 223 | kmem_bufctl_t free; |
225 | unsigned short nodeid; | 224 | unsigned short nodeid; |
226 | }; | 225 | }; |
227 | 226 | ||
228 | /* | 227 | /* |
@@ -242,9 +241,9 @@ struct slab { | |||
242 | * We assume struct slab_rcu can overlay struct slab when destroying. | 241 | * We assume struct slab_rcu can overlay struct slab when destroying. |
243 | */ | 242 | */ |
244 | struct slab_rcu { | 243 | struct slab_rcu { |
245 | struct rcu_head head; | 244 | struct rcu_head head; |
246 | kmem_cache_t *cachep; | 245 | kmem_cache_t *cachep; |
247 | void *addr; | 246 | void *addr; |
248 | }; | 247 | }; |
249 | 248 | ||
250 | /* | 249 | /* |
@@ -279,23 +278,23 @@ struct array_cache { | |||
279 | #define BOOT_CPUCACHE_ENTRIES 1 | 278 | #define BOOT_CPUCACHE_ENTRIES 1 |
280 | struct arraycache_init { | 279 | struct arraycache_init { |
281 | struct array_cache cache; | 280 | struct array_cache cache; |
282 | void * entries[BOOT_CPUCACHE_ENTRIES]; | 281 | void *entries[BOOT_CPUCACHE_ENTRIES]; |
283 | }; | 282 | }; |
284 | 283 | ||
285 | /* | 284 | /* |
286 | * The slab lists for all objects. | 285 | * The slab lists for all objects. |
287 | */ | 286 | */ |
288 | struct kmem_list3 { | 287 | struct kmem_list3 { |
289 | struct list_head slabs_partial; /* partial list first, better asm code */ | 288 | struct list_head slabs_partial; /* partial list first, better asm code */ |
290 | struct list_head slabs_full; | 289 | struct list_head slabs_full; |
291 | struct list_head slabs_free; | 290 | struct list_head slabs_free; |
292 | unsigned long free_objects; | 291 | unsigned long free_objects; |
293 | unsigned long next_reap; | 292 | unsigned long next_reap; |
294 | int free_touched; | 293 | int free_touched; |
295 | unsigned int free_limit; | 294 | unsigned int free_limit; |
296 | spinlock_t list_lock; | 295 | spinlock_t list_lock; |
297 | struct array_cache *shared; /* shared per node */ | 296 | struct array_cache *shared; /* shared per node */ |
298 | struct array_cache **alien; /* on other nodes */ | 297 | struct array_cache **alien; /* on other nodes */ |
299 | }; | 298 | }; |
300 | 299 | ||
301 | /* | 300 | /* |
@@ -367,63 +366,63 @@ static inline void kmem_list3_init(struct kmem_list3 *parent) | |||
367 | * | 366 | * |
368 | * manages a cache. | 367 | * manages a cache. |
369 | */ | 368 | */ |
370 | 369 | ||
371 | struct kmem_cache { | 370 | struct kmem_cache { |
372 | /* 1) per-cpu data, touched during every alloc/free */ | 371 | /* 1) per-cpu data, touched during every alloc/free */ |
373 | struct array_cache *array[NR_CPUS]; | 372 | struct array_cache *array[NR_CPUS]; |
374 | unsigned int batchcount; | 373 | unsigned int batchcount; |
375 | unsigned int limit; | 374 | unsigned int limit; |
376 | unsigned int shared; | 375 | unsigned int shared; |
377 | unsigned int objsize; | 376 | unsigned int objsize; |
378 | /* 2) touched by every alloc & free from the backend */ | 377 | /* 2) touched by every alloc & free from the backend */ |
379 | struct kmem_list3 *nodelists[MAX_NUMNODES]; | 378 | struct kmem_list3 *nodelists[MAX_NUMNODES]; |
380 | unsigned int flags; /* constant flags */ | 379 | unsigned int flags; /* constant flags */ |
381 | unsigned int num; /* # of objs per slab */ | 380 | unsigned int num; /* # of objs per slab */ |
382 | spinlock_t spinlock; | 381 | spinlock_t spinlock; |
383 | 382 | ||
384 | /* 3) cache_grow/shrink */ | 383 | /* 3) cache_grow/shrink */ |
385 | /* order of pgs per slab (2^n) */ | 384 | /* order of pgs per slab (2^n) */ |
386 | unsigned int gfporder; | 385 | unsigned int gfporder; |
387 | 386 | ||
388 | /* force GFP flags, e.g. GFP_DMA */ | 387 | /* force GFP flags, e.g. GFP_DMA */ |
389 | gfp_t gfpflags; | 388 | gfp_t gfpflags; |
390 | 389 | ||
391 | size_t colour; /* cache colouring range */ | 390 | size_t colour; /* cache colouring range */ |
392 | unsigned int colour_off; /* colour offset */ | 391 | unsigned int colour_off; /* colour offset */ |
393 | unsigned int colour_next; /* cache colouring */ | 392 | unsigned int colour_next; /* cache colouring */ |
394 | kmem_cache_t *slabp_cache; | 393 | kmem_cache_t *slabp_cache; |
395 | unsigned int slab_size; | 394 | unsigned int slab_size; |
396 | unsigned int dflags; /* dynamic flags */ | 395 | unsigned int dflags; /* dynamic flags */ |
397 | 396 | ||
398 | /* constructor func */ | 397 | /* constructor func */ |
399 | void (*ctor)(void *, kmem_cache_t *, unsigned long); | 398 | void (*ctor) (void *, kmem_cache_t *, unsigned long); |
400 | 399 | ||
401 | /* de-constructor func */ | 400 | /* de-constructor func */ |
402 | void (*dtor)(void *, kmem_cache_t *, unsigned long); | 401 | void (*dtor) (void *, kmem_cache_t *, unsigned long); |
403 | 402 | ||
404 | /* 4) cache creation/removal */ | 403 | /* 4) cache creation/removal */ |
405 | const char *name; | 404 | const char *name; |
406 | struct list_head next; | 405 | struct list_head next; |
407 | 406 | ||
408 | /* 5) statistics */ | 407 | /* 5) statistics */ |
409 | #if STATS | 408 | #if STATS |
410 | unsigned long num_active; | 409 | unsigned long num_active; |
411 | unsigned long num_allocations; | 410 | unsigned long num_allocations; |
412 | unsigned long high_mark; | 411 | unsigned long high_mark; |
413 | unsigned long grown; | 412 | unsigned long grown; |
414 | unsigned long reaped; | 413 | unsigned long reaped; |
415 | unsigned long errors; | 414 | unsigned long errors; |
416 | unsigned long max_freeable; | 415 | unsigned long max_freeable; |
417 | unsigned long node_allocs; | 416 | unsigned long node_allocs; |
418 | unsigned long node_frees; | 417 | unsigned long node_frees; |
419 | atomic_t allochit; | 418 | atomic_t allochit; |
420 | atomic_t allocmiss; | 419 | atomic_t allocmiss; |
421 | atomic_t freehit; | 420 | atomic_t freehit; |
422 | atomic_t freemiss; | 421 | atomic_t freemiss; |
423 | #endif | 422 | #endif |
424 | #if DEBUG | 423 | #if DEBUG |
425 | int dbghead; | 424 | int dbghead; |
426 | int reallen; | 425 | int reallen; |
427 | #endif | 426 | #endif |
428 | }; | 427 | }; |
429 | 428 | ||
@@ -523,14 +522,15 @@ static unsigned long *dbg_redzone2(kmem_cache_t *cachep, void *objp) | |||
523 | { | 522 | { |
524 | BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); | 523 | BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); |
525 | if (cachep->flags & SLAB_STORE_USER) | 524 | if (cachep->flags & SLAB_STORE_USER) |
526 | return (unsigned long*) (objp+cachep->objsize-2*BYTES_PER_WORD); | 525 | return (unsigned long *)(objp + cachep->objsize - |
527 | return (unsigned long*) (objp+cachep->objsize-BYTES_PER_WORD); | 526 | 2 * BYTES_PER_WORD); |
527 | return (unsigned long *)(objp + cachep->objsize - BYTES_PER_WORD); | ||
528 | } | 528 | } |
529 | 529 | ||
530 | static void **dbg_userword(kmem_cache_t *cachep, void *objp) | 530 | static void **dbg_userword(kmem_cache_t *cachep, void *objp) |
531 | { | 531 | { |
532 | BUG_ON(!(cachep->flags & SLAB_STORE_USER)); | 532 | BUG_ON(!(cachep->flags & SLAB_STORE_USER)); |
533 | return (void**)(objp+cachep->objsize-BYTES_PER_WORD); | 533 | return (void **)(objp + cachep->objsize - BYTES_PER_WORD); |
534 | } | 534 | } |
535 | 535 | ||
536 | #else | 536 | #else |
@@ -607,31 +607,31 @@ struct cache_names { | |||
607 | static struct cache_names __initdata cache_names[] = { | 607 | static struct cache_names __initdata cache_names[] = { |
608 | #define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, | 608 | #define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, |
609 | #include <linux/kmalloc_sizes.h> | 609 | #include <linux/kmalloc_sizes.h> |
610 | { NULL, } | 610 | {NULL,} |
611 | #undef CACHE | 611 | #undef CACHE |
612 | }; | 612 | }; |
613 | 613 | ||
614 | static struct arraycache_init initarray_cache __initdata = | 614 | static struct arraycache_init initarray_cache __initdata = |
615 | { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; | 615 | { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; |
616 | static struct arraycache_init initarray_generic = | 616 | static struct arraycache_init initarray_generic = |
617 | { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; | 617 | { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; |
618 | 618 | ||
619 | /* internal cache of cache description objs */ | 619 | /* internal cache of cache description objs */ |
620 | static kmem_cache_t cache_cache = { | 620 | static kmem_cache_t cache_cache = { |
621 | .batchcount = 1, | 621 | .batchcount = 1, |
622 | .limit = BOOT_CPUCACHE_ENTRIES, | 622 | .limit = BOOT_CPUCACHE_ENTRIES, |
623 | .shared = 1, | 623 | .shared = 1, |
624 | .objsize = sizeof(kmem_cache_t), | 624 | .objsize = sizeof(kmem_cache_t), |
625 | .flags = SLAB_NO_REAP, | 625 | .flags = SLAB_NO_REAP, |
626 | .spinlock = SPIN_LOCK_UNLOCKED, | 626 | .spinlock = SPIN_LOCK_UNLOCKED, |
627 | .name = "kmem_cache", | 627 | .name = "kmem_cache", |
628 | #if DEBUG | 628 | #if DEBUG |
629 | .reallen = sizeof(kmem_cache_t), | 629 | .reallen = sizeof(kmem_cache_t), |
630 | #endif | 630 | #endif |
631 | }; | 631 | }; |
632 | 632 | ||
633 | /* Guard access to the cache-chain. */ | 633 | /* Guard access to the cache-chain. */ |
634 | static struct semaphore cache_chain_sem; | 634 | static struct semaphore cache_chain_sem; |
635 | static struct list_head cache_chain; | 635 | static struct list_head cache_chain; |
636 | 636 | ||
637 | /* | 637 | /* |
@@ -655,9 +655,9 @@ static enum { | |||
655 | 655 | ||
656 | static DEFINE_PER_CPU(struct work_struct, reap_work); | 656 | static DEFINE_PER_CPU(struct work_struct, reap_work); |
657 | 657 | ||
658 | static void free_block(kmem_cache_t* cachep, void** objpp, int len, int node); | 658 | static void free_block(kmem_cache_t *cachep, void **objpp, int len, int node); |
659 | static void enable_cpucache (kmem_cache_t *cachep); | 659 | static void enable_cpucache(kmem_cache_t *cachep); |
660 | static void cache_reap (void *unused); | 660 | static void cache_reap(void *unused); |
661 | static int __node_shrink(kmem_cache_t *cachep, int node); | 661 | static int __node_shrink(kmem_cache_t *cachep, int node); |
662 | 662 | ||
663 | static inline struct array_cache *ac_data(kmem_cache_t *cachep) | 663 | static inline struct array_cache *ac_data(kmem_cache_t *cachep) |
@@ -671,9 +671,9 @@ static inline kmem_cache_t *__find_general_cachep(size_t size, gfp_t gfpflags) | |||
671 | 671 | ||
672 | #if DEBUG | 672 | #if DEBUG |
673 | /* This happens if someone tries to call | 673 | /* This happens if someone tries to call |
674 | * kmem_cache_create(), or __kmalloc(), before | 674 | * kmem_cache_create(), or __kmalloc(), before |
675 | * the generic caches are initialized. | 675 | * the generic caches are initialized. |
676 | */ | 676 | */ |
677 | BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); | 677 | BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); |
678 | #endif | 678 | #endif |
679 | while (size > csizep->cs_size) | 679 | while (size > csizep->cs_size) |
@@ -697,10 +697,10 @@ EXPORT_SYMBOL(kmem_find_general_cachep); | |||
697 | 697 | ||
698 | /* Cal the num objs, wastage, and bytes left over for a given slab size. */ | 698 | /* Cal the num objs, wastage, and bytes left over for a given slab size. */ |
699 | static void cache_estimate(unsigned long gfporder, size_t size, size_t align, | 699 | static void cache_estimate(unsigned long gfporder, size_t size, size_t align, |
700 | int flags, size_t *left_over, unsigned int *num) | 700 | int flags, size_t *left_over, unsigned int *num) |
701 | { | 701 | { |
702 | int i; | 702 | int i; |
703 | size_t wastage = PAGE_SIZE<<gfporder; | 703 | size_t wastage = PAGE_SIZE << gfporder; |
704 | size_t extra = 0; | 704 | size_t extra = 0; |
705 | size_t base = 0; | 705 | size_t base = 0; |
706 | 706 | ||
@@ -709,7 +709,7 @@ static void cache_estimate(unsigned long gfporder, size_t size, size_t align, | |||
709 | extra = sizeof(kmem_bufctl_t); | 709 | extra = sizeof(kmem_bufctl_t); |
710 | } | 710 | } |
711 | i = 0; | 711 | i = 0; |
712 | while (i*size + ALIGN(base+i*extra, align) <= wastage) | 712 | while (i * size + ALIGN(base + i * extra, align) <= wastage) |
713 | i++; | 713 | i++; |
714 | if (i > 0) | 714 | if (i > 0) |
715 | i--; | 715 | i--; |
@@ -718,8 +718,8 @@ static void cache_estimate(unsigned long gfporder, size_t size, size_t align, | |||
718 | i = SLAB_LIMIT; | 718 | i = SLAB_LIMIT; |
719 | 719 | ||
720 | *num = i; | 720 | *num = i; |
721 | wastage -= i*size; | 721 | wastage -= i * size; |
722 | wastage -= ALIGN(base+i*extra, align); | 722 | wastage -= ALIGN(base + i * extra, align); |
723 | *left_over = wastage; | 723 | *left_over = wastage; |
724 | } | 724 | } |
725 | 725 | ||
@@ -728,7 +728,7 @@ static void cache_estimate(unsigned long gfporder, size_t size, size_t align, | |||
728 | static void __slab_error(const char *function, kmem_cache_t *cachep, char *msg) | 728 | static void __slab_error(const char *function, kmem_cache_t *cachep, char *msg) |
729 | { | 729 | { |
730 | printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", | 730 | printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", |
731 | function, cachep->name, msg); | 731 | function, cachep->name, msg); |
732 | dump_stack(); | 732 | dump_stack(); |
733 | } | 733 | } |
734 | 734 | ||
@@ -755,9 +755,9 @@ static void __devinit start_cpu_timer(int cpu) | |||
755 | } | 755 | } |
756 | 756 | ||
757 | static struct array_cache *alloc_arraycache(int node, int entries, | 757 | static struct array_cache *alloc_arraycache(int node, int entries, |
758 | int batchcount) | 758 | int batchcount) |
759 | { | 759 | { |
760 | int memsize = sizeof(void*)*entries+sizeof(struct array_cache); | 760 | int memsize = sizeof(void *) * entries + sizeof(struct array_cache); |
761 | struct array_cache *nc = NULL; | 761 | struct array_cache *nc = NULL; |
762 | 762 | ||
763 | nc = kmalloc_node(memsize, GFP_KERNEL, node); | 763 | nc = kmalloc_node(memsize, GFP_KERNEL, node); |
@@ -775,7 +775,7 @@ static struct array_cache *alloc_arraycache(int node, int entries, | |||
775 | static inline struct array_cache **alloc_alien_cache(int node, int limit) | 775 | static inline struct array_cache **alloc_alien_cache(int node, int limit) |
776 | { | 776 | { |
777 | struct array_cache **ac_ptr; | 777 | struct array_cache **ac_ptr; |
778 | int memsize = sizeof(void*)*MAX_NUMNODES; | 778 | int memsize = sizeof(void *) * MAX_NUMNODES; |
779 | int i; | 779 | int i; |
780 | 780 | ||
781 | if (limit > 1) | 781 | if (limit > 1) |
@@ -789,7 +789,7 @@ static inline struct array_cache **alloc_alien_cache(int node, int limit) | |||
789 | } | 789 | } |
790 | ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d); | 790 | ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d); |
791 | if (!ac_ptr[i]) { | 791 | if (!ac_ptr[i]) { |
792 | for (i--; i <=0; i--) | 792 | for (i--; i <= 0; i--) |
793 | kfree(ac_ptr[i]); | 793 | kfree(ac_ptr[i]); |
794 | kfree(ac_ptr); | 794 | kfree(ac_ptr); |
795 | return NULL; | 795 | return NULL; |
@@ -807,12 +807,13 @@ static inline void free_alien_cache(struct array_cache **ac_ptr) | |||
807 | return; | 807 | return; |
808 | 808 | ||
809 | for_each_node(i) | 809 | for_each_node(i) |
810 | kfree(ac_ptr[i]); | 810 | kfree(ac_ptr[i]); |
811 | 811 | ||
812 | kfree(ac_ptr); | 812 | kfree(ac_ptr); |
813 | } | 813 | } |
814 | 814 | ||
815 | static inline void __drain_alien_cache(kmem_cache_t *cachep, struct array_cache *ac, int node) | 815 | static inline void __drain_alien_cache(kmem_cache_t *cachep, |
816 | struct array_cache *ac, int node) | ||
816 | { | 817 | { |
817 | struct kmem_list3 *rl3 = cachep->nodelists[node]; | 818 | struct kmem_list3 *rl3 = cachep->nodelists[node]; |
818 | 819 | ||
@@ -826,7 +827,7 @@ static inline void __drain_alien_cache(kmem_cache_t *cachep, struct array_cache | |||
826 | 827 | ||
827 | static void drain_alien_cache(kmem_cache_t *cachep, struct kmem_list3 *l3) | 828 | static void drain_alien_cache(kmem_cache_t *cachep, struct kmem_list3 *l3) |
828 | { | 829 | { |
829 | int i=0; | 830 | int i = 0; |
830 | struct array_cache *ac; | 831 | struct array_cache *ac; |
831 | unsigned long flags; | 832 | unsigned long flags; |
832 | 833 | ||
@@ -846,10 +847,10 @@ static void drain_alien_cache(kmem_cache_t *cachep, struct kmem_list3 *l3) | |||
846 | #endif | 847 | #endif |
847 | 848 | ||
848 | static int __devinit cpuup_callback(struct notifier_block *nfb, | 849 | static int __devinit cpuup_callback(struct notifier_block *nfb, |
849 | unsigned long action, void *hcpu) | 850 | unsigned long action, void *hcpu) |
850 | { | 851 | { |
851 | long cpu = (long)hcpu; | 852 | long cpu = (long)hcpu; |
852 | kmem_cache_t* cachep; | 853 | kmem_cache_t *cachep; |
853 | struct kmem_list3 *l3 = NULL; | 854 | struct kmem_list3 *l3 = NULL; |
854 | int node = cpu_to_node(cpu); | 855 | int node = cpu_to_node(cpu); |
855 | int memsize = sizeof(struct kmem_list3); | 856 | int memsize = sizeof(struct kmem_list3); |
@@ -871,27 +872,27 @@ static int __devinit cpuup_callback(struct notifier_block *nfb, | |||
871 | */ | 872 | */ |
872 | if (!cachep->nodelists[node]) { | 873 | if (!cachep->nodelists[node]) { |
873 | if (!(l3 = kmalloc_node(memsize, | 874 | if (!(l3 = kmalloc_node(memsize, |
874 | GFP_KERNEL, node))) | 875 | GFP_KERNEL, node))) |
875 | goto bad; | 876 | goto bad; |
876 | kmem_list3_init(l3); | 877 | kmem_list3_init(l3); |
877 | l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + | 878 | l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + |
878 | ((unsigned long)cachep)%REAPTIMEOUT_LIST3; | 879 | ((unsigned long)cachep) % REAPTIMEOUT_LIST3; |
879 | 880 | ||
880 | cachep->nodelists[node] = l3; | 881 | cachep->nodelists[node] = l3; |
881 | } | 882 | } |
882 | 883 | ||
883 | spin_lock_irq(&cachep->nodelists[node]->list_lock); | 884 | spin_lock_irq(&cachep->nodelists[node]->list_lock); |
884 | cachep->nodelists[node]->free_limit = | 885 | cachep->nodelists[node]->free_limit = |
885 | (1 + nr_cpus_node(node)) * | 886 | (1 + nr_cpus_node(node)) * |
886 | cachep->batchcount + cachep->num; | 887 | cachep->batchcount + cachep->num; |
887 | spin_unlock_irq(&cachep->nodelists[node]->list_lock); | 888 | spin_unlock_irq(&cachep->nodelists[node]->list_lock); |
888 | } | 889 | } |
889 | 890 | ||
890 | /* Now we can go ahead with allocating the shared array's | 891 | /* Now we can go ahead with allocating the shared array's |
891 | & array cache's */ | 892 | & array cache's */ |
892 | list_for_each_entry(cachep, &cache_chain, next) { | 893 | list_for_each_entry(cachep, &cache_chain, next) { |
893 | nc = alloc_arraycache(node, cachep->limit, | 894 | nc = alloc_arraycache(node, cachep->limit, |
894 | cachep->batchcount); | 895 | cachep->batchcount); |
895 | if (!nc) | 896 | if (!nc) |
896 | goto bad; | 897 | goto bad; |
897 | cachep->array[cpu] = nc; | 898 | cachep->array[cpu] = nc; |
@@ -900,12 +901,13 @@ static int __devinit cpuup_callback(struct notifier_block *nfb, | |||
900 | BUG_ON(!l3); | 901 | BUG_ON(!l3); |
901 | if (!l3->shared) { | 902 | if (!l3->shared) { |
902 | if (!(nc = alloc_arraycache(node, | 903 | if (!(nc = alloc_arraycache(node, |
903 | cachep->shared*cachep->batchcount, | 904 | cachep->shared * |
904 | 0xbaadf00d))) | 905 | cachep->batchcount, |
905 | goto bad; | 906 | 0xbaadf00d))) |
907 | goto bad; | ||
906 | 908 | ||
907 | /* we are serialised from CPU_DEAD or | 909 | /* we are serialised from CPU_DEAD or |
908 | CPU_UP_CANCELLED by the cpucontrol lock */ | 910 | CPU_UP_CANCELLED by the cpucontrol lock */ |
909 | l3->shared = nc; | 911 | l3->shared = nc; |
910 | } | 912 | } |
911 | } | 913 | } |
@@ -942,13 +944,13 @@ static int __devinit cpuup_callback(struct notifier_block *nfb, | |||
942 | free_block(cachep, nc->entry, nc->avail, node); | 944 | free_block(cachep, nc->entry, nc->avail, node); |
943 | 945 | ||
944 | if (!cpus_empty(mask)) { | 946 | if (!cpus_empty(mask)) { |
945 | spin_unlock(&l3->list_lock); | 947 | spin_unlock(&l3->list_lock); |
946 | goto unlock_cache; | 948 | goto unlock_cache; |
947 | } | 949 | } |
948 | 950 | ||
949 | if (l3->shared) { | 951 | if (l3->shared) { |
950 | free_block(cachep, l3->shared->entry, | 952 | free_block(cachep, l3->shared->entry, |
951 | l3->shared->avail, node); | 953 | l3->shared->avail, node); |
952 | kfree(l3->shared); | 954 | kfree(l3->shared); |
953 | l3->shared = NULL; | 955 | l3->shared = NULL; |
954 | } | 956 | } |
@@ -966,7 +968,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb, | |||
966 | } else { | 968 | } else { |
967 | spin_unlock(&l3->list_lock); | 969 | spin_unlock(&l3->list_lock); |
968 | } | 970 | } |
969 | unlock_cache: | 971 | unlock_cache: |
970 | spin_unlock_irq(&cachep->spinlock); | 972 | spin_unlock_irq(&cachep->spinlock); |
971 | kfree(nc); | 973 | kfree(nc); |
972 | } | 974 | } |
@@ -975,7 +977,7 @@ unlock_cache: | |||
975 | #endif | 977 | #endif |
976 | } | 978 | } |
977 | return NOTIFY_OK; | 979 | return NOTIFY_OK; |
978 | bad: | 980 | bad: |
979 | up(&cache_chain_sem); | 981 | up(&cache_chain_sem); |
980 | return NOTIFY_BAD; | 982 | return NOTIFY_BAD; |
981 | } | 983 | } |
@@ -985,8 +987,7 @@ static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 }; | |||
985 | /* | 987 | /* |
986 | * swap the static kmem_list3 with kmalloced memory | 988 | * swap the static kmem_list3 with kmalloced memory |
987 | */ | 989 | */ |
988 | static void init_list(kmem_cache_t *cachep, struct kmem_list3 *list, | 990 | static void init_list(kmem_cache_t *cachep, struct kmem_list3 *list, int nodeid) |
989 | int nodeid) | ||
990 | { | 991 | { |
991 | struct kmem_list3 *ptr; | 992 | struct kmem_list3 *ptr; |
992 | 993 | ||
@@ -1055,14 +1056,14 @@ void __init kmem_cache_init(void) | |||
1055 | cache_cache.objsize = ALIGN(cache_cache.objsize, cache_line_size()); | 1056 | cache_cache.objsize = ALIGN(cache_cache.objsize, cache_line_size()); |
1056 | 1057 | ||
1057 | cache_estimate(0, cache_cache.objsize, cache_line_size(), 0, | 1058 | cache_estimate(0, cache_cache.objsize, cache_line_size(), 0, |
1058 | &left_over, &cache_cache.num); | 1059 | &left_over, &cache_cache.num); |
1059 | if (!cache_cache.num) | 1060 | if (!cache_cache.num) |
1060 | BUG(); | 1061 | BUG(); |
1061 | 1062 | ||
1062 | cache_cache.colour = left_over/cache_cache.colour_off; | 1063 | cache_cache.colour = left_over / cache_cache.colour_off; |
1063 | cache_cache.colour_next = 0; | 1064 | cache_cache.colour_next = 0; |
1064 | cache_cache.slab_size = ALIGN(cache_cache.num*sizeof(kmem_bufctl_t) + | 1065 | cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) + |
1065 | sizeof(struct slab), cache_line_size()); | 1066 | sizeof(struct slab), cache_line_size()); |
1066 | 1067 | ||
1067 | /* 2+3) create the kmalloc caches */ | 1068 | /* 2+3) create the kmalloc caches */ |
1068 | sizes = malloc_sizes; | 1069 | sizes = malloc_sizes; |
@@ -1074,14 +1075,18 @@ void __init kmem_cache_init(void) | |||
1074 | */ | 1075 | */ |
1075 | 1076 | ||
1076 | sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, | 1077 | sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, |
1077 | sizes[INDEX_AC].cs_size, ARCH_KMALLOC_MINALIGN, | 1078 | sizes[INDEX_AC].cs_size, |
1078 | (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); | 1079 | ARCH_KMALLOC_MINALIGN, |
1080 | (ARCH_KMALLOC_FLAGS | | ||
1081 | SLAB_PANIC), NULL, NULL); | ||
1079 | 1082 | ||
1080 | if (INDEX_AC != INDEX_L3) | 1083 | if (INDEX_AC != INDEX_L3) |
1081 | sizes[INDEX_L3].cs_cachep = | 1084 | sizes[INDEX_L3].cs_cachep = |
1082 | kmem_cache_create(names[INDEX_L3].name, | 1085 | kmem_cache_create(names[INDEX_L3].name, |
1083 | sizes[INDEX_L3].cs_size, ARCH_KMALLOC_MINALIGN, | 1086 | sizes[INDEX_L3].cs_size, |
1084 | (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); | 1087 | ARCH_KMALLOC_MINALIGN, |
1088 | (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, | ||
1089 | NULL); | ||
1085 | 1090 | ||
1086 | while (sizes->cs_size != ULONG_MAX) { | 1091 | while (sizes->cs_size != ULONG_MAX) { |
1087 | /* | 1092 | /* |
@@ -1091,35 +1096,41 @@ void __init kmem_cache_init(void) | |||
1091 | * Note for systems short on memory removing the alignment will | 1096 | * Note for systems short on memory removing the alignment will |
1092 | * allow tighter packing of the smaller caches. | 1097 | * allow tighter packing of the smaller caches. |
1093 | */ | 1098 | */ |
1094 | if(!sizes->cs_cachep) | 1099 | if (!sizes->cs_cachep) |
1095 | sizes->cs_cachep = kmem_cache_create(names->name, | 1100 | sizes->cs_cachep = kmem_cache_create(names->name, |
1096 | sizes->cs_size, ARCH_KMALLOC_MINALIGN, | 1101 | sizes->cs_size, |
1097 | (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); | 1102 | ARCH_KMALLOC_MINALIGN, |
1103 | (ARCH_KMALLOC_FLAGS | ||
1104 | | SLAB_PANIC), | ||
1105 | NULL, NULL); | ||
1098 | 1106 | ||
1099 | /* Inc off-slab bufctl limit until the ceiling is hit. */ | 1107 | /* Inc off-slab bufctl limit until the ceiling is hit. */ |
1100 | if (!(OFF_SLAB(sizes->cs_cachep))) { | 1108 | if (!(OFF_SLAB(sizes->cs_cachep))) { |
1101 | offslab_limit = sizes->cs_size-sizeof(struct slab); | 1109 | offslab_limit = sizes->cs_size - sizeof(struct slab); |
1102 | offslab_limit /= sizeof(kmem_bufctl_t); | 1110 | offslab_limit /= sizeof(kmem_bufctl_t); |
1103 | } | 1111 | } |
1104 | 1112 | ||
1105 | sizes->cs_dmacachep = kmem_cache_create(names->name_dma, | 1113 | sizes->cs_dmacachep = kmem_cache_create(names->name_dma, |
1106 | sizes->cs_size, ARCH_KMALLOC_MINALIGN, | 1114 | sizes->cs_size, |
1107 | (ARCH_KMALLOC_FLAGS | SLAB_CACHE_DMA | SLAB_PANIC), | 1115 | ARCH_KMALLOC_MINALIGN, |
1108 | NULL, NULL); | 1116 | (ARCH_KMALLOC_FLAGS | |
1117 | SLAB_CACHE_DMA | | ||
1118 | SLAB_PANIC), NULL, | ||
1119 | NULL); | ||
1109 | 1120 | ||
1110 | sizes++; | 1121 | sizes++; |
1111 | names++; | 1122 | names++; |
1112 | } | 1123 | } |
1113 | /* 4) Replace the bootstrap head arrays */ | 1124 | /* 4) Replace the bootstrap head arrays */ |
1114 | { | 1125 | { |
1115 | void * ptr; | 1126 | void *ptr; |
1116 | 1127 | ||
1117 | ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); | 1128 | ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); |
1118 | 1129 | ||
1119 | local_irq_disable(); | 1130 | local_irq_disable(); |
1120 | BUG_ON(ac_data(&cache_cache) != &initarray_cache.cache); | 1131 | BUG_ON(ac_data(&cache_cache) != &initarray_cache.cache); |
1121 | memcpy(ptr, ac_data(&cache_cache), | 1132 | memcpy(ptr, ac_data(&cache_cache), |
1122 | sizeof(struct arraycache_init)); | 1133 | sizeof(struct arraycache_init)); |
1123 | cache_cache.array[smp_processor_id()] = ptr; | 1134 | cache_cache.array[smp_processor_id()] = ptr; |
1124 | local_irq_enable(); | 1135 | local_irq_enable(); |
1125 | 1136 | ||
@@ -1127,11 +1138,11 @@ void __init kmem_cache_init(void) | |||
1127 | 1138 | ||
1128 | local_irq_disable(); | 1139 | local_irq_disable(); |
1129 | BUG_ON(ac_data(malloc_sizes[INDEX_AC].cs_cachep) | 1140 | BUG_ON(ac_data(malloc_sizes[INDEX_AC].cs_cachep) |
1130 | != &initarray_generic.cache); | 1141 | != &initarray_generic.cache); |
1131 | memcpy(ptr, ac_data(malloc_sizes[INDEX_AC].cs_cachep), | 1142 | memcpy(ptr, ac_data(malloc_sizes[INDEX_AC].cs_cachep), |
1132 | sizeof(struct arraycache_init)); | 1143 | sizeof(struct arraycache_init)); |
1133 | malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = | 1144 | malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = |
1134 | ptr; | 1145 | ptr; |
1135 | local_irq_enable(); | 1146 | local_irq_enable(); |
1136 | } | 1147 | } |
1137 | /* 5) Replace the bootstrap kmem_list3's */ | 1148 | /* 5) Replace the bootstrap kmem_list3's */ |
@@ -1139,16 +1150,16 @@ void __init kmem_cache_init(void) | |||
1139 | int node; | 1150 | int node; |
1140 | /* Replace the static kmem_list3 structures for the boot cpu */ | 1151 | /* Replace the static kmem_list3 structures for the boot cpu */ |
1141 | init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], | 1152 | init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], |
1142 | numa_node_id()); | 1153 | numa_node_id()); |
1143 | 1154 | ||
1144 | for_each_online_node(node) { | 1155 | for_each_online_node(node) { |
1145 | init_list(malloc_sizes[INDEX_AC].cs_cachep, | 1156 | init_list(malloc_sizes[INDEX_AC].cs_cachep, |
1146 | &initkmem_list3[SIZE_AC+node], node); | 1157 | &initkmem_list3[SIZE_AC + node], node); |
1147 | 1158 | ||
1148 | if (INDEX_AC != INDEX_L3) { | 1159 | if (INDEX_AC != INDEX_L3) { |
1149 | init_list(malloc_sizes[INDEX_L3].cs_cachep, | 1160 | init_list(malloc_sizes[INDEX_L3].cs_cachep, |
1150 | &initkmem_list3[SIZE_L3+node], | 1161 | &initkmem_list3[SIZE_L3 + node], |
1151 | node); | 1162 | node); |
1152 | } | 1163 | } |
1153 | } | 1164 | } |
1154 | } | 1165 | } |
@@ -1158,7 +1169,7 @@ void __init kmem_cache_init(void) | |||
1158 | kmem_cache_t *cachep; | 1169 | kmem_cache_t *cachep; |
1159 | down(&cache_chain_sem); | 1170 | down(&cache_chain_sem); |
1160 | list_for_each_entry(cachep, &cache_chain, next) | 1171 | list_for_each_entry(cachep, &cache_chain, next) |
1161 | enable_cpucache(cachep); | 1172 | enable_cpucache(cachep); |
1162 | up(&cache_chain_sem); | 1173 | up(&cache_chain_sem); |
1163 | } | 1174 | } |
1164 | 1175 | ||
@@ -1184,7 +1195,7 @@ static int __init cpucache_init(void) | |||
1184 | * pages to gfp. | 1195 | * pages to gfp. |
1185 | */ | 1196 | */ |
1186 | for_each_online_cpu(cpu) | 1197 | for_each_online_cpu(cpu) |
1187 | start_cpu_timer(cpu); | 1198 | start_cpu_timer(cpu); |
1188 | 1199 | ||
1189 | return 0; | 1200 | return 0; |
1190 | } | 1201 | } |
@@ -1226,7 +1237,7 @@ static void *kmem_getpages(kmem_cache_t *cachep, gfp_t flags, int nodeid) | |||
1226 | */ | 1237 | */ |
1227 | static void kmem_freepages(kmem_cache_t *cachep, void *addr) | 1238 | static void kmem_freepages(kmem_cache_t *cachep, void *addr) |
1228 | { | 1239 | { |
1229 | unsigned long i = (1<<cachep->gfporder); | 1240 | unsigned long i = (1 << cachep->gfporder); |
1230 | struct page *page = virt_to_page(addr); | 1241 | struct page *page = virt_to_page(addr); |
1231 | const unsigned long nr_freed = i; | 1242 | const unsigned long nr_freed = i; |
1232 | 1243 | ||
@@ -1239,13 +1250,13 @@ static void kmem_freepages(kmem_cache_t *cachep, void *addr) | |||
1239 | if (current->reclaim_state) | 1250 | if (current->reclaim_state) |
1240 | current->reclaim_state->reclaimed_slab += nr_freed; | 1251 | current->reclaim_state->reclaimed_slab += nr_freed; |
1241 | free_pages((unsigned long)addr, cachep->gfporder); | 1252 | free_pages((unsigned long)addr, cachep->gfporder); |
1242 | if (cachep->flags & SLAB_RECLAIM_ACCOUNT) | 1253 | if (cachep->flags & SLAB_RECLAIM_ACCOUNT) |
1243 | atomic_sub(1<<cachep->gfporder, &slab_reclaim_pages); | 1254 | atomic_sub(1 << cachep->gfporder, &slab_reclaim_pages); |
1244 | } | 1255 | } |
1245 | 1256 | ||
1246 | static void kmem_rcu_free(struct rcu_head *head) | 1257 | static void kmem_rcu_free(struct rcu_head *head) |
1247 | { | 1258 | { |
1248 | struct slab_rcu *slab_rcu = (struct slab_rcu *) head; | 1259 | struct slab_rcu *slab_rcu = (struct slab_rcu *)head; |
1249 | kmem_cache_t *cachep = slab_rcu->cachep; | 1260 | kmem_cache_t *cachep = slab_rcu->cachep; |
1250 | 1261 | ||
1251 | kmem_freepages(cachep, slab_rcu->addr); | 1262 | kmem_freepages(cachep, slab_rcu->addr); |
@@ -1257,19 +1268,19 @@ static void kmem_rcu_free(struct rcu_head *head) | |||
1257 | 1268 | ||
1258 | #ifdef CONFIG_DEBUG_PAGEALLOC | 1269 | #ifdef CONFIG_DEBUG_PAGEALLOC |
1259 | static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr, | 1270 | static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr, |
1260 | unsigned long caller) | 1271 | unsigned long caller) |
1261 | { | 1272 | { |
1262 | int size = obj_reallen(cachep); | 1273 | int size = obj_reallen(cachep); |
1263 | 1274 | ||
1264 | addr = (unsigned long *)&((char*)addr)[obj_dbghead(cachep)]; | 1275 | addr = (unsigned long *)&((char *)addr)[obj_dbghead(cachep)]; |
1265 | 1276 | ||
1266 | if (size < 5*sizeof(unsigned long)) | 1277 | if (size < 5 * sizeof(unsigned long)) |
1267 | return; | 1278 | return; |
1268 | 1279 | ||
1269 | *addr++=0x12345678; | 1280 | *addr++ = 0x12345678; |
1270 | *addr++=caller; | 1281 | *addr++ = caller; |
1271 | *addr++=smp_processor_id(); | 1282 | *addr++ = smp_processor_id(); |
1272 | size -= 3*sizeof(unsigned long); | 1283 | size -= 3 * sizeof(unsigned long); |
1273 | { | 1284 | { |
1274 | unsigned long *sptr = &caller; | 1285 | unsigned long *sptr = &caller; |
1275 | unsigned long svalue; | 1286 | unsigned long svalue; |
@@ -1277,7 +1288,7 @@ static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr, | |||
1277 | while (!kstack_end(sptr)) { | 1288 | while (!kstack_end(sptr)) { |
1278 | svalue = *sptr++; | 1289 | svalue = *sptr++; |
1279 | if (kernel_text_address(svalue)) { | 1290 | if (kernel_text_address(svalue)) { |
1280 | *addr++=svalue; | 1291 | *addr++ = svalue; |
1281 | size -= sizeof(unsigned long); | 1292 | size -= sizeof(unsigned long); |
1282 | if (size <= sizeof(unsigned long)) | 1293 | if (size <= sizeof(unsigned long)) |
1283 | break; | 1294 | break; |
@@ -1285,25 +1296,25 @@ static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr, | |||
1285 | } | 1296 | } |
1286 | 1297 | ||
1287 | } | 1298 | } |
1288 | *addr++=0x87654321; | 1299 | *addr++ = 0x87654321; |
1289 | } | 1300 | } |
1290 | #endif | 1301 | #endif |
1291 | 1302 | ||
1292 | static void poison_obj(kmem_cache_t *cachep, void *addr, unsigned char val) | 1303 | static void poison_obj(kmem_cache_t *cachep, void *addr, unsigned char val) |
1293 | { | 1304 | { |
1294 | int size = obj_reallen(cachep); | 1305 | int size = obj_reallen(cachep); |
1295 | addr = &((char*)addr)[obj_dbghead(cachep)]; | 1306 | addr = &((char *)addr)[obj_dbghead(cachep)]; |
1296 | 1307 | ||
1297 | memset(addr, val, size); | 1308 | memset(addr, val, size); |
1298 | *(unsigned char *)(addr+size-1) = POISON_END; | 1309 | *(unsigned char *)(addr + size - 1) = POISON_END; |
1299 | } | 1310 | } |
1300 | 1311 | ||
1301 | static void dump_line(char *data, int offset, int limit) | 1312 | static void dump_line(char *data, int offset, int limit) |
1302 | { | 1313 | { |
1303 | int i; | 1314 | int i; |
1304 | printk(KERN_ERR "%03x:", offset); | 1315 | printk(KERN_ERR "%03x:", offset); |
1305 | for (i=0;i<limit;i++) { | 1316 | for (i = 0; i < limit; i++) { |
1306 | printk(" %02x", (unsigned char)data[offset+i]); | 1317 | printk(" %02x", (unsigned char)data[offset + i]); |
1307 | } | 1318 | } |
1308 | printk("\n"); | 1319 | printk("\n"); |
1309 | } | 1320 | } |
@@ -1318,24 +1329,24 @@ static void print_objinfo(kmem_cache_t *cachep, void *objp, int lines) | |||
1318 | 1329 | ||
1319 | if (cachep->flags & SLAB_RED_ZONE) { | 1330 | if (cachep->flags & SLAB_RED_ZONE) { |
1320 | printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n", | 1331 | printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n", |
1321 | *dbg_redzone1(cachep, objp), | 1332 | *dbg_redzone1(cachep, objp), |
1322 | *dbg_redzone2(cachep, objp)); | 1333 | *dbg_redzone2(cachep, objp)); |
1323 | } | 1334 | } |
1324 | 1335 | ||
1325 | if (cachep->flags & SLAB_STORE_USER) { | 1336 | if (cachep->flags & SLAB_STORE_USER) { |
1326 | printk(KERN_ERR "Last user: [<%p>]", | 1337 | printk(KERN_ERR "Last user: [<%p>]", |
1327 | *dbg_userword(cachep, objp)); | 1338 | *dbg_userword(cachep, objp)); |
1328 | print_symbol("(%s)", | 1339 | print_symbol("(%s)", |
1329 | (unsigned long)*dbg_userword(cachep, objp)); | 1340 | (unsigned long)*dbg_userword(cachep, objp)); |
1330 | printk("\n"); | 1341 | printk("\n"); |
1331 | } | 1342 | } |
1332 | realobj = (char*)objp+obj_dbghead(cachep); | 1343 | realobj = (char *)objp + obj_dbghead(cachep); |
1333 | size = obj_reallen(cachep); | 1344 | size = obj_reallen(cachep); |
1334 | for (i=0; i<size && lines;i+=16, lines--) { | 1345 | for (i = 0; i < size && lines; i += 16, lines--) { |
1335 | int limit; | 1346 | int limit; |
1336 | limit = 16; | 1347 | limit = 16; |
1337 | if (i+limit > size) | 1348 | if (i + limit > size) |
1338 | limit = size-i; | 1349 | limit = size - i; |
1339 | dump_line(realobj, i, limit); | 1350 | dump_line(realobj, i, limit); |
1340 | } | 1351 | } |
1341 | } | 1352 | } |
@@ -1346,27 +1357,28 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp) | |||
1346 | int size, i; | 1357 | int size, i; |
1347 | int lines = 0; | 1358 | int lines = 0; |
1348 | 1359 | ||
1349 | realobj = (char*)objp+obj_dbghead(cachep); | 1360 | realobj = (char *)objp + obj_dbghead(cachep); |
1350 | size = obj_reallen(cachep); | 1361 | size = obj_reallen(cachep); |
1351 | 1362 | ||
1352 | for (i=0;i<size;i++) { | 1363 | for (i = 0; i < size; i++) { |
1353 | char exp = POISON_FREE; | 1364 | char exp = POISON_FREE; |
1354 | if (i == size-1) | 1365 | if (i == size - 1) |
1355 | exp = POISON_END; | 1366 | exp = POISON_END; |
1356 | if (realobj[i] != exp) { | 1367 | if (realobj[i] != exp) { |
1357 | int limit; | 1368 | int limit; |
1358 | /* Mismatch ! */ | 1369 | /* Mismatch ! */ |
1359 | /* Print header */ | 1370 | /* Print header */ |
1360 | if (lines == 0) { | 1371 | if (lines == 0) { |
1361 | printk(KERN_ERR "Slab corruption: start=%p, len=%d\n", | 1372 | printk(KERN_ERR |
1362 | realobj, size); | 1373 | "Slab corruption: start=%p, len=%d\n", |
1374 | realobj, size); | ||
1363 | print_objinfo(cachep, objp, 0); | 1375 | print_objinfo(cachep, objp, 0); |
1364 | } | 1376 | } |
1365 | /* Hexdump the affected line */ | 1377 | /* Hexdump the affected line */ |
1366 | i = (i/16)*16; | 1378 | i = (i / 16) * 16; |
1367 | limit = 16; | 1379 | limit = 16; |
1368 | if (i+limit > size) | 1380 | if (i + limit > size) |
1369 | limit = size-i; | 1381 | limit = size - i; |
1370 | dump_line(realobj, i, limit); | 1382 | dump_line(realobj, i, limit); |
1371 | i += 16; | 1383 | i += 16; |
1372 | lines++; | 1384 | lines++; |
@@ -1382,19 +1394,19 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp) | |||
1382 | struct slab *slabp = page_get_slab(virt_to_page(objp)); | 1394 | struct slab *slabp = page_get_slab(virt_to_page(objp)); |
1383 | int objnr; | 1395 | int objnr; |
1384 | 1396 | ||
1385 | objnr = (objp-slabp->s_mem)/cachep->objsize; | 1397 | objnr = (objp - slabp->s_mem) / cachep->objsize; |
1386 | if (objnr) { | 1398 | if (objnr) { |
1387 | objp = slabp->s_mem+(objnr-1)*cachep->objsize; | 1399 | objp = slabp->s_mem + (objnr - 1) * cachep->objsize; |
1388 | realobj = (char*)objp+obj_dbghead(cachep); | 1400 | realobj = (char *)objp + obj_dbghead(cachep); |
1389 | printk(KERN_ERR "Prev obj: start=%p, len=%d\n", | 1401 | printk(KERN_ERR "Prev obj: start=%p, len=%d\n", |
1390 | realobj, size); | 1402 | realobj, size); |
1391 | print_objinfo(cachep, objp, 2); | 1403 | print_objinfo(cachep, objp, 2); |
1392 | } | 1404 | } |
1393 | if (objnr+1 < cachep->num) { | 1405 | if (objnr + 1 < cachep->num) { |
1394 | objp = slabp->s_mem+(objnr+1)*cachep->objsize; | 1406 | objp = slabp->s_mem + (objnr + 1) * cachep->objsize; |
1395 | realobj = (char*)objp+obj_dbghead(cachep); | 1407 | realobj = (char *)objp + obj_dbghead(cachep); |
1396 | printk(KERN_ERR "Next obj: start=%p, len=%d\n", | 1408 | printk(KERN_ERR "Next obj: start=%p, len=%d\n", |
1397 | realobj, size); | 1409 | realobj, size); |
1398 | print_objinfo(cachep, objp, 2); | 1410 | print_objinfo(cachep, objp, 2); |
1399 | } | 1411 | } |
1400 | } | 1412 | } |
@@ -1405,7 +1417,7 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp) | |||
1405 | * Before calling the slab must have been unlinked from the cache. | 1417 | * Before calling the slab must have been unlinked from the cache. |
1406 | * The cache-lock is not held/needed. | 1418 | * The cache-lock is not held/needed. |
1407 | */ | 1419 | */ |
1408 | static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp) | 1420 | static void slab_destroy(kmem_cache_t *cachep, struct slab *slabp) |
1409 | { | 1421 | { |
1410 | void *addr = slabp->s_mem - slabp->colouroff; | 1422 | void *addr = slabp->s_mem - slabp->colouroff; |
1411 | 1423 | ||
@@ -1416,8 +1428,11 @@ static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp) | |||
1416 | 1428 | ||
1417 | if (cachep->flags & SLAB_POISON) { | 1429 | if (cachep->flags & SLAB_POISON) { |
1418 | #ifdef CONFIG_DEBUG_PAGEALLOC | 1430 | #ifdef CONFIG_DEBUG_PAGEALLOC |
1419 | if ((cachep->objsize%PAGE_SIZE)==0 && OFF_SLAB(cachep)) | 1431 | if ((cachep->objsize % PAGE_SIZE) == 0 |
1420 | kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE,1); | 1432 | && OFF_SLAB(cachep)) |
1433 | kernel_map_pages(virt_to_page(objp), | ||
1434 | cachep->objsize / PAGE_SIZE, | ||
1435 | 1); | ||
1421 | else | 1436 | else |
1422 | check_poison_obj(cachep, objp); | 1437 | check_poison_obj(cachep, objp); |
1423 | #else | 1438 | #else |
@@ -1427,20 +1442,20 @@ static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp) | |||
1427 | if (cachep->flags & SLAB_RED_ZONE) { | 1442 | if (cachep->flags & SLAB_RED_ZONE) { |
1428 | if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) | 1443 | if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) |
1429 | slab_error(cachep, "start of a freed object " | 1444 | slab_error(cachep, "start of a freed object " |
1430 | "was overwritten"); | 1445 | "was overwritten"); |
1431 | if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) | 1446 | if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) |
1432 | slab_error(cachep, "end of a freed object " | 1447 | slab_error(cachep, "end of a freed object " |
1433 | "was overwritten"); | 1448 | "was overwritten"); |
1434 | } | 1449 | } |
1435 | if (cachep->dtor && !(cachep->flags & SLAB_POISON)) | 1450 | if (cachep->dtor && !(cachep->flags & SLAB_POISON)) |
1436 | (cachep->dtor)(objp+obj_dbghead(cachep), cachep, 0); | 1451 | (cachep->dtor) (objp + obj_dbghead(cachep), cachep, 0); |
1437 | } | 1452 | } |
1438 | #else | 1453 | #else |
1439 | if (cachep->dtor) { | 1454 | if (cachep->dtor) { |
1440 | int i; | 1455 | int i; |
1441 | for (i = 0; i < cachep->num; i++) { | 1456 | for (i = 0; i < cachep->num; i++) { |
1442 | void* objp = slabp->s_mem+cachep->objsize*i; | 1457 | void *objp = slabp->s_mem + cachep->objsize * i; |
1443 | (cachep->dtor)(objp, cachep, 0); | 1458 | (cachep->dtor) (objp, cachep, 0); |
1444 | } | 1459 | } |
1445 | } | 1460 | } |
1446 | #endif | 1461 | #endif |
@@ -1448,7 +1463,7 @@ static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp) | |||
1448 | if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { | 1463 | if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { |
1449 | struct slab_rcu *slab_rcu; | 1464 | struct slab_rcu *slab_rcu; |
1450 | 1465 | ||
1451 | slab_rcu = (struct slab_rcu *) slabp; | 1466 | slab_rcu = (struct slab_rcu *)slabp; |
1452 | slab_rcu->cachep = cachep; | 1467 | slab_rcu->cachep = cachep; |
1453 | slab_rcu->addr = addr; | 1468 | slab_rcu->addr = addr; |
1454 | call_rcu(&slab_rcu->head, kmem_rcu_free); | 1469 | call_rcu(&slab_rcu->head, kmem_rcu_free); |
@@ -1466,10 +1481,10 @@ static inline void set_up_list3s(kmem_cache_t *cachep, int index) | |||
1466 | int node; | 1481 | int node; |
1467 | 1482 | ||
1468 | for_each_online_node(node) { | 1483 | for_each_online_node(node) { |
1469 | cachep->nodelists[node] = &initkmem_list3[index+node]; | 1484 | cachep->nodelists[node] = &initkmem_list3[index + node]; |
1470 | cachep->nodelists[node]->next_reap = jiffies + | 1485 | cachep->nodelists[node]->next_reap = jiffies + |
1471 | REAPTIMEOUT_LIST3 + | 1486 | REAPTIMEOUT_LIST3 + |
1472 | ((unsigned long)cachep)%REAPTIMEOUT_LIST3; | 1487 | ((unsigned long)cachep) % REAPTIMEOUT_LIST3; |
1473 | } | 1488 | } |
1474 | } | 1489 | } |
1475 | 1490 | ||
@@ -1486,7 +1501,7 @@ static inline size_t calculate_slab_order(kmem_cache_t *cachep, size_t size, | |||
1486 | { | 1501 | { |
1487 | size_t left_over = 0; | 1502 | size_t left_over = 0; |
1488 | 1503 | ||
1489 | for ( ; ; cachep->gfporder++) { | 1504 | for (;; cachep->gfporder++) { |
1490 | unsigned int num; | 1505 | unsigned int num; |
1491 | size_t remainder; | 1506 | size_t remainder; |
1492 | 1507 | ||
@@ -1566,14 +1581,13 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1566 | * Sanity checks... these are all serious usage bugs. | 1581 | * Sanity checks... these are all serious usage bugs. |
1567 | */ | 1582 | */ |
1568 | if ((!name) || | 1583 | if ((!name) || |
1569 | in_interrupt() || | 1584 | in_interrupt() || |
1570 | (size < BYTES_PER_WORD) || | 1585 | (size < BYTES_PER_WORD) || |
1571 | (size > (1<<MAX_OBJ_ORDER)*PAGE_SIZE) || | 1586 | (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) { |
1572 | (dtor && !ctor)) { | 1587 | printk(KERN_ERR "%s: Early error in slab %s\n", |
1573 | printk(KERN_ERR "%s: Early error in slab %s\n", | 1588 | __FUNCTION__, name); |
1574 | __FUNCTION__, name); | 1589 | BUG(); |
1575 | BUG(); | 1590 | } |
1576 | } | ||
1577 | 1591 | ||
1578 | down(&cache_chain_sem); | 1592 | down(&cache_chain_sem); |
1579 | 1593 | ||
@@ -1593,11 +1607,11 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1593 | set_fs(old_fs); | 1607 | set_fs(old_fs); |
1594 | if (res) { | 1608 | if (res) { |
1595 | printk("SLAB: cache with size %d has lost its name\n", | 1609 | printk("SLAB: cache with size %d has lost its name\n", |
1596 | pc->objsize); | 1610 | pc->objsize); |
1597 | continue; | 1611 | continue; |
1598 | } | 1612 | } |
1599 | 1613 | ||
1600 | if (!strcmp(pc->name,name)) { | 1614 | if (!strcmp(pc->name, name)) { |
1601 | printk("kmem_cache_create: duplicate cache %s\n", name); | 1615 | printk("kmem_cache_create: duplicate cache %s\n", name); |
1602 | dump_stack(); | 1616 | dump_stack(); |
1603 | goto oops; | 1617 | goto oops; |
@@ -1609,10 +1623,9 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1609 | if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { | 1623 | if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { |
1610 | /* No constructor, but inital state check requested */ | 1624 | /* No constructor, but inital state check requested */ |
1611 | printk(KERN_ERR "%s: No con, but init state check " | 1625 | printk(KERN_ERR "%s: No con, but init state check " |
1612 | "requested - %s\n", __FUNCTION__, name); | 1626 | "requested - %s\n", __FUNCTION__, name); |
1613 | flags &= ~SLAB_DEBUG_INITIAL; | 1627 | flags &= ~SLAB_DEBUG_INITIAL; |
1614 | } | 1628 | } |
1615 | |||
1616 | #if FORCED_DEBUG | 1629 | #if FORCED_DEBUG |
1617 | /* | 1630 | /* |
1618 | * Enable redzoning and last user accounting, except for caches with | 1631 | * Enable redzoning and last user accounting, except for caches with |
@@ -1620,8 +1633,9 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1620 | * above the next power of two: caches with object sizes just above a | 1633 | * above the next power of two: caches with object sizes just above a |
1621 | * power of two have a significant amount of internal fragmentation. | 1634 | * power of two have a significant amount of internal fragmentation. |
1622 | */ | 1635 | */ |
1623 | if ((size < 4096 || fls(size-1) == fls(size-1+3*BYTES_PER_WORD))) | 1636 | if ((size < 4096 |
1624 | flags |= SLAB_RED_ZONE|SLAB_STORE_USER; | 1637 | || fls(size - 1) == fls(size - 1 + 3 * BYTES_PER_WORD))) |
1638 | flags |= SLAB_RED_ZONE | SLAB_STORE_USER; | ||
1625 | if (!(flags & SLAB_DESTROY_BY_RCU)) | 1639 | if (!(flags & SLAB_DESTROY_BY_RCU)) |
1626 | flags |= SLAB_POISON; | 1640 | flags |= SLAB_POISON; |
1627 | #endif | 1641 | #endif |
@@ -1642,9 +1656,9 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1642 | * unaligned accesses for some archs when redzoning is used, and makes | 1656 | * unaligned accesses for some archs when redzoning is used, and makes |
1643 | * sure any on-slab bufctl's are also correctly aligned. | 1657 | * sure any on-slab bufctl's are also correctly aligned. |
1644 | */ | 1658 | */ |
1645 | if (size & (BYTES_PER_WORD-1)) { | 1659 | if (size & (BYTES_PER_WORD - 1)) { |
1646 | size += (BYTES_PER_WORD-1); | 1660 | size += (BYTES_PER_WORD - 1); |
1647 | size &= ~(BYTES_PER_WORD-1); | 1661 | size &= ~(BYTES_PER_WORD - 1); |
1648 | } | 1662 | } |
1649 | 1663 | ||
1650 | /* calculate out the final buffer alignment: */ | 1664 | /* calculate out the final buffer alignment: */ |
@@ -1655,7 +1669,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1655 | * objects into one cacheline. | 1669 | * objects into one cacheline. |
1656 | */ | 1670 | */ |
1657 | ralign = cache_line_size(); | 1671 | ralign = cache_line_size(); |
1658 | while (size <= ralign/2) | 1672 | while (size <= ralign / 2) |
1659 | ralign /= 2; | 1673 | ralign /= 2; |
1660 | } else { | 1674 | } else { |
1661 | ralign = BYTES_PER_WORD; | 1675 | ralign = BYTES_PER_WORD; |
@@ -1664,13 +1678,13 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1664 | if (ralign < ARCH_SLAB_MINALIGN) { | 1678 | if (ralign < ARCH_SLAB_MINALIGN) { |
1665 | ralign = ARCH_SLAB_MINALIGN; | 1679 | ralign = ARCH_SLAB_MINALIGN; |
1666 | if (ralign > BYTES_PER_WORD) | 1680 | if (ralign > BYTES_PER_WORD) |
1667 | flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER); | 1681 | flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); |
1668 | } | 1682 | } |
1669 | /* 3) caller mandated alignment: disables debug if necessary */ | 1683 | /* 3) caller mandated alignment: disables debug if necessary */ |
1670 | if (ralign < align) { | 1684 | if (ralign < align) { |
1671 | ralign = align; | 1685 | ralign = align; |
1672 | if (ralign > BYTES_PER_WORD) | 1686 | if (ralign > BYTES_PER_WORD) |
1673 | flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER); | 1687 | flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER); |
1674 | } | 1688 | } |
1675 | /* 4) Store it. Note that the debug code below can reduce | 1689 | /* 4) Store it. Note that the debug code below can reduce |
1676 | * the alignment to BYTES_PER_WORD. | 1690 | * the alignment to BYTES_PER_WORD. |
@@ -1692,7 +1706,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1692 | 1706 | ||
1693 | /* add space for red zone words */ | 1707 | /* add space for red zone words */ |
1694 | cachep->dbghead += BYTES_PER_WORD; | 1708 | cachep->dbghead += BYTES_PER_WORD; |
1695 | size += 2*BYTES_PER_WORD; | 1709 | size += 2 * BYTES_PER_WORD; |
1696 | } | 1710 | } |
1697 | if (flags & SLAB_STORE_USER) { | 1711 | if (flags & SLAB_STORE_USER) { |
1698 | /* user store requires word alignment and | 1712 | /* user store requires word alignment and |
@@ -1703,7 +1717,8 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1703 | size += BYTES_PER_WORD; | 1717 | size += BYTES_PER_WORD; |
1704 | } | 1718 | } |
1705 | #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) | 1719 | #if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) |
1706 | if (size >= malloc_sizes[INDEX_L3+1].cs_size && cachep->reallen > cache_line_size() && size < PAGE_SIZE) { | 1720 | if (size >= malloc_sizes[INDEX_L3 + 1].cs_size |
1721 | && cachep->reallen > cache_line_size() && size < PAGE_SIZE) { | ||
1707 | cachep->dbghead += PAGE_SIZE - size; | 1722 | cachep->dbghead += PAGE_SIZE - size; |
1708 | size = PAGE_SIZE; | 1723 | size = PAGE_SIZE; |
1709 | } | 1724 | } |
@@ -1711,7 +1726,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1711 | #endif | 1726 | #endif |
1712 | 1727 | ||
1713 | /* Determine if the slab management is 'on' or 'off' slab. */ | 1728 | /* Determine if the slab management is 'on' or 'off' slab. */ |
1714 | if (size >= (PAGE_SIZE>>3)) | 1729 | if (size >= (PAGE_SIZE >> 3)) |
1715 | /* | 1730 | /* |
1716 | * Size is large, assume best to place the slab management obj | 1731 | * Size is large, assume best to place the slab management obj |
1717 | * off-slab (should allow better packing of objs). | 1732 | * off-slab (should allow better packing of objs). |
@@ -1728,7 +1743,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1728 | */ | 1743 | */ |
1729 | cachep->gfporder = 0; | 1744 | cachep->gfporder = 0; |
1730 | cache_estimate(cachep->gfporder, size, align, flags, | 1745 | cache_estimate(cachep->gfporder, size, align, flags, |
1731 | &left_over, &cachep->num); | 1746 | &left_over, &cachep->num); |
1732 | } else | 1747 | } else |
1733 | left_over = calculate_slab_order(cachep, size, align, flags); | 1748 | left_over = calculate_slab_order(cachep, size, align, flags); |
1734 | 1749 | ||
@@ -1738,8 +1753,8 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1738 | cachep = NULL; | 1753 | cachep = NULL; |
1739 | goto oops; | 1754 | goto oops; |
1740 | } | 1755 | } |
1741 | slab_size = ALIGN(cachep->num*sizeof(kmem_bufctl_t) | 1756 | slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t) |
1742 | + sizeof(struct slab), align); | 1757 | + sizeof(struct slab), align); |
1743 | 1758 | ||
1744 | /* | 1759 | /* |
1745 | * If the slab has been placed off-slab, and we have enough space then | 1760 | * If the slab has been placed off-slab, and we have enough space then |
@@ -1752,14 +1767,15 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1752 | 1767 | ||
1753 | if (flags & CFLGS_OFF_SLAB) { | 1768 | if (flags & CFLGS_OFF_SLAB) { |
1754 | /* really off slab. No need for manual alignment */ | 1769 | /* really off slab. No need for manual alignment */ |
1755 | slab_size = cachep->num*sizeof(kmem_bufctl_t)+sizeof(struct slab); | 1770 | slab_size = |
1771 | cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab); | ||
1756 | } | 1772 | } |
1757 | 1773 | ||
1758 | cachep->colour_off = cache_line_size(); | 1774 | cachep->colour_off = cache_line_size(); |
1759 | /* Offset must be a multiple of the alignment. */ | 1775 | /* Offset must be a multiple of the alignment. */ |
1760 | if (cachep->colour_off < align) | 1776 | if (cachep->colour_off < align) |
1761 | cachep->colour_off = align; | 1777 | cachep->colour_off = align; |
1762 | cachep->colour = left_over/cachep->colour_off; | 1778 | cachep->colour = left_over / cachep->colour_off; |
1763 | cachep->slab_size = slab_size; | 1779 | cachep->slab_size = slab_size; |
1764 | cachep->flags = flags; | 1780 | cachep->flags = flags; |
1765 | cachep->gfpflags = 0; | 1781 | cachep->gfpflags = 0; |
@@ -1786,7 +1802,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1786 | * the creation of further caches will BUG(). | 1802 | * the creation of further caches will BUG(). |
1787 | */ | 1803 | */ |
1788 | cachep->array[smp_processor_id()] = | 1804 | cachep->array[smp_processor_id()] = |
1789 | &initarray_generic.cache; | 1805 | &initarray_generic.cache; |
1790 | 1806 | ||
1791 | /* If the cache that's used by | 1807 | /* If the cache that's used by |
1792 | * kmalloc(sizeof(kmem_list3)) is the first cache, | 1808 | * kmalloc(sizeof(kmem_list3)) is the first cache, |
@@ -1800,8 +1816,7 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1800 | g_cpucache_up = PARTIAL_AC; | 1816 | g_cpucache_up = PARTIAL_AC; |
1801 | } else { | 1817 | } else { |
1802 | cachep->array[smp_processor_id()] = | 1818 | cachep->array[smp_processor_id()] = |
1803 | kmalloc(sizeof(struct arraycache_init), | 1819 | kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); |
1804 | GFP_KERNEL); | ||
1805 | 1820 | ||
1806 | if (g_cpucache_up == PARTIAL_AC) { | 1821 | if (g_cpucache_up == PARTIAL_AC) { |
1807 | set_up_list3s(cachep, SIZE_L3); | 1822 | set_up_list3s(cachep, SIZE_L3); |
@@ -1811,16 +1826,18 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1811 | for_each_online_node(node) { | 1826 | for_each_online_node(node) { |
1812 | 1827 | ||
1813 | cachep->nodelists[node] = | 1828 | cachep->nodelists[node] = |
1814 | kmalloc_node(sizeof(struct kmem_list3), | 1829 | kmalloc_node(sizeof |
1815 | GFP_KERNEL, node); | 1830 | (struct kmem_list3), |
1831 | GFP_KERNEL, node); | ||
1816 | BUG_ON(!cachep->nodelists[node]); | 1832 | BUG_ON(!cachep->nodelists[node]); |
1817 | kmem_list3_init(cachep->nodelists[node]); | 1833 | kmem_list3_init(cachep-> |
1834 | nodelists[node]); | ||
1818 | } | 1835 | } |
1819 | } | 1836 | } |
1820 | } | 1837 | } |
1821 | cachep->nodelists[numa_node_id()]->next_reap = | 1838 | cachep->nodelists[numa_node_id()]->next_reap = |
1822 | jiffies + REAPTIMEOUT_LIST3 + | 1839 | jiffies + REAPTIMEOUT_LIST3 + |
1823 | ((unsigned long)cachep)%REAPTIMEOUT_LIST3; | 1840 | ((unsigned long)cachep) % REAPTIMEOUT_LIST3; |
1824 | 1841 | ||
1825 | BUG_ON(!ac_data(cachep)); | 1842 | BUG_ON(!ac_data(cachep)); |
1826 | ac_data(cachep)->avail = 0; | 1843 | ac_data(cachep)->avail = 0; |
@@ -1829,15 +1846,15 @@ kmem_cache_create (const char *name, size_t size, size_t align, | |||
1829 | ac_data(cachep)->touched = 0; | 1846 | ac_data(cachep)->touched = 0; |
1830 | cachep->batchcount = 1; | 1847 | cachep->batchcount = 1; |
1831 | cachep->limit = BOOT_CPUCACHE_ENTRIES; | 1848 | cachep->limit = BOOT_CPUCACHE_ENTRIES; |
1832 | } | 1849 | } |
1833 | 1850 | ||
1834 | /* cache setup completed, link it into the list */ | 1851 | /* cache setup completed, link it into the list */ |
1835 | list_add(&cachep->next, &cache_chain); | 1852 | list_add(&cachep->next, &cache_chain); |
1836 | unlock_cpu_hotplug(); | 1853 | unlock_cpu_hotplug(); |
1837 | oops: | 1854 | oops: |
1838 | if (!cachep && (flags & SLAB_PANIC)) | 1855 | if (!cachep && (flags & SLAB_PANIC)) |
1839 | panic("kmem_cache_create(): failed to create slab `%s'\n", | 1856 | panic("kmem_cache_create(): failed to create slab `%s'\n", |
1840 | name); | 1857 | name); |
1841 | up(&cache_chain_sem); | 1858 | up(&cache_chain_sem); |
1842 | return cachep; | 1859 | return cachep; |
1843 | } | 1860 | } |
@@ -1880,7 +1897,7 @@ static inline void check_spinlock_acquired_node(kmem_cache_t *cachep, int node) | |||
1880 | /* | 1897 | /* |
1881 | * Waits for all CPUs to execute func(). | 1898 | * Waits for all CPUs to execute func(). |
1882 | */ | 1899 | */ |
1883 | static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg) | 1900 | static void smp_call_function_all_cpus(void (*func)(void *arg), void *arg) |
1884 | { | 1901 | { |
1885 | check_irq_on(); | 1902 | check_irq_on(); |
1886 | preempt_disable(); | 1903 | preempt_disable(); |
@@ -1895,12 +1912,12 @@ static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg) | |||
1895 | preempt_enable(); | 1912 | preempt_enable(); |
1896 | } | 1913 | } |
1897 | 1914 | ||
1898 | static void drain_array_locked(kmem_cache_t* cachep, | 1915 | static void drain_array_locked(kmem_cache_t *cachep, struct array_cache *ac, |
1899 | struct array_cache *ac, int force, int node); | 1916 | int force, int node); |
1900 | 1917 | ||
1901 | static void do_drain(void *arg) | 1918 | static void do_drain(void *arg) |
1902 | { | 1919 | { |
1903 | kmem_cache_t *cachep = (kmem_cache_t*)arg; | 1920 | kmem_cache_t *cachep = (kmem_cache_t *) arg; |
1904 | struct array_cache *ac; | 1921 | struct array_cache *ac; |
1905 | int node = numa_node_id(); | 1922 | int node = numa_node_id(); |
1906 | 1923 | ||
@@ -1920,7 +1937,7 @@ static void drain_cpu_caches(kmem_cache_t *cachep) | |||
1920 | smp_call_function_all_cpus(do_drain, cachep); | 1937 | smp_call_function_all_cpus(do_drain, cachep); |
1921 | check_irq_on(); | 1938 | check_irq_on(); |
1922 | spin_lock_irq(&cachep->spinlock); | 1939 | spin_lock_irq(&cachep->spinlock); |
1923 | for_each_online_node(node) { | 1940 | for_each_online_node(node) { |
1924 | l3 = cachep->nodelists[node]; | 1941 | l3 = cachep->nodelists[node]; |
1925 | if (l3) { | 1942 | if (l3) { |
1926 | spin_lock(&l3->list_lock); | 1943 | spin_lock(&l3->list_lock); |
@@ -1958,8 +1975,7 @@ static int __node_shrink(kmem_cache_t *cachep, int node) | |||
1958 | slab_destroy(cachep, slabp); | 1975 | slab_destroy(cachep, slabp); |
1959 | spin_lock_irq(&l3->list_lock); | 1976 | spin_lock_irq(&l3->list_lock); |
1960 | } | 1977 | } |
1961 | ret = !list_empty(&l3->slabs_full) || | 1978 | ret = !list_empty(&l3->slabs_full) || !list_empty(&l3->slabs_partial); |
1962 | !list_empty(&l3->slabs_partial); | ||
1963 | return ret; | 1979 | return ret; |
1964 | } | 1980 | } |
1965 | 1981 | ||
@@ -2015,7 +2031,7 @@ EXPORT_SYMBOL(kmem_cache_shrink); | |||
2015 | * The caller must guarantee that noone will allocate memory from the cache | 2031 | * The caller must guarantee that noone will allocate memory from the cache |
2016 | * during the kmem_cache_destroy(). | 2032 | * during the kmem_cache_destroy(). |
2017 | */ | 2033 | */ |
2018 | int kmem_cache_destroy(kmem_cache_t * cachep) | 2034 | int kmem_cache_destroy(kmem_cache_t *cachep) |
2019 | { | 2035 | { |
2020 | int i; | 2036 | int i; |
2021 | struct kmem_list3 *l3; | 2037 | struct kmem_list3 *l3; |
@@ -2037,7 +2053,7 @@ int kmem_cache_destroy(kmem_cache_t * cachep) | |||
2037 | if (__cache_shrink(cachep)) { | 2053 | if (__cache_shrink(cachep)) { |
2038 | slab_error(cachep, "Can't free all objects"); | 2054 | slab_error(cachep, "Can't free all objects"); |
2039 | down(&cache_chain_sem); | 2055 | down(&cache_chain_sem); |
2040 | list_add(&cachep->next,&cache_chain); | 2056 | list_add(&cachep->next, &cache_chain); |
2041 | up(&cache_chain_sem); | 2057 | up(&cache_chain_sem); |
2042 | unlock_cpu_hotplug(); | 2058 | unlock_cpu_hotplug(); |
2043 | return 1; | 2059 | return 1; |
@@ -2047,7 +2063,7 @@ int kmem_cache_destroy(kmem_cache_t * cachep) | |||
2047 | synchronize_rcu(); | 2063 | synchronize_rcu(); |
2048 | 2064 | ||
2049 | for_each_online_cpu(i) | 2065 | for_each_online_cpu(i) |
2050 | kfree(cachep->array[i]); | 2066 | kfree(cachep->array[i]); |
2051 | 2067 | ||
2052 | /* NUMA: free the list3 structures */ | 2068 | /* NUMA: free the list3 structures */ |
2053 | for_each_online_node(i) { | 2069 | for_each_online_node(i) { |
@@ -2066,39 +2082,39 @@ int kmem_cache_destroy(kmem_cache_t * cachep) | |||
2066 | EXPORT_SYMBOL(kmem_cache_destroy); | 2082 | EXPORT_SYMBOL(kmem_cache_destroy); |
2067 | 2083 | ||
2068 | /* Get the memory for a slab management obj. */ | 2084 | /* Get the memory for a slab management obj. */ |
2069 | static struct slab* alloc_slabmgmt(kmem_cache_t *cachep, void *objp, | 2085 | static struct slab *alloc_slabmgmt(kmem_cache_t *cachep, void *objp, |
2070 | int colour_off, gfp_t local_flags) | 2086 | int colour_off, gfp_t local_flags) |
2071 | { | 2087 | { |
2072 | struct slab *slabp; | 2088 | struct slab *slabp; |
2073 | 2089 | ||
2074 | if (OFF_SLAB(cachep)) { | 2090 | if (OFF_SLAB(cachep)) { |
2075 | /* Slab management obj is off-slab. */ | 2091 | /* Slab management obj is off-slab. */ |
2076 | slabp = kmem_cache_alloc(cachep->slabp_cache, local_flags); | 2092 | slabp = kmem_cache_alloc(cachep->slabp_cache, local_flags); |
2077 | if (!slabp) | 2093 | if (!slabp) |
2078 | return NULL; | 2094 | return NULL; |
2079 | } else { | 2095 | } else { |
2080 | slabp = objp+colour_off; | 2096 | slabp = objp + colour_off; |
2081 | colour_off += cachep->slab_size; | 2097 | colour_off += cachep->slab_size; |
2082 | } | 2098 | } |
2083 | slabp->inuse = 0; | 2099 | slabp->inuse = 0; |
2084 | slabp->colouroff = colour_off; | 2100 | slabp->colouroff = colour_off; |
2085 | slabp->s_mem = objp+colour_off; | 2101 | slabp->s_mem = objp + colour_off; |
2086 | 2102 | ||
2087 | return slabp; | 2103 | return slabp; |
2088 | } | 2104 | } |
2089 | 2105 | ||
2090 | static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp) | 2106 | static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp) |
2091 | { | 2107 | { |
2092 | return (kmem_bufctl_t *)(slabp+1); | 2108 | return (kmem_bufctl_t *) (slabp + 1); |
2093 | } | 2109 | } |
2094 | 2110 | ||
2095 | static void cache_init_objs(kmem_cache_t *cachep, | 2111 | static void cache_init_objs(kmem_cache_t *cachep, |
2096 | struct slab *slabp, unsigned long ctor_flags) | 2112 | struct slab *slabp, unsigned long ctor_flags) |
2097 | { | 2113 | { |
2098 | int i; | 2114 | int i; |
2099 | 2115 | ||
2100 | for (i = 0; i < cachep->num; i++) { | 2116 | for (i = 0; i < cachep->num; i++) { |
2101 | void *objp = slabp->s_mem+cachep->objsize*i; | 2117 | void *objp = slabp->s_mem + cachep->objsize * i; |
2102 | #if DEBUG | 2118 | #if DEBUG |
2103 | /* need to poison the objs? */ | 2119 | /* need to poison the objs? */ |
2104 | if (cachep->flags & SLAB_POISON) | 2120 | if (cachep->flags & SLAB_POISON) |
@@ -2116,25 +2132,28 @@ static void cache_init_objs(kmem_cache_t *cachep, | |||
2116 | * Otherwise, deadlock. They must also be threaded. | 2132 | * Otherwise, deadlock. They must also be threaded. |
2117 | */ | 2133 | */ |
2118 | if (cachep->ctor && !(cachep->flags & SLAB_POISON)) | 2134 | if (cachep->ctor && !(cachep->flags & SLAB_POISON)) |
2119 | cachep->ctor(objp+obj_dbghead(cachep), cachep, ctor_flags); | 2135 | cachep->ctor(objp + obj_dbghead(cachep), cachep, |
2136 | ctor_flags); | ||
2120 | 2137 | ||
2121 | if (cachep->flags & SLAB_RED_ZONE) { | 2138 | if (cachep->flags & SLAB_RED_ZONE) { |
2122 | if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) | 2139 | if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) |
2123 | slab_error(cachep, "constructor overwrote the" | 2140 | slab_error(cachep, "constructor overwrote the" |
2124 | " end of an object"); | 2141 | " end of an object"); |
2125 | if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) | 2142 | if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) |
2126 | slab_error(cachep, "constructor overwrote the" | 2143 | slab_error(cachep, "constructor overwrote the" |
2127 | " start of an object"); | 2144 | " start of an object"); |
2128 | } | 2145 | } |
2129 | if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) | 2146 | if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep) |
2130 | kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 0); | 2147 | && cachep->flags & SLAB_POISON) |
2148 | kernel_map_pages(virt_to_page(objp), | ||
2149 | cachep->objsize / PAGE_SIZE, 0); | ||
2131 | #else | 2150 | #else |
2132 | if (cachep->ctor) | 2151 | if (cachep->ctor) |
2133 | cachep->ctor(objp, cachep, ctor_flags); | 2152 | cachep->ctor(objp, cachep, ctor_flags); |
2134 | #endif | 2153 | #endif |
2135 | slab_bufctl(slabp)[i] = i+1; | 2154 | slab_bufctl(slabp)[i] = i + 1; |
2136 | } | 2155 | } |
2137 | slab_bufctl(slabp)[i-1] = BUFCTL_END; | 2156 | slab_bufctl(slabp)[i - 1] = BUFCTL_END; |
2138 | slabp->free = 0; | 2157 | slabp->free = 0; |
2139 | } | 2158 | } |
2140 | 2159 | ||
@@ -2170,17 +2189,17 @@ static void set_slab_attr(kmem_cache_t *cachep, struct slab *slabp, void *objp) | |||
2170 | */ | 2189 | */ |
2171 | static int cache_grow(kmem_cache_t *cachep, gfp_t flags, int nodeid) | 2190 | static int cache_grow(kmem_cache_t *cachep, gfp_t flags, int nodeid) |
2172 | { | 2191 | { |
2173 | struct slab *slabp; | 2192 | struct slab *slabp; |
2174 | void *objp; | 2193 | void *objp; |
2175 | size_t offset; | 2194 | size_t offset; |
2176 | gfp_t local_flags; | 2195 | gfp_t local_flags; |
2177 | unsigned long ctor_flags; | 2196 | unsigned long ctor_flags; |
2178 | struct kmem_list3 *l3; | 2197 | struct kmem_list3 *l3; |
2179 | 2198 | ||
2180 | /* Be lazy and only check for valid flags here, | 2199 | /* Be lazy and only check for valid flags here, |
2181 | * keeping it out of the critical path in kmem_cache_alloc(). | 2200 | * keeping it out of the critical path in kmem_cache_alloc(). |
2182 | */ | 2201 | */ |
2183 | if (flags & ~(SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW)) | 2202 | if (flags & ~(SLAB_DMA | SLAB_LEVEL_MASK | SLAB_NO_GROW)) |
2184 | BUG(); | 2203 | BUG(); |
2185 | if (flags & SLAB_NO_GROW) | 2204 | if (flags & SLAB_NO_GROW) |
2186 | return 0; | 2205 | return 0; |
@@ -2246,9 +2265,9 @@ static int cache_grow(kmem_cache_t *cachep, gfp_t flags, int nodeid) | |||
2246 | l3->free_objects += cachep->num; | 2265 | l3->free_objects += cachep->num; |
2247 | spin_unlock(&l3->list_lock); | 2266 | spin_unlock(&l3->list_lock); |
2248 | return 1; | 2267 | return 1; |
2249 | opps1: | 2268 | opps1: |
2250 | kmem_freepages(cachep, objp); | 2269 | kmem_freepages(cachep, objp); |
2251 | failed: | 2270 | failed: |
2252 | if (local_flags & __GFP_WAIT) | 2271 | if (local_flags & __GFP_WAIT) |
2253 | local_irq_disable(); | 2272 | local_irq_disable(); |
2254 | return 0; | 2273 | return 0; |
@@ -2268,18 +2287,19 @@ static void kfree_debugcheck(const void *objp) | |||
2268 | 2287 | ||
2269 | if (!virt_addr_valid(objp)) { | 2288 | if (!virt_addr_valid(objp)) { |
2270 | printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", | 2289 | printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", |
2271 | (unsigned long)objp); | 2290 | (unsigned long)objp); |
2272 | BUG(); | 2291 | BUG(); |
2273 | } | 2292 | } |
2274 | page = virt_to_page(objp); | 2293 | page = virt_to_page(objp); |
2275 | if (!PageSlab(page)) { | 2294 | if (!PageSlab(page)) { |
2276 | printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n", (unsigned long)objp); | 2295 | printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n", |
2296 | (unsigned long)objp); | ||
2277 | BUG(); | 2297 | BUG(); |
2278 | } | 2298 | } |
2279 | } | 2299 | } |
2280 | 2300 | ||
2281 | static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp, | 2301 | static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp, |
2282 | void *caller) | 2302 | void *caller) |
2283 | { | 2303 | { |
2284 | struct page *page; | 2304 | struct page *page; |
2285 | unsigned int objnr; | 2305 | unsigned int objnr; |
@@ -2290,20 +2310,26 @@ static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp, | |||
2290 | page = virt_to_page(objp); | 2310 | page = virt_to_page(objp); |
2291 | 2311 | ||
2292 | if (page_get_cache(page) != cachep) { | 2312 | if (page_get_cache(page) != cachep) { |
2293 | printk(KERN_ERR "mismatch in kmem_cache_free: expected cache %p, got %p\n", | 2313 | printk(KERN_ERR |
2294 | page_get_cache(page),cachep); | 2314 | "mismatch in kmem_cache_free: expected cache %p, got %p\n", |
2315 | page_get_cache(page), cachep); | ||
2295 | printk(KERN_ERR "%p is %s.\n", cachep, cachep->name); | 2316 | printk(KERN_ERR "%p is %s.\n", cachep, cachep->name); |
2296 | printk(KERN_ERR "%p is %s.\n", page_get_cache(page), page_get_cache(page)->name); | 2317 | printk(KERN_ERR "%p is %s.\n", page_get_cache(page), |
2318 | page_get_cache(page)->name); | ||
2297 | WARN_ON(1); | 2319 | WARN_ON(1); |
2298 | } | 2320 | } |
2299 | slabp = page_get_slab(page); | 2321 | slabp = page_get_slab(page); |
2300 | 2322 | ||
2301 | if (cachep->flags & SLAB_RED_ZONE) { | 2323 | if (cachep->flags & SLAB_RED_ZONE) { |
2302 | if (*dbg_redzone1(cachep, objp) != RED_ACTIVE || *dbg_redzone2(cachep, objp) != RED_ACTIVE) { | 2324 | if (*dbg_redzone1(cachep, objp) != RED_ACTIVE |
2303 | slab_error(cachep, "double free, or memory outside" | 2325 | || *dbg_redzone2(cachep, objp) != RED_ACTIVE) { |
2304 | " object was overwritten"); | 2326 | slab_error(cachep, |
2305 | printk(KERN_ERR "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", | 2327 | "double free, or memory outside" |
2306 | objp, *dbg_redzone1(cachep, objp), *dbg_redzone2(cachep, objp)); | 2328 | " object was overwritten"); |
2329 | printk(KERN_ERR | ||
2330 | "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", | ||
2331 | objp, *dbg_redzone1(cachep, objp), | ||
2332 | *dbg_redzone2(cachep, objp)); | ||
2307 | } | 2333 | } |
2308 | *dbg_redzone1(cachep, objp) = RED_INACTIVE; | 2334 | *dbg_redzone1(cachep, objp) = RED_INACTIVE; |
2309 | *dbg_redzone2(cachep, objp) = RED_INACTIVE; | 2335 | *dbg_redzone2(cachep, objp) = RED_INACTIVE; |
@@ -2311,30 +2337,31 @@ static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp, | |||
2311 | if (cachep->flags & SLAB_STORE_USER) | 2337 | if (cachep->flags & SLAB_STORE_USER) |
2312 | *dbg_userword(cachep, objp) = caller; | 2338 | *dbg_userword(cachep, objp) = caller; |
2313 | 2339 | ||
2314 | objnr = (objp-slabp->s_mem)/cachep->objsize; | 2340 | objnr = (objp - slabp->s_mem) / cachep->objsize; |
2315 | 2341 | ||
2316 | BUG_ON(objnr >= cachep->num); | 2342 | BUG_ON(objnr >= cachep->num); |
2317 | BUG_ON(objp != slabp->s_mem + objnr*cachep->objsize); | 2343 | BUG_ON(objp != slabp->s_mem + objnr * cachep->objsize); |
2318 | 2344 | ||
2319 | if (cachep->flags & SLAB_DEBUG_INITIAL) { | 2345 | if (cachep->flags & SLAB_DEBUG_INITIAL) { |
2320 | /* Need to call the slab's constructor so the | 2346 | /* Need to call the slab's constructor so the |
2321 | * caller can perform a verify of its state (debugging). | 2347 | * caller can perform a verify of its state (debugging). |
2322 | * Called without the cache-lock held. | 2348 | * Called without the cache-lock held. |
2323 | */ | 2349 | */ |
2324 | cachep->ctor(objp+obj_dbghead(cachep), | 2350 | cachep->ctor(objp + obj_dbghead(cachep), |
2325 | cachep, SLAB_CTOR_CONSTRUCTOR|SLAB_CTOR_VERIFY); | 2351 | cachep, SLAB_CTOR_CONSTRUCTOR | SLAB_CTOR_VERIFY); |
2326 | } | 2352 | } |
2327 | if (cachep->flags & SLAB_POISON && cachep->dtor) { | 2353 | if (cachep->flags & SLAB_POISON && cachep->dtor) { |
2328 | /* we want to cache poison the object, | 2354 | /* we want to cache poison the object, |
2329 | * call the destruction callback | 2355 | * call the destruction callback |
2330 | */ | 2356 | */ |
2331 | cachep->dtor(objp+obj_dbghead(cachep), cachep, 0); | 2357 | cachep->dtor(objp + obj_dbghead(cachep), cachep, 0); |
2332 | } | 2358 | } |
2333 | if (cachep->flags & SLAB_POISON) { | 2359 | if (cachep->flags & SLAB_POISON) { |
2334 | #ifdef CONFIG_DEBUG_PAGEALLOC | 2360 | #ifdef CONFIG_DEBUG_PAGEALLOC |
2335 | if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) { | 2361 | if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) { |
2336 | store_stackinfo(cachep, objp, (unsigned long)caller); | 2362 | store_stackinfo(cachep, objp, (unsigned long)caller); |
2337 | kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 0); | 2363 | kernel_map_pages(virt_to_page(objp), |
2364 | cachep->objsize / PAGE_SIZE, 0); | ||
2338 | } else { | 2365 | } else { |
2339 | poison_obj(cachep, objp, POISON_FREE); | 2366 | poison_obj(cachep, objp, POISON_FREE); |
2340 | } | 2367 | } |
@@ -2349,7 +2376,7 @@ static void check_slabp(kmem_cache_t *cachep, struct slab *slabp) | |||
2349 | { | 2376 | { |
2350 | kmem_bufctl_t i; | 2377 | kmem_bufctl_t i; |
2351 | int entries = 0; | 2378 | int entries = 0; |
2352 | 2379 | ||
2353 | /* Check slab's freelist to see if this obj is there. */ | 2380 | /* Check slab's freelist to see if this obj is there. */ |
2354 | for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { | 2381 | for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { |
2355 | entries++; | 2382 | entries++; |
@@ -2357,13 +2384,16 @@ static void check_slabp(kmem_cache_t *cachep, struct slab *slabp) | |||
2357 | goto bad; | 2384 | goto bad; |
2358 | } | 2385 | } |
2359 | if (entries != cachep->num - slabp->inuse) { | 2386 | if (entries != cachep->num - slabp->inuse) { |
2360 | bad: | 2387 | bad: |
2361 | printk(KERN_ERR "slab: Internal list corruption detected in cache '%s'(%d), slabp %p(%d). Hexdump:\n", | 2388 | printk(KERN_ERR |
2362 | cachep->name, cachep->num, slabp, slabp->inuse); | 2389 | "slab: Internal list corruption detected in cache '%s'(%d), slabp %p(%d). Hexdump:\n", |
2363 | for (i=0;i<sizeof(slabp)+cachep->num*sizeof(kmem_bufctl_t);i++) { | 2390 | cachep->name, cachep->num, slabp, slabp->inuse); |
2364 | if ((i%16)==0) | 2391 | for (i = 0; |
2392 | i < sizeof(slabp) + cachep->num * sizeof(kmem_bufctl_t); | ||
2393 | i++) { | ||
2394 | if ((i % 16) == 0) | ||
2365 | printk("\n%03x:", i); | 2395 | printk("\n%03x:", i); |
2366 | printk(" %02x", ((unsigned char*)slabp)[i]); | 2396 | printk(" %02x", ((unsigned char *)slabp)[i]); |
2367 | } | 2397 | } |
2368 | printk("\n"); | 2398 | printk("\n"); |
2369 | BUG(); | 2399 | BUG(); |
@@ -2383,7 +2413,7 @@ static void *cache_alloc_refill(kmem_cache_t *cachep, gfp_t flags) | |||
2383 | 2413 | ||
2384 | check_irq_off(); | 2414 | check_irq_off(); |
2385 | ac = ac_data(cachep); | 2415 | ac = ac_data(cachep); |
2386 | retry: | 2416 | retry: |
2387 | batchcount = ac->batchcount; | 2417 | batchcount = ac->batchcount; |
2388 | if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { | 2418 | if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { |
2389 | /* if there was little recent activity on this | 2419 | /* if there was little recent activity on this |
@@ -2405,8 +2435,8 @@ retry: | |||
2405 | shared_array->avail -= batchcount; | 2435 | shared_array->avail -= batchcount; |
2406 | ac->avail = batchcount; | 2436 | ac->avail = batchcount; |
2407 | memcpy(ac->entry, | 2437 | memcpy(ac->entry, |
2408 | &(shared_array->entry[shared_array->avail]), | 2438 | &(shared_array->entry[shared_array->avail]), |
2409 | sizeof(void*)*batchcount); | 2439 | sizeof(void *) * batchcount); |
2410 | shared_array->touched = 1; | 2440 | shared_array->touched = 1; |
2411 | goto alloc_done; | 2441 | goto alloc_done; |
2412 | } | 2442 | } |
@@ -2434,7 +2464,7 @@ retry: | |||
2434 | 2464 | ||
2435 | /* get obj pointer */ | 2465 | /* get obj pointer */ |
2436 | ac->entry[ac->avail++] = slabp->s_mem + | 2466 | ac->entry[ac->avail++] = slabp->s_mem + |
2437 | slabp->free*cachep->objsize; | 2467 | slabp->free * cachep->objsize; |
2438 | 2468 | ||
2439 | slabp->inuse++; | 2469 | slabp->inuse++; |
2440 | next = slab_bufctl(slabp)[slabp->free]; | 2470 | next = slab_bufctl(slabp)[slabp->free]; |
@@ -2442,7 +2472,7 @@ retry: | |||
2442 | slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; | 2472 | slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; |
2443 | WARN_ON(numa_node_id() != slabp->nodeid); | 2473 | WARN_ON(numa_node_id() != slabp->nodeid); |
2444 | #endif | 2474 | #endif |
2445 | slabp->free = next; | 2475 | slabp->free = next; |
2446 | } | 2476 | } |
2447 | check_slabp(cachep, slabp); | 2477 | check_slabp(cachep, slabp); |
2448 | 2478 | ||
@@ -2454,9 +2484,9 @@ retry: | |||
2454 | list_add(&slabp->list, &l3->slabs_partial); | 2484 | list_add(&slabp->list, &l3->slabs_partial); |
2455 | } | 2485 | } |
2456 | 2486 | ||
2457 | must_grow: | 2487 | must_grow: |
2458 | l3->free_objects -= ac->avail; | 2488 | l3->free_objects -= ac->avail; |
2459 | alloc_done: | 2489 | alloc_done: |
2460 | spin_unlock(&l3->list_lock); | 2490 | spin_unlock(&l3->list_lock); |
2461 | 2491 | ||
2462 | if (unlikely(!ac->avail)) { | 2492 | if (unlikely(!ac->avail)) { |
@@ -2468,7 +2498,7 @@ alloc_done: | |||
2468 | if (!x && ac->avail == 0) // no objects in sight? abort | 2498 | if (!x && ac->avail == 0) // no objects in sight? abort |
2469 | return NULL; | 2499 | return NULL; |
2470 | 2500 | ||
2471 | if (!ac->avail) // objects refilled by interrupt? | 2501 | if (!ac->avail) // objects refilled by interrupt? |
2472 | goto retry; | 2502 | goto retry; |
2473 | } | 2503 | } |
2474 | ac->touched = 1; | 2504 | ac->touched = 1; |
@@ -2485,16 +2515,16 @@ cache_alloc_debugcheck_before(kmem_cache_t *cachep, gfp_t flags) | |||
2485 | } | 2515 | } |
2486 | 2516 | ||
2487 | #if DEBUG | 2517 | #if DEBUG |
2488 | static void * | 2518 | static void *cache_alloc_debugcheck_after(kmem_cache_t *cachep, gfp_t flags, |
2489 | cache_alloc_debugcheck_after(kmem_cache_t *cachep, | 2519 | void *objp, void *caller) |
2490 | gfp_t flags, void *objp, void *caller) | ||
2491 | { | 2520 | { |
2492 | if (!objp) | 2521 | if (!objp) |
2493 | return objp; | 2522 | return objp; |
2494 | if (cachep->flags & SLAB_POISON) { | 2523 | if (cachep->flags & SLAB_POISON) { |
2495 | #ifdef CONFIG_DEBUG_PAGEALLOC | 2524 | #ifdef CONFIG_DEBUG_PAGEALLOC |
2496 | if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) | 2525 | if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) |
2497 | kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 1); | 2526 | kernel_map_pages(virt_to_page(objp), |
2527 | cachep->objsize / PAGE_SIZE, 1); | ||
2498 | else | 2528 | else |
2499 | check_poison_obj(cachep, objp); | 2529 | check_poison_obj(cachep, objp); |
2500 | #else | 2530 | #else |
@@ -2506,24 +2536,28 @@ cache_alloc_debugcheck_after(kmem_cache_t *cachep, | |||
2506 | *dbg_userword(cachep, objp) = caller; | 2536 | *dbg_userword(cachep, objp) = caller; |
2507 | 2537 | ||
2508 | if (cachep->flags & SLAB_RED_ZONE) { | 2538 | if (cachep->flags & SLAB_RED_ZONE) { |
2509 | if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || *dbg_redzone2(cachep, objp) != RED_INACTIVE) { | 2539 | if (*dbg_redzone1(cachep, objp) != RED_INACTIVE |
2510 | slab_error(cachep, "double free, or memory outside" | 2540 | || *dbg_redzone2(cachep, objp) != RED_INACTIVE) { |
2511 | " object was overwritten"); | 2541 | slab_error(cachep, |
2512 | printk(KERN_ERR "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", | 2542 | "double free, or memory outside" |
2513 | objp, *dbg_redzone1(cachep, objp), *dbg_redzone2(cachep, objp)); | 2543 | " object was overwritten"); |
2544 | printk(KERN_ERR | ||
2545 | "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", | ||
2546 | objp, *dbg_redzone1(cachep, objp), | ||
2547 | *dbg_redzone2(cachep, objp)); | ||
2514 | } | 2548 | } |
2515 | *dbg_redzone1(cachep, objp) = RED_ACTIVE; | 2549 | *dbg_redzone1(cachep, objp) = RED_ACTIVE; |
2516 | *dbg_redzone2(cachep, objp) = RED_ACTIVE; | 2550 | *dbg_redzone2(cachep, objp) = RED_ACTIVE; |
2517 | } | 2551 | } |
2518 | objp += obj_dbghead(cachep); | 2552 | objp += obj_dbghead(cachep); |
2519 | if (cachep->ctor && cachep->flags & SLAB_POISON) { | 2553 | if (cachep->ctor && cachep->flags & SLAB_POISON) { |
2520 | unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR; | 2554 | unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR; |
2521 | 2555 | ||
2522 | if (!(flags & __GFP_WAIT)) | 2556 | if (!(flags & __GFP_WAIT)) |
2523 | ctor_flags |= SLAB_CTOR_ATOMIC; | 2557 | ctor_flags |= SLAB_CTOR_ATOMIC; |
2524 | 2558 | ||
2525 | cachep->ctor(objp, cachep, ctor_flags); | 2559 | cachep->ctor(objp, cachep, ctor_flags); |
2526 | } | 2560 | } |
2527 | return objp; | 2561 | return objp; |
2528 | } | 2562 | } |
2529 | #else | 2563 | #else |
@@ -2532,7 +2566,7 @@ cache_alloc_debugcheck_after(kmem_cache_t *cachep, | |||
2532 | 2566 | ||
2533 | static inline void *____cache_alloc(kmem_cache_t *cachep, gfp_t flags) | 2567 | static inline void *____cache_alloc(kmem_cache_t *cachep, gfp_t flags) |
2534 | { | 2568 | { |
2535 | void* objp; | 2569 | void *objp; |
2536 | struct array_cache *ac; | 2570 | struct array_cache *ac; |
2537 | 2571 | ||
2538 | check_irq_off(); | 2572 | check_irq_off(); |
@@ -2551,7 +2585,7 @@ static inline void *____cache_alloc(kmem_cache_t *cachep, gfp_t flags) | |||
2551 | static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags) | 2585 | static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags) |
2552 | { | 2586 | { |
2553 | unsigned long save_flags; | 2587 | unsigned long save_flags; |
2554 | void* objp; | 2588 | void *objp; |
2555 | 2589 | ||
2556 | cache_alloc_debugcheck_before(cachep, flags); | 2590 | cache_alloc_debugcheck_before(cachep, flags); |
2557 | 2591 | ||
@@ -2559,7 +2593,7 @@ static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags) | |||
2559 | objp = ____cache_alloc(cachep, flags); | 2593 | objp = ____cache_alloc(cachep, flags); |
2560 | local_irq_restore(save_flags); | 2594 | local_irq_restore(save_flags); |
2561 | objp = cache_alloc_debugcheck_after(cachep, flags, objp, | 2595 | objp = cache_alloc_debugcheck_after(cachep, flags, objp, |
2562 | __builtin_return_address(0)); | 2596 | __builtin_return_address(0)); |
2563 | prefetchw(objp); | 2597 | prefetchw(objp); |
2564 | return objp; | 2598 | return objp; |
2565 | } | 2599 | } |
@@ -2571,74 +2605,75 @@ static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags) | |||
2571 | static void *__cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid) | 2605 | static void *__cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid) |
2572 | { | 2606 | { |
2573 | struct list_head *entry; | 2607 | struct list_head *entry; |
2574 | struct slab *slabp; | 2608 | struct slab *slabp; |
2575 | struct kmem_list3 *l3; | 2609 | struct kmem_list3 *l3; |
2576 | void *obj; | 2610 | void *obj; |
2577 | kmem_bufctl_t next; | 2611 | kmem_bufctl_t next; |
2578 | int x; | 2612 | int x; |
2579 | 2613 | ||
2580 | l3 = cachep->nodelists[nodeid]; | 2614 | l3 = cachep->nodelists[nodeid]; |
2581 | BUG_ON(!l3); | 2615 | BUG_ON(!l3); |
2582 | 2616 | ||
2583 | retry: | 2617 | retry: |
2584 | spin_lock(&l3->list_lock); | 2618 | spin_lock(&l3->list_lock); |
2585 | entry = l3->slabs_partial.next; | 2619 | entry = l3->slabs_partial.next; |
2586 | if (entry == &l3->slabs_partial) { | 2620 | if (entry == &l3->slabs_partial) { |
2587 | l3->free_touched = 1; | 2621 | l3->free_touched = 1; |
2588 | entry = l3->slabs_free.next; | 2622 | entry = l3->slabs_free.next; |
2589 | if (entry == &l3->slabs_free) | 2623 | if (entry == &l3->slabs_free) |
2590 | goto must_grow; | 2624 | goto must_grow; |
2591 | } | 2625 | } |
2592 | 2626 | ||
2593 | slabp = list_entry(entry, struct slab, list); | 2627 | slabp = list_entry(entry, struct slab, list); |
2594 | check_spinlock_acquired_node(cachep, nodeid); | 2628 | check_spinlock_acquired_node(cachep, nodeid); |
2595 | check_slabp(cachep, slabp); | 2629 | check_slabp(cachep, slabp); |
2596 | 2630 | ||
2597 | STATS_INC_NODEALLOCS(cachep); | 2631 | STATS_INC_NODEALLOCS(cachep); |
2598 | STATS_INC_ACTIVE(cachep); | 2632 | STATS_INC_ACTIVE(cachep); |
2599 | STATS_SET_HIGH(cachep); | 2633 | STATS_SET_HIGH(cachep); |
2600 | 2634 | ||
2601 | BUG_ON(slabp->inuse == cachep->num); | 2635 | BUG_ON(slabp->inuse == cachep->num); |
2602 | 2636 | ||
2603 | /* get obj pointer */ | 2637 | /* get obj pointer */ |
2604 | obj = slabp->s_mem + slabp->free*cachep->objsize; | 2638 | obj = slabp->s_mem + slabp->free * cachep->objsize; |
2605 | slabp->inuse++; | 2639 | slabp->inuse++; |
2606 | next = slab_bufctl(slabp)[slabp->free]; | 2640 | next = slab_bufctl(slabp)[slabp->free]; |
2607 | #if DEBUG | 2641 | #if DEBUG |
2608 | slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; | 2642 | slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; |
2609 | #endif | 2643 | #endif |
2610 | slabp->free = next; | 2644 | slabp->free = next; |
2611 | check_slabp(cachep, slabp); | 2645 | check_slabp(cachep, slabp); |
2612 | l3->free_objects--; | 2646 | l3->free_objects--; |
2613 | /* move slabp to correct slabp list: */ | 2647 | /* move slabp to correct slabp list: */ |
2614 | list_del(&slabp->list); | 2648 | list_del(&slabp->list); |
2615 | 2649 | ||
2616 | if (slabp->free == BUFCTL_END) { | 2650 | if (slabp->free == BUFCTL_END) { |
2617 | list_add(&slabp->list, &l3->slabs_full); | 2651 | list_add(&slabp->list, &l3->slabs_full); |
2618 | } else { | 2652 | } else { |
2619 | list_add(&slabp->list, &l3->slabs_partial); | 2653 | list_add(&slabp->list, &l3->slabs_partial); |
2620 | } | 2654 | } |
2621 | 2655 | ||
2622 | spin_unlock(&l3->list_lock); | 2656 | spin_unlock(&l3->list_lock); |
2623 | goto done; | 2657 | goto done; |
2624 | 2658 | ||
2625 | must_grow: | 2659 | must_grow: |
2626 | spin_unlock(&l3->list_lock); | 2660 | spin_unlock(&l3->list_lock); |
2627 | x = cache_grow(cachep, flags, nodeid); | 2661 | x = cache_grow(cachep, flags, nodeid); |
2628 | 2662 | ||
2629 | if (!x) | 2663 | if (!x) |
2630 | return NULL; | 2664 | return NULL; |
2631 | 2665 | ||
2632 | goto retry; | 2666 | goto retry; |
2633 | done: | 2667 | done: |
2634 | return obj; | 2668 | return obj; |
2635 | } | 2669 | } |
2636 | #endif | 2670 | #endif |
2637 | 2671 | ||
2638 | /* | 2672 | /* |
2639 | * Caller needs to acquire correct kmem_list's list_lock | 2673 | * Caller needs to acquire correct kmem_list's list_lock |
2640 | */ | 2674 | */ |
2641 | static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects, int node) | 2675 | static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects, |
2676 | int node) | ||
2642 | { | 2677 | { |
2643 | int i; | 2678 | int i; |
2644 | struct kmem_list3 *l3; | 2679 | struct kmem_list3 *l3; |
@@ -2661,7 +2696,7 @@ static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects, int n | |||
2661 | 2696 | ||
2662 | if (slab_bufctl(slabp)[objnr] != BUFCTL_FREE) { | 2697 | if (slab_bufctl(slabp)[objnr] != BUFCTL_FREE) { |
2663 | printk(KERN_ERR "slab: double free detected in cache " | 2698 | printk(KERN_ERR "slab: double free detected in cache " |
2664 | "'%s', objp %p\n", cachep->name, objp); | 2699 | "'%s', objp %p\n", cachep->name, objp); |
2665 | BUG(); | 2700 | BUG(); |
2666 | } | 2701 | } |
2667 | #endif | 2702 | #endif |
@@ -2705,20 +2740,19 @@ static void cache_flusharray(kmem_cache_t *cachep, struct array_cache *ac) | |||
2705 | spin_lock(&l3->list_lock); | 2740 | spin_lock(&l3->list_lock); |
2706 | if (l3->shared) { | 2741 | if (l3->shared) { |
2707 | struct array_cache *shared_array = l3->shared; | 2742 | struct array_cache *shared_array = l3->shared; |
2708 | int max = shared_array->limit-shared_array->avail; | 2743 | int max = shared_array->limit - shared_array->avail; |
2709 | if (max) { | 2744 | if (max) { |
2710 | if (batchcount > max) | 2745 | if (batchcount > max) |
2711 | batchcount = max; | 2746 | batchcount = max; |
2712 | memcpy(&(shared_array->entry[shared_array->avail]), | 2747 | memcpy(&(shared_array->entry[shared_array->avail]), |
2713 | ac->entry, | 2748 | ac->entry, sizeof(void *) * batchcount); |
2714 | sizeof(void*)*batchcount); | ||
2715 | shared_array->avail += batchcount; | 2749 | shared_array->avail += batchcount; |
2716 | goto free_done; | 2750 | goto free_done; |
2717 | } | 2751 | } |
2718 | } | 2752 | } |
2719 | 2753 | ||
2720 | free_block(cachep, ac->entry, batchcount, node); | 2754 | free_block(cachep, ac->entry, batchcount, node); |
2721 | free_done: | 2755 | free_done: |
2722 | #if STATS | 2756 | #if STATS |
2723 | { | 2757 | { |
2724 | int i = 0; | 2758 | int i = 0; |
@@ -2740,10 +2774,9 @@ free_done: | |||
2740 | spin_unlock(&l3->list_lock); | 2774 | spin_unlock(&l3->list_lock); |
2741 | ac->avail -= batchcount; | 2775 | ac->avail -= batchcount; |
2742 | memmove(ac->entry, &(ac->entry[batchcount]), | 2776 | memmove(ac->entry, &(ac->entry[batchcount]), |
2743 | sizeof(void*)*ac->avail); | 2777 | sizeof(void *) * ac->avail); |
2744 | } | 2778 | } |
2745 | 2779 | ||
2746 | |||
2747 | /* | 2780 | /* |
2748 | * __cache_free | 2781 | * __cache_free |
2749 | * Release an obj back to its cache. If the obj has a constructed | 2782 | * Release an obj back to its cache. If the obj has a constructed |
@@ -2768,7 +2801,8 @@ static inline void __cache_free(kmem_cache_t *cachep, void *objp) | |||
2768 | if (unlikely(slabp->nodeid != numa_node_id())) { | 2801 | if (unlikely(slabp->nodeid != numa_node_id())) { |
2769 | struct array_cache *alien = NULL; | 2802 | struct array_cache *alien = NULL; |
2770 | int nodeid = slabp->nodeid; | 2803 | int nodeid = slabp->nodeid; |
2771 | struct kmem_list3 *l3 = cachep->nodelists[numa_node_id()]; | 2804 | struct kmem_list3 *l3 = |
2805 | cachep->nodelists[numa_node_id()]; | ||
2772 | 2806 | ||
2773 | STATS_INC_NODEFREES(cachep); | 2807 | STATS_INC_NODEFREES(cachep); |
2774 | if (l3->alien && l3->alien[nodeid]) { | 2808 | if (l3->alien && l3->alien[nodeid]) { |
@@ -2776,15 +2810,15 @@ static inline void __cache_free(kmem_cache_t *cachep, void *objp) | |||
2776 | spin_lock(&alien->lock); | 2810 | spin_lock(&alien->lock); |
2777 | if (unlikely(alien->avail == alien->limit)) | 2811 | if (unlikely(alien->avail == alien->limit)) |
2778 | __drain_alien_cache(cachep, | 2812 | __drain_alien_cache(cachep, |
2779 | alien, nodeid); | 2813 | alien, nodeid); |
2780 | alien->entry[alien->avail++] = objp; | 2814 | alien->entry[alien->avail++] = objp; |
2781 | spin_unlock(&alien->lock); | 2815 | spin_unlock(&alien->lock); |
2782 | } else { | 2816 | } else { |
2783 | spin_lock(&(cachep->nodelists[nodeid])-> | 2817 | spin_lock(&(cachep->nodelists[nodeid])-> |
2784 | list_lock); | 2818 | list_lock); |
2785 | free_block(cachep, &objp, 1, nodeid); | 2819 | free_block(cachep, &objp, 1, nodeid); |
2786 | spin_unlock(&(cachep->nodelists[nodeid])-> | 2820 | spin_unlock(&(cachep->nodelists[nodeid])-> |
2787 | list_lock); | 2821 | list_lock); |
2788 | } | 2822 | } |
2789 | return; | 2823 | return; |
2790 | } | 2824 | } |
@@ -2831,9 +2865,9 @@ EXPORT_SYMBOL(kmem_cache_alloc); | |||
2831 | */ | 2865 | */ |
2832 | int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr) | 2866 | int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr) |
2833 | { | 2867 | { |
2834 | unsigned long addr = (unsigned long) ptr; | 2868 | unsigned long addr = (unsigned long)ptr; |
2835 | unsigned long min_addr = PAGE_OFFSET; | 2869 | unsigned long min_addr = PAGE_OFFSET; |
2836 | unsigned long align_mask = BYTES_PER_WORD-1; | 2870 | unsigned long align_mask = BYTES_PER_WORD - 1; |
2837 | unsigned long size = cachep->objsize; | 2871 | unsigned long size = cachep->objsize; |
2838 | struct page *page; | 2872 | struct page *page; |
2839 | 2873 | ||
@@ -2853,7 +2887,7 @@ int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr) | |||
2853 | if (unlikely(page_get_cache(page) != cachep)) | 2887 | if (unlikely(page_get_cache(page) != cachep)) |
2854 | goto out; | 2888 | goto out; |
2855 | return 1; | 2889 | return 1; |
2856 | out: | 2890 | out: |
2857 | return 0; | 2891 | return 0; |
2858 | } | 2892 | } |
2859 | 2893 | ||
@@ -2880,8 +2914,10 @@ void *kmem_cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid) | |||
2880 | 2914 | ||
2881 | if (unlikely(!cachep->nodelists[nodeid])) { | 2915 | if (unlikely(!cachep->nodelists[nodeid])) { |
2882 | /* Fall back to __cache_alloc if we run into trouble */ | 2916 | /* Fall back to __cache_alloc if we run into trouble */ |
2883 | printk(KERN_WARNING "slab: not allocating in inactive node %d for cache %s\n", nodeid, cachep->name); | 2917 | printk(KERN_WARNING |
2884 | return __cache_alloc(cachep,flags); | 2918 | "slab: not allocating in inactive node %d for cache %s\n", |
2919 | nodeid, cachep->name); | ||
2920 | return __cache_alloc(cachep, flags); | ||
2885 | } | 2921 | } |
2886 | 2922 | ||
2887 | cache_alloc_debugcheck_before(cachep, flags); | 2923 | cache_alloc_debugcheck_before(cachep, flags); |
@@ -2891,7 +2927,9 @@ void *kmem_cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid) | |||
2891 | else | 2927 | else |
2892 | ptr = __cache_alloc_node(cachep, flags, nodeid); | 2928 | ptr = __cache_alloc_node(cachep, flags, nodeid); |
2893 | local_irq_restore(save_flags); | 2929 | local_irq_restore(save_flags); |
2894 | ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, __builtin_return_address(0)); | 2930 | ptr = |
2931 | cache_alloc_debugcheck_after(cachep, flags, ptr, | ||
2932 | __builtin_return_address(0)); | ||
2895 | 2933 | ||
2896 | return ptr; | 2934 | return ptr; |
2897 | } | 2935 | } |
@@ -2957,7 +2995,7 @@ EXPORT_SYMBOL(__kmalloc); | |||
2957 | void *__alloc_percpu(size_t size) | 2995 | void *__alloc_percpu(size_t size) |
2958 | { | 2996 | { |
2959 | int i; | 2997 | int i; |
2960 | struct percpu_data *pdata = kmalloc(sizeof (*pdata), GFP_KERNEL); | 2998 | struct percpu_data *pdata = kmalloc(sizeof(*pdata), GFP_KERNEL); |
2961 | 2999 | ||
2962 | if (!pdata) | 3000 | if (!pdata) |
2963 | return NULL; | 3001 | return NULL; |
@@ -2981,9 +3019,9 @@ void *__alloc_percpu(size_t size) | |||
2981 | } | 3019 | } |
2982 | 3020 | ||
2983 | /* Catch derefs w/o wrappers */ | 3021 | /* Catch derefs w/o wrappers */ |
2984 | return (void *) (~(unsigned long) pdata); | 3022 | return (void *)(~(unsigned long)pdata); |
2985 | 3023 | ||
2986 | unwind_oom: | 3024 | unwind_oom: |
2987 | while (--i >= 0) { | 3025 | while (--i >= 0) { |
2988 | if (!cpu_possible(i)) | 3026 | if (!cpu_possible(i)) |
2989 | continue; | 3027 | continue; |
@@ -3046,7 +3084,7 @@ void kfree(const void *objp) | |||
3046 | local_irq_save(flags); | 3084 | local_irq_save(flags); |
3047 | kfree_debugcheck(objp); | 3085 | kfree_debugcheck(objp); |
3048 | c = page_get_cache(virt_to_page(objp)); | 3086 | c = page_get_cache(virt_to_page(objp)); |
3049 | __cache_free(c, (void*)objp); | 3087 | __cache_free(c, (void *)objp); |
3050 | local_irq_restore(flags); | 3088 | local_irq_restore(flags); |
3051 | } | 3089 | } |
3052 | EXPORT_SYMBOL(kfree); | 3090 | EXPORT_SYMBOL(kfree); |
@@ -3059,17 +3097,16 @@ EXPORT_SYMBOL(kfree); | |||
3059 | * Don't free memory not originally allocated by alloc_percpu() | 3097 | * Don't free memory not originally allocated by alloc_percpu() |
3060 | * The complemented objp is to check for that. | 3098 | * The complemented objp is to check for that. |
3061 | */ | 3099 | */ |
3062 | void | 3100 | void free_percpu(const void *objp) |
3063 | free_percpu(const void *objp) | ||
3064 | { | 3101 | { |
3065 | int i; | 3102 | int i; |
3066 | struct percpu_data *p = (struct percpu_data *) (~(unsigned long) objp); | 3103 | struct percpu_data *p = (struct percpu_data *)(~(unsigned long)objp); |
3067 | 3104 | ||
3068 | /* | 3105 | /* |
3069 | * We allocate for all cpus so we cannot use for online cpu here. | 3106 | * We allocate for all cpus so we cannot use for online cpu here. |
3070 | */ | 3107 | */ |
3071 | for_each_cpu(i) | 3108 | for_each_cpu(i) |
3072 | kfree(p->ptrs[i]); | 3109 | kfree(p->ptrs[i]); |
3073 | kfree(p); | 3110 | kfree(p); |
3074 | } | 3111 | } |
3075 | EXPORT_SYMBOL(free_percpu); | 3112 | EXPORT_SYMBOL(free_percpu); |
@@ -3103,44 +3140,44 @@ static int alloc_kmemlist(kmem_cache_t *cachep) | |||
3103 | if (!(new_alien = alloc_alien_cache(node, cachep->limit))) | 3140 | if (!(new_alien = alloc_alien_cache(node, cachep->limit))) |
3104 | goto fail; | 3141 | goto fail; |
3105 | #endif | 3142 | #endif |
3106 | if (!(new = alloc_arraycache(node, (cachep->shared* | 3143 | if (!(new = alloc_arraycache(node, (cachep->shared * |
3107 | cachep->batchcount), 0xbaadf00d))) | 3144 | cachep->batchcount), |
3145 | 0xbaadf00d))) | ||
3108 | goto fail; | 3146 | goto fail; |
3109 | if ((l3 = cachep->nodelists[node])) { | 3147 | if ((l3 = cachep->nodelists[node])) { |
3110 | 3148 | ||
3111 | spin_lock_irq(&l3->list_lock); | 3149 | spin_lock_irq(&l3->list_lock); |
3112 | 3150 | ||
3113 | if ((nc = cachep->nodelists[node]->shared)) | 3151 | if ((nc = cachep->nodelists[node]->shared)) |
3114 | free_block(cachep, nc->entry, | 3152 | free_block(cachep, nc->entry, nc->avail, node); |
3115 | nc->avail, node); | ||
3116 | 3153 | ||
3117 | l3->shared = new; | 3154 | l3->shared = new; |
3118 | if (!cachep->nodelists[node]->alien) { | 3155 | if (!cachep->nodelists[node]->alien) { |
3119 | l3->alien = new_alien; | 3156 | l3->alien = new_alien; |
3120 | new_alien = NULL; | 3157 | new_alien = NULL; |
3121 | } | 3158 | } |
3122 | l3->free_limit = (1 + nr_cpus_node(node))* | 3159 | l3->free_limit = (1 + nr_cpus_node(node)) * |
3123 | cachep->batchcount + cachep->num; | 3160 | cachep->batchcount + cachep->num; |
3124 | spin_unlock_irq(&l3->list_lock); | 3161 | spin_unlock_irq(&l3->list_lock); |
3125 | kfree(nc); | 3162 | kfree(nc); |
3126 | free_alien_cache(new_alien); | 3163 | free_alien_cache(new_alien); |
3127 | continue; | 3164 | continue; |
3128 | } | 3165 | } |
3129 | if (!(l3 = kmalloc_node(sizeof(struct kmem_list3), | 3166 | if (!(l3 = kmalloc_node(sizeof(struct kmem_list3), |
3130 | GFP_KERNEL, node))) | 3167 | GFP_KERNEL, node))) |
3131 | goto fail; | 3168 | goto fail; |
3132 | 3169 | ||
3133 | kmem_list3_init(l3); | 3170 | kmem_list3_init(l3); |
3134 | l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + | 3171 | l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + |
3135 | ((unsigned long)cachep)%REAPTIMEOUT_LIST3; | 3172 | ((unsigned long)cachep) % REAPTIMEOUT_LIST3; |
3136 | l3->shared = new; | 3173 | l3->shared = new; |
3137 | l3->alien = new_alien; | 3174 | l3->alien = new_alien; |
3138 | l3->free_limit = (1 + nr_cpus_node(node))* | 3175 | l3->free_limit = (1 + nr_cpus_node(node)) * |
3139 | cachep->batchcount + cachep->num; | 3176 | cachep->batchcount + cachep->num; |
3140 | cachep->nodelists[node] = l3; | 3177 | cachep->nodelists[node] = l3; |
3141 | } | 3178 | } |
3142 | return err; | 3179 | return err; |
3143 | fail: | 3180 | fail: |
3144 | err = -ENOMEM; | 3181 | err = -ENOMEM; |
3145 | return err; | 3182 | return err; |
3146 | } | 3183 | } |
@@ -3162,18 +3199,19 @@ static void do_ccupdate_local(void *info) | |||
3162 | new->new[smp_processor_id()] = old; | 3199 | new->new[smp_processor_id()] = old; |
3163 | } | 3200 | } |
3164 | 3201 | ||
3165 | |||
3166 | static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount, | 3202 | static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount, |
3167 | int shared) | 3203 | int shared) |
3168 | { | 3204 | { |
3169 | struct ccupdate_struct new; | 3205 | struct ccupdate_struct new; |
3170 | int i, err; | 3206 | int i, err; |
3171 | 3207 | ||
3172 | memset(&new.new,0,sizeof(new.new)); | 3208 | memset(&new.new, 0, sizeof(new.new)); |
3173 | for_each_online_cpu(i) { | 3209 | for_each_online_cpu(i) { |
3174 | new.new[i] = alloc_arraycache(cpu_to_node(i), limit, batchcount); | 3210 | new.new[i] = |
3211 | alloc_arraycache(cpu_to_node(i), limit, batchcount); | ||
3175 | if (!new.new[i]) { | 3212 | if (!new.new[i]) { |
3176 | for (i--; i >= 0; i--) kfree(new.new[i]); | 3213 | for (i--; i >= 0; i--) |
3214 | kfree(new.new[i]); | ||
3177 | return -ENOMEM; | 3215 | return -ENOMEM; |
3178 | } | 3216 | } |
3179 | } | 3217 | } |
@@ -3201,13 +3239,12 @@ static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount, | |||
3201 | err = alloc_kmemlist(cachep); | 3239 | err = alloc_kmemlist(cachep); |
3202 | if (err) { | 3240 | if (err) { |
3203 | printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n", | 3241 | printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n", |
3204 | cachep->name, -err); | 3242 | cachep->name, -err); |
3205 | BUG(); | 3243 | BUG(); |
3206 | } | 3244 | } |
3207 | return 0; | 3245 | return 0; |
3208 | } | 3246 | } |
3209 | 3247 | ||
3210 | |||
3211 | static void enable_cpucache(kmem_cache_t *cachep) | 3248 | static void enable_cpucache(kmem_cache_t *cachep) |
3212 | { | 3249 | { |
3213 | int err; | 3250 | int err; |
@@ -3254,14 +3291,14 @@ static void enable_cpucache(kmem_cache_t *cachep) | |||
3254 | if (limit > 32) | 3291 | if (limit > 32) |
3255 | limit = 32; | 3292 | limit = 32; |
3256 | #endif | 3293 | #endif |
3257 | err = do_tune_cpucache(cachep, limit, (limit+1)/2, shared); | 3294 | err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared); |
3258 | if (err) | 3295 | if (err) |
3259 | printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", | 3296 | printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", |
3260 | cachep->name, -err); | 3297 | cachep->name, -err); |
3261 | } | 3298 | } |
3262 | 3299 | ||
3263 | static void drain_array_locked(kmem_cache_t *cachep, | 3300 | static void drain_array_locked(kmem_cache_t *cachep, struct array_cache *ac, |
3264 | struct array_cache *ac, int force, int node) | 3301 | int force, int node) |
3265 | { | 3302 | { |
3266 | int tofree; | 3303 | int tofree; |
3267 | 3304 | ||
@@ -3269,14 +3306,14 @@ static void drain_array_locked(kmem_cache_t *cachep, | |||
3269 | if (ac->touched && !force) { | 3306 | if (ac->touched && !force) { |
3270 | ac->touched = 0; | 3307 | ac->touched = 0; |
3271 | } else if (ac->avail) { | 3308 | } else if (ac->avail) { |
3272 | tofree = force ? ac->avail : (ac->limit+4)/5; | 3309 | tofree = force ? ac->avail : (ac->limit + 4) / 5; |
3273 | if (tofree > ac->avail) { | 3310 | if (tofree > ac->avail) { |
3274 | tofree = (ac->avail+1)/2; | 3311 | tofree = (ac->avail + 1) / 2; |
3275 | } | 3312 | } |
3276 | free_block(cachep, ac->entry, tofree, node); | 3313 | free_block(cachep, ac->entry, tofree, node); |
3277 | ac->avail -= tofree; | 3314 | ac->avail -= tofree; |
3278 | memmove(ac->entry, &(ac->entry[tofree]), | 3315 | memmove(ac->entry, &(ac->entry[tofree]), |
3279 | sizeof(void*)*ac->avail); | 3316 | sizeof(void *) * ac->avail); |
3280 | } | 3317 | } |
3281 | } | 3318 | } |
3282 | 3319 | ||
@@ -3299,13 +3336,14 @@ static void cache_reap(void *unused) | |||
3299 | 3336 | ||
3300 | if (down_trylock(&cache_chain_sem)) { | 3337 | if (down_trylock(&cache_chain_sem)) { |
3301 | /* Give up. Setup the next iteration. */ | 3338 | /* Give up. Setup the next iteration. */ |
3302 | schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC); | 3339 | schedule_delayed_work(&__get_cpu_var(reap_work), |
3340 | REAPTIMEOUT_CPUC); | ||
3303 | return; | 3341 | return; |
3304 | } | 3342 | } |
3305 | 3343 | ||
3306 | list_for_each(walk, &cache_chain) { | 3344 | list_for_each(walk, &cache_chain) { |
3307 | kmem_cache_t *searchp; | 3345 | kmem_cache_t *searchp; |
3308 | struct list_head* p; | 3346 | struct list_head *p; |
3309 | int tofree; | 3347 | int tofree; |
3310 | struct slab *slabp; | 3348 | struct slab *slabp; |
3311 | 3349 | ||
@@ -3322,7 +3360,7 @@ static void cache_reap(void *unused) | |||
3322 | spin_lock_irq(&l3->list_lock); | 3360 | spin_lock_irq(&l3->list_lock); |
3323 | 3361 | ||
3324 | drain_array_locked(searchp, ac_data(searchp), 0, | 3362 | drain_array_locked(searchp, ac_data(searchp), 0, |
3325 | numa_node_id()); | 3363 | numa_node_id()); |
3326 | 3364 | ||
3327 | if (time_after(l3->next_reap, jiffies)) | 3365 | if (time_after(l3->next_reap, jiffies)) |
3328 | goto next_unlock; | 3366 | goto next_unlock; |
@@ -3331,14 +3369,16 @@ static void cache_reap(void *unused) | |||
3331 | 3369 | ||
3332 | if (l3->shared) | 3370 | if (l3->shared) |
3333 | drain_array_locked(searchp, l3->shared, 0, | 3371 | drain_array_locked(searchp, l3->shared, 0, |
3334 | numa_node_id()); | 3372 | numa_node_id()); |
3335 | 3373 | ||
3336 | if (l3->free_touched) { | 3374 | if (l3->free_touched) { |
3337 | l3->free_touched = 0; | 3375 | l3->free_touched = 0; |
3338 | goto next_unlock; | 3376 | goto next_unlock; |
3339 | } | 3377 | } |
3340 | 3378 | ||
3341 | tofree = (l3->free_limit+5*searchp->num-1)/(5*searchp->num); | 3379 | tofree = |
3380 | (l3->free_limit + 5 * searchp->num - | ||
3381 | 1) / (5 * searchp->num); | ||
3342 | do { | 3382 | do { |
3343 | p = l3->slabs_free.next; | 3383 | p = l3->slabs_free.next; |
3344 | if (p == &(l3->slabs_free)) | 3384 | if (p == &(l3->slabs_free)) |
@@ -3358,10 +3398,10 @@ static void cache_reap(void *unused) | |||
3358 | spin_unlock_irq(&l3->list_lock); | 3398 | spin_unlock_irq(&l3->list_lock); |
3359 | slab_destroy(searchp, slabp); | 3399 | slab_destroy(searchp, slabp); |
3360 | spin_lock_irq(&l3->list_lock); | 3400 | spin_lock_irq(&l3->list_lock); |
3361 | } while(--tofree > 0); | 3401 | } while (--tofree > 0); |
3362 | next_unlock: | 3402 | next_unlock: |
3363 | spin_unlock_irq(&l3->list_lock); | 3403 | spin_unlock_irq(&l3->list_lock); |
3364 | next: | 3404 | next: |
3365 | cond_resched(); | 3405 | cond_resched(); |
3366 | } | 3406 | } |
3367 | check_irq_on(); | 3407 | check_irq_on(); |
@@ -3418,7 +3458,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos) | |||
3418 | kmem_cache_t *cachep = p; | 3458 | kmem_cache_t *cachep = p; |
3419 | ++*pos; | 3459 | ++*pos; |
3420 | return cachep->next.next == &cache_chain ? NULL | 3460 | return cachep->next.next == &cache_chain ? NULL |
3421 | : list_entry(cachep->next.next, kmem_cache_t, next); | 3461 | : list_entry(cachep->next.next, kmem_cache_t, next); |
3422 | } | 3462 | } |
3423 | 3463 | ||
3424 | static void s_stop(struct seq_file *m, void *p) | 3464 | static void s_stop(struct seq_file *m, void *p) |
@@ -3430,11 +3470,11 @@ static int s_show(struct seq_file *m, void *p) | |||
3430 | { | 3470 | { |
3431 | kmem_cache_t *cachep = p; | 3471 | kmem_cache_t *cachep = p; |
3432 | struct list_head *q; | 3472 | struct list_head *q; |
3433 | struct slab *slabp; | 3473 | struct slab *slabp; |
3434 | unsigned long active_objs; | 3474 | unsigned long active_objs; |
3435 | unsigned long num_objs; | 3475 | unsigned long num_objs; |
3436 | unsigned long active_slabs = 0; | 3476 | unsigned long active_slabs = 0; |
3437 | unsigned long num_slabs, free_objects = 0, shared_avail = 0; | 3477 | unsigned long num_slabs, free_objects = 0, shared_avail = 0; |
3438 | const char *name; | 3478 | const char *name; |
3439 | char *error = NULL; | 3479 | char *error = NULL; |
3440 | int node; | 3480 | int node; |
@@ -3451,14 +3491,14 @@ static int s_show(struct seq_file *m, void *p) | |||
3451 | 3491 | ||
3452 | spin_lock(&l3->list_lock); | 3492 | spin_lock(&l3->list_lock); |
3453 | 3493 | ||
3454 | list_for_each(q,&l3->slabs_full) { | 3494 | list_for_each(q, &l3->slabs_full) { |
3455 | slabp = list_entry(q, struct slab, list); | 3495 | slabp = list_entry(q, struct slab, list); |
3456 | if (slabp->inuse != cachep->num && !error) | 3496 | if (slabp->inuse != cachep->num && !error) |
3457 | error = "slabs_full accounting error"; | 3497 | error = "slabs_full accounting error"; |
3458 | active_objs += cachep->num; | 3498 | active_objs += cachep->num; |
3459 | active_slabs++; | 3499 | active_slabs++; |
3460 | } | 3500 | } |
3461 | list_for_each(q,&l3->slabs_partial) { | 3501 | list_for_each(q, &l3->slabs_partial) { |
3462 | slabp = list_entry(q, struct slab, list); | 3502 | slabp = list_entry(q, struct slab, list); |
3463 | if (slabp->inuse == cachep->num && !error) | 3503 | if (slabp->inuse == cachep->num && !error) |
3464 | error = "slabs_partial inuse accounting error"; | 3504 | error = "slabs_partial inuse accounting error"; |
@@ -3467,7 +3507,7 @@ static int s_show(struct seq_file *m, void *p) | |||
3467 | active_objs += slabp->inuse; | 3507 | active_objs += slabp->inuse; |
3468 | active_slabs++; | 3508 | active_slabs++; |
3469 | } | 3509 | } |
3470 | list_for_each(q,&l3->slabs_free) { | 3510 | list_for_each(q, &l3->slabs_free) { |
3471 | slabp = list_entry(q, struct slab, list); | 3511 | slabp = list_entry(q, struct slab, list); |
3472 | if (slabp->inuse && !error) | 3512 | if (slabp->inuse && !error) |
3473 | error = "slabs_free/inuse accounting error"; | 3513 | error = "slabs_free/inuse accounting error"; |
@@ -3478,25 +3518,24 @@ static int s_show(struct seq_file *m, void *p) | |||
3478 | 3518 | ||
3479 | spin_unlock(&l3->list_lock); | 3519 | spin_unlock(&l3->list_lock); |
3480 | } | 3520 | } |
3481 | num_slabs+=active_slabs; | 3521 | num_slabs += active_slabs; |
3482 | num_objs = num_slabs*cachep->num; | 3522 | num_objs = num_slabs * cachep->num; |
3483 | if (num_objs - active_objs != free_objects && !error) | 3523 | if (num_objs - active_objs != free_objects && !error) |
3484 | error = "free_objects accounting error"; | 3524 | error = "free_objects accounting error"; |
3485 | 3525 | ||
3486 | name = cachep->name; | 3526 | name = cachep->name; |
3487 | if (error) | 3527 | if (error) |
3488 | printk(KERN_ERR "slab: cache %s error: %s\n", name, error); | 3528 | printk(KERN_ERR "slab: cache %s error: %s\n", name, error); |
3489 | 3529 | ||
3490 | seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", | 3530 | seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", |
3491 | name, active_objs, num_objs, cachep->objsize, | 3531 | name, active_objs, num_objs, cachep->objsize, |
3492 | cachep->num, (1<<cachep->gfporder)); | 3532 | cachep->num, (1 << cachep->gfporder)); |
3493 | seq_printf(m, " : tunables %4u %4u %4u", | 3533 | seq_printf(m, " : tunables %4u %4u %4u", |
3494 | cachep->limit, cachep->batchcount, | 3534 | cachep->limit, cachep->batchcount, cachep->shared); |
3495 | cachep->shared); | ||
3496 | seq_printf(m, " : slabdata %6lu %6lu %6lu", | 3535 | seq_printf(m, " : slabdata %6lu %6lu %6lu", |
3497 | active_slabs, num_slabs, shared_avail); | 3536 | active_slabs, num_slabs, shared_avail); |
3498 | #if STATS | 3537 | #if STATS |
3499 | { /* list3 stats */ | 3538 | { /* list3 stats */ |
3500 | unsigned long high = cachep->high_mark; | 3539 | unsigned long high = cachep->high_mark; |
3501 | unsigned long allocs = cachep->num_allocations; | 3540 | unsigned long allocs = cachep->num_allocations; |
3502 | unsigned long grown = cachep->grown; | 3541 | unsigned long grown = cachep->grown; |
@@ -3507,9 +3546,7 @@ static int s_show(struct seq_file *m, void *p) | |||
3507 | unsigned long node_frees = cachep->node_frees; | 3546 | unsigned long node_frees = cachep->node_frees; |
3508 | 3547 | ||
3509 | seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \ | 3548 | seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \ |
3510 | %4lu %4lu %4lu %4lu", | 3549 | %4lu %4lu %4lu %4lu", allocs, high, grown, reaped, errors, max_freeable, node_allocs, node_frees); |
3511 | allocs, high, grown, reaped, errors, | ||
3512 | max_freeable, node_allocs, node_frees); | ||
3513 | } | 3550 | } |
3514 | /* cpu stats */ | 3551 | /* cpu stats */ |
3515 | { | 3552 | { |
@@ -3519,7 +3556,7 @@ static int s_show(struct seq_file *m, void *p) | |||
3519 | unsigned long freemiss = atomic_read(&cachep->freemiss); | 3556 | unsigned long freemiss = atomic_read(&cachep->freemiss); |
3520 | 3557 | ||
3521 | seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", | 3558 | seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", |
3522 | allochit, allocmiss, freehit, freemiss); | 3559 | allochit, allocmiss, freehit, freemiss); |
3523 | } | 3560 | } |
3524 | #endif | 3561 | #endif |
3525 | seq_putc(m, '\n'); | 3562 | seq_putc(m, '\n'); |
@@ -3542,10 +3579,10 @@ static int s_show(struct seq_file *m, void *p) | |||
3542 | */ | 3579 | */ |
3543 | 3580 | ||
3544 | struct seq_operations slabinfo_op = { | 3581 | struct seq_operations slabinfo_op = { |
3545 | .start = s_start, | 3582 | .start = s_start, |
3546 | .next = s_next, | 3583 | .next = s_next, |
3547 | .stop = s_stop, | 3584 | .stop = s_stop, |
3548 | .show = s_show, | 3585 | .show = s_show, |
3549 | }; | 3586 | }; |
3550 | 3587 | ||
3551 | #define MAX_SLABINFO_WRITE 128 | 3588 | #define MAX_SLABINFO_WRITE 128 |
@@ -3556,18 +3593,18 @@ struct seq_operations slabinfo_op = { | |||
3556 | * @count: data length | 3593 | * @count: data length |
3557 | * @ppos: unused | 3594 | * @ppos: unused |
3558 | */ | 3595 | */ |
3559 | ssize_t slabinfo_write(struct file *file, const char __user *buffer, | 3596 | ssize_t slabinfo_write(struct file *file, const char __user * buffer, |
3560 | size_t count, loff_t *ppos) | 3597 | size_t count, loff_t *ppos) |
3561 | { | 3598 | { |
3562 | char kbuf[MAX_SLABINFO_WRITE+1], *tmp; | 3599 | char kbuf[MAX_SLABINFO_WRITE + 1], *tmp; |
3563 | int limit, batchcount, shared, res; | 3600 | int limit, batchcount, shared, res; |
3564 | struct list_head *p; | 3601 | struct list_head *p; |
3565 | 3602 | ||
3566 | if (count > MAX_SLABINFO_WRITE) | 3603 | if (count > MAX_SLABINFO_WRITE) |
3567 | return -EINVAL; | 3604 | return -EINVAL; |
3568 | if (copy_from_user(&kbuf, buffer, count)) | 3605 | if (copy_from_user(&kbuf, buffer, count)) |
3569 | return -EFAULT; | 3606 | return -EFAULT; |
3570 | kbuf[MAX_SLABINFO_WRITE] = '\0'; | 3607 | kbuf[MAX_SLABINFO_WRITE] = '\0'; |
3571 | 3608 | ||
3572 | tmp = strchr(kbuf, ' '); | 3609 | tmp = strchr(kbuf, ' '); |
3573 | if (!tmp) | 3610 | if (!tmp) |
@@ -3580,18 +3617,17 @@ ssize_t slabinfo_write(struct file *file, const char __user *buffer, | |||
3580 | /* Find the cache in the chain of caches. */ | 3617 | /* Find the cache in the chain of caches. */ |
3581 | down(&cache_chain_sem); | 3618 | down(&cache_chain_sem); |
3582 | res = -EINVAL; | 3619 | res = -EINVAL; |
3583 | list_for_each(p,&cache_chain) { | 3620 | list_for_each(p, &cache_chain) { |
3584 | kmem_cache_t *cachep = list_entry(p, kmem_cache_t, next); | 3621 | kmem_cache_t *cachep = list_entry(p, kmem_cache_t, next); |
3585 | 3622 | ||
3586 | if (!strcmp(cachep->name, kbuf)) { | 3623 | if (!strcmp(cachep->name, kbuf)) { |
3587 | if (limit < 1 || | 3624 | if (limit < 1 || |
3588 | batchcount < 1 || | 3625 | batchcount < 1 || |
3589 | batchcount > limit || | 3626 | batchcount > limit || shared < 0) { |
3590 | shared < 0) { | ||
3591 | res = 0; | 3627 | res = 0; |
3592 | } else { | 3628 | } else { |
3593 | res = do_tune_cpucache(cachep, limit, | 3629 | res = do_tune_cpucache(cachep, limit, |
3594 | batchcount, shared); | 3630 | batchcount, shared); |
3595 | } | 3631 | } |
3596 | break; | 3632 | break; |
3597 | } | 3633 | } |