aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mm/slab.c964
1 files changed, 500 insertions, 464 deletions
diff --git a/mm/slab.c b/mm/slab.c
index 2551b1eeadb3..f71d8be2f4e0 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -130,7 +130,6 @@
130#define FORCED_DEBUG 0 130#define FORCED_DEBUG 0
131#endif 131#endif
132 132
133
134/* Shouldn't this be in a header file somewhere? */ 133/* Shouldn't this be in a header file somewhere? */
135#define BYTES_PER_WORD sizeof(void *) 134#define BYTES_PER_WORD sizeof(void *)
136 135
@@ -217,12 +216,12 @@ static unsigned long offslab_limit;
217 * Slabs are chained into three list: fully used, partial, fully free slabs. 216 * Slabs are chained into three list: fully used, partial, fully free slabs.
218 */ 217 */
219struct slab { 218struct slab {
220 struct list_head list; 219 struct list_head list;
221 unsigned long colouroff; 220 unsigned long colouroff;
222 void *s_mem; /* including colour offset */ 221 void *s_mem; /* including colour offset */
223 unsigned int inuse; /* num of objs active in slab */ 222 unsigned int inuse; /* num of objs active in slab */
224 kmem_bufctl_t free; 223 kmem_bufctl_t free;
225 unsigned short nodeid; 224 unsigned short nodeid;
226}; 225};
227 226
228/* 227/*
@@ -242,9 +241,9 @@ struct slab {
242 * We assume struct slab_rcu can overlay struct slab when destroying. 241 * We assume struct slab_rcu can overlay struct slab when destroying.
243 */ 242 */
244struct slab_rcu { 243struct slab_rcu {
245 struct rcu_head head; 244 struct rcu_head head;
246 kmem_cache_t *cachep; 245 kmem_cache_t *cachep;
247 void *addr; 246 void *addr;
248}; 247};
249 248
250/* 249/*
@@ -279,23 +278,23 @@ struct array_cache {
279#define BOOT_CPUCACHE_ENTRIES 1 278#define BOOT_CPUCACHE_ENTRIES 1
280struct arraycache_init { 279struct arraycache_init {
281 struct array_cache cache; 280 struct array_cache cache;
282 void * entries[BOOT_CPUCACHE_ENTRIES]; 281 void *entries[BOOT_CPUCACHE_ENTRIES];
283}; 282};
284 283
285/* 284/*
286 * The slab lists for all objects. 285 * The slab lists for all objects.
287 */ 286 */
288struct kmem_list3 { 287struct kmem_list3 {
289 struct list_head slabs_partial; /* partial list first, better asm code */ 288 struct list_head slabs_partial; /* partial list first, better asm code */
290 struct list_head slabs_full; 289 struct list_head slabs_full;
291 struct list_head slabs_free; 290 struct list_head slabs_free;
292 unsigned long free_objects; 291 unsigned long free_objects;
293 unsigned long next_reap; 292 unsigned long next_reap;
294 int free_touched; 293 int free_touched;
295 unsigned int free_limit; 294 unsigned int free_limit;
296 spinlock_t list_lock; 295 spinlock_t list_lock;
297 struct array_cache *shared; /* shared per node */ 296 struct array_cache *shared; /* shared per node */
298 struct array_cache **alien; /* on other nodes */ 297 struct array_cache **alien; /* on other nodes */
299}; 298};
300 299
301/* 300/*
@@ -367,63 +366,63 @@ static inline void kmem_list3_init(struct kmem_list3 *parent)
367 * 366 *
368 * manages a cache. 367 * manages a cache.
369 */ 368 */
370 369
371struct kmem_cache { 370struct kmem_cache {
372/* 1) per-cpu data, touched during every alloc/free */ 371/* 1) per-cpu data, touched during every alloc/free */
373 struct array_cache *array[NR_CPUS]; 372 struct array_cache *array[NR_CPUS];
374 unsigned int batchcount; 373 unsigned int batchcount;
375 unsigned int limit; 374 unsigned int limit;
376 unsigned int shared; 375 unsigned int shared;
377 unsigned int objsize; 376 unsigned int objsize;
378/* 2) touched by every alloc & free from the backend */ 377/* 2) touched by every alloc & free from the backend */
379 struct kmem_list3 *nodelists[MAX_NUMNODES]; 378 struct kmem_list3 *nodelists[MAX_NUMNODES];
380 unsigned int flags; /* constant flags */ 379 unsigned int flags; /* constant flags */
381 unsigned int num; /* # of objs per slab */ 380 unsigned int num; /* # of objs per slab */
382 spinlock_t spinlock; 381 spinlock_t spinlock;
383 382
384/* 3) cache_grow/shrink */ 383/* 3) cache_grow/shrink */
385 /* order of pgs per slab (2^n) */ 384 /* order of pgs per slab (2^n) */
386 unsigned int gfporder; 385 unsigned int gfporder;
387 386
388 /* force GFP flags, e.g. GFP_DMA */ 387 /* force GFP flags, e.g. GFP_DMA */
389 gfp_t gfpflags; 388 gfp_t gfpflags;
390 389
391 size_t colour; /* cache colouring range */ 390 size_t colour; /* cache colouring range */
392 unsigned int colour_off; /* colour offset */ 391 unsigned int colour_off; /* colour offset */
393 unsigned int colour_next; /* cache colouring */ 392 unsigned int colour_next; /* cache colouring */
394 kmem_cache_t *slabp_cache; 393 kmem_cache_t *slabp_cache;
395 unsigned int slab_size; 394 unsigned int slab_size;
396 unsigned int dflags; /* dynamic flags */ 395 unsigned int dflags; /* dynamic flags */
397 396
398 /* constructor func */ 397 /* constructor func */
399 void (*ctor)(void *, kmem_cache_t *, unsigned long); 398 void (*ctor) (void *, kmem_cache_t *, unsigned long);
400 399
401 /* de-constructor func */ 400 /* de-constructor func */
402 void (*dtor)(void *, kmem_cache_t *, unsigned long); 401 void (*dtor) (void *, kmem_cache_t *, unsigned long);
403 402
404/* 4) cache creation/removal */ 403/* 4) cache creation/removal */
405 const char *name; 404 const char *name;
406 struct list_head next; 405 struct list_head next;
407 406
408/* 5) statistics */ 407/* 5) statistics */
409#if STATS 408#if STATS
410 unsigned long num_active; 409 unsigned long num_active;
411 unsigned long num_allocations; 410 unsigned long num_allocations;
412 unsigned long high_mark; 411 unsigned long high_mark;
413 unsigned long grown; 412 unsigned long grown;
414 unsigned long reaped; 413 unsigned long reaped;
415 unsigned long errors; 414 unsigned long errors;
416 unsigned long max_freeable; 415 unsigned long max_freeable;
417 unsigned long node_allocs; 416 unsigned long node_allocs;
418 unsigned long node_frees; 417 unsigned long node_frees;
419 atomic_t allochit; 418 atomic_t allochit;
420 atomic_t allocmiss; 419 atomic_t allocmiss;
421 atomic_t freehit; 420 atomic_t freehit;
422 atomic_t freemiss; 421 atomic_t freemiss;
423#endif 422#endif
424#if DEBUG 423#if DEBUG
425 int dbghead; 424 int dbghead;
426 int reallen; 425 int reallen;
427#endif 426#endif
428}; 427};
429 428
@@ -523,14 +522,15 @@ static unsigned long *dbg_redzone2(kmem_cache_t *cachep, void *objp)
523{ 522{
524 BUG_ON(!(cachep->flags & SLAB_RED_ZONE)); 523 BUG_ON(!(cachep->flags & SLAB_RED_ZONE));
525 if (cachep->flags & SLAB_STORE_USER) 524 if (cachep->flags & SLAB_STORE_USER)
526 return (unsigned long*) (objp+cachep->objsize-2*BYTES_PER_WORD); 525 return (unsigned long *)(objp + cachep->objsize -
527 return (unsigned long*) (objp+cachep->objsize-BYTES_PER_WORD); 526 2 * BYTES_PER_WORD);
527 return (unsigned long *)(objp + cachep->objsize - BYTES_PER_WORD);
528} 528}
529 529
530static void **dbg_userword(kmem_cache_t *cachep, void *objp) 530static void **dbg_userword(kmem_cache_t *cachep, void *objp)
531{ 531{
532 BUG_ON(!(cachep->flags & SLAB_STORE_USER)); 532 BUG_ON(!(cachep->flags & SLAB_STORE_USER));
533 return (void**)(objp+cachep->objsize-BYTES_PER_WORD); 533 return (void **)(objp + cachep->objsize - BYTES_PER_WORD);
534} 534}
535 535
536#else 536#else
@@ -607,31 +607,31 @@ struct cache_names {
607static struct cache_names __initdata cache_names[] = { 607static struct cache_names __initdata cache_names[] = {
608#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" }, 608#define CACHE(x) { .name = "size-" #x, .name_dma = "size-" #x "(DMA)" },
609#include <linux/kmalloc_sizes.h> 609#include <linux/kmalloc_sizes.h>
610 { NULL, } 610 {NULL,}
611#undef CACHE 611#undef CACHE
612}; 612};
613 613
614static struct arraycache_init initarray_cache __initdata = 614static struct arraycache_init initarray_cache __initdata =
615 { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 615 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
616static struct arraycache_init initarray_generic = 616static struct arraycache_init initarray_generic =
617 { { 0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 617 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
618 618
619/* internal cache of cache description objs */ 619/* internal cache of cache description objs */
620static kmem_cache_t cache_cache = { 620static kmem_cache_t cache_cache = {
621 .batchcount = 1, 621 .batchcount = 1,
622 .limit = BOOT_CPUCACHE_ENTRIES, 622 .limit = BOOT_CPUCACHE_ENTRIES,
623 .shared = 1, 623 .shared = 1,
624 .objsize = sizeof(kmem_cache_t), 624 .objsize = sizeof(kmem_cache_t),
625 .flags = SLAB_NO_REAP, 625 .flags = SLAB_NO_REAP,
626 .spinlock = SPIN_LOCK_UNLOCKED, 626 .spinlock = SPIN_LOCK_UNLOCKED,
627 .name = "kmem_cache", 627 .name = "kmem_cache",
628#if DEBUG 628#if DEBUG
629 .reallen = sizeof(kmem_cache_t), 629 .reallen = sizeof(kmem_cache_t),
630#endif 630#endif
631}; 631};
632 632
633/* Guard access to the cache-chain. */ 633/* Guard access to the cache-chain. */
634static struct semaphore cache_chain_sem; 634static struct semaphore cache_chain_sem;
635static struct list_head cache_chain; 635static struct list_head cache_chain;
636 636
637/* 637/*
@@ -655,9 +655,9 @@ static enum {
655 655
656static DEFINE_PER_CPU(struct work_struct, reap_work); 656static DEFINE_PER_CPU(struct work_struct, reap_work);
657 657
658static void free_block(kmem_cache_t* cachep, void** objpp, int len, int node); 658static void free_block(kmem_cache_t *cachep, void **objpp, int len, int node);
659static void enable_cpucache (kmem_cache_t *cachep); 659static void enable_cpucache(kmem_cache_t *cachep);
660static void cache_reap (void *unused); 660static void cache_reap(void *unused);
661static int __node_shrink(kmem_cache_t *cachep, int node); 661static int __node_shrink(kmem_cache_t *cachep, int node);
662 662
663static inline struct array_cache *ac_data(kmem_cache_t *cachep) 663static inline struct array_cache *ac_data(kmem_cache_t *cachep)
@@ -671,9 +671,9 @@ static inline kmem_cache_t *__find_general_cachep(size_t size, gfp_t gfpflags)
671 671
672#if DEBUG 672#if DEBUG
673 /* This happens if someone tries to call 673 /* This happens if someone tries to call
674 * kmem_cache_create(), or __kmalloc(), before 674 * kmem_cache_create(), or __kmalloc(), before
675 * the generic caches are initialized. 675 * the generic caches are initialized.
676 */ 676 */
677 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); 677 BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL);
678#endif 678#endif
679 while (size > csizep->cs_size) 679 while (size > csizep->cs_size)
@@ -697,10 +697,10 @@ EXPORT_SYMBOL(kmem_find_general_cachep);
697 697
698/* Cal the num objs, wastage, and bytes left over for a given slab size. */ 698/* Cal the num objs, wastage, and bytes left over for a given slab size. */
699static void cache_estimate(unsigned long gfporder, size_t size, size_t align, 699static void cache_estimate(unsigned long gfporder, size_t size, size_t align,
700 int flags, size_t *left_over, unsigned int *num) 700 int flags, size_t *left_over, unsigned int *num)
701{ 701{
702 int i; 702 int i;
703 size_t wastage = PAGE_SIZE<<gfporder; 703 size_t wastage = PAGE_SIZE << gfporder;
704 size_t extra = 0; 704 size_t extra = 0;
705 size_t base = 0; 705 size_t base = 0;
706 706
@@ -709,7 +709,7 @@ static void cache_estimate(unsigned long gfporder, size_t size, size_t align,
709 extra = sizeof(kmem_bufctl_t); 709 extra = sizeof(kmem_bufctl_t);
710 } 710 }
711 i = 0; 711 i = 0;
712 while (i*size + ALIGN(base+i*extra, align) <= wastage) 712 while (i * size + ALIGN(base + i * extra, align) <= wastage)
713 i++; 713 i++;
714 if (i > 0) 714 if (i > 0)
715 i--; 715 i--;
@@ -718,8 +718,8 @@ static void cache_estimate(unsigned long gfporder, size_t size, size_t align,
718 i = SLAB_LIMIT; 718 i = SLAB_LIMIT;
719 719
720 *num = i; 720 *num = i;
721 wastage -= i*size; 721 wastage -= i * size;
722 wastage -= ALIGN(base+i*extra, align); 722 wastage -= ALIGN(base + i * extra, align);
723 *left_over = wastage; 723 *left_over = wastage;
724} 724}
725 725
@@ -728,7 +728,7 @@ static void cache_estimate(unsigned long gfporder, size_t size, size_t align,
728static void __slab_error(const char *function, kmem_cache_t *cachep, char *msg) 728static void __slab_error(const char *function, kmem_cache_t *cachep, char *msg)
729{ 729{
730 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n", 730 printk(KERN_ERR "slab error in %s(): cache `%s': %s\n",
731 function, cachep->name, msg); 731 function, cachep->name, msg);
732 dump_stack(); 732 dump_stack();
733} 733}
734 734
@@ -755,9 +755,9 @@ static void __devinit start_cpu_timer(int cpu)
755} 755}
756 756
757static struct array_cache *alloc_arraycache(int node, int entries, 757static struct array_cache *alloc_arraycache(int node, int entries,
758 int batchcount) 758 int batchcount)
759{ 759{
760 int memsize = sizeof(void*)*entries+sizeof(struct array_cache); 760 int memsize = sizeof(void *) * entries + sizeof(struct array_cache);
761 struct array_cache *nc = NULL; 761 struct array_cache *nc = NULL;
762 762
763 nc = kmalloc_node(memsize, GFP_KERNEL, node); 763 nc = kmalloc_node(memsize, GFP_KERNEL, node);
@@ -775,7 +775,7 @@ static struct array_cache *alloc_arraycache(int node, int entries,
775static inline struct array_cache **alloc_alien_cache(int node, int limit) 775static inline struct array_cache **alloc_alien_cache(int node, int limit)
776{ 776{
777 struct array_cache **ac_ptr; 777 struct array_cache **ac_ptr;
778 int memsize = sizeof(void*)*MAX_NUMNODES; 778 int memsize = sizeof(void *) * MAX_NUMNODES;
779 int i; 779 int i;
780 780
781 if (limit > 1) 781 if (limit > 1)
@@ -789,7 +789,7 @@ static inline struct array_cache **alloc_alien_cache(int node, int limit)
789 } 789 }
790 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d); 790 ac_ptr[i] = alloc_arraycache(node, limit, 0xbaadf00d);
791 if (!ac_ptr[i]) { 791 if (!ac_ptr[i]) {
792 for (i--; i <=0; i--) 792 for (i--; i <= 0; i--)
793 kfree(ac_ptr[i]); 793 kfree(ac_ptr[i]);
794 kfree(ac_ptr); 794 kfree(ac_ptr);
795 return NULL; 795 return NULL;
@@ -807,12 +807,13 @@ static inline void free_alien_cache(struct array_cache **ac_ptr)
807 return; 807 return;
808 808
809 for_each_node(i) 809 for_each_node(i)
810 kfree(ac_ptr[i]); 810 kfree(ac_ptr[i]);
811 811
812 kfree(ac_ptr); 812 kfree(ac_ptr);
813} 813}
814 814
815static inline void __drain_alien_cache(kmem_cache_t *cachep, struct array_cache *ac, int node) 815static inline void __drain_alien_cache(kmem_cache_t *cachep,
816 struct array_cache *ac, int node)
816{ 817{
817 struct kmem_list3 *rl3 = cachep->nodelists[node]; 818 struct kmem_list3 *rl3 = cachep->nodelists[node];
818 819
@@ -826,7 +827,7 @@ static inline void __drain_alien_cache(kmem_cache_t *cachep, struct array_cache
826 827
827static void drain_alien_cache(kmem_cache_t *cachep, struct kmem_list3 *l3) 828static void drain_alien_cache(kmem_cache_t *cachep, struct kmem_list3 *l3)
828{ 829{
829 int i=0; 830 int i = 0;
830 struct array_cache *ac; 831 struct array_cache *ac;
831 unsigned long flags; 832 unsigned long flags;
832 833
@@ -846,10 +847,10 @@ static void drain_alien_cache(kmem_cache_t *cachep, struct kmem_list3 *l3)
846#endif 847#endif
847 848
848static int __devinit cpuup_callback(struct notifier_block *nfb, 849static int __devinit cpuup_callback(struct notifier_block *nfb,
849 unsigned long action, void *hcpu) 850 unsigned long action, void *hcpu)
850{ 851{
851 long cpu = (long)hcpu; 852 long cpu = (long)hcpu;
852 kmem_cache_t* cachep; 853 kmem_cache_t *cachep;
853 struct kmem_list3 *l3 = NULL; 854 struct kmem_list3 *l3 = NULL;
854 int node = cpu_to_node(cpu); 855 int node = cpu_to_node(cpu);
855 int memsize = sizeof(struct kmem_list3); 856 int memsize = sizeof(struct kmem_list3);
@@ -871,27 +872,27 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
871 */ 872 */
872 if (!cachep->nodelists[node]) { 873 if (!cachep->nodelists[node]) {
873 if (!(l3 = kmalloc_node(memsize, 874 if (!(l3 = kmalloc_node(memsize,
874 GFP_KERNEL, node))) 875 GFP_KERNEL, node)))
875 goto bad; 876 goto bad;
876 kmem_list3_init(l3); 877 kmem_list3_init(l3);
877 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + 878 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
878 ((unsigned long)cachep)%REAPTIMEOUT_LIST3; 879 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
879 880
880 cachep->nodelists[node] = l3; 881 cachep->nodelists[node] = l3;
881 } 882 }
882 883
883 spin_lock_irq(&cachep->nodelists[node]->list_lock); 884 spin_lock_irq(&cachep->nodelists[node]->list_lock);
884 cachep->nodelists[node]->free_limit = 885 cachep->nodelists[node]->free_limit =
885 (1 + nr_cpus_node(node)) * 886 (1 + nr_cpus_node(node)) *
886 cachep->batchcount + cachep->num; 887 cachep->batchcount + cachep->num;
887 spin_unlock_irq(&cachep->nodelists[node]->list_lock); 888 spin_unlock_irq(&cachep->nodelists[node]->list_lock);
888 } 889 }
889 890
890 /* Now we can go ahead with allocating the shared array's 891 /* Now we can go ahead with allocating the shared array's
891 & array cache's */ 892 & array cache's */
892 list_for_each_entry(cachep, &cache_chain, next) { 893 list_for_each_entry(cachep, &cache_chain, next) {
893 nc = alloc_arraycache(node, cachep->limit, 894 nc = alloc_arraycache(node, cachep->limit,
894 cachep->batchcount); 895 cachep->batchcount);
895 if (!nc) 896 if (!nc)
896 goto bad; 897 goto bad;
897 cachep->array[cpu] = nc; 898 cachep->array[cpu] = nc;
@@ -900,12 +901,13 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
900 BUG_ON(!l3); 901 BUG_ON(!l3);
901 if (!l3->shared) { 902 if (!l3->shared) {
902 if (!(nc = alloc_arraycache(node, 903 if (!(nc = alloc_arraycache(node,
903 cachep->shared*cachep->batchcount, 904 cachep->shared *
904 0xbaadf00d))) 905 cachep->batchcount,
905 goto bad; 906 0xbaadf00d)))
907 goto bad;
906 908
907 /* we are serialised from CPU_DEAD or 909 /* we are serialised from CPU_DEAD or
908 CPU_UP_CANCELLED by the cpucontrol lock */ 910 CPU_UP_CANCELLED by the cpucontrol lock */
909 l3->shared = nc; 911 l3->shared = nc;
910 } 912 }
911 } 913 }
@@ -942,13 +944,13 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
942 free_block(cachep, nc->entry, nc->avail, node); 944 free_block(cachep, nc->entry, nc->avail, node);
943 945
944 if (!cpus_empty(mask)) { 946 if (!cpus_empty(mask)) {
945 spin_unlock(&l3->list_lock); 947 spin_unlock(&l3->list_lock);
946 goto unlock_cache; 948 goto unlock_cache;
947 } 949 }
948 950
949 if (l3->shared) { 951 if (l3->shared) {
950 free_block(cachep, l3->shared->entry, 952 free_block(cachep, l3->shared->entry,
951 l3->shared->avail, node); 953 l3->shared->avail, node);
952 kfree(l3->shared); 954 kfree(l3->shared);
953 l3->shared = NULL; 955 l3->shared = NULL;
954 } 956 }
@@ -966,7 +968,7 @@ static int __devinit cpuup_callback(struct notifier_block *nfb,
966 } else { 968 } else {
967 spin_unlock(&l3->list_lock); 969 spin_unlock(&l3->list_lock);
968 } 970 }
969unlock_cache: 971 unlock_cache:
970 spin_unlock_irq(&cachep->spinlock); 972 spin_unlock_irq(&cachep->spinlock);
971 kfree(nc); 973 kfree(nc);
972 } 974 }
@@ -975,7 +977,7 @@ unlock_cache:
975#endif 977#endif
976 } 978 }
977 return NOTIFY_OK; 979 return NOTIFY_OK;
978bad: 980 bad:
979 up(&cache_chain_sem); 981 up(&cache_chain_sem);
980 return NOTIFY_BAD; 982 return NOTIFY_BAD;
981} 983}
@@ -985,8 +987,7 @@ static struct notifier_block cpucache_notifier = { &cpuup_callback, NULL, 0 };
985/* 987/*
986 * swap the static kmem_list3 with kmalloced memory 988 * swap the static kmem_list3 with kmalloced memory
987 */ 989 */
988static void init_list(kmem_cache_t *cachep, struct kmem_list3 *list, 990static void init_list(kmem_cache_t *cachep, struct kmem_list3 *list, int nodeid)
989 int nodeid)
990{ 991{
991 struct kmem_list3 *ptr; 992 struct kmem_list3 *ptr;
992 993
@@ -1055,14 +1056,14 @@ void __init kmem_cache_init(void)
1055 cache_cache.objsize = ALIGN(cache_cache.objsize, cache_line_size()); 1056 cache_cache.objsize = ALIGN(cache_cache.objsize, cache_line_size());
1056 1057
1057 cache_estimate(0, cache_cache.objsize, cache_line_size(), 0, 1058 cache_estimate(0, cache_cache.objsize, cache_line_size(), 0,
1058 &left_over, &cache_cache.num); 1059 &left_over, &cache_cache.num);
1059 if (!cache_cache.num) 1060 if (!cache_cache.num)
1060 BUG(); 1061 BUG();
1061 1062
1062 cache_cache.colour = left_over/cache_cache.colour_off; 1063 cache_cache.colour = left_over / cache_cache.colour_off;
1063 cache_cache.colour_next = 0; 1064 cache_cache.colour_next = 0;
1064 cache_cache.slab_size = ALIGN(cache_cache.num*sizeof(kmem_bufctl_t) + 1065 cache_cache.slab_size = ALIGN(cache_cache.num * sizeof(kmem_bufctl_t) +
1065 sizeof(struct slab), cache_line_size()); 1066 sizeof(struct slab), cache_line_size());
1066 1067
1067 /* 2+3) create the kmalloc caches */ 1068 /* 2+3) create the kmalloc caches */
1068 sizes = malloc_sizes; 1069 sizes = malloc_sizes;
@@ -1074,14 +1075,18 @@ void __init kmem_cache_init(void)
1074 */ 1075 */
1075 1076
1076 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, 1077 sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name,
1077 sizes[INDEX_AC].cs_size, ARCH_KMALLOC_MINALIGN, 1078 sizes[INDEX_AC].cs_size,
1078 (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); 1079 ARCH_KMALLOC_MINALIGN,
1080 (ARCH_KMALLOC_FLAGS |
1081 SLAB_PANIC), NULL, NULL);
1079 1082
1080 if (INDEX_AC != INDEX_L3) 1083 if (INDEX_AC != INDEX_L3)
1081 sizes[INDEX_L3].cs_cachep = 1084 sizes[INDEX_L3].cs_cachep =
1082 kmem_cache_create(names[INDEX_L3].name, 1085 kmem_cache_create(names[INDEX_L3].name,
1083 sizes[INDEX_L3].cs_size, ARCH_KMALLOC_MINALIGN, 1086 sizes[INDEX_L3].cs_size,
1084 (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); 1087 ARCH_KMALLOC_MINALIGN,
1088 (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL,
1089 NULL);
1085 1090
1086 while (sizes->cs_size != ULONG_MAX) { 1091 while (sizes->cs_size != ULONG_MAX) {
1087 /* 1092 /*
@@ -1091,35 +1096,41 @@ void __init kmem_cache_init(void)
1091 * Note for systems short on memory removing the alignment will 1096 * Note for systems short on memory removing the alignment will
1092 * allow tighter packing of the smaller caches. 1097 * allow tighter packing of the smaller caches.
1093 */ 1098 */
1094 if(!sizes->cs_cachep) 1099 if (!sizes->cs_cachep)
1095 sizes->cs_cachep = kmem_cache_create(names->name, 1100 sizes->cs_cachep = kmem_cache_create(names->name,
1096 sizes->cs_size, ARCH_KMALLOC_MINALIGN, 1101 sizes->cs_size,
1097 (ARCH_KMALLOC_FLAGS | SLAB_PANIC), NULL, NULL); 1102 ARCH_KMALLOC_MINALIGN,
1103 (ARCH_KMALLOC_FLAGS
1104 | SLAB_PANIC),
1105 NULL, NULL);
1098 1106
1099 /* Inc off-slab bufctl limit until the ceiling is hit. */ 1107 /* Inc off-slab bufctl limit until the ceiling is hit. */
1100 if (!(OFF_SLAB(sizes->cs_cachep))) { 1108 if (!(OFF_SLAB(sizes->cs_cachep))) {
1101 offslab_limit = sizes->cs_size-sizeof(struct slab); 1109 offslab_limit = sizes->cs_size - sizeof(struct slab);
1102 offslab_limit /= sizeof(kmem_bufctl_t); 1110 offslab_limit /= sizeof(kmem_bufctl_t);
1103 } 1111 }
1104 1112
1105 sizes->cs_dmacachep = kmem_cache_create(names->name_dma, 1113 sizes->cs_dmacachep = kmem_cache_create(names->name_dma,
1106 sizes->cs_size, ARCH_KMALLOC_MINALIGN, 1114 sizes->cs_size,
1107 (ARCH_KMALLOC_FLAGS | SLAB_CACHE_DMA | SLAB_PANIC), 1115 ARCH_KMALLOC_MINALIGN,
1108 NULL, NULL); 1116 (ARCH_KMALLOC_FLAGS |
1117 SLAB_CACHE_DMA |
1118 SLAB_PANIC), NULL,
1119 NULL);
1109 1120
1110 sizes++; 1121 sizes++;
1111 names++; 1122 names++;
1112 } 1123 }
1113 /* 4) Replace the bootstrap head arrays */ 1124 /* 4) Replace the bootstrap head arrays */
1114 { 1125 {
1115 void * ptr; 1126 void *ptr;
1116 1127
1117 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); 1128 ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
1118 1129
1119 local_irq_disable(); 1130 local_irq_disable();
1120 BUG_ON(ac_data(&cache_cache) != &initarray_cache.cache); 1131 BUG_ON(ac_data(&cache_cache) != &initarray_cache.cache);
1121 memcpy(ptr, ac_data(&cache_cache), 1132 memcpy(ptr, ac_data(&cache_cache),
1122 sizeof(struct arraycache_init)); 1133 sizeof(struct arraycache_init));
1123 cache_cache.array[smp_processor_id()] = ptr; 1134 cache_cache.array[smp_processor_id()] = ptr;
1124 local_irq_enable(); 1135 local_irq_enable();
1125 1136
@@ -1127,11 +1138,11 @@ void __init kmem_cache_init(void)
1127 1138
1128 local_irq_disable(); 1139 local_irq_disable();
1129 BUG_ON(ac_data(malloc_sizes[INDEX_AC].cs_cachep) 1140 BUG_ON(ac_data(malloc_sizes[INDEX_AC].cs_cachep)
1130 != &initarray_generic.cache); 1141 != &initarray_generic.cache);
1131 memcpy(ptr, ac_data(malloc_sizes[INDEX_AC].cs_cachep), 1142 memcpy(ptr, ac_data(malloc_sizes[INDEX_AC].cs_cachep),
1132 sizeof(struct arraycache_init)); 1143 sizeof(struct arraycache_init));
1133 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = 1144 malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] =
1134 ptr; 1145 ptr;
1135 local_irq_enable(); 1146 local_irq_enable();
1136 } 1147 }
1137 /* 5) Replace the bootstrap kmem_list3's */ 1148 /* 5) Replace the bootstrap kmem_list3's */
@@ -1139,16 +1150,16 @@ void __init kmem_cache_init(void)
1139 int node; 1150 int node;
1140 /* Replace the static kmem_list3 structures for the boot cpu */ 1151 /* Replace the static kmem_list3 structures for the boot cpu */
1141 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], 1152 init_list(&cache_cache, &initkmem_list3[CACHE_CACHE],
1142 numa_node_id()); 1153 numa_node_id());
1143 1154
1144 for_each_online_node(node) { 1155 for_each_online_node(node) {
1145 init_list(malloc_sizes[INDEX_AC].cs_cachep, 1156 init_list(malloc_sizes[INDEX_AC].cs_cachep,
1146 &initkmem_list3[SIZE_AC+node], node); 1157 &initkmem_list3[SIZE_AC + node], node);
1147 1158
1148 if (INDEX_AC != INDEX_L3) { 1159 if (INDEX_AC != INDEX_L3) {
1149 init_list(malloc_sizes[INDEX_L3].cs_cachep, 1160 init_list(malloc_sizes[INDEX_L3].cs_cachep,
1150 &initkmem_list3[SIZE_L3+node], 1161 &initkmem_list3[SIZE_L3 + node],
1151 node); 1162 node);
1152 } 1163 }
1153 } 1164 }
1154 } 1165 }
@@ -1158,7 +1169,7 @@ void __init kmem_cache_init(void)
1158 kmem_cache_t *cachep; 1169 kmem_cache_t *cachep;
1159 down(&cache_chain_sem); 1170 down(&cache_chain_sem);
1160 list_for_each_entry(cachep, &cache_chain, next) 1171 list_for_each_entry(cachep, &cache_chain, next)
1161 enable_cpucache(cachep); 1172 enable_cpucache(cachep);
1162 up(&cache_chain_sem); 1173 up(&cache_chain_sem);
1163 } 1174 }
1164 1175
@@ -1184,7 +1195,7 @@ static int __init cpucache_init(void)
1184 * pages to gfp. 1195 * pages to gfp.
1185 */ 1196 */
1186 for_each_online_cpu(cpu) 1197 for_each_online_cpu(cpu)
1187 start_cpu_timer(cpu); 1198 start_cpu_timer(cpu);
1188 1199
1189 return 0; 1200 return 0;
1190} 1201}
@@ -1226,7 +1237,7 @@ static void *kmem_getpages(kmem_cache_t *cachep, gfp_t flags, int nodeid)
1226 */ 1237 */
1227static void kmem_freepages(kmem_cache_t *cachep, void *addr) 1238static void kmem_freepages(kmem_cache_t *cachep, void *addr)
1228{ 1239{
1229 unsigned long i = (1<<cachep->gfporder); 1240 unsigned long i = (1 << cachep->gfporder);
1230 struct page *page = virt_to_page(addr); 1241 struct page *page = virt_to_page(addr);
1231 const unsigned long nr_freed = i; 1242 const unsigned long nr_freed = i;
1232 1243
@@ -1239,13 +1250,13 @@ static void kmem_freepages(kmem_cache_t *cachep, void *addr)
1239 if (current->reclaim_state) 1250 if (current->reclaim_state)
1240 current->reclaim_state->reclaimed_slab += nr_freed; 1251 current->reclaim_state->reclaimed_slab += nr_freed;
1241 free_pages((unsigned long)addr, cachep->gfporder); 1252 free_pages((unsigned long)addr, cachep->gfporder);
1242 if (cachep->flags & SLAB_RECLAIM_ACCOUNT) 1253 if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1243 atomic_sub(1<<cachep->gfporder, &slab_reclaim_pages); 1254 atomic_sub(1 << cachep->gfporder, &slab_reclaim_pages);
1244} 1255}
1245 1256
1246static void kmem_rcu_free(struct rcu_head *head) 1257static void kmem_rcu_free(struct rcu_head *head)
1247{ 1258{
1248 struct slab_rcu *slab_rcu = (struct slab_rcu *) head; 1259 struct slab_rcu *slab_rcu = (struct slab_rcu *)head;
1249 kmem_cache_t *cachep = slab_rcu->cachep; 1260 kmem_cache_t *cachep = slab_rcu->cachep;
1250 1261
1251 kmem_freepages(cachep, slab_rcu->addr); 1262 kmem_freepages(cachep, slab_rcu->addr);
@@ -1257,19 +1268,19 @@ static void kmem_rcu_free(struct rcu_head *head)
1257 1268
1258#ifdef CONFIG_DEBUG_PAGEALLOC 1269#ifdef CONFIG_DEBUG_PAGEALLOC
1259static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr, 1270static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr,
1260 unsigned long caller) 1271 unsigned long caller)
1261{ 1272{
1262 int size = obj_reallen(cachep); 1273 int size = obj_reallen(cachep);
1263 1274
1264 addr = (unsigned long *)&((char*)addr)[obj_dbghead(cachep)]; 1275 addr = (unsigned long *)&((char *)addr)[obj_dbghead(cachep)];
1265 1276
1266 if (size < 5*sizeof(unsigned long)) 1277 if (size < 5 * sizeof(unsigned long))
1267 return; 1278 return;
1268 1279
1269 *addr++=0x12345678; 1280 *addr++ = 0x12345678;
1270 *addr++=caller; 1281 *addr++ = caller;
1271 *addr++=smp_processor_id(); 1282 *addr++ = smp_processor_id();
1272 size -= 3*sizeof(unsigned long); 1283 size -= 3 * sizeof(unsigned long);
1273 { 1284 {
1274 unsigned long *sptr = &caller; 1285 unsigned long *sptr = &caller;
1275 unsigned long svalue; 1286 unsigned long svalue;
@@ -1277,7 +1288,7 @@ static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr,
1277 while (!kstack_end(sptr)) { 1288 while (!kstack_end(sptr)) {
1278 svalue = *sptr++; 1289 svalue = *sptr++;
1279 if (kernel_text_address(svalue)) { 1290 if (kernel_text_address(svalue)) {
1280 *addr++=svalue; 1291 *addr++ = svalue;
1281 size -= sizeof(unsigned long); 1292 size -= sizeof(unsigned long);
1282 if (size <= sizeof(unsigned long)) 1293 if (size <= sizeof(unsigned long))
1283 break; 1294 break;
@@ -1285,25 +1296,25 @@ static void store_stackinfo(kmem_cache_t *cachep, unsigned long *addr,
1285 } 1296 }
1286 1297
1287 } 1298 }
1288 *addr++=0x87654321; 1299 *addr++ = 0x87654321;
1289} 1300}
1290#endif 1301#endif
1291 1302
1292static void poison_obj(kmem_cache_t *cachep, void *addr, unsigned char val) 1303static void poison_obj(kmem_cache_t *cachep, void *addr, unsigned char val)
1293{ 1304{
1294 int size = obj_reallen(cachep); 1305 int size = obj_reallen(cachep);
1295 addr = &((char*)addr)[obj_dbghead(cachep)]; 1306 addr = &((char *)addr)[obj_dbghead(cachep)];
1296 1307
1297 memset(addr, val, size); 1308 memset(addr, val, size);
1298 *(unsigned char *)(addr+size-1) = POISON_END; 1309 *(unsigned char *)(addr + size - 1) = POISON_END;
1299} 1310}
1300 1311
1301static void dump_line(char *data, int offset, int limit) 1312static void dump_line(char *data, int offset, int limit)
1302{ 1313{
1303 int i; 1314 int i;
1304 printk(KERN_ERR "%03x:", offset); 1315 printk(KERN_ERR "%03x:", offset);
1305 for (i=0;i<limit;i++) { 1316 for (i = 0; i < limit; i++) {
1306 printk(" %02x", (unsigned char)data[offset+i]); 1317 printk(" %02x", (unsigned char)data[offset + i]);
1307 } 1318 }
1308 printk("\n"); 1319 printk("\n");
1309} 1320}
@@ -1318,24 +1329,24 @@ static void print_objinfo(kmem_cache_t *cachep, void *objp, int lines)
1318 1329
1319 if (cachep->flags & SLAB_RED_ZONE) { 1330 if (cachep->flags & SLAB_RED_ZONE) {
1320 printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n", 1331 printk(KERN_ERR "Redzone: 0x%lx/0x%lx.\n",
1321 *dbg_redzone1(cachep, objp), 1332 *dbg_redzone1(cachep, objp),
1322 *dbg_redzone2(cachep, objp)); 1333 *dbg_redzone2(cachep, objp));
1323 } 1334 }
1324 1335
1325 if (cachep->flags & SLAB_STORE_USER) { 1336 if (cachep->flags & SLAB_STORE_USER) {
1326 printk(KERN_ERR "Last user: [<%p>]", 1337 printk(KERN_ERR "Last user: [<%p>]",
1327 *dbg_userword(cachep, objp)); 1338 *dbg_userword(cachep, objp));
1328 print_symbol("(%s)", 1339 print_symbol("(%s)",
1329 (unsigned long)*dbg_userword(cachep, objp)); 1340 (unsigned long)*dbg_userword(cachep, objp));
1330 printk("\n"); 1341 printk("\n");
1331 } 1342 }
1332 realobj = (char*)objp+obj_dbghead(cachep); 1343 realobj = (char *)objp + obj_dbghead(cachep);
1333 size = obj_reallen(cachep); 1344 size = obj_reallen(cachep);
1334 for (i=0; i<size && lines;i+=16, lines--) { 1345 for (i = 0; i < size && lines; i += 16, lines--) {
1335 int limit; 1346 int limit;
1336 limit = 16; 1347 limit = 16;
1337 if (i+limit > size) 1348 if (i + limit > size)
1338 limit = size-i; 1349 limit = size - i;
1339 dump_line(realobj, i, limit); 1350 dump_line(realobj, i, limit);
1340 } 1351 }
1341} 1352}
@@ -1346,27 +1357,28 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp)
1346 int size, i; 1357 int size, i;
1347 int lines = 0; 1358 int lines = 0;
1348 1359
1349 realobj = (char*)objp+obj_dbghead(cachep); 1360 realobj = (char *)objp + obj_dbghead(cachep);
1350 size = obj_reallen(cachep); 1361 size = obj_reallen(cachep);
1351 1362
1352 for (i=0;i<size;i++) { 1363 for (i = 0; i < size; i++) {
1353 char exp = POISON_FREE; 1364 char exp = POISON_FREE;
1354 if (i == size-1) 1365 if (i == size - 1)
1355 exp = POISON_END; 1366 exp = POISON_END;
1356 if (realobj[i] != exp) { 1367 if (realobj[i] != exp) {
1357 int limit; 1368 int limit;
1358 /* Mismatch ! */ 1369 /* Mismatch ! */
1359 /* Print header */ 1370 /* Print header */
1360 if (lines == 0) { 1371 if (lines == 0) {
1361 printk(KERN_ERR "Slab corruption: start=%p, len=%d\n", 1372 printk(KERN_ERR
1362 realobj, size); 1373 "Slab corruption: start=%p, len=%d\n",
1374 realobj, size);
1363 print_objinfo(cachep, objp, 0); 1375 print_objinfo(cachep, objp, 0);
1364 } 1376 }
1365 /* Hexdump the affected line */ 1377 /* Hexdump the affected line */
1366 i = (i/16)*16; 1378 i = (i / 16) * 16;
1367 limit = 16; 1379 limit = 16;
1368 if (i+limit > size) 1380 if (i + limit > size)
1369 limit = size-i; 1381 limit = size - i;
1370 dump_line(realobj, i, limit); 1382 dump_line(realobj, i, limit);
1371 i += 16; 1383 i += 16;
1372 lines++; 1384 lines++;
@@ -1382,19 +1394,19 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp)
1382 struct slab *slabp = page_get_slab(virt_to_page(objp)); 1394 struct slab *slabp = page_get_slab(virt_to_page(objp));
1383 int objnr; 1395 int objnr;
1384 1396
1385 objnr = (objp-slabp->s_mem)/cachep->objsize; 1397 objnr = (objp - slabp->s_mem) / cachep->objsize;
1386 if (objnr) { 1398 if (objnr) {
1387 objp = slabp->s_mem+(objnr-1)*cachep->objsize; 1399 objp = slabp->s_mem + (objnr - 1) * cachep->objsize;
1388 realobj = (char*)objp+obj_dbghead(cachep); 1400 realobj = (char *)objp + obj_dbghead(cachep);
1389 printk(KERN_ERR "Prev obj: start=%p, len=%d\n", 1401 printk(KERN_ERR "Prev obj: start=%p, len=%d\n",
1390 realobj, size); 1402 realobj, size);
1391 print_objinfo(cachep, objp, 2); 1403 print_objinfo(cachep, objp, 2);
1392 } 1404 }
1393 if (objnr+1 < cachep->num) { 1405 if (objnr + 1 < cachep->num) {
1394 objp = slabp->s_mem+(objnr+1)*cachep->objsize; 1406 objp = slabp->s_mem + (objnr + 1) * cachep->objsize;
1395 realobj = (char*)objp+obj_dbghead(cachep); 1407 realobj = (char *)objp + obj_dbghead(cachep);
1396 printk(KERN_ERR "Next obj: start=%p, len=%d\n", 1408 printk(KERN_ERR "Next obj: start=%p, len=%d\n",
1397 realobj, size); 1409 realobj, size);
1398 print_objinfo(cachep, objp, 2); 1410 print_objinfo(cachep, objp, 2);
1399 } 1411 }
1400 } 1412 }
@@ -1405,7 +1417,7 @@ static void check_poison_obj(kmem_cache_t *cachep, void *objp)
1405 * Before calling the slab must have been unlinked from the cache. 1417 * Before calling the slab must have been unlinked from the cache.
1406 * The cache-lock is not held/needed. 1418 * The cache-lock is not held/needed.
1407 */ 1419 */
1408static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp) 1420static void slab_destroy(kmem_cache_t *cachep, struct slab *slabp)
1409{ 1421{
1410 void *addr = slabp->s_mem - slabp->colouroff; 1422 void *addr = slabp->s_mem - slabp->colouroff;
1411 1423
@@ -1416,8 +1428,11 @@ static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp)
1416 1428
1417 if (cachep->flags & SLAB_POISON) { 1429 if (cachep->flags & SLAB_POISON) {
1418#ifdef CONFIG_DEBUG_PAGEALLOC 1430#ifdef CONFIG_DEBUG_PAGEALLOC
1419 if ((cachep->objsize%PAGE_SIZE)==0 && OFF_SLAB(cachep)) 1431 if ((cachep->objsize % PAGE_SIZE) == 0
1420 kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE,1); 1432 && OFF_SLAB(cachep))
1433 kernel_map_pages(virt_to_page(objp),
1434 cachep->objsize / PAGE_SIZE,
1435 1);
1421 else 1436 else
1422 check_poison_obj(cachep, objp); 1437 check_poison_obj(cachep, objp);
1423#else 1438#else
@@ -1427,20 +1442,20 @@ static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp)
1427 if (cachep->flags & SLAB_RED_ZONE) { 1442 if (cachep->flags & SLAB_RED_ZONE) {
1428 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 1443 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
1429 slab_error(cachep, "start of a freed object " 1444 slab_error(cachep, "start of a freed object "
1430 "was overwritten"); 1445 "was overwritten");
1431 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 1446 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
1432 slab_error(cachep, "end of a freed object " 1447 slab_error(cachep, "end of a freed object "
1433 "was overwritten"); 1448 "was overwritten");
1434 } 1449 }
1435 if (cachep->dtor && !(cachep->flags & SLAB_POISON)) 1450 if (cachep->dtor && !(cachep->flags & SLAB_POISON))
1436 (cachep->dtor)(objp+obj_dbghead(cachep), cachep, 0); 1451 (cachep->dtor) (objp + obj_dbghead(cachep), cachep, 0);
1437 } 1452 }
1438#else 1453#else
1439 if (cachep->dtor) { 1454 if (cachep->dtor) {
1440 int i; 1455 int i;
1441 for (i = 0; i < cachep->num; i++) { 1456 for (i = 0; i < cachep->num; i++) {
1442 void* objp = slabp->s_mem+cachep->objsize*i; 1457 void *objp = slabp->s_mem + cachep->objsize * i;
1443 (cachep->dtor)(objp, cachep, 0); 1458 (cachep->dtor) (objp, cachep, 0);
1444 } 1459 }
1445 } 1460 }
1446#endif 1461#endif
@@ -1448,7 +1463,7 @@ static void slab_destroy (kmem_cache_t *cachep, struct slab *slabp)
1448 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) { 1463 if (unlikely(cachep->flags & SLAB_DESTROY_BY_RCU)) {
1449 struct slab_rcu *slab_rcu; 1464 struct slab_rcu *slab_rcu;
1450 1465
1451 slab_rcu = (struct slab_rcu *) slabp; 1466 slab_rcu = (struct slab_rcu *)slabp;
1452 slab_rcu->cachep = cachep; 1467 slab_rcu->cachep = cachep;
1453 slab_rcu->addr = addr; 1468 slab_rcu->addr = addr;
1454 call_rcu(&slab_rcu->head, kmem_rcu_free); 1469 call_rcu(&slab_rcu->head, kmem_rcu_free);
@@ -1466,10 +1481,10 @@ static inline void set_up_list3s(kmem_cache_t *cachep, int index)
1466 int node; 1481 int node;
1467 1482
1468 for_each_online_node(node) { 1483 for_each_online_node(node) {
1469 cachep->nodelists[node] = &initkmem_list3[index+node]; 1484 cachep->nodelists[node] = &initkmem_list3[index + node];
1470 cachep->nodelists[node]->next_reap = jiffies + 1485 cachep->nodelists[node]->next_reap = jiffies +
1471 REAPTIMEOUT_LIST3 + 1486 REAPTIMEOUT_LIST3 +
1472 ((unsigned long)cachep)%REAPTIMEOUT_LIST3; 1487 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1473 } 1488 }
1474} 1489}
1475 1490
@@ -1486,7 +1501,7 @@ static inline size_t calculate_slab_order(kmem_cache_t *cachep, size_t size,
1486{ 1501{
1487 size_t left_over = 0; 1502 size_t left_over = 0;
1488 1503
1489 for ( ; ; cachep->gfporder++) { 1504 for (;; cachep->gfporder++) {
1490 unsigned int num; 1505 unsigned int num;
1491 size_t remainder; 1506 size_t remainder;
1492 1507
@@ -1566,14 +1581,13 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1566 * Sanity checks... these are all serious usage bugs. 1581 * Sanity checks... these are all serious usage bugs.
1567 */ 1582 */
1568 if ((!name) || 1583 if ((!name) ||
1569 in_interrupt() || 1584 in_interrupt() ||
1570 (size < BYTES_PER_WORD) || 1585 (size < BYTES_PER_WORD) ||
1571 (size > (1<<MAX_OBJ_ORDER)*PAGE_SIZE) || 1586 (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) {
1572 (dtor && !ctor)) { 1587 printk(KERN_ERR "%s: Early error in slab %s\n",
1573 printk(KERN_ERR "%s: Early error in slab %s\n", 1588 __FUNCTION__, name);
1574 __FUNCTION__, name); 1589 BUG();
1575 BUG(); 1590 }
1576 }
1577 1591
1578 down(&cache_chain_sem); 1592 down(&cache_chain_sem);
1579 1593
@@ -1593,11 +1607,11 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1593 set_fs(old_fs); 1607 set_fs(old_fs);
1594 if (res) { 1608 if (res) {
1595 printk("SLAB: cache with size %d has lost its name\n", 1609 printk("SLAB: cache with size %d has lost its name\n",
1596 pc->objsize); 1610 pc->objsize);
1597 continue; 1611 continue;
1598 } 1612 }
1599 1613
1600 if (!strcmp(pc->name,name)) { 1614 if (!strcmp(pc->name, name)) {
1601 printk("kmem_cache_create: duplicate cache %s\n", name); 1615 printk("kmem_cache_create: duplicate cache %s\n", name);
1602 dump_stack(); 1616 dump_stack();
1603 goto oops; 1617 goto oops;
@@ -1609,10 +1623,9 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1609 if ((flags & SLAB_DEBUG_INITIAL) && !ctor) { 1623 if ((flags & SLAB_DEBUG_INITIAL) && !ctor) {
1610 /* No constructor, but inital state check requested */ 1624 /* No constructor, but inital state check requested */
1611 printk(KERN_ERR "%s: No con, but init state check " 1625 printk(KERN_ERR "%s: No con, but init state check "
1612 "requested - %s\n", __FUNCTION__, name); 1626 "requested - %s\n", __FUNCTION__, name);
1613 flags &= ~SLAB_DEBUG_INITIAL; 1627 flags &= ~SLAB_DEBUG_INITIAL;
1614 } 1628 }
1615
1616#if FORCED_DEBUG 1629#if FORCED_DEBUG
1617 /* 1630 /*
1618 * Enable redzoning and last user accounting, except for caches with 1631 * Enable redzoning and last user accounting, except for caches with
@@ -1620,8 +1633,9 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1620 * above the next power of two: caches with object sizes just above a 1633 * above the next power of two: caches with object sizes just above a
1621 * power of two have a significant amount of internal fragmentation. 1634 * power of two have a significant amount of internal fragmentation.
1622 */ 1635 */
1623 if ((size < 4096 || fls(size-1) == fls(size-1+3*BYTES_PER_WORD))) 1636 if ((size < 4096
1624 flags |= SLAB_RED_ZONE|SLAB_STORE_USER; 1637 || fls(size - 1) == fls(size - 1 + 3 * BYTES_PER_WORD)))
1638 flags |= SLAB_RED_ZONE | SLAB_STORE_USER;
1625 if (!(flags & SLAB_DESTROY_BY_RCU)) 1639 if (!(flags & SLAB_DESTROY_BY_RCU))
1626 flags |= SLAB_POISON; 1640 flags |= SLAB_POISON;
1627#endif 1641#endif
@@ -1642,9 +1656,9 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1642 * unaligned accesses for some archs when redzoning is used, and makes 1656 * unaligned accesses for some archs when redzoning is used, and makes
1643 * sure any on-slab bufctl's are also correctly aligned. 1657 * sure any on-slab bufctl's are also correctly aligned.
1644 */ 1658 */
1645 if (size & (BYTES_PER_WORD-1)) { 1659 if (size & (BYTES_PER_WORD - 1)) {
1646 size += (BYTES_PER_WORD-1); 1660 size += (BYTES_PER_WORD - 1);
1647 size &= ~(BYTES_PER_WORD-1); 1661 size &= ~(BYTES_PER_WORD - 1);
1648 } 1662 }
1649 1663
1650 /* calculate out the final buffer alignment: */ 1664 /* calculate out the final buffer alignment: */
@@ -1655,7 +1669,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1655 * objects into one cacheline. 1669 * objects into one cacheline.
1656 */ 1670 */
1657 ralign = cache_line_size(); 1671 ralign = cache_line_size();
1658 while (size <= ralign/2) 1672 while (size <= ralign / 2)
1659 ralign /= 2; 1673 ralign /= 2;
1660 } else { 1674 } else {
1661 ralign = BYTES_PER_WORD; 1675 ralign = BYTES_PER_WORD;
@@ -1664,13 +1678,13 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1664 if (ralign < ARCH_SLAB_MINALIGN) { 1678 if (ralign < ARCH_SLAB_MINALIGN) {
1665 ralign = ARCH_SLAB_MINALIGN; 1679 ralign = ARCH_SLAB_MINALIGN;
1666 if (ralign > BYTES_PER_WORD) 1680 if (ralign > BYTES_PER_WORD)
1667 flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER); 1681 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
1668 } 1682 }
1669 /* 3) caller mandated alignment: disables debug if necessary */ 1683 /* 3) caller mandated alignment: disables debug if necessary */
1670 if (ralign < align) { 1684 if (ralign < align) {
1671 ralign = align; 1685 ralign = align;
1672 if (ralign > BYTES_PER_WORD) 1686 if (ralign > BYTES_PER_WORD)
1673 flags &= ~(SLAB_RED_ZONE|SLAB_STORE_USER); 1687 flags &= ~(SLAB_RED_ZONE | SLAB_STORE_USER);
1674 } 1688 }
1675 /* 4) Store it. Note that the debug code below can reduce 1689 /* 4) Store it. Note that the debug code below can reduce
1676 * the alignment to BYTES_PER_WORD. 1690 * the alignment to BYTES_PER_WORD.
@@ -1692,7 +1706,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1692 1706
1693 /* add space for red zone words */ 1707 /* add space for red zone words */
1694 cachep->dbghead += BYTES_PER_WORD; 1708 cachep->dbghead += BYTES_PER_WORD;
1695 size += 2*BYTES_PER_WORD; 1709 size += 2 * BYTES_PER_WORD;
1696 } 1710 }
1697 if (flags & SLAB_STORE_USER) { 1711 if (flags & SLAB_STORE_USER) {
1698 /* user store requires word alignment and 1712 /* user store requires word alignment and
@@ -1703,7 +1717,8 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1703 size += BYTES_PER_WORD; 1717 size += BYTES_PER_WORD;
1704 } 1718 }
1705#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC) 1719#if FORCED_DEBUG && defined(CONFIG_DEBUG_PAGEALLOC)
1706 if (size >= malloc_sizes[INDEX_L3+1].cs_size && cachep->reallen > cache_line_size() && size < PAGE_SIZE) { 1720 if (size >= malloc_sizes[INDEX_L3 + 1].cs_size
1721 && cachep->reallen > cache_line_size() && size < PAGE_SIZE) {
1707 cachep->dbghead += PAGE_SIZE - size; 1722 cachep->dbghead += PAGE_SIZE - size;
1708 size = PAGE_SIZE; 1723 size = PAGE_SIZE;
1709 } 1724 }
@@ -1711,7 +1726,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1711#endif 1726#endif
1712 1727
1713 /* Determine if the slab management is 'on' or 'off' slab. */ 1728 /* Determine if the slab management is 'on' or 'off' slab. */
1714 if (size >= (PAGE_SIZE>>3)) 1729 if (size >= (PAGE_SIZE >> 3))
1715 /* 1730 /*
1716 * Size is large, assume best to place the slab management obj 1731 * Size is large, assume best to place the slab management obj
1717 * off-slab (should allow better packing of objs). 1732 * off-slab (should allow better packing of objs).
@@ -1728,7 +1743,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1728 */ 1743 */
1729 cachep->gfporder = 0; 1744 cachep->gfporder = 0;
1730 cache_estimate(cachep->gfporder, size, align, flags, 1745 cache_estimate(cachep->gfporder, size, align, flags,
1731 &left_over, &cachep->num); 1746 &left_over, &cachep->num);
1732 } else 1747 } else
1733 left_over = calculate_slab_order(cachep, size, align, flags); 1748 left_over = calculate_slab_order(cachep, size, align, flags);
1734 1749
@@ -1738,8 +1753,8 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1738 cachep = NULL; 1753 cachep = NULL;
1739 goto oops; 1754 goto oops;
1740 } 1755 }
1741 slab_size = ALIGN(cachep->num*sizeof(kmem_bufctl_t) 1756 slab_size = ALIGN(cachep->num * sizeof(kmem_bufctl_t)
1742 + sizeof(struct slab), align); 1757 + sizeof(struct slab), align);
1743 1758
1744 /* 1759 /*
1745 * If the slab has been placed off-slab, and we have enough space then 1760 * If the slab has been placed off-slab, and we have enough space then
@@ -1752,14 +1767,15 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1752 1767
1753 if (flags & CFLGS_OFF_SLAB) { 1768 if (flags & CFLGS_OFF_SLAB) {
1754 /* really off slab. No need for manual alignment */ 1769 /* really off slab. No need for manual alignment */
1755 slab_size = cachep->num*sizeof(kmem_bufctl_t)+sizeof(struct slab); 1770 slab_size =
1771 cachep->num * sizeof(kmem_bufctl_t) + sizeof(struct slab);
1756 } 1772 }
1757 1773
1758 cachep->colour_off = cache_line_size(); 1774 cachep->colour_off = cache_line_size();
1759 /* Offset must be a multiple of the alignment. */ 1775 /* Offset must be a multiple of the alignment. */
1760 if (cachep->colour_off < align) 1776 if (cachep->colour_off < align)
1761 cachep->colour_off = align; 1777 cachep->colour_off = align;
1762 cachep->colour = left_over/cachep->colour_off; 1778 cachep->colour = left_over / cachep->colour_off;
1763 cachep->slab_size = slab_size; 1779 cachep->slab_size = slab_size;
1764 cachep->flags = flags; 1780 cachep->flags = flags;
1765 cachep->gfpflags = 0; 1781 cachep->gfpflags = 0;
@@ -1786,7 +1802,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1786 * the creation of further caches will BUG(). 1802 * the creation of further caches will BUG().
1787 */ 1803 */
1788 cachep->array[smp_processor_id()] = 1804 cachep->array[smp_processor_id()] =
1789 &initarray_generic.cache; 1805 &initarray_generic.cache;
1790 1806
1791 /* If the cache that's used by 1807 /* If the cache that's used by
1792 * kmalloc(sizeof(kmem_list3)) is the first cache, 1808 * kmalloc(sizeof(kmem_list3)) is the first cache,
@@ -1800,8 +1816,7 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1800 g_cpucache_up = PARTIAL_AC; 1816 g_cpucache_up = PARTIAL_AC;
1801 } else { 1817 } else {
1802 cachep->array[smp_processor_id()] = 1818 cachep->array[smp_processor_id()] =
1803 kmalloc(sizeof(struct arraycache_init), 1819 kmalloc(sizeof(struct arraycache_init), GFP_KERNEL);
1804 GFP_KERNEL);
1805 1820
1806 if (g_cpucache_up == PARTIAL_AC) { 1821 if (g_cpucache_up == PARTIAL_AC) {
1807 set_up_list3s(cachep, SIZE_L3); 1822 set_up_list3s(cachep, SIZE_L3);
@@ -1811,16 +1826,18 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1811 for_each_online_node(node) { 1826 for_each_online_node(node) {
1812 1827
1813 cachep->nodelists[node] = 1828 cachep->nodelists[node] =
1814 kmalloc_node(sizeof(struct kmem_list3), 1829 kmalloc_node(sizeof
1815 GFP_KERNEL, node); 1830 (struct kmem_list3),
1831 GFP_KERNEL, node);
1816 BUG_ON(!cachep->nodelists[node]); 1832 BUG_ON(!cachep->nodelists[node]);
1817 kmem_list3_init(cachep->nodelists[node]); 1833 kmem_list3_init(cachep->
1834 nodelists[node]);
1818 } 1835 }
1819 } 1836 }
1820 } 1837 }
1821 cachep->nodelists[numa_node_id()]->next_reap = 1838 cachep->nodelists[numa_node_id()]->next_reap =
1822 jiffies + REAPTIMEOUT_LIST3 + 1839 jiffies + REAPTIMEOUT_LIST3 +
1823 ((unsigned long)cachep)%REAPTIMEOUT_LIST3; 1840 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
1824 1841
1825 BUG_ON(!ac_data(cachep)); 1842 BUG_ON(!ac_data(cachep));
1826 ac_data(cachep)->avail = 0; 1843 ac_data(cachep)->avail = 0;
@@ -1829,15 +1846,15 @@ kmem_cache_create (const char *name, size_t size, size_t align,
1829 ac_data(cachep)->touched = 0; 1846 ac_data(cachep)->touched = 0;
1830 cachep->batchcount = 1; 1847 cachep->batchcount = 1;
1831 cachep->limit = BOOT_CPUCACHE_ENTRIES; 1848 cachep->limit = BOOT_CPUCACHE_ENTRIES;
1832 } 1849 }
1833 1850
1834 /* cache setup completed, link it into the list */ 1851 /* cache setup completed, link it into the list */
1835 list_add(&cachep->next, &cache_chain); 1852 list_add(&cachep->next, &cache_chain);
1836 unlock_cpu_hotplug(); 1853 unlock_cpu_hotplug();
1837oops: 1854 oops:
1838 if (!cachep && (flags & SLAB_PANIC)) 1855 if (!cachep && (flags & SLAB_PANIC))
1839 panic("kmem_cache_create(): failed to create slab `%s'\n", 1856 panic("kmem_cache_create(): failed to create slab `%s'\n",
1840 name); 1857 name);
1841 up(&cache_chain_sem); 1858 up(&cache_chain_sem);
1842 return cachep; 1859 return cachep;
1843} 1860}
@@ -1880,7 +1897,7 @@ static inline void check_spinlock_acquired_node(kmem_cache_t *cachep, int node)
1880/* 1897/*
1881 * Waits for all CPUs to execute func(). 1898 * Waits for all CPUs to execute func().
1882 */ 1899 */
1883static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg) 1900static void smp_call_function_all_cpus(void (*func)(void *arg), void *arg)
1884{ 1901{
1885 check_irq_on(); 1902 check_irq_on();
1886 preempt_disable(); 1903 preempt_disable();
@@ -1895,12 +1912,12 @@ static void smp_call_function_all_cpus(void (*func) (void *arg), void *arg)
1895 preempt_enable(); 1912 preempt_enable();
1896} 1913}
1897 1914
1898static void drain_array_locked(kmem_cache_t* cachep, 1915static void drain_array_locked(kmem_cache_t *cachep, struct array_cache *ac,
1899 struct array_cache *ac, int force, int node); 1916 int force, int node);
1900 1917
1901static void do_drain(void *arg) 1918static void do_drain(void *arg)
1902{ 1919{
1903 kmem_cache_t *cachep = (kmem_cache_t*)arg; 1920 kmem_cache_t *cachep = (kmem_cache_t *) arg;
1904 struct array_cache *ac; 1921 struct array_cache *ac;
1905 int node = numa_node_id(); 1922 int node = numa_node_id();
1906 1923
@@ -1920,7 +1937,7 @@ static void drain_cpu_caches(kmem_cache_t *cachep)
1920 smp_call_function_all_cpus(do_drain, cachep); 1937 smp_call_function_all_cpus(do_drain, cachep);
1921 check_irq_on(); 1938 check_irq_on();
1922 spin_lock_irq(&cachep->spinlock); 1939 spin_lock_irq(&cachep->spinlock);
1923 for_each_online_node(node) { 1940 for_each_online_node(node) {
1924 l3 = cachep->nodelists[node]; 1941 l3 = cachep->nodelists[node];
1925 if (l3) { 1942 if (l3) {
1926 spin_lock(&l3->list_lock); 1943 spin_lock(&l3->list_lock);
@@ -1958,8 +1975,7 @@ static int __node_shrink(kmem_cache_t *cachep, int node)
1958 slab_destroy(cachep, slabp); 1975 slab_destroy(cachep, slabp);
1959 spin_lock_irq(&l3->list_lock); 1976 spin_lock_irq(&l3->list_lock);
1960 } 1977 }
1961 ret = !list_empty(&l3->slabs_full) || 1978 ret = !list_empty(&l3->slabs_full) || !list_empty(&l3->slabs_partial);
1962 !list_empty(&l3->slabs_partial);
1963 return ret; 1979 return ret;
1964} 1980}
1965 1981
@@ -2015,7 +2031,7 @@ EXPORT_SYMBOL(kmem_cache_shrink);
2015 * The caller must guarantee that noone will allocate memory from the cache 2031 * The caller must guarantee that noone will allocate memory from the cache
2016 * during the kmem_cache_destroy(). 2032 * during the kmem_cache_destroy().
2017 */ 2033 */
2018int kmem_cache_destroy(kmem_cache_t * cachep) 2034int kmem_cache_destroy(kmem_cache_t *cachep)
2019{ 2035{
2020 int i; 2036 int i;
2021 struct kmem_list3 *l3; 2037 struct kmem_list3 *l3;
@@ -2037,7 +2053,7 @@ int kmem_cache_destroy(kmem_cache_t * cachep)
2037 if (__cache_shrink(cachep)) { 2053 if (__cache_shrink(cachep)) {
2038 slab_error(cachep, "Can't free all objects"); 2054 slab_error(cachep, "Can't free all objects");
2039 down(&cache_chain_sem); 2055 down(&cache_chain_sem);
2040 list_add(&cachep->next,&cache_chain); 2056 list_add(&cachep->next, &cache_chain);
2041 up(&cache_chain_sem); 2057 up(&cache_chain_sem);
2042 unlock_cpu_hotplug(); 2058 unlock_cpu_hotplug();
2043 return 1; 2059 return 1;
@@ -2047,7 +2063,7 @@ int kmem_cache_destroy(kmem_cache_t * cachep)
2047 synchronize_rcu(); 2063 synchronize_rcu();
2048 2064
2049 for_each_online_cpu(i) 2065 for_each_online_cpu(i)
2050 kfree(cachep->array[i]); 2066 kfree(cachep->array[i]);
2051 2067
2052 /* NUMA: free the list3 structures */ 2068 /* NUMA: free the list3 structures */
2053 for_each_online_node(i) { 2069 for_each_online_node(i) {
@@ -2066,39 +2082,39 @@ int kmem_cache_destroy(kmem_cache_t * cachep)
2066EXPORT_SYMBOL(kmem_cache_destroy); 2082EXPORT_SYMBOL(kmem_cache_destroy);
2067 2083
2068/* Get the memory for a slab management obj. */ 2084/* Get the memory for a slab management obj. */
2069static struct slab* alloc_slabmgmt(kmem_cache_t *cachep, void *objp, 2085static struct slab *alloc_slabmgmt(kmem_cache_t *cachep, void *objp,
2070 int colour_off, gfp_t local_flags) 2086 int colour_off, gfp_t local_flags)
2071{ 2087{
2072 struct slab *slabp; 2088 struct slab *slabp;
2073 2089
2074 if (OFF_SLAB(cachep)) { 2090 if (OFF_SLAB(cachep)) {
2075 /* Slab management obj is off-slab. */ 2091 /* Slab management obj is off-slab. */
2076 slabp = kmem_cache_alloc(cachep->slabp_cache, local_flags); 2092 slabp = kmem_cache_alloc(cachep->slabp_cache, local_flags);
2077 if (!slabp) 2093 if (!slabp)
2078 return NULL; 2094 return NULL;
2079 } else { 2095 } else {
2080 slabp = objp+colour_off; 2096 slabp = objp + colour_off;
2081 colour_off += cachep->slab_size; 2097 colour_off += cachep->slab_size;
2082 } 2098 }
2083 slabp->inuse = 0; 2099 slabp->inuse = 0;
2084 slabp->colouroff = colour_off; 2100 slabp->colouroff = colour_off;
2085 slabp->s_mem = objp+colour_off; 2101 slabp->s_mem = objp + colour_off;
2086 2102
2087 return slabp; 2103 return slabp;
2088} 2104}
2089 2105
2090static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp) 2106static inline kmem_bufctl_t *slab_bufctl(struct slab *slabp)
2091{ 2107{
2092 return (kmem_bufctl_t *)(slabp+1); 2108 return (kmem_bufctl_t *) (slabp + 1);
2093} 2109}
2094 2110
2095static void cache_init_objs(kmem_cache_t *cachep, 2111static void cache_init_objs(kmem_cache_t *cachep,
2096 struct slab *slabp, unsigned long ctor_flags) 2112 struct slab *slabp, unsigned long ctor_flags)
2097{ 2113{
2098 int i; 2114 int i;
2099 2115
2100 for (i = 0; i < cachep->num; i++) { 2116 for (i = 0; i < cachep->num; i++) {
2101 void *objp = slabp->s_mem+cachep->objsize*i; 2117 void *objp = slabp->s_mem + cachep->objsize * i;
2102#if DEBUG 2118#if DEBUG
2103 /* need to poison the objs? */ 2119 /* need to poison the objs? */
2104 if (cachep->flags & SLAB_POISON) 2120 if (cachep->flags & SLAB_POISON)
@@ -2116,25 +2132,28 @@ static void cache_init_objs(kmem_cache_t *cachep,
2116 * Otherwise, deadlock. They must also be threaded. 2132 * Otherwise, deadlock. They must also be threaded.
2117 */ 2133 */
2118 if (cachep->ctor && !(cachep->flags & SLAB_POISON)) 2134 if (cachep->ctor && !(cachep->flags & SLAB_POISON))
2119 cachep->ctor(objp+obj_dbghead(cachep), cachep, ctor_flags); 2135 cachep->ctor(objp + obj_dbghead(cachep), cachep,
2136 ctor_flags);
2120 2137
2121 if (cachep->flags & SLAB_RED_ZONE) { 2138 if (cachep->flags & SLAB_RED_ZONE) {
2122 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) 2139 if (*dbg_redzone2(cachep, objp) != RED_INACTIVE)
2123 slab_error(cachep, "constructor overwrote the" 2140 slab_error(cachep, "constructor overwrote the"
2124 " end of an object"); 2141 " end of an object");
2125 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE) 2142 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
2126 slab_error(cachep, "constructor overwrote the" 2143 slab_error(cachep, "constructor overwrote the"
2127 " start of an object"); 2144 " start of an object");
2128 } 2145 }
2129 if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep) && cachep->flags & SLAB_POISON) 2146 if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)
2130 kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 0); 2147 && cachep->flags & SLAB_POISON)
2148 kernel_map_pages(virt_to_page(objp),
2149 cachep->objsize / PAGE_SIZE, 0);
2131#else 2150#else
2132 if (cachep->ctor) 2151 if (cachep->ctor)
2133 cachep->ctor(objp, cachep, ctor_flags); 2152 cachep->ctor(objp, cachep, ctor_flags);
2134#endif 2153#endif
2135 slab_bufctl(slabp)[i] = i+1; 2154 slab_bufctl(slabp)[i] = i + 1;
2136 } 2155 }
2137 slab_bufctl(slabp)[i-1] = BUFCTL_END; 2156 slab_bufctl(slabp)[i - 1] = BUFCTL_END;
2138 slabp->free = 0; 2157 slabp->free = 0;
2139} 2158}
2140 2159
@@ -2170,17 +2189,17 @@ static void set_slab_attr(kmem_cache_t *cachep, struct slab *slabp, void *objp)
2170 */ 2189 */
2171static int cache_grow(kmem_cache_t *cachep, gfp_t flags, int nodeid) 2190static int cache_grow(kmem_cache_t *cachep, gfp_t flags, int nodeid)
2172{ 2191{
2173 struct slab *slabp; 2192 struct slab *slabp;
2174 void *objp; 2193 void *objp;
2175 size_t offset; 2194 size_t offset;
2176 gfp_t local_flags; 2195 gfp_t local_flags;
2177 unsigned long ctor_flags; 2196 unsigned long ctor_flags;
2178 struct kmem_list3 *l3; 2197 struct kmem_list3 *l3;
2179 2198
2180 /* Be lazy and only check for valid flags here, 2199 /* Be lazy and only check for valid flags here,
2181 * keeping it out of the critical path in kmem_cache_alloc(). 2200 * keeping it out of the critical path in kmem_cache_alloc().
2182 */ 2201 */
2183 if (flags & ~(SLAB_DMA|SLAB_LEVEL_MASK|SLAB_NO_GROW)) 2202 if (flags & ~(SLAB_DMA | SLAB_LEVEL_MASK | SLAB_NO_GROW))
2184 BUG(); 2203 BUG();
2185 if (flags & SLAB_NO_GROW) 2204 if (flags & SLAB_NO_GROW)
2186 return 0; 2205 return 0;
@@ -2246,9 +2265,9 @@ static int cache_grow(kmem_cache_t *cachep, gfp_t flags, int nodeid)
2246 l3->free_objects += cachep->num; 2265 l3->free_objects += cachep->num;
2247 spin_unlock(&l3->list_lock); 2266 spin_unlock(&l3->list_lock);
2248 return 1; 2267 return 1;
2249opps1: 2268 opps1:
2250 kmem_freepages(cachep, objp); 2269 kmem_freepages(cachep, objp);
2251failed: 2270 failed:
2252 if (local_flags & __GFP_WAIT) 2271 if (local_flags & __GFP_WAIT)
2253 local_irq_disable(); 2272 local_irq_disable();
2254 return 0; 2273 return 0;
@@ -2268,18 +2287,19 @@ static void kfree_debugcheck(const void *objp)
2268 2287
2269 if (!virt_addr_valid(objp)) { 2288 if (!virt_addr_valid(objp)) {
2270 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n", 2289 printk(KERN_ERR "kfree_debugcheck: out of range ptr %lxh.\n",
2271 (unsigned long)objp); 2290 (unsigned long)objp);
2272 BUG(); 2291 BUG();
2273 } 2292 }
2274 page = virt_to_page(objp); 2293 page = virt_to_page(objp);
2275 if (!PageSlab(page)) { 2294 if (!PageSlab(page)) {
2276 printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n", (unsigned long)objp); 2295 printk(KERN_ERR "kfree_debugcheck: bad ptr %lxh.\n",
2296 (unsigned long)objp);
2277 BUG(); 2297 BUG();
2278 } 2298 }
2279} 2299}
2280 2300
2281static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp, 2301static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp,
2282 void *caller) 2302 void *caller)
2283{ 2303{
2284 struct page *page; 2304 struct page *page;
2285 unsigned int objnr; 2305 unsigned int objnr;
@@ -2290,20 +2310,26 @@ static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp,
2290 page = virt_to_page(objp); 2310 page = virt_to_page(objp);
2291 2311
2292 if (page_get_cache(page) != cachep) { 2312 if (page_get_cache(page) != cachep) {
2293 printk(KERN_ERR "mismatch in kmem_cache_free: expected cache %p, got %p\n", 2313 printk(KERN_ERR
2294 page_get_cache(page),cachep); 2314 "mismatch in kmem_cache_free: expected cache %p, got %p\n",
2315 page_get_cache(page), cachep);
2295 printk(KERN_ERR "%p is %s.\n", cachep, cachep->name); 2316 printk(KERN_ERR "%p is %s.\n", cachep, cachep->name);
2296 printk(KERN_ERR "%p is %s.\n", page_get_cache(page), page_get_cache(page)->name); 2317 printk(KERN_ERR "%p is %s.\n", page_get_cache(page),
2318 page_get_cache(page)->name);
2297 WARN_ON(1); 2319 WARN_ON(1);
2298 } 2320 }
2299 slabp = page_get_slab(page); 2321 slabp = page_get_slab(page);
2300 2322
2301 if (cachep->flags & SLAB_RED_ZONE) { 2323 if (cachep->flags & SLAB_RED_ZONE) {
2302 if (*dbg_redzone1(cachep, objp) != RED_ACTIVE || *dbg_redzone2(cachep, objp) != RED_ACTIVE) { 2324 if (*dbg_redzone1(cachep, objp) != RED_ACTIVE
2303 slab_error(cachep, "double free, or memory outside" 2325 || *dbg_redzone2(cachep, objp) != RED_ACTIVE) {
2304 " object was overwritten"); 2326 slab_error(cachep,
2305 printk(KERN_ERR "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", 2327 "double free, or memory outside"
2306 objp, *dbg_redzone1(cachep, objp), *dbg_redzone2(cachep, objp)); 2328 " object was overwritten");
2329 printk(KERN_ERR
2330 "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n",
2331 objp, *dbg_redzone1(cachep, objp),
2332 *dbg_redzone2(cachep, objp));
2307 } 2333 }
2308 *dbg_redzone1(cachep, objp) = RED_INACTIVE; 2334 *dbg_redzone1(cachep, objp) = RED_INACTIVE;
2309 *dbg_redzone2(cachep, objp) = RED_INACTIVE; 2335 *dbg_redzone2(cachep, objp) = RED_INACTIVE;
@@ -2311,30 +2337,31 @@ static void *cache_free_debugcheck(kmem_cache_t *cachep, void *objp,
2311 if (cachep->flags & SLAB_STORE_USER) 2337 if (cachep->flags & SLAB_STORE_USER)
2312 *dbg_userword(cachep, objp) = caller; 2338 *dbg_userword(cachep, objp) = caller;
2313 2339
2314 objnr = (objp-slabp->s_mem)/cachep->objsize; 2340 objnr = (objp - slabp->s_mem) / cachep->objsize;
2315 2341
2316 BUG_ON(objnr >= cachep->num); 2342 BUG_ON(objnr >= cachep->num);
2317 BUG_ON(objp != slabp->s_mem + objnr*cachep->objsize); 2343 BUG_ON(objp != slabp->s_mem + objnr * cachep->objsize);
2318 2344
2319 if (cachep->flags & SLAB_DEBUG_INITIAL) { 2345 if (cachep->flags & SLAB_DEBUG_INITIAL) {
2320 /* Need to call the slab's constructor so the 2346 /* Need to call the slab's constructor so the
2321 * caller can perform a verify of its state (debugging). 2347 * caller can perform a verify of its state (debugging).
2322 * Called without the cache-lock held. 2348 * Called without the cache-lock held.
2323 */ 2349 */
2324 cachep->ctor(objp+obj_dbghead(cachep), 2350 cachep->ctor(objp + obj_dbghead(cachep),
2325 cachep, SLAB_CTOR_CONSTRUCTOR|SLAB_CTOR_VERIFY); 2351 cachep, SLAB_CTOR_CONSTRUCTOR | SLAB_CTOR_VERIFY);
2326 } 2352 }
2327 if (cachep->flags & SLAB_POISON && cachep->dtor) { 2353 if (cachep->flags & SLAB_POISON && cachep->dtor) {
2328 /* we want to cache poison the object, 2354 /* we want to cache poison the object,
2329 * call the destruction callback 2355 * call the destruction callback
2330 */ 2356 */
2331 cachep->dtor(objp+obj_dbghead(cachep), cachep, 0); 2357 cachep->dtor(objp + obj_dbghead(cachep), cachep, 0);
2332 } 2358 }
2333 if (cachep->flags & SLAB_POISON) { 2359 if (cachep->flags & SLAB_POISON) {
2334#ifdef CONFIG_DEBUG_PAGEALLOC 2360#ifdef CONFIG_DEBUG_PAGEALLOC
2335 if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) { 2361 if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) {
2336 store_stackinfo(cachep, objp, (unsigned long)caller); 2362 store_stackinfo(cachep, objp, (unsigned long)caller);
2337 kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 0); 2363 kernel_map_pages(virt_to_page(objp),
2364 cachep->objsize / PAGE_SIZE, 0);
2338 } else { 2365 } else {
2339 poison_obj(cachep, objp, POISON_FREE); 2366 poison_obj(cachep, objp, POISON_FREE);
2340 } 2367 }
@@ -2349,7 +2376,7 @@ static void check_slabp(kmem_cache_t *cachep, struct slab *slabp)
2349{ 2376{
2350 kmem_bufctl_t i; 2377 kmem_bufctl_t i;
2351 int entries = 0; 2378 int entries = 0;
2352 2379
2353 /* Check slab's freelist to see if this obj is there. */ 2380 /* Check slab's freelist to see if this obj is there. */
2354 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) { 2381 for (i = slabp->free; i != BUFCTL_END; i = slab_bufctl(slabp)[i]) {
2355 entries++; 2382 entries++;
@@ -2357,13 +2384,16 @@ static void check_slabp(kmem_cache_t *cachep, struct slab *slabp)
2357 goto bad; 2384 goto bad;
2358 } 2385 }
2359 if (entries != cachep->num - slabp->inuse) { 2386 if (entries != cachep->num - slabp->inuse) {
2360bad: 2387 bad:
2361 printk(KERN_ERR "slab: Internal list corruption detected in cache '%s'(%d), slabp %p(%d). Hexdump:\n", 2388 printk(KERN_ERR
2362 cachep->name, cachep->num, slabp, slabp->inuse); 2389 "slab: Internal list corruption detected in cache '%s'(%d), slabp %p(%d). Hexdump:\n",
2363 for (i=0;i<sizeof(slabp)+cachep->num*sizeof(kmem_bufctl_t);i++) { 2390 cachep->name, cachep->num, slabp, slabp->inuse);
2364 if ((i%16)==0) 2391 for (i = 0;
2392 i < sizeof(slabp) + cachep->num * sizeof(kmem_bufctl_t);
2393 i++) {
2394 if ((i % 16) == 0)
2365 printk("\n%03x:", i); 2395 printk("\n%03x:", i);
2366 printk(" %02x", ((unsigned char*)slabp)[i]); 2396 printk(" %02x", ((unsigned char *)slabp)[i]);
2367 } 2397 }
2368 printk("\n"); 2398 printk("\n");
2369 BUG(); 2399 BUG();
@@ -2383,7 +2413,7 @@ static void *cache_alloc_refill(kmem_cache_t *cachep, gfp_t flags)
2383 2413
2384 check_irq_off(); 2414 check_irq_off();
2385 ac = ac_data(cachep); 2415 ac = ac_data(cachep);
2386retry: 2416 retry:
2387 batchcount = ac->batchcount; 2417 batchcount = ac->batchcount;
2388 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) { 2418 if (!ac->touched && batchcount > BATCHREFILL_LIMIT) {
2389 /* if there was little recent activity on this 2419 /* if there was little recent activity on this
@@ -2405,8 +2435,8 @@ retry:
2405 shared_array->avail -= batchcount; 2435 shared_array->avail -= batchcount;
2406 ac->avail = batchcount; 2436 ac->avail = batchcount;
2407 memcpy(ac->entry, 2437 memcpy(ac->entry,
2408 &(shared_array->entry[shared_array->avail]), 2438 &(shared_array->entry[shared_array->avail]),
2409 sizeof(void*)*batchcount); 2439 sizeof(void *) * batchcount);
2410 shared_array->touched = 1; 2440 shared_array->touched = 1;
2411 goto alloc_done; 2441 goto alloc_done;
2412 } 2442 }
@@ -2434,7 +2464,7 @@ retry:
2434 2464
2435 /* get obj pointer */ 2465 /* get obj pointer */
2436 ac->entry[ac->avail++] = slabp->s_mem + 2466 ac->entry[ac->avail++] = slabp->s_mem +
2437 slabp->free*cachep->objsize; 2467 slabp->free * cachep->objsize;
2438 2468
2439 slabp->inuse++; 2469 slabp->inuse++;
2440 next = slab_bufctl(slabp)[slabp->free]; 2470 next = slab_bufctl(slabp)[slabp->free];
@@ -2442,7 +2472,7 @@ retry:
2442 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; 2472 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2443 WARN_ON(numa_node_id() != slabp->nodeid); 2473 WARN_ON(numa_node_id() != slabp->nodeid);
2444#endif 2474#endif
2445 slabp->free = next; 2475 slabp->free = next;
2446 } 2476 }
2447 check_slabp(cachep, slabp); 2477 check_slabp(cachep, slabp);
2448 2478
@@ -2454,9 +2484,9 @@ retry:
2454 list_add(&slabp->list, &l3->slabs_partial); 2484 list_add(&slabp->list, &l3->slabs_partial);
2455 } 2485 }
2456 2486
2457must_grow: 2487 must_grow:
2458 l3->free_objects -= ac->avail; 2488 l3->free_objects -= ac->avail;
2459alloc_done: 2489 alloc_done:
2460 spin_unlock(&l3->list_lock); 2490 spin_unlock(&l3->list_lock);
2461 2491
2462 if (unlikely(!ac->avail)) { 2492 if (unlikely(!ac->avail)) {
@@ -2468,7 +2498,7 @@ alloc_done:
2468 if (!x && ac->avail == 0) // no objects in sight? abort 2498 if (!x && ac->avail == 0) // no objects in sight? abort
2469 return NULL; 2499 return NULL;
2470 2500
2471 if (!ac->avail) // objects refilled by interrupt? 2501 if (!ac->avail) // objects refilled by interrupt?
2472 goto retry; 2502 goto retry;
2473 } 2503 }
2474 ac->touched = 1; 2504 ac->touched = 1;
@@ -2485,16 +2515,16 @@ cache_alloc_debugcheck_before(kmem_cache_t *cachep, gfp_t flags)
2485} 2515}
2486 2516
2487#if DEBUG 2517#if DEBUG
2488static void * 2518static void *cache_alloc_debugcheck_after(kmem_cache_t *cachep, gfp_t flags,
2489cache_alloc_debugcheck_after(kmem_cache_t *cachep, 2519 void *objp, void *caller)
2490 gfp_t flags, void *objp, void *caller)
2491{ 2520{
2492 if (!objp) 2521 if (!objp)
2493 return objp; 2522 return objp;
2494 if (cachep->flags & SLAB_POISON) { 2523 if (cachep->flags & SLAB_POISON) {
2495#ifdef CONFIG_DEBUG_PAGEALLOC 2524#ifdef CONFIG_DEBUG_PAGEALLOC
2496 if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep)) 2525 if ((cachep->objsize % PAGE_SIZE) == 0 && OFF_SLAB(cachep))
2497 kernel_map_pages(virt_to_page(objp), cachep->objsize/PAGE_SIZE, 1); 2526 kernel_map_pages(virt_to_page(objp),
2527 cachep->objsize / PAGE_SIZE, 1);
2498 else 2528 else
2499 check_poison_obj(cachep, objp); 2529 check_poison_obj(cachep, objp);
2500#else 2530#else
@@ -2506,24 +2536,28 @@ cache_alloc_debugcheck_after(kmem_cache_t *cachep,
2506 *dbg_userword(cachep, objp) = caller; 2536 *dbg_userword(cachep, objp) = caller;
2507 2537
2508 if (cachep->flags & SLAB_RED_ZONE) { 2538 if (cachep->flags & SLAB_RED_ZONE) {
2509 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE || *dbg_redzone2(cachep, objp) != RED_INACTIVE) { 2539 if (*dbg_redzone1(cachep, objp) != RED_INACTIVE
2510 slab_error(cachep, "double free, or memory outside" 2540 || *dbg_redzone2(cachep, objp) != RED_INACTIVE) {
2511 " object was overwritten"); 2541 slab_error(cachep,
2512 printk(KERN_ERR "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n", 2542 "double free, or memory outside"
2513 objp, *dbg_redzone1(cachep, objp), *dbg_redzone2(cachep, objp)); 2543 " object was overwritten");
2544 printk(KERN_ERR
2545 "%p: redzone 1: 0x%lx, redzone 2: 0x%lx.\n",
2546 objp, *dbg_redzone1(cachep, objp),
2547 *dbg_redzone2(cachep, objp));
2514 } 2548 }
2515 *dbg_redzone1(cachep, objp) = RED_ACTIVE; 2549 *dbg_redzone1(cachep, objp) = RED_ACTIVE;
2516 *dbg_redzone2(cachep, objp) = RED_ACTIVE; 2550 *dbg_redzone2(cachep, objp) = RED_ACTIVE;
2517 } 2551 }
2518 objp += obj_dbghead(cachep); 2552 objp += obj_dbghead(cachep);
2519 if (cachep->ctor && cachep->flags & SLAB_POISON) { 2553 if (cachep->ctor && cachep->flags & SLAB_POISON) {
2520 unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR; 2554 unsigned long ctor_flags = SLAB_CTOR_CONSTRUCTOR;
2521 2555
2522 if (!(flags & __GFP_WAIT)) 2556 if (!(flags & __GFP_WAIT))
2523 ctor_flags |= SLAB_CTOR_ATOMIC; 2557 ctor_flags |= SLAB_CTOR_ATOMIC;
2524 2558
2525 cachep->ctor(objp, cachep, ctor_flags); 2559 cachep->ctor(objp, cachep, ctor_flags);
2526 } 2560 }
2527 return objp; 2561 return objp;
2528} 2562}
2529#else 2563#else
@@ -2532,7 +2566,7 @@ cache_alloc_debugcheck_after(kmem_cache_t *cachep,
2532 2566
2533static inline void *____cache_alloc(kmem_cache_t *cachep, gfp_t flags) 2567static inline void *____cache_alloc(kmem_cache_t *cachep, gfp_t flags)
2534{ 2568{
2535 void* objp; 2569 void *objp;
2536 struct array_cache *ac; 2570 struct array_cache *ac;
2537 2571
2538 check_irq_off(); 2572 check_irq_off();
@@ -2551,7 +2585,7 @@ static inline void *____cache_alloc(kmem_cache_t *cachep, gfp_t flags)
2551static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags) 2585static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags)
2552{ 2586{
2553 unsigned long save_flags; 2587 unsigned long save_flags;
2554 void* objp; 2588 void *objp;
2555 2589
2556 cache_alloc_debugcheck_before(cachep, flags); 2590 cache_alloc_debugcheck_before(cachep, flags);
2557 2591
@@ -2559,7 +2593,7 @@ static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags)
2559 objp = ____cache_alloc(cachep, flags); 2593 objp = ____cache_alloc(cachep, flags);
2560 local_irq_restore(save_flags); 2594 local_irq_restore(save_flags);
2561 objp = cache_alloc_debugcheck_after(cachep, flags, objp, 2595 objp = cache_alloc_debugcheck_after(cachep, flags, objp,
2562 __builtin_return_address(0)); 2596 __builtin_return_address(0));
2563 prefetchw(objp); 2597 prefetchw(objp);
2564 return objp; 2598 return objp;
2565} 2599}
@@ -2571,74 +2605,75 @@ static inline void *__cache_alloc(kmem_cache_t *cachep, gfp_t flags)
2571static void *__cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid) 2605static void *__cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid)
2572{ 2606{
2573 struct list_head *entry; 2607 struct list_head *entry;
2574 struct slab *slabp; 2608 struct slab *slabp;
2575 struct kmem_list3 *l3; 2609 struct kmem_list3 *l3;
2576 void *obj; 2610 void *obj;
2577 kmem_bufctl_t next; 2611 kmem_bufctl_t next;
2578 int x; 2612 int x;
2579 2613
2580 l3 = cachep->nodelists[nodeid]; 2614 l3 = cachep->nodelists[nodeid];
2581 BUG_ON(!l3); 2615 BUG_ON(!l3);
2582 2616
2583retry: 2617 retry:
2584 spin_lock(&l3->list_lock); 2618 spin_lock(&l3->list_lock);
2585 entry = l3->slabs_partial.next; 2619 entry = l3->slabs_partial.next;
2586 if (entry == &l3->slabs_partial) { 2620 if (entry == &l3->slabs_partial) {
2587 l3->free_touched = 1; 2621 l3->free_touched = 1;
2588 entry = l3->slabs_free.next; 2622 entry = l3->slabs_free.next;
2589 if (entry == &l3->slabs_free) 2623 if (entry == &l3->slabs_free)
2590 goto must_grow; 2624 goto must_grow;
2591 } 2625 }
2592 2626
2593 slabp = list_entry(entry, struct slab, list); 2627 slabp = list_entry(entry, struct slab, list);
2594 check_spinlock_acquired_node(cachep, nodeid); 2628 check_spinlock_acquired_node(cachep, nodeid);
2595 check_slabp(cachep, slabp); 2629 check_slabp(cachep, slabp);
2596 2630
2597 STATS_INC_NODEALLOCS(cachep); 2631 STATS_INC_NODEALLOCS(cachep);
2598 STATS_INC_ACTIVE(cachep); 2632 STATS_INC_ACTIVE(cachep);
2599 STATS_SET_HIGH(cachep); 2633 STATS_SET_HIGH(cachep);
2600 2634
2601 BUG_ON(slabp->inuse == cachep->num); 2635 BUG_ON(slabp->inuse == cachep->num);
2602 2636
2603 /* get obj pointer */ 2637 /* get obj pointer */
2604 obj = slabp->s_mem + slabp->free*cachep->objsize; 2638 obj = slabp->s_mem + slabp->free * cachep->objsize;
2605 slabp->inuse++; 2639 slabp->inuse++;
2606 next = slab_bufctl(slabp)[slabp->free]; 2640 next = slab_bufctl(slabp)[slabp->free];
2607#if DEBUG 2641#if DEBUG
2608 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE; 2642 slab_bufctl(slabp)[slabp->free] = BUFCTL_FREE;
2609#endif 2643#endif
2610 slabp->free = next; 2644 slabp->free = next;
2611 check_slabp(cachep, slabp); 2645 check_slabp(cachep, slabp);
2612 l3->free_objects--; 2646 l3->free_objects--;
2613 /* move slabp to correct slabp list: */ 2647 /* move slabp to correct slabp list: */
2614 list_del(&slabp->list); 2648 list_del(&slabp->list);
2615 2649
2616 if (slabp->free == BUFCTL_END) { 2650 if (slabp->free == BUFCTL_END) {
2617 list_add(&slabp->list, &l3->slabs_full); 2651 list_add(&slabp->list, &l3->slabs_full);
2618 } else { 2652 } else {
2619 list_add(&slabp->list, &l3->slabs_partial); 2653 list_add(&slabp->list, &l3->slabs_partial);
2620 } 2654 }
2621 2655
2622 spin_unlock(&l3->list_lock); 2656 spin_unlock(&l3->list_lock);
2623 goto done; 2657 goto done;
2624 2658
2625must_grow: 2659 must_grow:
2626 spin_unlock(&l3->list_lock); 2660 spin_unlock(&l3->list_lock);
2627 x = cache_grow(cachep, flags, nodeid); 2661 x = cache_grow(cachep, flags, nodeid);
2628 2662
2629 if (!x) 2663 if (!x)
2630 return NULL; 2664 return NULL;
2631 2665
2632 goto retry; 2666 goto retry;
2633done: 2667 done:
2634 return obj; 2668 return obj;
2635} 2669}
2636#endif 2670#endif
2637 2671
2638/* 2672/*
2639 * Caller needs to acquire correct kmem_list's list_lock 2673 * Caller needs to acquire correct kmem_list's list_lock
2640 */ 2674 */
2641static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects, int node) 2675static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects,
2676 int node)
2642{ 2677{
2643 int i; 2678 int i;
2644 struct kmem_list3 *l3; 2679 struct kmem_list3 *l3;
@@ -2661,7 +2696,7 @@ static void free_block(kmem_cache_t *cachep, void **objpp, int nr_objects, int n
2661 2696
2662 if (slab_bufctl(slabp)[objnr] != BUFCTL_FREE) { 2697 if (slab_bufctl(slabp)[objnr] != BUFCTL_FREE) {
2663 printk(KERN_ERR "slab: double free detected in cache " 2698 printk(KERN_ERR "slab: double free detected in cache "
2664 "'%s', objp %p\n", cachep->name, objp); 2699 "'%s', objp %p\n", cachep->name, objp);
2665 BUG(); 2700 BUG();
2666 } 2701 }
2667#endif 2702#endif
@@ -2705,20 +2740,19 @@ static void cache_flusharray(kmem_cache_t *cachep, struct array_cache *ac)
2705 spin_lock(&l3->list_lock); 2740 spin_lock(&l3->list_lock);
2706 if (l3->shared) { 2741 if (l3->shared) {
2707 struct array_cache *shared_array = l3->shared; 2742 struct array_cache *shared_array = l3->shared;
2708 int max = shared_array->limit-shared_array->avail; 2743 int max = shared_array->limit - shared_array->avail;
2709 if (max) { 2744 if (max) {
2710 if (batchcount > max) 2745 if (batchcount > max)
2711 batchcount = max; 2746 batchcount = max;
2712 memcpy(&(shared_array->entry[shared_array->avail]), 2747 memcpy(&(shared_array->entry[shared_array->avail]),
2713 ac->entry, 2748 ac->entry, sizeof(void *) * batchcount);
2714 sizeof(void*)*batchcount);
2715 shared_array->avail += batchcount; 2749 shared_array->avail += batchcount;
2716 goto free_done; 2750 goto free_done;
2717 } 2751 }
2718 } 2752 }
2719 2753
2720 free_block(cachep, ac->entry, batchcount, node); 2754 free_block(cachep, ac->entry, batchcount, node);
2721free_done: 2755 free_done:
2722#if STATS 2756#if STATS
2723 { 2757 {
2724 int i = 0; 2758 int i = 0;
@@ -2740,10 +2774,9 @@ free_done:
2740 spin_unlock(&l3->list_lock); 2774 spin_unlock(&l3->list_lock);
2741 ac->avail -= batchcount; 2775 ac->avail -= batchcount;
2742 memmove(ac->entry, &(ac->entry[batchcount]), 2776 memmove(ac->entry, &(ac->entry[batchcount]),
2743 sizeof(void*)*ac->avail); 2777 sizeof(void *) * ac->avail);
2744} 2778}
2745 2779
2746
2747/* 2780/*
2748 * __cache_free 2781 * __cache_free
2749 * Release an obj back to its cache. If the obj has a constructed 2782 * Release an obj back to its cache. If the obj has a constructed
@@ -2768,7 +2801,8 @@ static inline void __cache_free(kmem_cache_t *cachep, void *objp)
2768 if (unlikely(slabp->nodeid != numa_node_id())) { 2801 if (unlikely(slabp->nodeid != numa_node_id())) {
2769 struct array_cache *alien = NULL; 2802 struct array_cache *alien = NULL;
2770 int nodeid = slabp->nodeid; 2803 int nodeid = slabp->nodeid;
2771 struct kmem_list3 *l3 = cachep->nodelists[numa_node_id()]; 2804 struct kmem_list3 *l3 =
2805 cachep->nodelists[numa_node_id()];
2772 2806
2773 STATS_INC_NODEFREES(cachep); 2807 STATS_INC_NODEFREES(cachep);
2774 if (l3->alien && l3->alien[nodeid]) { 2808 if (l3->alien && l3->alien[nodeid]) {
@@ -2776,15 +2810,15 @@ static inline void __cache_free(kmem_cache_t *cachep, void *objp)
2776 spin_lock(&alien->lock); 2810 spin_lock(&alien->lock);
2777 if (unlikely(alien->avail == alien->limit)) 2811 if (unlikely(alien->avail == alien->limit))
2778 __drain_alien_cache(cachep, 2812 __drain_alien_cache(cachep,
2779 alien, nodeid); 2813 alien, nodeid);
2780 alien->entry[alien->avail++] = objp; 2814 alien->entry[alien->avail++] = objp;
2781 spin_unlock(&alien->lock); 2815 spin_unlock(&alien->lock);
2782 } else { 2816 } else {
2783 spin_lock(&(cachep->nodelists[nodeid])-> 2817 spin_lock(&(cachep->nodelists[nodeid])->
2784 list_lock); 2818 list_lock);
2785 free_block(cachep, &objp, 1, nodeid); 2819 free_block(cachep, &objp, 1, nodeid);
2786 spin_unlock(&(cachep->nodelists[nodeid])-> 2820 spin_unlock(&(cachep->nodelists[nodeid])->
2787 list_lock); 2821 list_lock);
2788 } 2822 }
2789 return; 2823 return;
2790 } 2824 }
@@ -2831,9 +2865,9 @@ EXPORT_SYMBOL(kmem_cache_alloc);
2831 */ 2865 */
2832int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr) 2866int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr)
2833{ 2867{
2834 unsigned long addr = (unsigned long) ptr; 2868 unsigned long addr = (unsigned long)ptr;
2835 unsigned long min_addr = PAGE_OFFSET; 2869 unsigned long min_addr = PAGE_OFFSET;
2836 unsigned long align_mask = BYTES_PER_WORD-1; 2870 unsigned long align_mask = BYTES_PER_WORD - 1;
2837 unsigned long size = cachep->objsize; 2871 unsigned long size = cachep->objsize;
2838 struct page *page; 2872 struct page *page;
2839 2873
@@ -2853,7 +2887,7 @@ int fastcall kmem_ptr_validate(kmem_cache_t *cachep, void *ptr)
2853 if (unlikely(page_get_cache(page) != cachep)) 2887 if (unlikely(page_get_cache(page) != cachep))
2854 goto out; 2888 goto out;
2855 return 1; 2889 return 1;
2856out: 2890 out:
2857 return 0; 2891 return 0;
2858} 2892}
2859 2893
@@ -2880,8 +2914,10 @@ void *kmem_cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid)
2880 2914
2881 if (unlikely(!cachep->nodelists[nodeid])) { 2915 if (unlikely(!cachep->nodelists[nodeid])) {
2882 /* Fall back to __cache_alloc if we run into trouble */ 2916 /* Fall back to __cache_alloc if we run into trouble */
2883 printk(KERN_WARNING "slab: not allocating in inactive node %d for cache %s\n", nodeid, cachep->name); 2917 printk(KERN_WARNING
2884 return __cache_alloc(cachep,flags); 2918 "slab: not allocating in inactive node %d for cache %s\n",
2919 nodeid, cachep->name);
2920 return __cache_alloc(cachep, flags);
2885 } 2921 }
2886 2922
2887 cache_alloc_debugcheck_before(cachep, flags); 2923 cache_alloc_debugcheck_before(cachep, flags);
@@ -2891,7 +2927,9 @@ void *kmem_cache_alloc_node(kmem_cache_t *cachep, gfp_t flags, int nodeid)
2891 else 2927 else
2892 ptr = __cache_alloc_node(cachep, flags, nodeid); 2928 ptr = __cache_alloc_node(cachep, flags, nodeid);
2893 local_irq_restore(save_flags); 2929 local_irq_restore(save_flags);
2894 ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, __builtin_return_address(0)); 2930 ptr =
2931 cache_alloc_debugcheck_after(cachep, flags, ptr,
2932 __builtin_return_address(0));
2895 2933
2896 return ptr; 2934 return ptr;
2897} 2935}
@@ -2957,7 +2995,7 @@ EXPORT_SYMBOL(__kmalloc);
2957void *__alloc_percpu(size_t size) 2995void *__alloc_percpu(size_t size)
2958{ 2996{
2959 int i; 2997 int i;
2960 struct percpu_data *pdata = kmalloc(sizeof (*pdata), GFP_KERNEL); 2998 struct percpu_data *pdata = kmalloc(sizeof(*pdata), GFP_KERNEL);
2961 2999
2962 if (!pdata) 3000 if (!pdata)
2963 return NULL; 3001 return NULL;
@@ -2981,9 +3019,9 @@ void *__alloc_percpu(size_t size)
2981 } 3019 }
2982 3020
2983 /* Catch derefs w/o wrappers */ 3021 /* Catch derefs w/o wrappers */
2984 return (void *) (~(unsigned long) pdata); 3022 return (void *)(~(unsigned long)pdata);
2985 3023
2986unwind_oom: 3024 unwind_oom:
2987 while (--i >= 0) { 3025 while (--i >= 0) {
2988 if (!cpu_possible(i)) 3026 if (!cpu_possible(i))
2989 continue; 3027 continue;
@@ -3046,7 +3084,7 @@ void kfree(const void *objp)
3046 local_irq_save(flags); 3084 local_irq_save(flags);
3047 kfree_debugcheck(objp); 3085 kfree_debugcheck(objp);
3048 c = page_get_cache(virt_to_page(objp)); 3086 c = page_get_cache(virt_to_page(objp));
3049 __cache_free(c, (void*)objp); 3087 __cache_free(c, (void *)objp);
3050 local_irq_restore(flags); 3088 local_irq_restore(flags);
3051} 3089}
3052EXPORT_SYMBOL(kfree); 3090EXPORT_SYMBOL(kfree);
@@ -3059,17 +3097,16 @@ EXPORT_SYMBOL(kfree);
3059 * Don't free memory not originally allocated by alloc_percpu() 3097 * Don't free memory not originally allocated by alloc_percpu()
3060 * The complemented objp is to check for that. 3098 * The complemented objp is to check for that.
3061 */ 3099 */
3062void 3100void free_percpu(const void *objp)
3063free_percpu(const void *objp)
3064{ 3101{
3065 int i; 3102 int i;
3066 struct percpu_data *p = (struct percpu_data *) (~(unsigned long) objp); 3103 struct percpu_data *p = (struct percpu_data *)(~(unsigned long)objp);
3067 3104
3068 /* 3105 /*
3069 * We allocate for all cpus so we cannot use for online cpu here. 3106 * We allocate for all cpus so we cannot use for online cpu here.
3070 */ 3107 */
3071 for_each_cpu(i) 3108 for_each_cpu(i)
3072 kfree(p->ptrs[i]); 3109 kfree(p->ptrs[i]);
3073 kfree(p); 3110 kfree(p);
3074} 3111}
3075EXPORT_SYMBOL(free_percpu); 3112EXPORT_SYMBOL(free_percpu);
@@ -3103,44 +3140,44 @@ static int alloc_kmemlist(kmem_cache_t *cachep)
3103 if (!(new_alien = alloc_alien_cache(node, cachep->limit))) 3140 if (!(new_alien = alloc_alien_cache(node, cachep->limit)))
3104 goto fail; 3141 goto fail;
3105#endif 3142#endif
3106 if (!(new = alloc_arraycache(node, (cachep->shared* 3143 if (!(new = alloc_arraycache(node, (cachep->shared *
3107 cachep->batchcount), 0xbaadf00d))) 3144 cachep->batchcount),
3145 0xbaadf00d)))
3108 goto fail; 3146 goto fail;
3109 if ((l3 = cachep->nodelists[node])) { 3147 if ((l3 = cachep->nodelists[node])) {
3110 3148
3111 spin_lock_irq(&l3->list_lock); 3149 spin_lock_irq(&l3->list_lock);
3112 3150
3113 if ((nc = cachep->nodelists[node]->shared)) 3151 if ((nc = cachep->nodelists[node]->shared))
3114 free_block(cachep, nc->entry, 3152 free_block(cachep, nc->entry, nc->avail, node);
3115 nc->avail, node);
3116 3153
3117 l3->shared = new; 3154 l3->shared = new;
3118 if (!cachep->nodelists[node]->alien) { 3155 if (!cachep->nodelists[node]->alien) {
3119 l3->alien = new_alien; 3156 l3->alien = new_alien;
3120 new_alien = NULL; 3157 new_alien = NULL;
3121 } 3158 }
3122 l3->free_limit = (1 + nr_cpus_node(node))* 3159 l3->free_limit = (1 + nr_cpus_node(node)) *
3123 cachep->batchcount + cachep->num; 3160 cachep->batchcount + cachep->num;
3124 spin_unlock_irq(&l3->list_lock); 3161 spin_unlock_irq(&l3->list_lock);
3125 kfree(nc); 3162 kfree(nc);
3126 free_alien_cache(new_alien); 3163 free_alien_cache(new_alien);
3127 continue; 3164 continue;
3128 } 3165 }
3129 if (!(l3 = kmalloc_node(sizeof(struct kmem_list3), 3166 if (!(l3 = kmalloc_node(sizeof(struct kmem_list3),
3130 GFP_KERNEL, node))) 3167 GFP_KERNEL, node)))
3131 goto fail; 3168 goto fail;
3132 3169
3133 kmem_list3_init(l3); 3170 kmem_list3_init(l3);
3134 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 + 3171 l3->next_reap = jiffies + REAPTIMEOUT_LIST3 +
3135 ((unsigned long)cachep)%REAPTIMEOUT_LIST3; 3172 ((unsigned long)cachep) % REAPTIMEOUT_LIST3;
3136 l3->shared = new; 3173 l3->shared = new;
3137 l3->alien = new_alien; 3174 l3->alien = new_alien;
3138 l3->free_limit = (1 + nr_cpus_node(node))* 3175 l3->free_limit = (1 + nr_cpus_node(node)) *
3139 cachep->batchcount + cachep->num; 3176 cachep->batchcount + cachep->num;
3140 cachep->nodelists[node] = l3; 3177 cachep->nodelists[node] = l3;
3141 } 3178 }
3142 return err; 3179 return err;
3143fail: 3180 fail:
3144 err = -ENOMEM; 3181 err = -ENOMEM;
3145 return err; 3182 return err;
3146} 3183}
@@ -3162,18 +3199,19 @@ static void do_ccupdate_local(void *info)
3162 new->new[smp_processor_id()] = old; 3199 new->new[smp_processor_id()] = old;
3163} 3200}
3164 3201
3165
3166static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount, 3202static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount,
3167 int shared) 3203 int shared)
3168{ 3204{
3169 struct ccupdate_struct new; 3205 struct ccupdate_struct new;
3170 int i, err; 3206 int i, err;
3171 3207
3172 memset(&new.new,0,sizeof(new.new)); 3208 memset(&new.new, 0, sizeof(new.new));
3173 for_each_online_cpu(i) { 3209 for_each_online_cpu(i) {
3174 new.new[i] = alloc_arraycache(cpu_to_node(i), limit, batchcount); 3210 new.new[i] =
3211 alloc_arraycache(cpu_to_node(i), limit, batchcount);
3175 if (!new.new[i]) { 3212 if (!new.new[i]) {
3176 for (i--; i >= 0; i--) kfree(new.new[i]); 3213 for (i--; i >= 0; i--)
3214 kfree(new.new[i]);
3177 return -ENOMEM; 3215 return -ENOMEM;
3178 } 3216 }
3179 } 3217 }
@@ -3201,13 +3239,12 @@ static int do_tune_cpucache(kmem_cache_t *cachep, int limit, int batchcount,
3201 err = alloc_kmemlist(cachep); 3239 err = alloc_kmemlist(cachep);
3202 if (err) { 3240 if (err) {
3203 printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n", 3241 printk(KERN_ERR "alloc_kmemlist failed for %s, error %d.\n",
3204 cachep->name, -err); 3242 cachep->name, -err);
3205 BUG(); 3243 BUG();
3206 } 3244 }
3207 return 0; 3245 return 0;
3208} 3246}
3209 3247
3210
3211static void enable_cpucache(kmem_cache_t *cachep) 3248static void enable_cpucache(kmem_cache_t *cachep)
3212{ 3249{
3213 int err; 3250 int err;
@@ -3254,14 +3291,14 @@ static void enable_cpucache(kmem_cache_t *cachep)
3254 if (limit > 32) 3291 if (limit > 32)
3255 limit = 32; 3292 limit = 32;
3256#endif 3293#endif
3257 err = do_tune_cpucache(cachep, limit, (limit+1)/2, shared); 3294 err = do_tune_cpucache(cachep, limit, (limit + 1) / 2, shared);
3258 if (err) 3295 if (err)
3259 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n", 3296 printk(KERN_ERR "enable_cpucache failed for %s, error %d.\n",
3260 cachep->name, -err); 3297 cachep->name, -err);
3261} 3298}
3262 3299
3263static void drain_array_locked(kmem_cache_t *cachep, 3300static void drain_array_locked(kmem_cache_t *cachep, struct array_cache *ac,
3264 struct array_cache *ac, int force, int node) 3301 int force, int node)
3265{ 3302{
3266 int tofree; 3303 int tofree;
3267 3304
@@ -3269,14 +3306,14 @@ static void drain_array_locked(kmem_cache_t *cachep,
3269 if (ac->touched && !force) { 3306 if (ac->touched && !force) {
3270 ac->touched = 0; 3307 ac->touched = 0;
3271 } else if (ac->avail) { 3308 } else if (ac->avail) {
3272 tofree = force ? ac->avail : (ac->limit+4)/5; 3309 tofree = force ? ac->avail : (ac->limit + 4) / 5;
3273 if (tofree > ac->avail) { 3310 if (tofree > ac->avail) {
3274 tofree = (ac->avail+1)/2; 3311 tofree = (ac->avail + 1) / 2;
3275 } 3312 }
3276 free_block(cachep, ac->entry, tofree, node); 3313 free_block(cachep, ac->entry, tofree, node);
3277 ac->avail -= tofree; 3314 ac->avail -= tofree;
3278 memmove(ac->entry, &(ac->entry[tofree]), 3315 memmove(ac->entry, &(ac->entry[tofree]),
3279 sizeof(void*)*ac->avail); 3316 sizeof(void *) * ac->avail);
3280 } 3317 }
3281} 3318}
3282 3319
@@ -3299,13 +3336,14 @@ static void cache_reap(void *unused)
3299 3336
3300 if (down_trylock(&cache_chain_sem)) { 3337 if (down_trylock(&cache_chain_sem)) {
3301 /* Give up. Setup the next iteration. */ 3338 /* Give up. Setup the next iteration. */
3302 schedule_delayed_work(&__get_cpu_var(reap_work), REAPTIMEOUT_CPUC); 3339 schedule_delayed_work(&__get_cpu_var(reap_work),
3340 REAPTIMEOUT_CPUC);
3303 return; 3341 return;
3304 } 3342 }
3305 3343
3306 list_for_each(walk, &cache_chain) { 3344 list_for_each(walk, &cache_chain) {
3307 kmem_cache_t *searchp; 3345 kmem_cache_t *searchp;
3308 struct list_head* p; 3346 struct list_head *p;
3309 int tofree; 3347 int tofree;
3310 struct slab *slabp; 3348 struct slab *slabp;
3311 3349
@@ -3322,7 +3360,7 @@ static void cache_reap(void *unused)
3322 spin_lock_irq(&l3->list_lock); 3360 spin_lock_irq(&l3->list_lock);
3323 3361
3324 drain_array_locked(searchp, ac_data(searchp), 0, 3362 drain_array_locked(searchp, ac_data(searchp), 0,
3325 numa_node_id()); 3363 numa_node_id());
3326 3364
3327 if (time_after(l3->next_reap, jiffies)) 3365 if (time_after(l3->next_reap, jiffies))
3328 goto next_unlock; 3366 goto next_unlock;
@@ -3331,14 +3369,16 @@ static void cache_reap(void *unused)
3331 3369
3332 if (l3->shared) 3370 if (l3->shared)
3333 drain_array_locked(searchp, l3->shared, 0, 3371 drain_array_locked(searchp, l3->shared, 0,
3334 numa_node_id()); 3372 numa_node_id());
3335 3373
3336 if (l3->free_touched) { 3374 if (l3->free_touched) {
3337 l3->free_touched = 0; 3375 l3->free_touched = 0;
3338 goto next_unlock; 3376 goto next_unlock;
3339 } 3377 }
3340 3378
3341 tofree = (l3->free_limit+5*searchp->num-1)/(5*searchp->num); 3379 tofree =
3380 (l3->free_limit + 5 * searchp->num -
3381 1) / (5 * searchp->num);
3342 do { 3382 do {
3343 p = l3->slabs_free.next; 3383 p = l3->slabs_free.next;
3344 if (p == &(l3->slabs_free)) 3384 if (p == &(l3->slabs_free))
@@ -3358,10 +3398,10 @@ static void cache_reap(void *unused)
3358 spin_unlock_irq(&l3->list_lock); 3398 spin_unlock_irq(&l3->list_lock);
3359 slab_destroy(searchp, slabp); 3399 slab_destroy(searchp, slabp);
3360 spin_lock_irq(&l3->list_lock); 3400 spin_lock_irq(&l3->list_lock);
3361 } while(--tofree > 0); 3401 } while (--tofree > 0);
3362next_unlock: 3402 next_unlock:
3363 spin_unlock_irq(&l3->list_lock); 3403 spin_unlock_irq(&l3->list_lock);
3364next: 3404 next:
3365 cond_resched(); 3405 cond_resched();
3366 } 3406 }
3367 check_irq_on(); 3407 check_irq_on();
@@ -3418,7 +3458,7 @@ static void *s_next(struct seq_file *m, void *p, loff_t *pos)
3418 kmem_cache_t *cachep = p; 3458 kmem_cache_t *cachep = p;
3419 ++*pos; 3459 ++*pos;
3420 return cachep->next.next == &cache_chain ? NULL 3460 return cachep->next.next == &cache_chain ? NULL
3421 : list_entry(cachep->next.next, kmem_cache_t, next); 3461 : list_entry(cachep->next.next, kmem_cache_t, next);
3422} 3462}
3423 3463
3424static void s_stop(struct seq_file *m, void *p) 3464static void s_stop(struct seq_file *m, void *p)
@@ -3430,11 +3470,11 @@ static int s_show(struct seq_file *m, void *p)
3430{ 3470{
3431 kmem_cache_t *cachep = p; 3471 kmem_cache_t *cachep = p;
3432 struct list_head *q; 3472 struct list_head *q;
3433 struct slab *slabp; 3473 struct slab *slabp;
3434 unsigned long active_objs; 3474 unsigned long active_objs;
3435 unsigned long num_objs; 3475 unsigned long num_objs;
3436 unsigned long active_slabs = 0; 3476 unsigned long active_slabs = 0;
3437 unsigned long num_slabs, free_objects = 0, shared_avail = 0; 3477 unsigned long num_slabs, free_objects = 0, shared_avail = 0;
3438 const char *name; 3478 const char *name;
3439 char *error = NULL; 3479 char *error = NULL;
3440 int node; 3480 int node;
@@ -3451,14 +3491,14 @@ static int s_show(struct seq_file *m, void *p)
3451 3491
3452 spin_lock(&l3->list_lock); 3492 spin_lock(&l3->list_lock);
3453 3493
3454 list_for_each(q,&l3->slabs_full) { 3494 list_for_each(q, &l3->slabs_full) {
3455 slabp = list_entry(q, struct slab, list); 3495 slabp = list_entry(q, struct slab, list);
3456 if (slabp->inuse != cachep->num && !error) 3496 if (slabp->inuse != cachep->num && !error)
3457 error = "slabs_full accounting error"; 3497 error = "slabs_full accounting error";
3458 active_objs += cachep->num; 3498 active_objs += cachep->num;
3459 active_slabs++; 3499 active_slabs++;
3460 } 3500 }
3461 list_for_each(q,&l3->slabs_partial) { 3501 list_for_each(q, &l3->slabs_partial) {
3462 slabp = list_entry(q, struct slab, list); 3502 slabp = list_entry(q, struct slab, list);
3463 if (slabp->inuse == cachep->num && !error) 3503 if (slabp->inuse == cachep->num && !error)
3464 error = "slabs_partial inuse accounting error"; 3504 error = "slabs_partial inuse accounting error";
@@ -3467,7 +3507,7 @@ static int s_show(struct seq_file *m, void *p)
3467 active_objs += slabp->inuse; 3507 active_objs += slabp->inuse;
3468 active_slabs++; 3508 active_slabs++;
3469 } 3509 }
3470 list_for_each(q,&l3->slabs_free) { 3510 list_for_each(q, &l3->slabs_free) {
3471 slabp = list_entry(q, struct slab, list); 3511 slabp = list_entry(q, struct slab, list);
3472 if (slabp->inuse && !error) 3512 if (slabp->inuse && !error)
3473 error = "slabs_free/inuse accounting error"; 3513 error = "slabs_free/inuse accounting error";
@@ -3478,25 +3518,24 @@ static int s_show(struct seq_file *m, void *p)
3478 3518
3479 spin_unlock(&l3->list_lock); 3519 spin_unlock(&l3->list_lock);
3480 } 3520 }
3481 num_slabs+=active_slabs; 3521 num_slabs += active_slabs;
3482 num_objs = num_slabs*cachep->num; 3522 num_objs = num_slabs * cachep->num;
3483 if (num_objs - active_objs != free_objects && !error) 3523 if (num_objs - active_objs != free_objects && !error)
3484 error = "free_objects accounting error"; 3524 error = "free_objects accounting error";
3485 3525
3486 name = cachep->name; 3526 name = cachep->name;
3487 if (error) 3527 if (error)
3488 printk(KERN_ERR "slab: cache %s error: %s\n", name, error); 3528 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
3489 3529
3490 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 3530 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
3491 name, active_objs, num_objs, cachep->objsize, 3531 name, active_objs, num_objs, cachep->objsize,
3492 cachep->num, (1<<cachep->gfporder)); 3532 cachep->num, (1 << cachep->gfporder));
3493 seq_printf(m, " : tunables %4u %4u %4u", 3533 seq_printf(m, " : tunables %4u %4u %4u",
3494 cachep->limit, cachep->batchcount, 3534 cachep->limit, cachep->batchcount, cachep->shared);
3495 cachep->shared);
3496 seq_printf(m, " : slabdata %6lu %6lu %6lu", 3535 seq_printf(m, " : slabdata %6lu %6lu %6lu",
3497 active_slabs, num_slabs, shared_avail); 3536 active_slabs, num_slabs, shared_avail);
3498#if STATS 3537#if STATS
3499 { /* list3 stats */ 3538 { /* list3 stats */
3500 unsigned long high = cachep->high_mark; 3539 unsigned long high = cachep->high_mark;
3501 unsigned long allocs = cachep->num_allocations; 3540 unsigned long allocs = cachep->num_allocations;
3502 unsigned long grown = cachep->grown; 3541 unsigned long grown = cachep->grown;
@@ -3507,9 +3546,7 @@ static int s_show(struct seq_file *m, void *p)
3507 unsigned long node_frees = cachep->node_frees; 3546 unsigned long node_frees = cachep->node_frees;
3508 3547
3509 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \ 3548 seq_printf(m, " : globalstat %7lu %6lu %5lu %4lu \
3510 %4lu %4lu %4lu %4lu", 3549 %4lu %4lu %4lu %4lu", allocs, high, grown, reaped, errors, max_freeable, node_allocs, node_frees);
3511 allocs, high, grown, reaped, errors,
3512 max_freeable, node_allocs, node_frees);
3513 } 3550 }
3514 /* cpu stats */ 3551 /* cpu stats */
3515 { 3552 {
@@ -3519,7 +3556,7 @@ static int s_show(struct seq_file *m, void *p)
3519 unsigned long freemiss = atomic_read(&cachep->freemiss); 3556 unsigned long freemiss = atomic_read(&cachep->freemiss);
3520 3557
3521 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu", 3558 seq_printf(m, " : cpustat %6lu %6lu %6lu %6lu",
3522 allochit, allocmiss, freehit, freemiss); 3559 allochit, allocmiss, freehit, freemiss);
3523 } 3560 }
3524#endif 3561#endif
3525 seq_putc(m, '\n'); 3562 seq_putc(m, '\n');
@@ -3542,10 +3579,10 @@ static int s_show(struct seq_file *m, void *p)
3542 */ 3579 */
3543 3580
3544struct seq_operations slabinfo_op = { 3581struct seq_operations slabinfo_op = {
3545 .start = s_start, 3582 .start = s_start,
3546 .next = s_next, 3583 .next = s_next,
3547 .stop = s_stop, 3584 .stop = s_stop,
3548 .show = s_show, 3585 .show = s_show,
3549}; 3586};
3550 3587
3551#define MAX_SLABINFO_WRITE 128 3588#define MAX_SLABINFO_WRITE 128
@@ -3556,18 +3593,18 @@ struct seq_operations slabinfo_op = {
3556 * @count: data length 3593 * @count: data length
3557 * @ppos: unused 3594 * @ppos: unused
3558 */ 3595 */
3559ssize_t slabinfo_write(struct file *file, const char __user *buffer, 3596ssize_t slabinfo_write(struct file *file, const char __user * buffer,
3560 size_t count, loff_t *ppos) 3597 size_t count, loff_t *ppos)
3561{ 3598{
3562 char kbuf[MAX_SLABINFO_WRITE+1], *tmp; 3599 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
3563 int limit, batchcount, shared, res; 3600 int limit, batchcount, shared, res;
3564 struct list_head *p; 3601 struct list_head *p;
3565 3602
3566 if (count > MAX_SLABINFO_WRITE) 3603 if (count > MAX_SLABINFO_WRITE)
3567 return -EINVAL; 3604 return -EINVAL;
3568 if (copy_from_user(&kbuf, buffer, count)) 3605 if (copy_from_user(&kbuf, buffer, count))
3569 return -EFAULT; 3606 return -EFAULT;
3570 kbuf[MAX_SLABINFO_WRITE] = '\0'; 3607 kbuf[MAX_SLABINFO_WRITE] = '\0';
3571 3608
3572 tmp = strchr(kbuf, ' '); 3609 tmp = strchr(kbuf, ' ');
3573 if (!tmp) 3610 if (!tmp)
@@ -3580,18 +3617,17 @@ ssize_t slabinfo_write(struct file *file, const char __user *buffer,
3580 /* Find the cache in the chain of caches. */ 3617 /* Find the cache in the chain of caches. */
3581 down(&cache_chain_sem); 3618 down(&cache_chain_sem);
3582 res = -EINVAL; 3619 res = -EINVAL;
3583 list_for_each(p,&cache_chain) { 3620 list_for_each(p, &cache_chain) {
3584 kmem_cache_t *cachep = list_entry(p, kmem_cache_t, next); 3621 kmem_cache_t *cachep = list_entry(p, kmem_cache_t, next);
3585 3622
3586 if (!strcmp(cachep->name, kbuf)) { 3623 if (!strcmp(cachep->name, kbuf)) {
3587 if (limit < 1 || 3624 if (limit < 1 ||
3588 batchcount < 1 || 3625 batchcount < 1 ||
3589 batchcount > limit || 3626 batchcount > limit || shared < 0) {
3590 shared < 0) {
3591 res = 0; 3627 res = 0;
3592 } else { 3628 } else {
3593 res = do_tune_cpucache(cachep, limit, 3629 res = do_tune_cpucache(cachep, limit,
3594 batchcount, shared); 3630 batchcount, shared);
3595 } 3631 }
3596 break; 3632 break;
3597 } 3633 }