aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/mm_types.h7
-rw-r--r--include/linux/slab.h9
-rw-r--r--include/linux/slab_def.h6
-rw-r--r--mm/slab.c289
-rw-r--r--mm/slab.h53
-rw-r--r--mm/slab_common.c182
-rw-r--r--mm/slob.c46
-rw-r--r--mm/slub.c301
8 files changed, 402 insertions, 491 deletions
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 7d9ebb7cc98..f8f5162a357 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -128,10 +128,7 @@ struct page {
128 }; 128 };
129 129
130 struct list_head list; /* slobs list of pages */ 130 struct list_head list; /* slobs list of pages */
131 struct { /* slab fields */ 131 struct slab *slab_page; /* slab fields */
132 struct kmem_cache *slab_cache;
133 struct slab *slab_page;
134 };
135 }; 132 };
136 133
137 /* Remainder is not double word aligned */ 134 /* Remainder is not double word aligned */
@@ -146,7 +143,7 @@ struct page {
146#if USE_SPLIT_PTLOCKS 143#if USE_SPLIT_PTLOCKS
147 spinlock_t ptl; 144 spinlock_t ptl;
148#endif 145#endif
149 struct kmem_cache *slab; /* SLUB: Pointer to slab */ 146 struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */
150 struct page *first_page; /* Compound tail pages */ 147 struct page *first_page; /* Compound tail pages */
151 }; 148 };
152 149
diff --git a/include/linux/slab.h b/include/linux/slab.h
index 83d1a1454b7..743a1041512 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -128,7 +128,6 @@ struct kmem_cache *kmem_cache_create(const char *, size_t, size_t,
128void kmem_cache_destroy(struct kmem_cache *); 128void kmem_cache_destroy(struct kmem_cache *);
129int kmem_cache_shrink(struct kmem_cache *); 129int kmem_cache_shrink(struct kmem_cache *);
130void kmem_cache_free(struct kmem_cache *, void *); 130void kmem_cache_free(struct kmem_cache *, void *);
131unsigned int kmem_cache_size(struct kmem_cache *);
132 131
133/* 132/*
134 * Please use this macro to create slab caches. Simply specify the 133 * Please use this macro to create slab caches. Simply specify the
@@ -388,6 +387,14 @@ static inline void *kzalloc_node(size_t size, gfp_t flags, int node)
388 return kmalloc_node(size, flags | __GFP_ZERO, node); 387 return kmalloc_node(size, flags | __GFP_ZERO, node);
389} 388}
390 389
390/*
391 * Determine the size of a slab object
392 */
393static inline unsigned int kmem_cache_size(struct kmem_cache *s)
394{
395 return s->object_size;
396}
397
391void __init kmem_cache_init_late(void); 398void __init kmem_cache_init_late(void);
392 399
393#endif /* _LINUX_SLAB_H */ 400#endif /* _LINUX_SLAB_H */
diff --git a/include/linux/slab_def.h b/include/linux/slab_def.h
index cc290f0bdb3..45c0356fdc8 100644
--- a/include/linux/slab_def.h
+++ b/include/linux/slab_def.h
@@ -89,9 +89,13 @@ struct kmem_cache {
89 * (see kmem_cache_init()) 89 * (see kmem_cache_init())
90 * We still use [NR_CPUS] and not [1] or [0] because cache_cache 90 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
91 * is statically defined, so we reserve the max number of cpus. 91 * is statically defined, so we reserve the max number of cpus.
92 *
93 * We also need to guarantee that the list is able to accomodate a
94 * pointer for each node since "nodelists" uses the remainder of
95 * available pointers.
92 */ 96 */
93 struct kmem_list3 **nodelists; 97 struct kmem_list3 **nodelists;
94 struct array_cache *array[NR_CPUS]; 98 struct array_cache *array[NR_CPUS + MAX_NUMNODES];
95 /* 99 /*
96 * Do not add fields after array[] 100 * Do not add fields after array[]
97 */ 101 */
diff --git a/mm/slab.c b/mm/slab.c
index 33d3363658d..2c3a2e0394d 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -162,23 +162,6 @@
162 */ 162 */
163static bool pfmemalloc_active __read_mostly; 163static bool pfmemalloc_active __read_mostly;
164 164
165/* Legal flag mask for kmem_cache_create(). */
166#if DEBUG
167# define CREATE_MASK (SLAB_RED_ZONE | \
168 SLAB_POISON | SLAB_HWCACHE_ALIGN | \
169 SLAB_CACHE_DMA | \
170 SLAB_STORE_USER | \
171 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
172 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
173 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
174#else
175# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \
176 SLAB_CACHE_DMA | \
177 SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \
178 SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \
179 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE | SLAB_NOTRACK)
180#endif
181
182/* 165/*
183 * kmem_bufctl_t: 166 * kmem_bufctl_t:
184 * 167 *
@@ -564,15 +547,11 @@ static struct cache_names __initdata cache_names[] = {
564#undef CACHE 547#undef CACHE
565}; 548};
566 549
567static struct arraycache_init initarray_cache __initdata =
568 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
569static struct arraycache_init initarray_generic = 550static struct arraycache_init initarray_generic =
570 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; 551 { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} };
571 552
572/* internal cache of cache description objs */ 553/* internal cache of cache description objs */
573static struct kmem_list3 *kmem_cache_nodelists[MAX_NUMNODES];
574static struct kmem_cache kmem_cache_boot = { 554static struct kmem_cache kmem_cache_boot = {
575 .nodelists = kmem_cache_nodelists,
576 .batchcount = 1, 555 .batchcount = 1,
577 .limit = BOOT_CPUCACHE_ENTRIES, 556 .limit = BOOT_CPUCACHE_ENTRIES,
578 .shared = 1, 557 .shared = 1,
@@ -1577,28 +1556,33 @@ static void __init set_up_list3s(struct kmem_cache *cachep, int index)
1577} 1556}
1578 1557
1579/* 1558/*
1559 * The memory after the last cpu cache pointer is used for the
1560 * the nodelists pointer.
1561 */
1562static void setup_nodelists_pointer(struct kmem_cache *cachep)
1563{
1564 cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids];
1565}
1566
1567/*
1580 * Initialisation. Called after the page allocator have been initialised and 1568 * Initialisation. Called after the page allocator have been initialised and
1581 * before smp_init(). 1569 * before smp_init().
1582 */ 1570 */
1583void __init kmem_cache_init(void) 1571void __init kmem_cache_init(void)
1584{ 1572{
1585 size_t left_over;
1586 struct cache_sizes *sizes; 1573 struct cache_sizes *sizes;
1587 struct cache_names *names; 1574 struct cache_names *names;
1588 int i; 1575 int i;
1589 int order;
1590 int node;
1591 1576
1592 kmem_cache = &kmem_cache_boot; 1577 kmem_cache = &kmem_cache_boot;
1578 setup_nodelists_pointer(kmem_cache);
1593 1579
1594 if (num_possible_nodes() == 1) 1580 if (num_possible_nodes() == 1)
1595 use_alien_caches = 0; 1581 use_alien_caches = 0;
1596 1582
1597 for (i = 0; i < NUM_INIT_LISTS; i++) { 1583 for (i = 0; i < NUM_INIT_LISTS; i++)
1598 kmem_list3_init(&initkmem_list3[i]); 1584 kmem_list3_init(&initkmem_list3[i]);
1599 if (i < MAX_NUMNODES) 1585
1600 kmem_cache->nodelists[i] = NULL;
1601 }
1602 set_up_list3s(kmem_cache, CACHE_CACHE); 1586 set_up_list3s(kmem_cache, CACHE_CACHE);
1603 1587
1604 /* 1588 /*
@@ -1629,37 +1613,16 @@ void __init kmem_cache_init(void)
1629 * 6) Resize the head arrays of the kmalloc caches to their final sizes. 1613 * 6) Resize the head arrays of the kmalloc caches to their final sizes.
1630 */ 1614 */
1631 1615
1632 node = numa_mem_id();
1633
1634 /* 1) create the kmem_cache */ 1616 /* 1) create the kmem_cache */
1635 INIT_LIST_HEAD(&slab_caches);
1636 list_add(&kmem_cache->list, &slab_caches);
1637 kmem_cache->colour_off = cache_line_size();
1638 kmem_cache->array[smp_processor_id()] = &initarray_cache.cache;
1639 kmem_cache->nodelists[node] = &initkmem_list3[CACHE_CACHE + node];
1640 1617
1641 /* 1618 /*
1642 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids 1619 * struct kmem_cache size depends on nr_node_ids & nr_cpu_ids
1643 */ 1620 */
1644 kmem_cache->size = offsetof(struct kmem_cache, array[nr_cpu_ids]) + 1621 create_boot_cache(kmem_cache, "kmem_cache",
1645 nr_node_ids * sizeof(struct kmem_list3 *); 1622 offsetof(struct kmem_cache, array[nr_cpu_ids]) +
1646 kmem_cache->object_size = kmem_cache->size; 1623 nr_node_ids * sizeof(struct kmem_list3 *),
1647 kmem_cache->size = ALIGN(kmem_cache->object_size, 1624 SLAB_HWCACHE_ALIGN);
1648 cache_line_size()); 1625 list_add(&kmem_cache->list, &slab_caches);
1649 kmem_cache->reciprocal_buffer_size =
1650 reciprocal_value(kmem_cache->size);
1651
1652 for (order = 0; order < MAX_ORDER; order++) {
1653 cache_estimate(order, kmem_cache->size,
1654 cache_line_size(), 0, &left_over, &kmem_cache->num);
1655 if (kmem_cache->num)
1656 break;
1657 }
1658 BUG_ON(!kmem_cache->num);
1659 kmem_cache->gfporder = order;
1660 kmem_cache->colour = left_over / kmem_cache->colour_off;
1661 kmem_cache->slab_size = ALIGN(kmem_cache->num * sizeof(kmem_bufctl_t) +
1662 sizeof(struct slab), cache_line_size());
1663 1626
1664 /* 2+3) create the kmalloc caches */ 1627 /* 2+3) create the kmalloc caches */
1665 sizes = malloc_sizes; 1628 sizes = malloc_sizes;
@@ -1671,23 +1634,13 @@ void __init kmem_cache_init(void)
1671 * bug. 1634 * bug.
1672 */ 1635 */
1673 1636
1674 sizes[INDEX_AC].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1637 sizes[INDEX_AC].cs_cachep = create_kmalloc_cache(names[INDEX_AC].name,
1675 sizes[INDEX_AC].cs_cachep->name = names[INDEX_AC].name; 1638 sizes[INDEX_AC].cs_size, ARCH_KMALLOC_FLAGS);
1676 sizes[INDEX_AC].cs_cachep->size = sizes[INDEX_AC].cs_size; 1639
1677 sizes[INDEX_AC].cs_cachep->object_size = sizes[INDEX_AC].cs_size; 1640 if (INDEX_AC != INDEX_L3)
1678 sizes[INDEX_AC].cs_cachep->align = ARCH_KMALLOC_MINALIGN; 1641 sizes[INDEX_L3].cs_cachep =
1679 __kmem_cache_create(sizes[INDEX_AC].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC); 1642 create_kmalloc_cache(names[INDEX_L3].name,
1680 list_add(&sizes[INDEX_AC].cs_cachep->list, &slab_caches); 1643 sizes[INDEX_L3].cs_size, ARCH_KMALLOC_FLAGS);
1681
1682 if (INDEX_AC != INDEX_L3) {
1683 sizes[INDEX_L3].cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
1684 sizes[INDEX_L3].cs_cachep->name = names[INDEX_L3].name;
1685 sizes[INDEX_L3].cs_cachep->size = sizes[INDEX_L3].cs_size;
1686 sizes[INDEX_L3].cs_cachep->object_size = sizes[INDEX_L3].cs_size;
1687 sizes[INDEX_L3].cs_cachep->align = ARCH_KMALLOC_MINALIGN;
1688 __kmem_cache_create(sizes[INDEX_L3].cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
1689 list_add(&sizes[INDEX_L3].cs_cachep->list, &slab_caches);
1690 }
1691 1644
1692 slab_early_init = 0; 1645 slab_early_init = 0;
1693 1646
@@ -1699,24 +1652,14 @@ void __init kmem_cache_init(void)
1699 * Note for systems short on memory removing the alignment will 1652 * Note for systems short on memory removing the alignment will
1700 * allow tighter packing of the smaller caches. 1653 * allow tighter packing of the smaller caches.
1701 */ 1654 */
1702 if (!sizes->cs_cachep) { 1655 if (!sizes->cs_cachep)
1703 sizes->cs_cachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1656 sizes->cs_cachep = create_kmalloc_cache(names->name,
1704 sizes->cs_cachep->name = names->name; 1657 sizes->cs_size, ARCH_KMALLOC_FLAGS);
1705 sizes->cs_cachep->size = sizes->cs_size; 1658
1706 sizes->cs_cachep->object_size = sizes->cs_size;
1707 sizes->cs_cachep->align = ARCH_KMALLOC_MINALIGN;
1708 __kmem_cache_create(sizes->cs_cachep, ARCH_KMALLOC_FLAGS|SLAB_PANIC);
1709 list_add(&sizes->cs_cachep->list, &slab_caches);
1710 }
1711#ifdef CONFIG_ZONE_DMA 1659#ifdef CONFIG_ZONE_DMA
1712 sizes->cs_dmacachep = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); 1660 sizes->cs_dmacachep = create_kmalloc_cache(
1713 sizes->cs_dmacachep->name = names->name_dma; 1661 names->name_dma, sizes->cs_size,
1714 sizes->cs_dmacachep->size = sizes->cs_size; 1662 SLAB_CACHE_DMA|ARCH_KMALLOC_FLAGS);
1715 sizes->cs_dmacachep->object_size = sizes->cs_size;
1716 sizes->cs_dmacachep->align = ARCH_KMALLOC_MINALIGN;
1717 __kmem_cache_create(sizes->cs_dmacachep,
1718 ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC);
1719 list_add(&sizes->cs_dmacachep->list, &slab_caches);
1720#endif 1663#endif
1721 sizes++; 1664 sizes++;
1722 names++; 1665 names++;
@@ -1727,7 +1670,6 @@ void __init kmem_cache_init(void)
1727 1670
1728 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT); 1671 ptr = kmalloc(sizeof(struct arraycache_init), GFP_NOWAIT);
1729 1672
1730 BUG_ON(cpu_cache_get(kmem_cache) != &initarray_cache.cache);
1731 memcpy(ptr, cpu_cache_get(kmem_cache), 1673 memcpy(ptr, cpu_cache_get(kmem_cache),
1732 sizeof(struct arraycache_init)); 1674 sizeof(struct arraycache_init));
1733 /* 1675 /*
@@ -2282,7 +2224,15 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2282 2224
2283 if (slab_state == DOWN) { 2225 if (slab_state == DOWN) {
2284 /* 2226 /*
2285 * Note: the first kmem_cache_create must create the cache 2227 * Note: Creation of first cache (kmem_cache).
2228 * The setup_list3s is taken care
2229 * of by the caller of __kmem_cache_create
2230 */
2231 cachep->array[smp_processor_id()] = &initarray_generic.cache;
2232 slab_state = PARTIAL;
2233 } else if (slab_state == PARTIAL) {
2234 /*
2235 * Note: the second kmem_cache_create must create the cache
2286 * that's used by kmalloc(24), otherwise the creation of 2236 * that's used by kmalloc(24), otherwise the creation of
2287 * further caches will BUG(). 2237 * further caches will BUG().
2288 */ 2238 */
@@ -2290,7 +2240,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2290 2240
2291 /* 2241 /*
2292 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is 2242 * If the cache that's used by kmalloc(sizeof(kmem_list3)) is
2293 * the first cache, then we need to set up all its list3s, 2243 * the second cache, then we need to set up all its list3s,
2294 * otherwise the creation of further caches will BUG(). 2244 * otherwise the creation of further caches will BUG().
2295 */ 2245 */
2296 set_up_list3s(cachep, SIZE_AC); 2246 set_up_list3s(cachep, SIZE_AC);
@@ -2299,6 +2249,7 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2299 else 2249 else
2300 slab_state = PARTIAL_ARRAYCACHE; 2250 slab_state = PARTIAL_ARRAYCACHE;
2301 } else { 2251 } else {
2252 /* Remaining boot caches */
2302 cachep->array[smp_processor_id()] = 2253 cachep->array[smp_processor_id()] =
2303 kmalloc(sizeof(struct arraycache_init), gfp); 2254 kmalloc(sizeof(struct arraycache_init), gfp);
2304 2255
@@ -2331,11 +2282,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep, gfp_t gfp)
2331 2282
2332/** 2283/**
2333 * __kmem_cache_create - Create a cache. 2284 * __kmem_cache_create - Create a cache.
2334 * @name: A string which is used in /proc/slabinfo to identify this cache. 2285 * @cachep: cache management descriptor
2335 * @size: The size of objects to be created in this cache.
2336 * @align: The required alignment for the objects.
2337 * @flags: SLAB flags 2286 * @flags: SLAB flags
2338 * @ctor: A constructor for the objects.
2339 * 2287 *
2340 * Returns a ptr to the cache on success, NULL on failure. 2288 * Returns a ptr to the cache on success, NULL on failure.
2341 * Cannot be called within a int, but can be interrupted. 2289 * Cannot be called within a int, but can be interrupted.
@@ -2378,11 +2326,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2378 if (flags & SLAB_DESTROY_BY_RCU) 2326 if (flags & SLAB_DESTROY_BY_RCU)
2379 BUG_ON(flags & SLAB_POISON); 2327 BUG_ON(flags & SLAB_POISON);
2380#endif 2328#endif
2381 /*
2382 * Always checks flags, a caller might be expecting debug support which
2383 * isn't available.
2384 */
2385 BUG_ON(flags & ~CREATE_MASK);
2386 2329
2387 /* 2330 /*
2388 * Check that size is in terms of words. This is needed to avoid 2331 * Check that size is in terms of words. This is needed to avoid
@@ -2394,22 +2337,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2394 size &= ~(BYTES_PER_WORD - 1); 2337 size &= ~(BYTES_PER_WORD - 1);
2395 } 2338 }
2396 2339
2397 /* calculate the final buffer alignment: */
2398
2399 /* 1) arch recommendation: can be overridden for debug */
2400 if (flags & SLAB_HWCACHE_ALIGN) {
2401 /*
2402 * Default alignment: as specified by the arch code. Except if
2403 * an object is really small, then squeeze multiple objects into
2404 * one cacheline.
2405 */
2406 ralign = cache_line_size();
2407 while (size <= ralign / 2)
2408 ralign /= 2;
2409 } else {
2410 ralign = BYTES_PER_WORD;
2411 }
2412
2413 /* 2340 /*
2414 * Redzoning and user store require word alignment or possibly larger. 2341 * Redzoning and user store require word alignment or possibly larger.
2415 * Note this will be overridden by architecture or caller mandated 2342 * Note this will be overridden by architecture or caller mandated
@@ -2426,10 +2353,6 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2426 size &= ~(REDZONE_ALIGN - 1); 2353 size &= ~(REDZONE_ALIGN - 1);
2427 } 2354 }
2428 2355
2429 /* 2) arch mandated alignment */
2430 if (ralign < ARCH_SLAB_MINALIGN) {
2431 ralign = ARCH_SLAB_MINALIGN;
2432 }
2433 /* 3) caller mandated alignment */ 2356 /* 3) caller mandated alignment */
2434 if (ralign < cachep->align) { 2357 if (ralign < cachep->align) {
2435 ralign = cachep->align; 2358 ralign = cachep->align;
@@ -2447,7 +2370,7 @@ __kmem_cache_create (struct kmem_cache *cachep, unsigned long flags)
2447 else 2370 else
2448 gfp = GFP_NOWAIT; 2371 gfp = GFP_NOWAIT;
2449 2372
2450 cachep->nodelists = (struct kmem_list3 **)&cachep->array[nr_cpu_ids]; 2373 setup_nodelists_pointer(cachep);
2451#if DEBUG 2374#if DEBUG
2452 2375
2453 /* 2376 /*
@@ -3969,12 +3892,6 @@ void kfree(const void *objp)
3969} 3892}
3970EXPORT_SYMBOL(kfree); 3893EXPORT_SYMBOL(kfree);
3971 3894
3972unsigned int kmem_cache_size(struct kmem_cache *cachep)
3973{
3974 return cachep->object_size;
3975}
3976EXPORT_SYMBOL(kmem_cache_size);
3977
3978/* 3895/*
3979 * This initializes kmem_list3 or resizes various caches for all nodes. 3896 * This initializes kmem_list3 or resizes various caches for all nodes.
3980 */ 3897 */
@@ -4276,54 +4193,8 @@ out:
4276} 4193}
4277 4194
4278#ifdef CONFIG_SLABINFO 4195#ifdef CONFIG_SLABINFO
4279 4196void get_slabinfo(struct kmem_cache *cachep, struct slabinfo *sinfo)
4280static void print_slabinfo_header(struct seq_file *m)
4281{ 4197{
4282 /*
4283 * Output format version, so at least we can change it
4284 * without _too_ many complaints.
4285 */
4286#if STATS
4287 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
4288#else
4289 seq_puts(m, "slabinfo - version: 2.1\n");
4290#endif
4291 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
4292 "<objperslab> <pagesperslab>");
4293 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
4294 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
4295#if STATS
4296 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
4297 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
4298 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
4299#endif
4300 seq_putc(m, '\n');
4301}
4302
4303static void *s_start(struct seq_file *m, loff_t *pos)
4304{
4305 loff_t n = *pos;
4306
4307 mutex_lock(&slab_mutex);
4308 if (!n)
4309 print_slabinfo_header(m);
4310
4311 return seq_list_start(&slab_caches, *pos);
4312}
4313
4314static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4315{
4316 return seq_list_next(p, &slab_caches, pos);
4317}
4318
4319static void s_stop(struct seq_file *m, void *p)
4320{
4321 mutex_unlock(&slab_mutex);
4322}
4323
4324static int s_show(struct seq_file *m, void *p)
4325{
4326 struct kmem_cache *cachep = list_entry(p, struct kmem_cache, list);
4327 struct slab *slabp; 4198 struct slab *slabp;
4328 unsigned long active_objs; 4199 unsigned long active_objs;
4329 unsigned long num_objs; 4200 unsigned long num_objs;
@@ -4378,13 +4249,20 @@ static int s_show(struct seq_file *m, void *p)
4378 if (error) 4249 if (error)
4379 printk(KERN_ERR "slab: cache %s error: %s\n", name, error); 4250 printk(KERN_ERR "slab: cache %s error: %s\n", name, error);
4380 4251
4381 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", 4252 sinfo->active_objs = active_objs;
4382 name, active_objs, num_objs, cachep->size, 4253 sinfo->num_objs = num_objs;
4383 cachep->num, (1 << cachep->gfporder)); 4254 sinfo->active_slabs = active_slabs;
4384 seq_printf(m, " : tunables %4u %4u %4u", 4255 sinfo->num_slabs = num_slabs;
4385 cachep->limit, cachep->batchcount, cachep->shared); 4256 sinfo->shared_avail = shared_avail;
4386 seq_printf(m, " : slabdata %6lu %6lu %6lu", 4257 sinfo->limit = cachep->limit;
4387 active_slabs, num_slabs, shared_avail); 4258 sinfo->batchcount = cachep->batchcount;
4259 sinfo->shared = cachep->shared;
4260 sinfo->objects_per_slab = cachep->num;
4261 sinfo->cache_order = cachep->gfporder;
4262}
4263
4264void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *cachep)
4265{
4388#if STATS 4266#if STATS
4389 { /* list3 stats */ 4267 { /* list3 stats */
4390 unsigned long high = cachep->high_mark; 4268 unsigned long high = cachep->high_mark;
@@ -4414,31 +4292,8 @@ static int s_show(struct seq_file *m, void *p)
4414 allochit, allocmiss, freehit, freemiss); 4292 allochit, allocmiss, freehit, freemiss);
4415 } 4293 }
4416#endif 4294#endif
4417 seq_putc(m, '\n');
4418 return 0;
4419} 4295}
4420 4296
4421/*
4422 * slabinfo_op - iterator that generates /proc/slabinfo
4423 *
4424 * Output layout:
4425 * cache-name
4426 * num-active-objs
4427 * total-objs
4428 * object size
4429 * num-active-slabs
4430 * total-slabs
4431 * num-pages-per-slab
4432 * + further values on SMP and with statistics enabled
4433 */
4434
4435static const struct seq_operations slabinfo_op = {
4436 .start = s_start,
4437 .next = s_next,
4438 .stop = s_stop,
4439 .show = s_show,
4440};
4441
4442#define MAX_SLABINFO_WRITE 128 4297#define MAX_SLABINFO_WRITE 128
4443/** 4298/**
4444 * slabinfo_write - Tuning for the slab allocator 4299 * slabinfo_write - Tuning for the slab allocator
@@ -4447,7 +4302,7 @@ static const struct seq_operations slabinfo_op = {
4447 * @count: data length 4302 * @count: data length
4448 * @ppos: unused 4303 * @ppos: unused
4449 */ 4304 */
4450static ssize_t slabinfo_write(struct file *file, const char __user *buffer, 4305ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4451 size_t count, loff_t *ppos) 4306 size_t count, loff_t *ppos)
4452{ 4307{
4453 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp; 4308 char kbuf[MAX_SLABINFO_WRITE + 1], *tmp;
@@ -4490,19 +4345,6 @@ static ssize_t slabinfo_write(struct file *file, const char __user *buffer,
4490 return res; 4345 return res;
4491} 4346}
4492 4347
4493static int slabinfo_open(struct inode *inode, struct file *file)
4494{
4495 return seq_open(file, &slabinfo_op);
4496}
4497
4498static const struct file_operations proc_slabinfo_operations = {
4499 .open = slabinfo_open,
4500 .read = seq_read,
4501 .write = slabinfo_write,
4502 .llseek = seq_lseek,
4503 .release = seq_release,
4504};
4505
4506#ifdef CONFIG_DEBUG_SLAB_LEAK 4348#ifdef CONFIG_DEBUG_SLAB_LEAK
4507 4349
4508static void *leaks_start(struct seq_file *m, loff_t *pos) 4350static void *leaks_start(struct seq_file *m, loff_t *pos)
@@ -4631,6 +4473,16 @@ static int leaks_show(struct seq_file *m, void *p)
4631 return 0; 4473 return 0;
4632} 4474}
4633 4475
4476static void *s_next(struct seq_file *m, void *p, loff_t *pos)
4477{
4478 return seq_list_next(p, &slab_caches, pos);
4479}
4480
4481static void s_stop(struct seq_file *m, void *p)
4482{
4483 mutex_unlock(&slab_mutex);
4484}
4485
4634static const struct seq_operations slabstats_op = { 4486static const struct seq_operations slabstats_op = {
4635 .start = leaks_start, 4487 .start = leaks_start,
4636 .next = s_next, 4488 .next = s_next,
@@ -4665,7 +4517,6 @@ static const struct file_operations proc_slabstats_operations = {
4665 4517
4666static int __init slab_proc_init(void) 4518static int __init slab_proc_init(void)
4667{ 4519{
4668 proc_create("slabinfo",S_IWUSR|S_IRUSR,NULL,&proc_slabinfo_operations);
4669#ifdef CONFIG_DEBUG_SLAB_LEAK 4520#ifdef CONFIG_DEBUG_SLAB_LEAK
4670 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations); 4521 proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4671#endif 4522#endif
diff --git a/mm/slab.h b/mm/slab.h
index 7deeb449a30..1cb9c9ee0e6 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -32,9 +32,17 @@ extern struct list_head slab_caches;
32/* The slab cache that manages slab cache information */ 32/* The slab cache that manages slab cache information */
33extern struct kmem_cache *kmem_cache; 33extern struct kmem_cache *kmem_cache;
34 34
35unsigned long calculate_alignment(unsigned long flags,
36 unsigned long align, unsigned long size);
37
35/* Functions provided by the slab allocators */ 38/* Functions provided by the slab allocators */
36extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags); 39extern int __kmem_cache_create(struct kmem_cache *, unsigned long flags);
37 40
41extern struct kmem_cache *create_kmalloc_cache(const char *name, size_t size,
42 unsigned long flags);
43extern void create_boot_cache(struct kmem_cache *, const char *name,
44 size_t size, unsigned long flags);
45
38#ifdef CONFIG_SLUB 46#ifdef CONFIG_SLUB
39struct kmem_cache *__kmem_cache_alias(const char *name, size_t size, 47struct kmem_cache *__kmem_cache_alias(const char *name, size_t size,
40 size_t align, unsigned long flags, void (*ctor)(void *)); 48 size_t align, unsigned long flags, void (*ctor)(void *));
@@ -45,6 +53,51 @@ static inline struct kmem_cache *__kmem_cache_alias(const char *name, size_t siz
45#endif 53#endif
46 54
47 55
56/* Legal flag mask for kmem_cache_create(), for various configurations */
57#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | SLAB_PANIC | \
58 SLAB_DESTROY_BY_RCU | SLAB_DEBUG_OBJECTS )
59
60#if defined(CONFIG_DEBUG_SLAB)
61#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
62#elif defined(CONFIG_SLUB_DEBUG)
63#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
64 SLAB_TRACE | SLAB_DEBUG_FREE)
65#else
66#define SLAB_DEBUG_FLAGS (0)
67#endif
68
69#if defined(CONFIG_SLAB)
70#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
71 SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | SLAB_NOTRACK)
72#elif defined(CONFIG_SLUB)
73#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
74 SLAB_TEMPORARY | SLAB_NOTRACK)
75#else
76#define SLAB_CACHE_FLAGS (0)
77#endif
78
79#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
80
48int __kmem_cache_shutdown(struct kmem_cache *); 81int __kmem_cache_shutdown(struct kmem_cache *);
49 82
83struct seq_file;
84struct file;
85
86struct slabinfo {
87 unsigned long active_objs;
88 unsigned long num_objs;
89 unsigned long active_slabs;
90 unsigned long num_slabs;
91 unsigned long shared_avail;
92 unsigned int limit;
93 unsigned int batchcount;
94 unsigned int shared;
95 unsigned int objects_per_slab;
96 unsigned int cache_order;
97};
98
99void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
100void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
101ssize_t slabinfo_write(struct file *file, const char __user *buffer,
102 size_t count, loff_t *ppos);
50#endif 103#endif
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 069a24e6440..a8e76d79ee6 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -13,6 +13,8 @@
13#include <linux/module.h> 13#include <linux/module.h>
14#include <linux/cpu.h> 14#include <linux/cpu.h>
15#include <linux/uaccess.h> 15#include <linux/uaccess.h>
16#include <linux/seq_file.h>
17#include <linux/proc_fs.h>
16#include <asm/cacheflush.h> 18#include <asm/cacheflush.h>
17#include <asm/tlbflush.h> 19#include <asm/tlbflush.h>
18#include <asm/page.h> 20#include <asm/page.h>
@@ -71,6 +73,34 @@ static inline int kmem_cache_sanity_check(const char *name, size_t size)
71#endif 73#endif
72 74
73/* 75/*
76 * Figure out what the alignment of the objects will be given a set of
77 * flags, a user specified alignment and the size of the objects.
78 */
79unsigned long calculate_alignment(unsigned long flags,
80 unsigned long align, unsigned long size)
81{
82 /*
83 * If the user wants hardware cache aligned objects then follow that
84 * suggestion if the object is sufficiently large.
85 *
86 * The hardware cache alignment cannot override the specified
87 * alignment though. If that is greater then use it.
88 */
89 if (flags & SLAB_HWCACHE_ALIGN) {
90 unsigned long ralign = cache_line_size();
91 while (size <= ralign / 2)
92 ralign /= 2;
93 align = max(align, ralign);
94 }
95
96 if (align < ARCH_SLAB_MINALIGN)
97 align = ARCH_SLAB_MINALIGN;
98
99 return ALIGN(align, sizeof(void *));
100}
101
102
103/*
74 * kmem_cache_create - Create a cache. 104 * kmem_cache_create - Create a cache.
75 * @name: A string which is used in /proc/slabinfo to identify this cache. 105 * @name: A string which is used in /proc/slabinfo to identify this cache.
76 * @size: The size of objects to be created in this cache. 106 * @size: The size of objects to be created in this cache.
@@ -107,6 +137,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
107 if (!kmem_cache_sanity_check(name, size) == 0) 137 if (!kmem_cache_sanity_check(name, size) == 0)
108 goto out_locked; 138 goto out_locked;
109 139
140 /*
141 * Some allocators will constraint the set of valid flags to a subset
142 * of all flags. We expect them to define CACHE_CREATE_MASK in this
143 * case, and we'll just provide them with a sanitized version of the
144 * passed flags.
145 */
146 flags &= CACHE_CREATE_MASK;
110 147
111 s = __kmem_cache_alias(name, size, align, flags, ctor); 148 s = __kmem_cache_alias(name, size, align, flags, ctor);
112 if (s) 149 if (s)
@@ -115,7 +152,7 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align
115 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL); 152 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
116 if (s) { 153 if (s) {
117 s->object_size = s->size = size; 154 s->object_size = s->size = size;
118 s->align = align; 155 s->align = calculate_alignment(flags, align, size);
119 s->ctor = ctor; 156 s->ctor = ctor;
120 s->name = kstrdup(name, GFP_KERNEL); 157 s->name = kstrdup(name, GFP_KERNEL);
121 if (!s->name) { 158 if (!s->name) {
@@ -192,3 +229,146 @@ int slab_is_available(void)
192{ 229{
193 return slab_state >= UP; 230 return slab_state >= UP;
194} 231}
232
233#ifndef CONFIG_SLOB
234/* Create a cache during boot when no slab services are available yet */
235void __init create_boot_cache(struct kmem_cache *s, const char *name, size_t size,
236 unsigned long flags)
237{
238 int err;
239
240 s->name = name;
241 s->size = s->object_size = size;
242 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
243 err = __kmem_cache_create(s, flags);
244
245 if (err)
246 panic("Creation of kmalloc slab %s size=%zd failed. Reason %d\n",
247 name, size, err);
248
249 s->refcount = -1; /* Exempt from merging for now */
250}
251
252struct kmem_cache *__init create_kmalloc_cache(const char *name, size_t size,
253 unsigned long flags)
254{
255 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
256
257 if (!s)
258 panic("Out of memory when creating slab %s\n", name);
259
260 create_boot_cache(s, name, size, flags);
261 list_add(&s->list, &slab_caches);
262 s->refcount = 1;
263 return s;
264}
265
266#endif /* !CONFIG_SLOB */
267
268
269#ifdef CONFIG_SLABINFO
270static void print_slabinfo_header(struct seq_file *m)
271{
272 /*
273 * Output format version, so at least we can change it
274 * without _too_ many complaints.
275 */
276#ifdef CONFIG_DEBUG_SLAB
277 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
278#else
279 seq_puts(m, "slabinfo - version: 2.1\n");
280#endif
281 seq_puts(m, "# name <active_objs> <num_objs> <objsize> "
282 "<objperslab> <pagesperslab>");
283 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
284 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
285#ifdef CONFIG_DEBUG_SLAB
286 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> "
287 "<error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
288 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
289#endif
290 seq_putc(m, '\n');
291}
292
293static void *s_start(struct seq_file *m, loff_t *pos)
294{
295 loff_t n = *pos;
296
297 mutex_lock(&slab_mutex);
298 if (!n)
299 print_slabinfo_header(m);
300
301 return seq_list_start(&slab_caches, *pos);
302}
303
304static void *s_next(struct seq_file *m, void *p, loff_t *pos)
305{
306 return seq_list_next(p, &slab_caches, pos);
307}
308
309static void s_stop(struct seq_file *m, void *p)
310{
311 mutex_unlock(&slab_mutex);
312}
313
314static int s_show(struct seq_file *m, void *p)
315{
316 struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
317 struct slabinfo sinfo;
318
319 memset(&sinfo, 0, sizeof(sinfo));
320 get_slabinfo(s, &sinfo);
321
322 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
323 s->name, sinfo.active_objs, sinfo.num_objs, s->size,
324 sinfo.objects_per_slab, (1 << sinfo.cache_order));
325
326 seq_printf(m, " : tunables %4u %4u %4u",
327 sinfo.limit, sinfo.batchcount, sinfo.shared);
328 seq_printf(m, " : slabdata %6lu %6lu %6lu",
329 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
330 slabinfo_show_stats(m, s);
331 seq_putc(m, '\n');
332 return 0;
333}
334
335/*
336 * slabinfo_op - iterator that generates /proc/slabinfo
337 *
338 * Output layout:
339 * cache-name
340 * num-active-objs
341 * total-objs
342 * object size
343 * num-active-slabs
344 * total-slabs
345 * num-pages-per-slab
346 * + further values on SMP and with statistics enabled
347 */
348static const struct seq_operations slabinfo_op = {
349 .start = s_start,
350 .next = s_next,
351 .stop = s_stop,
352 .show = s_show,
353};
354
355static int slabinfo_open(struct inode *inode, struct file *file)
356{
357 return seq_open(file, &slabinfo_op);
358}
359
360static const struct file_operations proc_slabinfo_operations = {
361 .open = slabinfo_open,
362 .read = seq_read,
363 .write = slabinfo_write,
364 .llseek = seq_lseek,
365 .release = seq_release,
366};
367
368static int __init slab_proc_init(void)
369{
370 proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations);
371 return 0;
372}
373module_init(slab_proc_init);
374#endif /* CONFIG_SLABINFO */
diff --git a/mm/slob.c b/mm/slob.c
index 1e921c5e957..795bab7d391 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -28,9 +28,8 @@
28 * from kmalloc are prepended with a 4-byte header with the kmalloc size. 28 * from kmalloc are prepended with a 4-byte header with the kmalloc size.
29 * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls 29 * If kmalloc is asked for objects of PAGE_SIZE or larger, it calls
30 * alloc_pages() directly, allocating compound pages so the page order 30 * alloc_pages() directly, allocating compound pages so the page order
31 * does not have to be separately tracked, and also stores the exact 31 * does not have to be separately tracked.
32 * allocation size in page->private so that it can be used to accurately 32 * These objects are detected in kfree() because PageSlab()
33 * provide ksize(). These objects are detected in kfree() because slob_page()
34 * is false for them. 33 * is false for them.
35 * 34 *
36 * SLAB is emulated on top of SLOB by simply calling constructors and 35 * SLAB is emulated on top of SLOB by simply calling constructors and
@@ -124,7 +123,6 @@ static inline void clear_slob_page_free(struct page *sp)
124 123
125#define SLOB_UNIT sizeof(slob_t) 124#define SLOB_UNIT sizeof(slob_t)
126#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT) 125#define SLOB_UNITS(size) (((size) + SLOB_UNIT - 1)/SLOB_UNIT)
127#define SLOB_ALIGN L1_CACHE_BYTES
128 126
129/* 127/*
130 * struct slob_rcu is inserted at the tail of allocated slob blocks, which 128 * struct slob_rcu is inserted at the tail of allocated slob blocks, which
@@ -455,11 +453,6 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
455 if (likely(order)) 453 if (likely(order))
456 gfp |= __GFP_COMP; 454 gfp |= __GFP_COMP;
457 ret = slob_new_pages(gfp, order, node); 455 ret = slob_new_pages(gfp, order, node);
458 if (ret) {
459 struct page *page;
460 page = virt_to_page(ret);
461 page->private = size;
462 }
463 456
464 trace_kmalloc_node(caller, ret, 457 trace_kmalloc_node(caller, ret,
465 size, PAGE_SIZE << order, gfp, node); 458 size, PAGE_SIZE << order, gfp, node);
@@ -506,7 +499,7 @@ void kfree(const void *block)
506 unsigned int *m = (unsigned int *)(block - align); 499 unsigned int *m = (unsigned int *)(block - align);
507 slob_free(m, *m + align); 500 slob_free(m, *m + align);
508 } else 501 } else
509 put_page(sp); 502 __free_pages(sp, compound_order(sp));
510} 503}
511EXPORT_SYMBOL(kfree); 504EXPORT_SYMBOL(kfree);
512 505
@@ -514,37 +507,30 @@ EXPORT_SYMBOL(kfree);
514size_t ksize(const void *block) 507size_t ksize(const void *block)
515{ 508{
516 struct page *sp; 509 struct page *sp;
510 int align;
511 unsigned int *m;
517 512
518 BUG_ON(!block); 513 BUG_ON(!block);
519 if (unlikely(block == ZERO_SIZE_PTR)) 514 if (unlikely(block == ZERO_SIZE_PTR))
520 return 0; 515 return 0;
521 516
522 sp = virt_to_page(block); 517 sp = virt_to_page(block);
523 if (PageSlab(sp)) { 518 if (unlikely(!PageSlab(sp)))
524 int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN); 519 return PAGE_SIZE << compound_order(sp);
525 unsigned int *m = (unsigned int *)(block - align); 520
526 return SLOB_UNITS(*m) * SLOB_UNIT; 521 align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
527 } else 522 m = (unsigned int *)(block - align);
528 return sp->private; 523 return SLOB_UNITS(*m) * SLOB_UNIT;
529} 524}
530EXPORT_SYMBOL(ksize); 525EXPORT_SYMBOL(ksize);
531 526
532int __kmem_cache_create(struct kmem_cache *c, unsigned long flags) 527int __kmem_cache_create(struct kmem_cache *c, unsigned long flags)
533{ 528{
534 size_t align = c->size;
535
536 if (flags & SLAB_DESTROY_BY_RCU) { 529 if (flags & SLAB_DESTROY_BY_RCU) {
537 /* leave room for rcu footer at the end of object */ 530 /* leave room for rcu footer at the end of object */
538 c->size += sizeof(struct slob_rcu); 531 c->size += sizeof(struct slob_rcu);
539 } 532 }
540 c->flags = flags; 533 c->flags = flags;
541 /* ignore alignment unless it's forced */
542 c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0;
543 if (c->align < ARCH_SLAB_MINALIGN)
544 c->align = ARCH_SLAB_MINALIGN;
545 if (c->align < align)
546 c->align = align;
547
548 return 0; 534 return 0;
549} 535}
550 536
@@ -558,12 +544,12 @@ void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
558 544
559 if (c->size < PAGE_SIZE) { 545 if (c->size < PAGE_SIZE) {
560 b = slob_alloc(c->size, flags, c->align, node); 546 b = slob_alloc(c->size, flags, c->align, node);
561 trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, 547 trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
562 SLOB_UNITS(c->size) * SLOB_UNIT, 548 SLOB_UNITS(c->size) * SLOB_UNIT,
563 flags, node); 549 flags, node);
564 } else { 550 } else {
565 b = slob_new_pages(flags, get_order(c->size), node); 551 b = slob_new_pages(flags, get_order(c->size), node);
566 trace_kmem_cache_alloc_node(_RET_IP_, b, c->size, 552 trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
567 PAGE_SIZE << get_order(c->size), 553 PAGE_SIZE << get_order(c->size),
568 flags, node); 554 flags, node);
569 } 555 }
@@ -608,12 +594,6 @@ void kmem_cache_free(struct kmem_cache *c, void *b)
608} 594}
609EXPORT_SYMBOL(kmem_cache_free); 595EXPORT_SYMBOL(kmem_cache_free);
610 596
611unsigned int kmem_cache_size(struct kmem_cache *c)
612{
613 return c->size;
614}
615EXPORT_SYMBOL(kmem_cache_size);
616
617int __kmem_cache_shutdown(struct kmem_cache *c) 597int __kmem_cache_shutdown(struct kmem_cache *c)
618{ 598{
619 /* No way to check for remaining objects */ 599 /* No way to check for remaining objects */
diff --git a/mm/slub.c b/mm/slub.c
index 487f0bdd53c..87f9f32bf0c 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -112,9 +112,6 @@
112 * the fast path and disables lockless freelists. 112 * the fast path and disables lockless freelists.
113 */ 113 */
114 114
115#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
116 SLAB_TRACE | SLAB_DEBUG_FREE)
117
118static inline int kmem_cache_debug(struct kmem_cache *s) 115static inline int kmem_cache_debug(struct kmem_cache *s)
119{ 116{
120#ifdef CONFIG_SLUB_DEBUG 117#ifdef CONFIG_SLUB_DEBUG
@@ -179,8 +176,6 @@ static inline int kmem_cache_debug(struct kmem_cache *s)
179#define __OBJECT_POISON 0x80000000UL /* Poison object */ 176#define __OBJECT_POISON 0x80000000UL /* Poison object */
180#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ 177#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */
181 178
182static int kmem_size = sizeof(struct kmem_cache);
183
184#ifdef CONFIG_SMP 179#ifdef CONFIG_SMP
185static struct notifier_block slab_notifier; 180static struct notifier_block slab_notifier;
186#endif 181#endif
@@ -1092,11 +1087,11 @@ static noinline struct kmem_cache_node *free_debug_processing(
1092 if (!check_object(s, page, object, SLUB_RED_ACTIVE)) 1087 if (!check_object(s, page, object, SLUB_RED_ACTIVE))
1093 goto out; 1088 goto out;
1094 1089
1095 if (unlikely(s != page->slab)) { 1090 if (unlikely(s != page->slab_cache)) {
1096 if (!PageSlab(page)) { 1091 if (!PageSlab(page)) {
1097 slab_err(s, page, "Attempt to free object(0x%p) " 1092 slab_err(s, page, "Attempt to free object(0x%p) "
1098 "outside of slab", object); 1093 "outside of slab", object);
1099 } else if (!page->slab) { 1094 } else if (!page->slab_cache) {
1100 printk(KERN_ERR 1095 printk(KERN_ERR
1101 "SLUB <none>: no slab for object 0x%p.\n", 1096 "SLUB <none>: no slab for object 0x%p.\n",
1102 object); 1097 object);
@@ -1357,7 +1352,7 @@ static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
1357 goto out; 1352 goto out;
1358 1353
1359 inc_slabs_node(s, page_to_nid(page), page->objects); 1354 inc_slabs_node(s, page_to_nid(page), page->objects);
1360 page->slab = s; 1355 page->slab_cache = s;
1361 __SetPageSlab(page); 1356 __SetPageSlab(page);
1362 if (page->pfmemalloc) 1357 if (page->pfmemalloc)
1363 SetPageSlabPfmemalloc(page); 1358 SetPageSlabPfmemalloc(page);
@@ -1424,7 +1419,7 @@ static void rcu_free_slab(struct rcu_head *h)
1424 else 1419 else
1425 page = container_of((struct list_head *)h, struct page, lru); 1420 page = container_of((struct list_head *)h, struct page, lru);
1426 1421
1427 __free_slab(page->slab, page); 1422 __free_slab(page->slab_cache, page);
1428} 1423}
1429 1424
1430static void free_slab(struct kmem_cache *s, struct page *page) 1425static void free_slab(struct kmem_cache *s, struct page *page)
@@ -1872,12 +1867,14 @@ redo:
1872/* 1867/*
1873 * Unfreeze all the cpu partial slabs. 1868 * Unfreeze all the cpu partial slabs.
1874 * 1869 *
1875 * This function must be called with interrupt disabled. 1870 * This function must be called with interrupts disabled
1871 * for the cpu using c (or some other guarantee must be there
1872 * to guarantee no concurrent accesses).
1876 */ 1873 */
1877static void unfreeze_partials(struct kmem_cache *s) 1874static void unfreeze_partials(struct kmem_cache *s,
1875 struct kmem_cache_cpu *c)
1878{ 1876{
1879 struct kmem_cache_node *n = NULL, *n2 = NULL; 1877 struct kmem_cache_node *n = NULL, *n2 = NULL;
1880 struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
1881 struct page *page, *discard_page = NULL; 1878 struct page *page, *discard_page = NULL;
1882 1879
1883 while ((page = c->partial)) { 1880 while ((page = c->partial)) {
@@ -1963,7 +1960,7 @@ static int put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
1963 * set to the per node partial list. 1960 * set to the per node partial list.
1964 */ 1961 */
1965 local_irq_save(flags); 1962 local_irq_save(flags);
1966 unfreeze_partials(s); 1963 unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
1967 local_irq_restore(flags); 1964 local_irq_restore(flags);
1968 oldpage = NULL; 1965 oldpage = NULL;
1969 pobjects = 0; 1966 pobjects = 0;
@@ -2006,7 +2003,7 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
2006 if (c->page) 2003 if (c->page)
2007 flush_slab(s, c); 2004 flush_slab(s, c);
2008 2005
2009 unfreeze_partials(s); 2006 unfreeze_partials(s, c);
2010 } 2007 }
2011} 2008}
2012 2009
@@ -2459,7 +2456,6 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2459 void *prior; 2456 void *prior;
2460 void **object = (void *)x; 2457 void **object = (void *)x;
2461 int was_frozen; 2458 int was_frozen;
2462 int inuse;
2463 struct page new; 2459 struct page new;
2464 unsigned long counters; 2460 unsigned long counters;
2465 struct kmem_cache_node *n = NULL; 2461 struct kmem_cache_node *n = NULL;
@@ -2472,13 +2468,17 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2472 return; 2468 return;
2473 2469
2474 do { 2470 do {
2471 if (unlikely(n)) {
2472 spin_unlock_irqrestore(&n->list_lock, flags);
2473 n = NULL;
2474 }
2475 prior = page->freelist; 2475 prior = page->freelist;
2476 counters = page->counters; 2476 counters = page->counters;
2477 set_freepointer(s, object, prior); 2477 set_freepointer(s, object, prior);
2478 new.counters = counters; 2478 new.counters = counters;
2479 was_frozen = new.frozen; 2479 was_frozen = new.frozen;
2480 new.inuse--; 2480 new.inuse--;
2481 if ((!new.inuse || !prior) && !was_frozen && !n) { 2481 if ((!new.inuse || !prior) && !was_frozen) {
2482 2482
2483 if (!kmem_cache_debug(s) && !prior) 2483 if (!kmem_cache_debug(s) && !prior)
2484 2484
@@ -2503,7 +2503,6 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2503 2503
2504 } 2504 }
2505 } 2505 }
2506 inuse = new.inuse;
2507 2506
2508 } while (!cmpxchg_double_slab(s, page, 2507 } while (!cmpxchg_double_slab(s, page,
2509 prior, counters, 2508 prior, counters,
@@ -2529,25 +2528,17 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
2529 return; 2528 return;
2530 } 2529 }
2531 2530
2531 if (unlikely(!new.inuse && n->nr_partial > s->min_partial))
2532 goto slab_empty;
2533
2532 /* 2534 /*
2533 * was_frozen may have been set after we acquired the list_lock in 2535 * Objects left in the slab. If it was not on the partial list before
2534 * an earlier loop. So we need to check it here again. 2536 * then add it.
2535 */ 2537 */
2536 if (was_frozen) 2538 if (kmem_cache_debug(s) && unlikely(!prior)) {
2537 stat(s, FREE_FROZEN); 2539 remove_full(s, page);
2538 else { 2540 add_partial(n, page, DEACTIVATE_TO_TAIL);
2539 if (unlikely(!inuse && n->nr_partial > s->min_partial)) 2541 stat(s, FREE_ADD_PARTIAL);
2540 goto slab_empty;
2541
2542 /*
2543 * Objects left in the slab. If it was not on the partial list before
2544 * then add it.
2545 */
2546 if (unlikely(!prior)) {
2547 remove_full(s, page);
2548 add_partial(n, page, DEACTIVATE_TO_TAIL);
2549 stat(s, FREE_ADD_PARTIAL);
2550 }
2551 } 2542 }
2552 spin_unlock_irqrestore(&n->list_lock, flags); 2543 spin_unlock_irqrestore(&n->list_lock, flags);
2553 return; 2544 return;
@@ -2623,9 +2614,9 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
2623 2614
2624 page = virt_to_head_page(x); 2615 page = virt_to_head_page(x);
2625 2616
2626 if (kmem_cache_debug(s) && page->slab != s) { 2617 if (kmem_cache_debug(s) && page->slab_cache != s) {
2627 pr_err("kmem_cache_free: Wrong slab cache. %s but object" 2618 pr_err("kmem_cache_free: Wrong slab cache. %s but object"
2628 " is from %s\n", page->slab->name, s->name); 2619 " is from %s\n", page->slab_cache->name, s->name);
2629 WARN_ON_ONCE(1); 2620 WARN_ON_ONCE(1);
2630 return; 2621 return;
2631 } 2622 }
@@ -2769,32 +2760,6 @@ static inline int calculate_order(int size, int reserved)
2769 return -ENOSYS; 2760 return -ENOSYS;
2770} 2761}
2771 2762
2772/*
2773 * Figure out what the alignment of the objects will be.
2774 */
2775static unsigned long calculate_alignment(unsigned long flags,
2776 unsigned long align, unsigned long size)
2777{
2778 /*
2779 * If the user wants hardware cache aligned objects then follow that
2780 * suggestion if the object is sufficiently large.
2781 *
2782 * The hardware cache alignment cannot override the specified
2783 * alignment though. If that is greater then use it.
2784 */
2785 if (flags & SLAB_HWCACHE_ALIGN) {
2786 unsigned long ralign = cache_line_size();
2787 while (size <= ralign / 2)
2788 ralign /= 2;
2789 align = max(align, ralign);
2790 }
2791
2792 if (align < ARCH_SLAB_MINALIGN)
2793 align = ARCH_SLAB_MINALIGN;
2794
2795 return ALIGN(align, sizeof(void *));
2796}
2797
2798static void 2763static void
2799init_kmem_cache_node(struct kmem_cache_node *n) 2764init_kmem_cache_node(struct kmem_cache_node *n)
2800{ 2765{
@@ -2928,7 +2893,6 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
2928{ 2893{
2929 unsigned long flags = s->flags; 2894 unsigned long flags = s->flags;
2930 unsigned long size = s->object_size; 2895 unsigned long size = s->object_size;
2931 unsigned long align = s->align;
2932 int order; 2896 int order;
2933 2897
2934 /* 2898 /*
@@ -3000,19 +2964,11 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
3000#endif 2964#endif
3001 2965
3002 /* 2966 /*
3003 * Determine the alignment based on various parameters that the
3004 * user specified and the dynamic determination of cache line size
3005 * on bootup.
3006 */
3007 align = calculate_alignment(flags, align, s->object_size);
3008 s->align = align;
3009
3010 /*
3011 * SLUB stores one object immediately after another beginning from 2967 * SLUB stores one object immediately after another beginning from
3012 * offset 0. In order to align the objects we have to simply size 2968 * offset 0. In order to align the objects we have to simply size
3013 * each object to conform to the alignment. 2969 * each object to conform to the alignment.
3014 */ 2970 */
3015 size = ALIGN(size, align); 2971 size = ALIGN(size, s->align);
3016 s->size = size; 2972 s->size = size;
3017 if (forced_order >= 0) 2973 if (forced_order >= 0)
3018 order = forced_order; 2974 order = forced_order;
@@ -3041,7 +2997,6 @@ static int calculate_sizes(struct kmem_cache *s, int forced_order)
3041 s->max = s->oo; 2997 s->max = s->oo;
3042 2998
3043 return !!oo_objects(s->oo); 2999 return !!oo_objects(s->oo);
3044
3045} 3000}
3046 3001
3047static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) 3002static int kmem_cache_open(struct kmem_cache *s, unsigned long flags)
@@ -3127,15 +3082,6 @@ error:
3127 return -EINVAL; 3082 return -EINVAL;
3128} 3083}
3129 3084
3130/*
3131 * Determine the size of a slab object
3132 */
3133unsigned int kmem_cache_size(struct kmem_cache *s)
3134{
3135 return s->object_size;
3136}
3137EXPORT_SYMBOL(kmem_cache_size);
3138
3139static void list_slab_objects(struct kmem_cache *s, struct page *page, 3085static void list_slab_objects(struct kmem_cache *s, struct page *page,
3140 const char *text) 3086 const char *text)
3141{ 3087{
@@ -3261,32 +3207,6 @@ static int __init setup_slub_nomerge(char *str)
3261 3207
3262__setup("slub_nomerge", setup_slub_nomerge); 3208__setup("slub_nomerge", setup_slub_nomerge);
3263 3209
3264static struct kmem_cache *__init create_kmalloc_cache(const char *name,
3265 int size, unsigned int flags)
3266{
3267 struct kmem_cache *s;
3268
3269 s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
3270
3271 s->name = name;
3272 s->size = s->object_size = size;
3273 s->align = ARCH_KMALLOC_MINALIGN;
3274
3275 /*
3276 * This function is called with IRQs disabled during early-boot on
3277 * single CPU so there's no need to take slab_mutex here.
3278 */
3279 if (kmem_cache_open(s, flags))
3280 goto panic;
3281
3282 list_add(&s->list, &slab_caches);
3283 return s;
3284
3285panic:
3286 panic("Creation of kmalloc slab %s size=%d failed.\n", name, size);
3287 return NULL;
3288}
3289
3290/* 3210/*
3291 * Conversion table for small slabs sizes / 8 to the index in the 3211 * Conversion table for small slabs sizes / 8 to the index in the
3292 * kmalloc array. This is necessary for slabs < 192 since we have non power 3212 * kmalloc array. This is necessary for slabs < 192 since we have non power
@@ -3424,7 +3344,7 @@ size_t ksize(const void *object)
3424 return PAGE_SIZE << compound_order(page); 3344 return PAGE_SIZE << compound_order(page);
3425 } 3345 }
3426 3346
3427 return slab_ksize(page->slab); 3347 return slab_ksize(page->slab_cache);
3428} 3348}
3429EXPORT_SYMBOL(ksize); 3349EXPORT_SYMBOL(ksize);
3430 3350
@@ -3449,8 +3369,8 @@ bool verify_mem_not_deleted(const void *x)
3449 } 3369 }
3450 3370
3451 slab_lock(page); 3371 slab_lock(page);
3452 if (on_freelist(page->slab, page, object)) { 3372 if (on_freelist(page->slab_cache, page, object)) {
3453 object_err(page->slab, page, object, "Object is on free-list"); 3373 object_err(page->slab_cache, page, object, "Object is on free-list");
3454 rv = false; 3374 rv = false;
3455 } else { 3375 } else {
3456 rv = true; 3376 rv = true;
@@ -3481,7 +3401,7 @@ void kfree(const void *x)
3481 __free_pages(page, compound_order(page)); 3401 __free_pages(page, compound_order(page));
3482 return; 3402 return;
3483 } 3403 }
3484 slab_free(page->slab, page, object, _RET_IP_); 3404 slab_free(page->slab_cache, page, object, _RET_IP_);
3485} 3405}
3486EXPORT_SYMBOL(kfree); 3406EXPORT_SYMBOL(kfree);
3487 3407
@@ -3676,15 +3596,16 @@ static int slab_memory_callback(struct notifier_block *self,
3676 3596
3677/* 3597/*
3678 * Used for early kmem_cache structures that were allocated using 3598 * Used for early kmem_cache structures that were allocated using
3679 * the page allocator 3599 * the page allocator. Allocate them properly then fix up the pointers
3600 * that may be pointing to the wrong kmem_cache structure.
3680 */ 3601 */
3681 3602
3682static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s) 3603static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
3683{ 3604{
3684 int node; 3605 int node;
3606 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
3685 3607
3686 list_add(&s->list, &slab_caches); 3608 memcpy(s, static_cache, kmem_cache->object_size);
3687 s->refcount = -1;
3688 3609
3689 for_each_node_state(node, N_NORMAL_MEMORY) { 3610 for_each_node_state(node, N_NORMAL_MEMORY) {
3690 struct kmem_cache_node *n = get_node(s, node); 3611 struct kmem_cache_node *n = get_node(s, node);
@@ -3692,78 +3613,52 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s)
3692 3613
3693 if (n) { 3614 if (n) {
3694 list_for_each_entry(p, &n->partial, lru) 3615 list_for_each_entry(p, &n->partial, lru)
3695 p->slab = s; 3616 p->slab_cache = s;
3696 3617
3697#ifdef CONFIG_SLUB_DEBUG 3618#ifdef CONFIG_SLUB_DEBUG
3698 list_for_each_entry(p, &n->full, lru) 3619 list_for_each_entry(p, &n->full, lru)
3699 p->slab = s; 3620 p->slab_cache = s;
3700#endif 3621#endif
3701 } 3622 }
3702 } 3623 }
3624 list_add(&s->list, &slab_caches);
3625 return s;
3703} 3626}
3704 3627
3705void __init kmem_cache_init(void) 3628void __init kmem_cache_init(void)
3706{ 3629{
3630 static __initdata struct kmem_cache boot_kmem_cache,
3631 boot_kmem_cache_node;
3707 int i; 3632 int i;
3708 int caches = 0; 3633 int caches = 2;
3709 struct kmem_cache *temp_kmem_cache;
3710 int order;
3711 struct kmem_cache *temp_kmem_cache_node;
3712 unsigned long kmalloc_size;
3713 3634
3714 if (debug_guardpage_minorder()) 3635 if (debug_guardpage_minorder())
3715 slub_max_order = 0; 3636 slub_max_order = 0;
3716 3637
3717 kmem_size = offsetof(struct kmem_cache, node) + 3638 kmem_cache_node = &boot_kmem_cache_node;
3718 nr_node_ids * sizeof(struct kmem_cache_node *); 3639 kmem_cache = &boot_kmem_cache;
3719
3720 /* Allocate two kmem_caches from the page allocator */
3721 kmalloc_size = ALIGN(kmem_size, cache_line_size());
3722 order = get_order(2 * kmalloc_size);
3723 kmem_cache = (void *)__get_free_pages(GFP_NOWAIT | __GFP_ZERO, order);
3724
3725 /*
3726 * Must first have the slab cache available for the allocations of the
3727 * struct kmem_cache_node's. There is special bootstrap code in
3728 * kmem_cache_open for slab_state == DOWN.
3729 */
3730 kmem_cache_node = (void *)kmem_cache + kmalloc_size;
3731 3640
3732 kmem_cache_node->name = "kmem_cache_node"; 3641 create_boot_cache(kmem_cache_node, "kmem_cache_node",
3733 kmem_cache_node->size = kmem_cache_node->object_size = 3642 sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN);
3734 sizeof(struct kmem_cache_node);
3735 kmem_cache_open(kmem_cache_node, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
3736 3643
3737 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI); 3644 hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
3738 3645
3739 /* Able to allocate the per node structures */ 3646 /* Able to allocate the per node structures */
3740 slab_state = PARTIAL; 3647 slab_state = PARTIAL;
3741 3648
3742 temp_kmem_cache = kmem_cache; 3649 create_boot_cache(kmem_cache, "kmem_cache",
3743 kmem_cache->name = "kmem_cache"; 3650 offsetof(struct kmem_cache, node) +
3744 kmem_cache->size = kmem_cache->object_size = kmem_size; 3651 nr_node_ids * sizeof(struct kmem_cache_node *),
3745 kmem_cache_open(kmem_cache, SLAB_HWCACHE_ALIGN | SLAB_PANIC); 3652 SLAB_HWCACHE_ALIGN);
3746 3653
3747 kmem_cache = kmem_cache_alloc(kmem_cache, GFP_NOWAIT); 3654 kmem_cache = bootstrap(&boot_kmem_cache);
3748 memcpy(kmem_cache, temp_kmem_cache, kmem_size);
3749 3655
3750 /* 3656 /*
3751 * Allocate kmem_cache_node properly from the kmem_cache slab. 3657 * Allocate kmem_cache_node properly from the kmem_cache slab.
3752 * kmem_cache_node is separately allocated so no need to 3658 * kmem_cache_node is separately allocated so no need to
3753 * update any list pointers. 3659 * update any list pointers.
3754 */ 3660 */
3755 temp_kmem_cache_node = kmem_cache_node; 3661 kmem_cache_node = bootstrap(&boot_kmem_cache_node);
3756
3757 kmem_cache_node = kmem_cache_alloc(kmem_cache, GFP_NOWAIT);
3758 memcpy(kmem_cache_node, temp_kmem_cache_node, kmem_size);
3759
3760 kmem_cache_bootstrap_fixup(kmem_cache_node);
3761
3762 caches++;
3763 kmem_cache_bootstrap_fixup(kmem_cache);
3764 caches++;
3765 /* Free temporary boot structure */
3766 free_pages((unsigned long)temp_kmem_cache, order);
3767 3662
3768 /* Now we can use the kmem_cache to allocate kmalloc slabs */ 3663 /* Now we can use the kmem_cache to allocate kmalloc slabs */
3769 3664
@@ -3964,6 +3859,10 @@ int __kmem_cache_create(struct kmem_cache *s, unsigned long flags)
3964 if (err) 3859 if (err)
3965 return err; 3860 return err;
3966 3861
3862 /* Mutex is not taken during early boot */
3863 if (slab_state <= UP)
3864 return 0;
3865
3967 mutex_unlock(&slab_mutex); 3866 mutex_unlock(&slab_mutex);
3968 err = sysfs_slab_add(s); 3867 err = sysfs_slab_add(s);
3969 mutex_lock(&slab_mutex); 3868 mutex_lock(&slab_mutex);
@@ -5265,13 +5164,8 @@ static int sysfs_slab_add(struct kmem_cache *s)
5265{ 5164{
5266 int err; 5165 int err;
5267 const char *name; 5166 const char *name;
5268 int unmergeable; 5167 int unmergeable = slab_unmergeable(s);
5269 5168
5270 if (slab_state < FULL)
5271 /* Defer until later */
5272 return 0;
5273
5274 unmergeable = slab_unmergeable(s);
5275 if (unmergeable) { 5169 if (unmergeable) {
5276 /* 5170 /*
5277 * Slabcache can never be merged so we can use the name proper. 5171 * Slabcache can never be merged so we can use the name proper.
@@ -5405,49 +5299,14 @@ __initcall(slab_sysfs_init);
5405 * The /proc/slabinfo ABI 5299 * The /proc/slabinfo ABI
5406 */ 5300 */
5407#ifdef CONFIG_SLABINFO 5301#ifdef CONFIG_SLABINFO
5408static void print_slabinfo_header(struct seq_file *m) 5302void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
5409{
5410 seq_puts(m, "slabinfo - version: 2.1\n");
5411 seq_puts(m, "# name <active_objs> <num_objs> <object_size> "
5412 "<objperslab> <pagesperslab>");
5413 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
5414 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
5415 seq_putc(m, '\n');
5416}
5417
5418static void *s_start(struct seq_file *m, loff_t *pos)
5419{
5420 loff_t n = *pos;
5421
5422 mutex_lock(&slab_mutex);
5423 if (!n)
5424 print_slabinfo_header(m);
5425
5426 return seq_list_start(&slab_caches, *pos);
5427}
5428
5429static void *s_next(struct seq_file *m, void *p, loff_t *pos)
5430{
5431 return seq_list_next(p, &slab_caches, pos);
5432}
5433
5434static void s_stop(struct seq_file *m, void *p)
5435{
5436 mutex_unlock(&slab_mutex);
5437}
5438
5439static int s_show(struct seq_file *m, void *p)
5440{ 5303{
5441 unsigned long nr_partials = 0; 5304 unsigned long nr_partials = 0;
5442 unsigned long nr_slabs = 0; 5305 unsigned long nr_slabs = 0;
5443 unsigned long nr_inuse = 0;
5444 unsigned long nr_objs = 0; 5306 unsigned long nr_objs = 0;
5445 unsigned long nr_free = 0; 5307 unsigned long nr_free = 0;
5446 struct kmem_cache *s;
5447 int node; 5308 int node;
5448 5309
5449 s = list_entry(p, struct kmem_cache, list);
5450
5451 for_each_online_node(node) { 5310 for_each_online_node(node) {
5452 struct kmem_cache_node *n = get_node(s, node); 5311 struct kmem_cache_node *n = get_node(s, node);
5453 5312
@@ -5460,41 +5319,21 @@ static int s_show(struct seq_file *m, void *p)
5460 nr_free += count_partial(n, count_free); 5319 nr_free += count_partial(n, count_free);
5461 } 5320 }
5462 5321
5463 nr_inuse = nr_objs - nr_free; 5322 sinfo->active_objs = nr_objs - nr_free;
5464 5323 sinfo->num_objs = nr_objs;
5465 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d", s->name, nr_inuse, 5324 sinfo->active_slabs = nr_slabs;
5466 nr_objs, s->size, oo_objects(s->oo), 5325 sinfo->num_slabs = nr_slabs;
5467 (1 << oo_order(s->oo))); 5326 sinfo->objects_per_slab = oo_objects(s->oo);
5468 seq_printf(m, " : tunables %4u %4u %4u", 0, 0, 0); 5327 sinfo->cache_order = oo_order(s->oo);
5469 seq_printf(m, " : slabdata %6lu %6lu %6lu", nr_slabs, nr_slabs,
5470 0UL);
5471 seq_putc(m, '\n');
5472 return 0;
5473} 5328}
5474 5329
5475static const struct seq_operations slabinfo_op = { 5330void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
5476 .start = s_start,
5477 .next = s_next,
5478 .stop = s_stop,
5479 .show = s_show,
5480};
5481
5482static int slabinfo_open(struct inode *inode, struct file *file)
5483{ 5331{
5484 return seq_open(file, &slabinfo_op);
5485} 5332}
5486 5333
5487static const struct file_operations proc_slabinfo_operations = { 5334ssize_t slabinfo_write(struct file *file, const char __user *buffer,
5488 .open = slabinfo_open, 5335 size_t count, loff_t *ppos)
5489 .read = seq_read,
5490 .llseek = seq_lseek,
5491 .release = seq_release,
5492};
5493
5494static int __init slab_proc_init(void)
5495{ 5336{
5496 proc_create("slabinfo", S_IRUSR, NULL, &proc_slabinfo_operations); 5337 return -EIO;
5497 return 0;
5498} 5338}
5499module_init(slab_proc_init);
5500#endif /* CONFIG_SLABINFO */ 5339#endif /* CONFIG_SLABINFO */
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-27 21:52:15 -0400